diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py index bb00ce822382..a48d0fa5f2c5 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py @@ -15,6 +15,8 @@ """Wrapper for streaming results.""" __CROSS_SYNC_OUTPUT__ = "google.cloud.spanner_v1.streamed" +from typing import NoReturn + from google.protobuf.struct_pb2 import ListValue, Value from google.cloud import exceptions @@ -62,6 +64,7 @@ def __init__( self._field_decoders = None self._lazy_decode = lazy_decode # Return protobuf values self._done = False + self._error = None @property def fields(self): @@ -272,6 +275,25 @@ def _merge_values(self, values): if remaining_values > full_values_count: self._append_to_current_row(values[batch_end:]) + def _set_terminal_error(self, message: str) -> NoReturn: + """Record a sticky terminal InternalServerError and raise it.""" + self._error = exceptions.InternalServerError(message) + self._done = True + raise self._error + + def _validate_stream_termination(self) -> None: + """Validate that the stream did not end mid-chunk or mid-row. + + :raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the + stream ended with a pending chunked value or an incomplete row. + """ + if self._pending_chunk is not None: + self._set_terminal_error( + "Stream ended while waiting for continuation of chunked value." + ) + if self._current_row: + self._set_terminal_error("Stream ended with an incomplete row.") + @CrossSync.convert async def _consume_next(self): """Consume the next partial result set from the stream. @@ -287,12 +309,20 @@ async def _consume_next(self): if response_pb.HasField("stats"): # last response self._stats = response.stats + if (response_pb.values or response_pb.chunked_value) and not self.fields: + self._set_terminal_error("Missing row_type in PartialResultSet metadata.") + if self._pending_chunk is not None or response_pb.chunked_value: values = list(response_pb.values) - if self._pending_chunk is not None: - values[0] = self._merge_chunk(values[0]) - if response_pb.chunked_value: - self._pending_chunk = values.pop() + if values: + if self._pending_chunk is not None: + values[0] = self._merge_chunk(values[0]) + if response_pb.chunked_value: + self._pending_chunk = values.pop() + elif response_pb.chunked_value: + self._set_terminal_error( + "Received chunked_value=True with empty values." + ) else: values = response_pb.values @@ -303,18 +333,24 @@ async def _consume_next(self): @CrossSync.convert(sync_name="__iter__") async def __aiter__(self): + if self._error is not None: + raise self._error while True: if not self._rows: if self._done: + self._validate_stream_termination() return try: await self._consume_next() except StopAsyncIteration: + self._done = True + self._validate_stream_termination() return iter_rows, self._rows = self._rows, [] for row in iter_rows: yield row if self._done: + self._validate_stream_termination() return def decode_row(self, row: []) -> []: @@ -366,6 +402,8 @@ async def one_or_none(self): :raises: :exc:`ValueError`: If there are multiple results. :raises: :exc:`RuntimeError`: If consumption has already occurred, in whole or in part. + :raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the + stream ended with a pending chunked value or an incomplete row. """ # Sanity check: Has consumption of this query already started? # If it has, then this is an exception. @@ -379,8 +417,12 @@ async def one_or_none(self): try: await self._consume_next() except StopAsyncIteration: + self._done = True break + if self._done: + self._validate_stream_termination() + if len(self._rows) == 1: return self._rows.pop() if not self._rows: diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py index 917f6f5f320f..68e9314cc111 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py @@ -17,6 +17,8 @@ """Wrapper for streaming results.""" +from typing import NoReturn + from google.protobuf.struct_pb2 import ListValue, Value from google.cloud import exceptions @@ -54,6 +56,7 @@ def __init__( self._field_decoders = None self._lazy_decode = lazy_decode self._done = False + self._error = None @property def fields(self): @@ -242,6 +245,24 @@ def _merge_values(self, values): if remaining_values > full_values_count: self._append_to_current_row(values[batch_end:]) + def _set_terminal_error(self, message: str) -> NoReturn: + """Record a sticky terminal InternalServerError and raise it.""" + self._error = exceptions.InternalServerError(message) + self._done = True + raise self._error + + def _validate_stream_termination(self) -> None: + """Validate that the stream did not end mid-chunk or mid-row. + + :raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the + stream ended with a pending chunked value or an incomplete row.""" + if self._pending_chunk is not None: + self._set_terminal_error( + "Stream ended while waiting for continuation of chunked value." + ) + if self._current_row: + self._set_terminal_error("Stream ended with an incomplete row.") + def _consume_next(self): """Consume the next partial result set from the stream. @@ -252,12 +273,19 @@ def _consume_next(self): self._metadata = response_pb.metadata if response_pb.HasField("stats"): self._stats = response.stats + if (response_pb.values or response_pb.chunked_value) and (not self.fields): + self._set_terminal_error("Missing row_type in PartialResultSet metadata.") if self._pending_chunk is not None or response_pb.chunked_value: values = list(response_pb.values) - if self._pending_chunk is not None: - values[0] = self._merge_chunk(values[0]) - if response_pb.chunked_value: - self._pending_chunk = values.pop() + if values: + if self._pending_chunk is not None: + values[0] = self._merge_chunk(values[0]) + if response_pb.chunked_value: + self._pending_chunk = values.pop() + elif response_pb.chunked_value: + self._set_terminal_error( + "Received chunked_value=True with empty values." + ) else: values = response_pb.values self._merge_values(values) @@ -265,18 +293,24 @@ def _consume_next(self): self._done = True def __iter__(self): + if self._error is not None: + raise self._error while True: if not self._rows: if self._done: + self._validate_stream_termination() return try: self._consume_next() except StopIteration: + self._done = True + self._validate_stream_termination() return iter_rows, self._rows = (self._rows, []) for row in iter_rows: yield row if self._done: + self._validate_stream_termination() return def decode_row(self, row: []) -> []: @@ -322,7 +356,9 @@ def one_or_none(self): :raises: :exc:`ValueError`: If there are multiple results. :raises: :exc:`RuntimeError`: If consumption has already occurred, - in whole or in part.""" + in whole or in part. + :raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the + stream ended with a pending chunked value or an incomplete row.""" if self._metadata is not None: raise RuntimeError( "Can not call `.one` or `.one_or_none` after stream consumption has already started." @@ -331,7 +367,10 @@ def one_or_none(self): try: self._consume_next() except StopIteration: + self._done = True break + if self._done: + self._validate_stream_termination() if len(self._rows) == 1: return self._rows.pop() if not self._rows: diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py index 1eae77319b43..f8b776821016 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py @@ -1071,6 +1071,29 @@ def test_dbapi_fetchall_mid_stream_aborted_issue_19079(self): finally: connection.close() + def test_streamed_result_set_invalid_termination_raises_internal_error(self): + for sql, fields, chunks, expected_message in _invalid_termination_scenarios(): + add_execute_streaming_sql_results( + sql, _make_partial_result_sets(fields, chunks) + ) + with self.database.snapshot(multi_use=True) as snapshot: + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + list(snapshot.execute_sql(sql)) + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + snapshot.execute_sql(sql).one_or_none() + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + snapshot.execute_sql(sql).one() + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + snapshot.execute_sql(sql).to_dict_list() + class TestBasicsAsync(AsyncMockServerTestBase): @asynccontextmanager @@ -1199,6 +1222,89 @@ async def test_async_execute_streaming_sql_last_flag_trailing_error_drained(self rows = [row async for row in results] self.assertEqual([[10], [20]], rows) + async def test_async_streamed_result_set_invalid_termination_raises_internal_error( + self, + ): + async with self._async_database() as database: + for sql, fields, chunks, expected_message in _invalid_termination_scenarios( + "async_" + ): + add_execute_streaming_sql_results( + sql, _make_partial_result_sets(fields, chunks) + ) + async with database.snapshot(multi_use=True) as snapshot: + results = await snapshot.execute_sql(sql) + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + _ = [row async for row in results] + one_or_none_results = await snapshot.execute_sql(sql) + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + await one_or_none_results.one_or_none() + one_results = await snapshot.execute_sql(sql) + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + await one_results.one() + dict_results = await snapshot.execute_sql(sql) + with self.assertRaisesRegex( + exceptions.InternalServerError, expected_message + ): + await dict_results.to_dict_list() + + +def _invalid_termination_scenarios(prefix: str = ""): + fields = [("id", TypeCode.INT64), ("payload", TypeCode.STRING)] + return [ + ( + f"select id, payload from {prefix}invalid_chunk_eof", + fields, + [{"values": ["1", "partial "], "chunked_value": True, "last": False}], + "Stream ended while waiting for continuation of chunked value.", + ), + ( + f"select id, payload from {prefix}invalid_chunk_last", + fields, + [{"values": ["1", "partial "], "chunked_value": True, "last": True}], + "Stream ended while waiting for continuation of chunked value.", + ), + ( + f"select id, payload from {prefix}invalid_chunk_empty_last", + fields, + [ + {"values": ["1", "partial "], "chunked_value": True, "last": False}, + {"values": [], "chunked_value": False, "last": True}, + ], + "Stream ended while waiting for continuation of chunked value.", + ), + ( + f"select id, payload from {prefix}invalid_empty_values_chunked", + fields, + [{"values": [], "chunked_value": True, "last": False}], + "Received chunked_value=True with empty values.", + ), + ( + f"select id, payload from {prefix}invalid_row_eof", + fields, + [{"values": ["1", "ok", "2"], "chunked_value": False, "last": False}], + "Stream ended with an incomplete row.", + ), + ( + f"select id, payload from {prefix}invalid_row_last", + fields, + [{"values": ["1", "ok", "2"], "chunked_value": False, "last": True}], + "Stream ended with an incomplete row.", + ), + ( + f"select id, payload from {prefix}invalid_missing_row_type", + [], + [{"values": ["1"], "chunked_value": False, "last": True}], + "Missing row_type in PartialResultSet metadata.", + ), + ] + def _execute_query(transaction: Transaction, sql: str): rows = transaction.execute_sql(sql, last_statement=True) diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py b/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py index 38c7b5125a75..6915640612b4 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py @@ -798,7 +798,7 @@ async def test_merge_values_empty_and_partial(self): VALUES = [self._make_value(bare) for bare in BARE] streamed._current_row = [] streamed._merge_values(VALUES) - self.assertEqual([i async for i in streamed], []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BARE) @CrossSync.pytest @@ -844,7 +844,7 @@ async def test_merge_values_empty_and_filled_plus(self): VALUES = [self._make_value(bare) for bare in BARE] streamed._current_row = [] streamed._merge_values(VALUES) - self.assertEqual([i async for i in streamed], [BARE[0:3], BARE[3:6]]) + self.assertEqual(streamed._rows, [BARE[0:3], BARE[3:6]]) self.assertEqual(streamed._current_row, BARE[6:]) @CrossSync.pytest @@ -862,7 +862,7 @@ async def test_merge_values_partial_and_empty(self): BEFORE = ["Phred Phlyntstone"] streamed._current_row[:] = BEFORE streamed._merge_values([]) - self.assertEqual([i async for i in streamed], []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BEFORE) @CrossSync.pytest @@ -882,7 +882,7 @@ async def test_merge_values_partial_and_partial(self): MERGED = [42] TO_MERGE = [self._make_value(item) for item in MERGED] streamed._merge_values(TO_MERGE) - self.assertEqual([i async for i in streamed], []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BEFORE + MERGED) @CrossSync.pytest @@ -923,7 +923,7 @@ async def test_merge_values_partial_and_filled_plus(self): TO_MERGE = [self._make_value(item) for item in MERGED] VALUES = BEFORE + MERGED streamed._merge_values(TO_MERGE) - self.assertEqual([i async for i in streamed], [VALUES[0:3], VALUES[3:6]]) + self.assertEqual(streamed._rows, [VALUES[0:3], VALUES[3:6]]) self.assertEqual(streamed._current_row, VALUES[6:]) @CrossSync.pytest @@ -999,7 +999,7 @@ async def test_consume_next_first_set_partial(self): source = CrossSync.Mock(_transaction_id=None, spec=["_transaction_id"]) streamed = self._make_one(iterator, source=source) await streamed._consume_next() - self.assertEqual([i async for i in streamed], []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BARE) self.assertEqual(streamed.metadata, metadata) @@ -1021,7 +1021,7 @@ async def test_consume_next_first_set_partial_existing_txn_id(self): source = CrossSync.Mock(_transaction_id=TXN_ID, spec=["_transaction_id"]) streamed = self._make_one(iterator, source=source) await streamed._consume_next() - self.assertEqual([i async for i in streamed], []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BARE) self.assertEqual(streamed.metadata, metadata) self.assertEqual(source._transaction_id, TXN_ID) @@ -1041,7 +1041,7 @@ async def test_consume_next_w_partial_result(self): streamed = self._make_one(iterator) streamed._metadata = self._make_result_set_metadata(FIELDS) await streamed._consume_next() - self.assertEqual([i async for i in streamed], []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, []) self.assertEqual(streamed._pending_chunk, VALUES[0]) @@ -1071,7 +1071,7 @@ async def test_consume_next_w_pending_chunk(self): streamed._pending_chunk = self._make_value("Phred ") await streamed._consume_next() self.assertEqual( - [i async for i in streamed], + streamed._rows, [["Phred Phlyntstone", BARE[1], BARE[2]], [BARE[3], BARE[4], BARE[5]]], ) self.assertEqual(streamed._current_row, [BARE[6]]) @@ -1112,6 +1112,7 @@ async def test___iter___empty(self): async def test___iter___one_result_set_partial(self): from google.protobuf.struct_pb2 import Value + from google.cloud import exceptions from google.cloud.spanner_v1 import TypeCode FIELDS = [ @@ -1127,9 +1128,11 @@ async def test___iter___one_result_set_partial(self): result_set = self._make_partial_result_set(VALUES, metadata=metadata) iterator = _MockCancellableIterator(result_set) streamed = self._make_one(iterator) - found = [i async for i in streamed] - self.assertEqual(found, []) - self.assertEqual([i async for i in streamed], []) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended with an incomplete row.", + ): + _ = [i async for i in streamed] self.assertEqual(streamed._current_row, BARE) self.assertEqual(streamed.metadata, metadata) @@ -2921,6 +2924,417 @@ async def test_merge_struct_null_last(self): self.assertTrue(res.list_value.values[0].HasField("null_value")) self.assertEqual(res.list_value.values[1].string_value, "v1") + @CrossSync.pytest + async def test___iter___chunked_value_eof_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + _ = [row async for row in streamed] + + @CrossSync.pytest + async def test___iter___chunked_value_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [ + self._make_value(1), + self._make_value("Alice"), + self._make_value(2), + self._make_value("Bob "), + ], + metadata=metadata, + chunked_value=True, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + first_rows = [] + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + async for row in streamed: + first_rows.append(row) + self.assertEqual(first_rows, [[1, "Alice"]]) + # Re-iteration after failure must yield 0 rows and still raise. + second_rows = [] + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + async for row in streamed: + second_rows.append(row) + self.assertEqual(second_rows, []) + + @CrossSync.pytest + async def test___iter___chunked_value_followed_by_empty_last_chunk_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + chunk2 = self._make_partial_result_set([], chunked_value=False, last=True) + iterator = _MockCancellableIterator(chunk1, chunk2) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + _ = [row async for row in streamed] + + @CrossSync.pytest + async def test___iter___empty_values_with_chunked_value_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value("Alice")], + metadata=metadata, + chunked_value=False, + last=False, + ) + chunk2 = self._make_partial_result_set([], chunked_value=True, last=False) + iterator = _MockCancellableIterator(chunk1, chunk2) + streamed = self._make_one(iterator) + first_rows = [] + with pytest.raises( + exceptions.InternalServerError, + match="Received chunked_value=True with empty values.", + ): + async for row in streamed: + first_rows.append(row) + self.assertEqual(first_rows, [["Alice"]]) + # Sticky terminal error must still raise on re-iteration rather than returning []. + second_rows = [] + with pytest.raises( + exceptions.InternalServerError, + match="Received chunked_value=True with empty values.", + ): + async for row in streamed: + second_rows.append(row) + self.assertEqual(second_rows, []) + + @CrossSync.pytest + async def test___iter___incomplete_row_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value(1), self._make_value("Alice"), self._make_value(2)], + metadata=metadata, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + first_rows = [] + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended with an incomplete row.", + ): + async for row in streamed: + first_rows.append(row) + self.assertEqual(first_rows, [[1, "Alice"]]) + second_rows = [] + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended with an incomplete row.", + ): + async for row in streamed: + second_rows.append(row) + self.assertEqual(second_rows, []) + + @CrossSync.pytest + async def test___iter___missing_row_type_with_values_error(self): + from google.cloud import exceptions + + metadata = self._make_result_set_metadata([]) + partial_result_set = self._make_partial_result_set( + [self._make_value(1)], metadata=metadata, last=True + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Missing row_type in PartialResultSet metadata.", + ): + _ = [row async for row in streamed] + with pytest.raises( + exceptions.InternalServerError, + match="Missing row_type in PartialResultSet metadata.", + ): + _ = [row async for row in streamed] + + @CrossSync.pytest + async def test___iter___missing_row_type_with_chunked_value_error(self): + from google.cloud import exceptions + + partial_result_set = self._make_partial_result_set( + [], chunked_value=True, last=False + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Missing row_type in PartialResultSet metadata.", + ): + _ = [row async for row in streamed] + + @CrossSync.pytest + async def test___iter___zero_columns_empty_values(self): + metadata = self._make_result_set_metadata([]) + stats = self._make_result_set_stats(row_count_exact=3) + partial_result_set = self._make_partial_result_set( + [], metadata=metadata, stats=stats, last=True + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + found = [row async for row in streamed] + self.assertEqual(found, []) + self.assertEqual(streamed.stats, stats) + + @CrossSync.pytest + async def test___iter___chunked_value_with_intermediate_empty_chunk(self): + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value("hello ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + chunk2 = self._make_partial_result_set([], chunked_value=False, last=False) + chunk3 = self._make_partial_result_set( + [self._make_value("world")], chunked_value=False, last=True + ) + iterator = _MockCancellableIterator(chunk1, chunk2, chunk3) + streamed = self._make_one(iterator) + found = [row async for row in streamed] + self.assertEqual(found, [["hello world"]]) + + @CrossSync.pytest + async def test_one_or_none_chunked_value_eof_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + await streamed.one_or_none() + # Sticky terminal error must propagate across subsequent consumers. + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + _ = [row async for row in streamed] + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + await streamed.to_dict_list() + + @CrossSync.pytest + async def test_one_or_none_chunked_value_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("complete"), self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + await streamed.one_or_none() + + @CrossSync.pytest + async def test_one_or_none_incomplete_row_eof_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value(1), self._make_value("Alice")], + metadata=metadata, + last=False, + ) + chunk2 = self._make_partial_result_set([self._make_value(2)], last=False) + iterator = _MockCancellableIterator(chunk1, chunk2) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended with an incomplete row.", + ): + await streamed.one_or_none() + + @CrossSync.pytest + async def test_one_or_none_incomplete_row_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value(1), self._make_value("Alice"), self._make_value(2)], + metadata=metadata, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended with an incomplete row.", + ): + await streamed.one_or_none() + + @CrossSync.pytest + async def test_one_incomplete_row_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value(1)], metadata=metadata, last=False + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended with an incomplete row.", + ): + await streamed.one() + + @CrossSync.pytest + async def test_to_dict_list_chunked_value_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("Alice"), self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + await streamed.to_dict_list() + + @CrossSync.pytest + async def test_one_or_none_after_terminal_error_raises_runtime_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + + chunked_stream = self._make_one( + _MockCancellableIterator( + self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + ) + ) + with pytest.raises( + exceptions.InternalServerError, + match="Stream ended while waiting for continuation of chunked value.", + ): + await chunked_stream.one_or_none() + with pytest.raises( + RuntimeError, + match="Can not call `.one` or `.one_or_none` after stream consumption has already started.", + ): + await chunked_stream.one_or_none() + + empty_chunked_stream = self._make_one( + _MockCancellableIterator( + self._make_partial_result_set( + [], metadata=metadata, chunked_value=True, last=False + ) + ) + ) + with pytest.raises( + exceptions.InternalServerError, + match="Received chunked_value=True with empty values.", + ): + _ = [row async for row in empty_chunked_stream] + with pytest.raises( + RuntimeError, + match="Can not call `.one` or `.one_or_none` after stream consumption has already started.", + ): + await empty_chunked_stream.one_or_none() + class _MockCancellableIterator(object): cancel_calls = 0 diff --git a/packages/google-cloud-spanner/tests/unit/test_streamed.py b/packages/google-cloud-spanner/tests/unit/test_streamed.py index 072b956dddf2..10dbb2afbec0 100644 --- a/packages/google-cloud-spanner/tests/unit/test_streamed.py +++ b/packages/google-cloud-spanner/tests/unit/test_streamed.py @@ -733,7 +733,7 @@ def test_merge_values_empty_and_partial(self): VALUES = [self._make_value(bare) for bare in BARE] streamed._current_row = [] streamed._merge_values(VALUES) - self.assertEqual(list(streamed), []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BARE) def test_merge_values_empty_and_filled(self): @@ -777,7 +777,7 @@ def test_merge_values_empty_and_filled_plus(self): VALUES = [self._make_value(bare) for bare in BARE] streamed._current_row = [] streamed._merge_values(VALUES) - self.assertEqual(list(streamed), [BARE[0:3], BARE[3:6]]) + self.assertEqual(streamed._rows, [BARE[0:3], BARE[3:6]]) self.assertEqual(streamed._current_row, BARE[6:]) def test_merge_values_partial_and_empty(self): @@ -794,7 +794,7 @@ def test_merge_values_partial_and_empty(self): BEFORE = ["Phred Phlyntstone"] streamed._current_row[:] = BEFORE streamed._merge_values([]) - self.assertEqual(list(streamed), []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BEFORE) def test_merge_values_partial_and_partial(self): @@ -813,7 +813,7 @@ def test_merge_values_partial_and_partial(self): MERGED = [42] TO_MERGE = [self._make_value(item) for item in MERGED] streamed._merge_values(TO_MERGE) - self.assertEqual(list(streamed), []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BEFORE + MERGED) def test_merge_values_partial_and_filled(self): @@ -852,7 +852,7 @@ def test_merge_values_partial_and_filled_plus(self): TO_MERGE = [self._make_value(item) for item in MERGED] VALUES = BEFORE + MERGED streamed._merge_values(TO_MERGE) - self.assertEqual(list(streamed), [VALUES[0:3], VALUES[3:6]]) + self.assertEqual(streamed._rows, [VALUES[0:3], VALUES[3:6]]) self.assertEqual(streamed._current_row, VALUES[6:]) def test_one_or_none_no_value(self): @@ -920,7 +920,7 @@ def test_consume_next_first_set_partial(self): source = mock.Mock(_transaction_id=None, spec=["_transaction_id"]) streamed = self._make_one(iterator, source=source) streamed._consume_next() - self.assertEqual(list(streamed), []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BARE) self.assertEqual(streamed.metadata, metadata) @@ -941,7 +941,7 @@ def test_consume_next_first_set_partial_existing_txn_id(self): source = mock.Mock(_transaction_id=TXN_ID, spec=["_transaction_id"]) streamed = self._make_one(iterator, source=source) streamed._consume_next() - self.assertEqual(list(streamed), []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, BARE) self.assertEqual(streamed.metadata, metadata) self.assertEqual(source._transaction_id, TXN_ID) @@ -960,7 +960,7 @@ def test_consume_next_w_partial_result(self): streamed = self._make_one(iterator) streamed._metadata = self._make_result_set_metadata(FIELDS) streamed._consume_next() - self.assertEqual(list(streamed), []) + self.assertEqual(streamed._rows, []) self.assertEqual(streamed._current_row, []) self.assertEqual(streamed._pending_chunk, VALUES[0]) @@ -989,7 +989,7 @@ def test_consume_next_w_pending_chunk(self): streamed._pending_chunk = self._make_value("Phred ") streamed._consume_next() self.assertEqual( - list(streamed), + streamed._rows, [["Phred Phlyntstone", BARE[1], BARE[2]], [BARE[3], BARE[4], BARE[5]]], ) self.assertEqual(streamed._current_row, [BARE[6]]) @@ -1027,6 +1027,7 @@ def test___iter___empty(self): def test___iter___one_result_set_partial(self): from google.protobuf.struct_pb2 import Value + from google.cloud import exceptions from google.cloud.spanner_v1 import TypeCode FIELDS = [ @@ -1042,9 +1043,11 @@ def test___iter___one_result_set_partial(self): result_set = self._make_partial_result_set(VALUES, metadata=metadata) iterator = _MockCancellableIterator(result_set) streamed = self._make_one(iterator) - found = list(streamed) - self.assertEqual(found, []) - self.assertEqual(list(streamed), []) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended with an incomplete row.", + ): + list(streamed) self.assertEqual(streamed._current_row, BARE) self.assertEqual(streamed.metadata, metadata) @@ -2753,6 +2756,401 @@ def test_merge_struct_null_last(self): self.assertTrue(res.list_value.values[0].HasField("null_value")) self.assertEqual(res.list_value.values[1].string_value, "v1") + def test___iter___chunked_value_eof_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + list(streamed) + + def test___iter___chunked_value_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [ + self._make_value(1), + self._make_value("Alice"), + self._make_value(2), + self._make_value("Bob "), + ], + metadata=metadata, + chunked_value=True, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + first_rows = [] + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + for row in streamed: + first_rows.append(row) + self.assertEqual(first_rows, [[1, "Alice"]]) + # Re-iteration after failure must yield 0 rows and still raise. + second_rows = [] + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + for row in streamed: + second_rows.append(row) + self.assertEqual(second_rows, []) + + def test___iter___chunked_value_followed_by_empty_last_chunk_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + chunk2 = self._make_partial_result_set([], chunked_value=False, last=True) + iterator = _MockCancellableIterator(chunk1, chunk2) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + list(streamed) + + def test___iter___empty_values_with_chunked_value_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value("Alice")], + metadata=metadata, + chunked_value=False, + last=False, + ) + chunk2 = self._make_partial_result_set([], chunked_value=True, last=False) + iterator = _MockCancellableIterator(chunk1, chunk2) + streamed = self._make_one(iterator) + first_rows = [] + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Received chunked_value=True with empty values.", + ): + for row in streamed: + first_rows.append(row) + self.assertEqual(first_rows, [["Alice"]]) + # Sticky terminal error must still raise on re-iteration rather than returning []. + second_rows = [] + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Received chunked_value=True with empty values.", + ): + for row in streamed: + second_rows.append(row) + self.assertEqual(second_rows, []) + + def test___iter___incomplete_row_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value(1), self._make_value("Alice"), self._make_value(2)], + metadata=metadata, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + first_rows = [] + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended with an incomplete row.", + ): + for row in streamed: + first_rows.append(row) + self.assertEqual(first_rows, [[1, "Alice"]]) + second_rows = [] + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended with an incomplete row.", + ): + for row in streamed: + second_rows.append(row) + self.assertEqual(second_rows, []) + + def test___iter___missing_row_type_with_values_error(self): + from google.cloud import exceptions + + metadata = self._make_result_set_metadata([]) + partial_result_set = self._make_partial_result_set( + [self._make_value(1)], metadata=metadata, last=True + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Missing row_type in PartialResultSet metadata.", + ): + list(streamed) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Missing row_type in PartialResultSet metadata.", + ): + list(streamed) + + def test___iter___missing_row_type_with_chunked_value_error(self): + from google.cloud import exceptions + + partial_result_set = self._make_partial_result_set( + [], chunked_value=True, last=False + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Missing row_type in PartialResultSet metadata.", + ): + list(streamed) + + def test___iter___zero_columns_empty_values(self): + metadata = self._make_result_set_metadata([]) + stats = self._make_result_set_stats(row_count_exact=3) + partial_result_set = self._make_partial_result_set( + [], metadata=metadata, stats=stats, last=True + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + found = list(streamed) + self.assertEqual(found, []) + self.assertEqual(streamed.stats, stats) + + def test___iter___chunked_value_with_intermediate_empty_chunk(self): + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value("hello ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + chunk2 = self._make_partial_result_set([], chunked_value=False, last=False) + chunk3 = self._make_partial_result_set( + [self._make_value("world")], chunked_value=False, last=True + ) + iterator = _MockCancellableIterator(chunk1, chunk2, chunk3) + streamed = self._make_one(iterator) + found = list(streamed) + self.assertEqual(found, [["hello world"]]) + + def test_one_or_none_chunked_value_eof_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + streamed.one_or_none() + # Sticky terminal error must propagate across subsequent consumers. + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + list(streamed) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + streamed.to_dict_list() + + def test_one_or_none_chunked_value_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("complete"), self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + streamed.one_or_none() + + def test_one_or_none_incomplete_row_eof_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + chunk1 = self._make_partial_result_set( + [self._make_value(1), self._make_value("Alice")], + metadata=metadata, + last=False, + ) + chunk2 = self._make_partial_result_set([self._make_value(2)], last=False) + iterator = _MockCancellableIterator(chunk1, chunk2) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended with an incomplete row.", + ): + streamed.one_or_none() + + def test_one_or_none_incomplete_row_last_flag_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value(1), self._make_value("Alice"), self._make_value(2)], + metadata=metadata, + last=True, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended with an incomplete row.", + ): + streamed.one_or_none() + + def test_one_incomplete_row_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [ + self._make_scalar_field("id", TypeCode.INT64), + self._make_scalar_field("name", TypeCode.STRING), + ] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value(1)], metadata=metadata, last=False + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended with an incomplete row.", + ): + streamed.one() + + def test_to_dict_list_chunked_value_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + partial_result_set = self._make_partial_result_set( + [self._make_value("Alice"), self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + iterator = _MockCancellableIterator(partial_result_set) + streamed = self._make_one(iterator) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + streamed.to_dict_list() + + def test_one_or_none_after_terminal_error_raises_runtime_error(self): + from google.cloud import exceptions + from google.cloud.spanner_v1 import TypeCode + + fields = [self._make_scalar_field("name", TypeCode.STRING)] + metadata = self._make_result_set_metadata(fields) + + chunked_stream = self._make_one( + _MockCancellableIterator( + self._make_partial_result_set( + [self._make_value("partial ")], + metadata=metadata, + chunked_value=True, + last=False, + ) + ) + ) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Stream ended while waiting for continuation of chunked value.", + ): + chunked_stream.one_or_none() + with self.assertRaisesRegex( + RuntimeError, + "Can not call `.one` or `.one_or_none` after stream consumption has already started.", + ): + chunked_stream.one_or_none() + + empty_chunked_stream = self._make_one( + _MockCancellableIterator( + self._make_partial_result_set( + [], metadata=metadata, chunked_value=True, last=False + ) + ) + ) + with self.assertRaisesRegex( + exceptions.InternalServerError, + "Received chunked_value=True with empty values.", + ): + list(empty_chunked_stream) + with self.assertRaisesRegex( + RuntimeError, + "Can not call `.one` or `.one_or_none` after stream consumption has already started.", + ): + empty_chunked_stream.one_or_none() + class _MockCancellableIterator(object): cancel_calls = 0