diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py index bb00ce822382..48dd9c109f84 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/streamed.py @@ -62,6 +62,7 @@ def __init__( self._field_decoders = None self._lazy_decode = lazy_decode # Return protobuf values self._done = False + self._closed = False @property def fields(self): @@ -301,21 +302,49 @@ async def _consume_next(self): if response_pb.last: self._done = True + @CrossSync.convert + async def _close(self) -> None: + """Close the underlying response iterator idempotently.""" + if self._closed: + return + self._closed = True + self._rows = [] + iterator = self._response_iterator + if iterator is None: + return + if CrossSync.is_async: + close_methods = ("aclose", "close", "cancel") + else: + close_methods = ("close", "cancel") + try: + for method_name in close_methods: + close_method = getattr(iterator, method_name, None) + if close_method is not None: + await CrossSync.run_if_async(close_method) + return + except Exception: + pass + @CrossSync.convert(sync_name="__iter__") async def __aiter__(self): - while True: - if not self._rows: - if self._done: + try: + while True: + if self._closed: return - try: - await self._consume_next() - except StopAsyncIteration: + if not self._rows: + try: + await self._consume_next() + except StopAsyncIteration: + return + iter_rows, self._rows = self._rows, [] + for row in iter_rows: + if self._closed: + return + yield row + if self._done: return - iter_rows, self._rows = self._rows, [] - for row in iter_rows: - yield row - if self._done: - return + finally: + await self._close() def decode_row(self, row: []) -> []: """Decodes a row from protobuf values to Python objects. This function @@ -375,17 +404,23 @@ async def one_or_none(self): "stream consumption has already started." ) - while not self._done and len(self._rows) < 2: - try: - await self._consume_next() - except StopAsyncIteration: - break - - if len(self._rows) == 1: - return self._rows.pop() - if not self._rows: + if self._closed: return None - raise ValueError("Expected one result; got more.") + + try: + while not self._done and len(self._rows) < 2: + try: + await self._consume_next() + except StopAsyncIteration: + break + + if len(self._rows) == 1: + return self._rows.pop() + if not self._rows: + return None + raise ValueError("Expected one result; got more.") + finally: + await self._close() @CrossSync.convert async def to_dict_list(self): @@ -399,12 +434,15 @@ async def to_dict_list(self): """ rows = [] column_names = None - async for row in self: - if column_names is None: - column_names = [ - column.name for column in self._metadata.row_type.fields - ] - rows.append(dict(zip(column_names, row))) + try: + async for row in self: + if column_names is None: + column_names = [ + column.name for column in self._metadata.row_type.fields + ] + rows.append(dict(zip(column_names, row))) + finally: + await self._close() return rows diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py index 917f6f5f320f..c962b3074862 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py @@ -20,6 +20,7 @@ from google.protobuf.struct_pb2 import ListValue, Value from google.cloud import exceptions +from google.cloud.aio._cross_sync import CrossSync from google.cloud.spanner_v1._helpers import _get_type_decoder, _parse_nullable from google.cloud.spanner_v1.types.result_set import PartialResultSet, ResultSetMetadata from google.cloud.spanner_v1.types.type import TypeCode @@ -54,6 +55,7 @@ def __init__( self._field_decoders = None self._lazy_decode = lazy_decode self._done = False + self._closed = False @property def fields(self): @@ -264,20 +266,44 @@ def _consume_next(self): if response_pb.last: self._done = True + def _close(self) -> None: + """Close the underlying response iterator idempotently.""" + if self._closed: + return + self._closed = True + self._rows = [] + iterator = self._response_iterator + if iterator is None: + return + close_methods = ("close", "cancel") + try: + for method_name in close_methods: + close_method = getattr(iterator, method_name, None) + if close_method is not None: + CrossSync._Sync_Impl.run_if_async(close_method) + return + except Exception: + pass + def __iter__(self): - while True: - if not self._rows: - if self._done: + try: + while True: + if self._closed: return - try: - self._consume_next() - except StopIteration: + if not self._rows: + try: + self._consume_next() + except StopIteration: + return + iter_rows, self._rows = (self._rows, []) + for row in iter_rows: + if self._closed: + return + yield row + if self._done: return - iter_rows, self._rows = (self._rows, []) - for row in iter_rows: - yield row - if self._done: - return + finally: + self._close() def decode_row(self, row: []) -> []: """Decodes a row from protobuf values to Python objects. This function @@ -327,16 +353,21 @@ def one_or_none(self): raise RuntimeError( "Can not call `.one` or `.one_or_none` after stream consumption has already started." ) - while not self._done and len(self._rows) < 2: - try: - self._consume_next() - except StopIteration: - break - if len(self._rows) == 1: - return self._rows.pop() - if not self._rows: + if self._closed: return None - raise ValueError("Expected one result; got more.") + try: + while not self._done and len(self._rows) < 2: + try: + self._consume_next() + except StopIteration: + break + if len(self._rows) == 1: + return self._rows.pop() + if not self._rows: + return None + raise ValueError("Expected one result; got more.") + finally: + self._close() def to_dict_list(self): """Return the result of a query as a list of dictionaries. @@ -348,12 +379,15 @@ def to_dict_list(self): :returns: result rows as a list of dictionaries""" rows = [] column_names = None - for row in self: - if column_names is None: - column_names = [ - column.name for column in self._metadata.row_type.fields - ] - rows.append(dict(zip(column_names, row))) + try: + for row in self: + if column_names is None: + column_names = [ + column.name for column in self._metadata.row_type.fields + ] + rows.append(dict(zip(column_names, row))) + finally: + self._close() return rows diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py index f9659e71a4af..acc93246e62b 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py @@ -424,3 +424,63 @@ def test_operation_latency_streaming_read(self): self.assertIn("operation_latencies", metrics) op_lat_point = metrics["operation_latencies"][0] self.assertEqual(op_lat_point.count, 1) + + def test_operation_latency_recorded_on_streamed_result_set_close(self): + fields = [("col1", TypeCode.INT64)] + partials = _make_partial_result_sets( + fields, + [ + {"values": ["1"], "resume_token": b"token-1", "last": False}, + {"values": ["2"], "last": True}, + ], + ) + add_execute_streaming_sql_results("select 1", partials) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with ( + mock.patch( + "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client._get_spanner_emulator_host", + return_value=None, + ), + ): + _, database = self._setup_client_and_database() + with database.snapshot() as snapshot: + results = snapshot.execute_sql("select 1") + rows = iter(results) + self.assertEqual(next(rows), [1]) + + # Stream is still mid-flight; operation metrics must not be recorded yet. + metrics_before = self._extract_metrics( + reader, method="Spanner.ExecuteStreamingSql" + ) + self.assertNotIn("operation_latencies", metrics_before) + self.assertNotIn("operation_count", metrics_before) + + # Closing the StreamedResultSet records completion metrics immediately + # even while `results` and `rows` remain alive in local scope. + results._close() + metrics_after = self._extract_metrics( + reader, method="Spanner.ExecuteStreamingSql" + ) + self.assertIn("operation_latencies", metrics_after) + self.assertEqual(metrics_after["operation_latencies"][0].count, 1) + self.assertIn("operation_count", metrics_after) + self.assertEqual(metrics_after["operation_count"][0].value, 1) + + # Idempotent: closing again does not double-record. + results._close() + rows.close() + metrics_final = self._extract_metrics( + reader, method="Spanner.ExecuteStreamingSql" + ) + self.assertEqual(metrics_final["operation_latencies"][0].count, 1) + self.assertEqual(metrics_final["operation_count"][0].value, 1) diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py index dd8ee834c37b..1fec2d66a07a 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py @@ -2238,6 +2238,71 @@ async def test_partition_query_w_leader_enabled(self): f"Expected {leader_header} in {metadata}", ) + async def test_streamed_result_set_records_metrics_completion_on_close(self): + from google.protobuf.struct_pb2 import Value + + from google.cloud.spanner_v1.metrics.metrics_tracer_factory import ( + MetricsTracerFactory, + ) + + class CancellableStream: + def __init__(self, items): + self._items = iter(items) + self.cancel = mock.Mock() + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._items) + except StopIteration: + raise StopAsyncIteration + + fields = [StructType.Field(name="col", type_=Type(code=TypeCode.INT64))] + metadata_pb = ResultSetMetadata(row_type=StructType(fields=fields)) + first_partial = PartialResultSet( + metadata=metadata_pb, resume_token=b"tok-1", last=False + ) + first_partial.values.append(Value(string_value="1")) + second_partial = PartialResultSet(last=True) + second_partial.values.append(Value(string_value="2")) + + raw_stream = CancellableStream([first_partial, second_partial]) + database = _Database() + database.spanner_api.execute_streaming_sql = mock.Mock(return_value=raw_stream) + session = _Session(database) + snapshot = self._make_snapshot(session, multi_use=False) + + mock_tracer = mock.Mock() + with mock.patch.object( + MetricsTracerFactory, "create_metrics_tracer", return_value=mock_tracer + ): + result_set = await snapshot.execute_sql(SQL_QUERY) + async_iter = result_set.__aiter__() + first_row = await async_iter.__anext__() + + self.assertEqual(first_row, [1]) + mock_tracer.record_operation_start.assert_called_once() + mock_tracer.record_operation_completion.assert_not_called() + raw_stream.cancel.assert_not_called() + + await result_set._close() + mock_tracer.record_operation_completion.assert_called_once() + raw_stream.cancel.assert_called_once() + await async_iter.aclose() + mock_tracer.record_operation_completion.assert_called_once() + + mock_tracer.reset_mock() + database.spanner_api.execute_streaming_sql.reset_mock() + unstarted_snapshot = self._make_snapshot(session, multi_use=False) + unstarted = await unstarted_snapshot.execute_sql(SQL_QUERY) + await unstarted._close() + self.assertEqual([row async for row in unstarted], []) + database.spanner_api.execute_streaming_sql.assert_not_called() + mock_tracer.record_operation_start.assert_not_called() + mock_tracer.record_operation_completion.assert_not_called() + class _Database(object): def __init__(self): diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py b/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py index 38c7b5125a75..b83eca419842 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_streamed.py @@ -32,6 +32,7 @@ def wrapper(*args, **kwargs): try: return loop.run_until_complete(testMethod(*args, **kwargs)) finally: + loop.run_until_complete(loop.shutdown_asyncgens()) loop.close() setattr(self, self._testMethodName, wrapper) @@ -2921,6 +2922,312 @@ async def test_merge_struct_null_last(self): self.assertTrue(res.list_value.values[0].HasField("null_value")) self.assertEqual(res.list_value.values[1].string_value, "v1") + def _make_int_partial(self, values, *, with_metadata=False, last=False): + from google.cloud.spanner_v1 import TypeCode + + metadata = None + if with_metadata: + fields = [self._make_scalar_field("id", TypeCode.INT64)] + metadata = self._make_result_set_metadata(fields) + return self._make_partial_result_set( + [self._make_value(value) for value in values], + metadata=metadata, + last=last, + ) + + @CrossSync.pytest + async def test_close_is_private_and_no_public_close(self): + iterator = _MockCancellableIterator() + result_set = self._make_one(iterator) + self.assertFalse(hasattr(result_set, "close")) + self.assertTrue(hasattr(result_set, "_close")) + + @CrossSync.pytest + async def test_close_with_none_iterator(self): + result_set = self._make_one(None) + await result_set._close() + self.assertTrue(result_set._closed) + self.assertEqual([row async for row in result_set], []) + + @CrossSync.pytest + async def test_close_without_close_method(self): + class PlainIterator: + pass + + result_set = self._make_one(PlainIterator()) + await result_set._close() + self.assertTrue(result_set._closed) + self.assertEqual([row async for row in result_set], []) + + @CrossSync.pytest + async def test_close_idempotent_with_generator(self): + generator_closed = [] + + async def generator(): + try: + yield self._make_partial_result_set([], last=False) + finally: + generator_closed.append(True) + + response_iterator = generator() + await response_iterator.__anext__() + result_set = self._make_one(response_iterator) + + await result_set._close() + await result_set._close() + + self.assertEqual(generator_closed, [True]) + + @CrossSync.pytest + async def test_close_supports_sync_close_method(self): + class SyncCloseIterator: + def __init__(self): + self.close_calls = 0 + + def close(self): + self.close_calls += 1 + + iterator = SyncCloseIterator() + result_set = self._make_one(iterator) + + await result_set._close() + await result_set._close() + + self.assertEqual(iterator.close_calls, 1) + + @CrossSync.pytest + async def test_close_falls_back_to_cancel(self): + class CancelOnlyIterator: + def __init__(self): + self.cancel_calls = 0 + + def cancel(self): + self.cancel_calls += 1 + + iterator = CancelOnlyIterator() + result_set = self._make_one(iterator) + + await result_set._close() + await result_set._close() + + self.assertEqual(iterator.cancel_calls, 1) + + @CrossSync.pytest + async def test_close_suppresses_iterator_close_error(self): + class FailingCloseIterator: + async def aclose(self): + raise RuntimeError("close failure") + + result_set = self._make_one(FailingCloseIterator()) + await result_set._close() + self.assertTrue(result_set._closed) + self.assertEqual([row async for row in result_set], []) + + class FailingCancelIterator: + def cancel(self): + raise RuntimeError("cancel failure") + + cancel_result_set = self._make_one(FailingCancelIterator()) + await cancel_result_set._close() + self.assertTrue(cancel_result_set._closed) + self.assertEqual([row async for row in cancel_result_set], []) + + @CrossSync.pytest + async def test_close_clears_buffered_rows_and_stops_iteration(self): + partial = self._make_int_partial([1, 2, 3], with_metadata=True, last=False) + result_set = self._make_one(_MockCancellableIterator(partial)) + + collected = [] + async for row in result_set: + collected.append(row) + await result_set._close() + + self.assertEqual(collected, [[1]]) + self.assertEqual(result_set._rows, []) + self.assertEqual([row async for row in result_set], []) + self.assertIsNotNone(result_set.metadata) + self.assertEqual(len(result_set.fields), 1) + self.assertEqual(result_set.decode_row(partial._pb.values[:1]), [1]) + + closed_before_consume = self._make_one(_MockCancellableIterator(partial)) + await closed_before_consume._close() + self.assertIsNone(await closed_before_consume.one_or_none()) + + @CrossSync.pytest + async def test_close_method_precedence(self): + calls = [] + + class DualCloseIterator: + async def aclose(self): + calls.append("aclose") + + def close(self): + calls.append("close") + + def cancel(self): + calls.append("cancel") + + result_set = self._make_one(DualCloseIterator()) + await result_set._close() + if CrossSync.is_async: + self.assertEqual(calls, ["aclose"]) + else: + self.assertEqual(calls, ["close"]) + + @CrossSync.pytest + async def test_stats_preserved_after_completion(self): + from google.cloud.spanner_v1.types import ResultSetStats + + partial = self._make_int_partial([1], with_metadata=True, last=True) + partial.stats = ResultSetStats(query_stats={"rows_returned": "1"}) + stats_result_set = self._make_one(_MockCancellableIterator(partial)) + self.assertEqual([row async for row in stats_result_set], [[1]]) + self.assertEqual(stats_result_set.stats.query_stats["rows_returned"], "1") + + @CrossSync.pytest + async def test_iter_closes_on_normal_completion(self): + generator_closed = [] + partial = self._make_int_partial([1], with_metadata=True, last=True) + + async def generator(): + try: + yield partial + finally: + generator_closed.append(True) + + result_set = self._make_one(generator()) + rows = [row async for row in result_set] + + self.assertEqual(rows, [[1]]) + self.assertEqual(generator_closed, [True]) + self.assertTrue(result_set._closed) + + @CrossSync.pytest + async def test_iter_closes_on_mid_stream_error(self): + from google.cloud.exceptions import NotFound + + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + + async def generator(): + try: + yield first_partial + raise NotFound("row missing") + finally: + generator_closed.append(True) + + result_set = self._make_one(generator()) + collected_rows = [] + with pytest.raises(NotFound, match="row missing"): + async for row in result_set: + collected_rows.append(row) + + self.assertEqual(collected_rows, [[1]]) + self.assertEqual(generator_closed, [True]) + self.assertTrue(result_set._closed) + + @CrossSync.pytest + async def test_iter_closes_on_early_break(self): + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + second_partial = self._make_int_partial([2], last=True) + + async def generator(): + try: + yield first_partial + yield second_partial + finally: + generator_closed.append(True) + + result_set = self._make_one(generator()) + collected_rows = [] + if CrossSync.is_async: + rows = result_set.__aiter__() + try: + async for row in rows: + collected_rows.append(row) + break + finally: + await rows.aclose() + else: + for row in result_set: + collected_rows.append(row) + break + + self.assertEqual(collected_rows, [[1]]) + self.assertEqual(generator_closed, [True]) + self.assertTrue(result_set._closed) + + @CrossSync.pytest + async def test_one_and_one_or_none_close_across_outcomes(self): + from google.cloud.exceptions import NotFound + + empty_partial = self._make_int_partial([], with_metadata=True, last=True) + empty_iter = _MockCancellableIterator(empty_partial) + empty_result_set = self._make_one(empty_iter) + self.assertIsNone(await empty_result_set.one_or_none()) + self.assertEqual(empty_iter.cancel_calls, 1) + + not_found_iter = _MockCancellableIterator(empty_partial) + not_found_result_set = self._make_one(not_found_iter) + with pytest.raises(NotFound, match="No rows matched the given query."): + await not_found_result_set.one() + self.assertEqual(not_found_iter.cancel_calls, 1) + + one_partial = self._make_int_partial([7], with_metadata=True, last=True) + single_iter = _MockCancellableIterator(one_partial) + single_result_set = self._make_one(single_iter) + self.assertEqual(await single_result_set.one(), [7]) + self.assertEqual(single_iter.cancel_calls, 1) + + multi_partial = self._make_int_partial([7, 8], with_metadata=True, last=True) + multi_iter = _MockCancellableIterator(multi_partial) + multi_result_set = self._make_one(multi_iter) + with pytest.raises(ValueError, match="Expected one result; got more."): + await multi_result_set.one_or_none() + self.assertEqual(multi_iter.cancel_calls, 1) + + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + + async def failing_generator(): + try: + yield first_partial + raise NotFound("one failed") + finally: + generator_closed.append(True) + + error_result_set = self._make_one(failing_generator()) + with pytest.raises(NotFound, match="one failed"): + await error_result_set.one() + self.assertEqual(generator_closed, [True]) + + @CrossSync.pytest + async def test_to_dict_list_closes_on_completion_and_error(self): + from google.cloud.exceptions import NotFound + + partial = self._make_int_partial([1], with_metadata=True, last=True) + ok_iter = _MockCancellableIterator(partial) + ok_result_set = self._make_one(ok_iter) + self.assertEqual(await ok_result_set.to_dict_list(), [{"id": 1}]) + self.assertEqual(ok_iter.cancel_calls, 1) + + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + + async def failing_generator(): + try: + yield first_partial + raise NotFound("stream failed") + finally: + generator_closed.append(True) + + failing_result_set = self._make_one(failing_generator()) + with pytest.raises(NotFound, match="stream failed"): + await failing_result_set.to_dict_list() + self.assertEqual(generator_closed, [True]) + self.assertTrue(failing_result_set._closed) + class _MockCancellableIterator(object): cancel_calls = 0 @@ -2928,6 +3235,9 @@ class _MockCancellableIterator(object): def __init__(self, *values): self.iter_values = iter(values) + def cancel(self): + self.cancel_calls += 1 + def __aiter__(self): return self diff --git a/packages/google-cloud-spanner/tests/unit/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/test_snapshot.py index b236ec2d8424..2c6508a56d2b 100644 --- a/packages/google-cloud-spanner/tests/unit/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/test_snapshot.py @@ -3164,6 +3164,76 @@ def test__build_transaction_options_w_exact_staleness(self): ), ) + def test_streamed_result_set_records_metrics_completion_on_close(self): + from google.protobuf.struct_pb2 import Value + + from google.cloud.spanner_v1 import ( + PartialResultSet, + ResultSetMetadata, + StructType, + Type, + TypeCode, + ) + from google.cloud.spanner_v1.metrics.metrics_tracer_factory import ( + MetricsTracerFactory, + ) + + class CancellableStream: + def __init__(self, items): + self._items = iter(items) + self.cancel = mock.Mock() + + def __iter__(self): + return self + + def __next__(self): + return next(self._items) + + fields = [StructType.Field(name="col", type_=Type(code=TypeCode.INT64))] + metadata_pb = ResultSetMetadata(row_type=StructType(fields=fields)) + first_partial = PartialResultSet( + metadata=metadata_pb, resume_token=b"tok-1", last=False + ) + first_partial.values.append(Value(string_value="1")) + second_partial = PartialResultSet(last=True) + second_partial.values.append(Value(string_value="2")) + + raw_stream = CancellableStream([first_partial, second_partial]) + database = _Database() + database.spanner_api = build_spanner_api() + database.spanner_api.execute_streaming_sql = mock.Mock(return_value=raw_stream) + session = _Session(database) + snapshot = self._make_one(session=session, multi_use=False) + + mock_tracer = mock.Mock() + with mock.patch.object( + MetricsTracerFactory, "create_metrics_tracer", return_value=mock_tracer + ): + result_set = snapshot.execute_sql(SQL_QUERY) + result_iter = iter(result_set) + first_row = next(result_iter) + + self.assertEqual(first_row, [1]) + mock_tracer.record_operation_start.assert_called_once() + mock_tracer.record_operation_completion.assert_not_called() + raw_stream.cancel.assert_not_called() + + result_set._close() + mock_tracer.record_operation_completion.assert_called_once() + raw_stream.cancel.assert_called_once() + result_iter.close() + mock_tracer.record_operation_completion.assert_called_once() + + mock_tracer.reset_mock() + database.spanner_api.execute_streaming_sql.reset_mock() + unstarted_snapshot = self._make_one(session=session, multi_use=False) + unstarted = unstarted_snapshot.execute_sql(SQL_QUERY) + unstarted._close() + self.assertEqual(list(unstarted), []) + database.spanner_api.execute_streaming_sql.assert_not_called() + mock_tracer.record_operation_start.assert_not_called() + mock_tracer.record_operation_completion.assert_not_called() + class _Client(object): NTH_CLIENT = AtomicCounter() diff --git a/packages/google-cloud-spanner/tests/unit/test_streamed.py b/packages/google-cloud-spanner/tests/unit/test_streamed.py index 072b956dddf2..80ce4dd182f1 100644 --- a/packages/google-cloud-spanner/tests/unit/test_streamed.py +++ b/packages/google-cloud-spanner/tests/unit/test_streamed.py @@ -2753,6 +2753,285 @@ def test_merge_struct_null_last(self): self.assertTrue(res.list_value.values[0].HasField("null_value")) self.assertEqual(res.list_value.values[1].string_value, "v1") + def _make_int_partial(self, values, *, with_metadata=False, last=False): + from google.cloud.spanner_v1 import TypeCode + + metadata = None + if with_metadata: + fields = [self._make_scalar_field("id", TypeCode.INT64)] + metadata = self._make_result_set_metadata(fields) + return self._make_partial_result_set( + [self._make_value(value) for value in values], + metadata=metadata, + last=last, + ) + + def test_close_is_private_and_no_public_close(self): + iterator = _MockCancellableIterator() + result_set = self._make_one(iterator) + self.assertFalse(hasattr(result_set, "close")) + self.assertTrue(hasattr(result_set, "_close")) + + def test_close_with_none_iterator(self): + result_set = self._make_one(None) + result_set._close() + self.assertTrue(result_set._closed) + self.assertEqual(list(result_set), []) + + def test_close_without_close_method(self): + class PlainIterator: + pass + + result_set = self._make_one(PlainIterator()) + result_set._close() + self.assertTrue(result_set._closed) + self.assertEqual(list(result_set), []) + + def test_close_idempotent_with_generator(self): + generator_closed = [] + + def generator(): + try: + yield self._make_partial_result_set([], last=False) + finally: + generator_closed.append(True) + + response_iterator = generator() + next(response_iterator) + result_set = self._make_one(response_iterator) + + result_set._close() + result_set._close() + + self.assertEqual(generator_closed, [True]) + + def test_close_supports_sync_close_method(self): + class SyncCloseIterator: + def __init__(self): + self.close_calls = 0 + + def close(self): + self.close_calls += 1 + + iterator = SyncCloseIterator() + result_set = self._make_one(iterator) + + result_set._close() + result_set._close() + + self.assertEqual(iterator.close_calls, 1) + + def test_close_falls_back_to_cancel(self): + class CancelOnlyIterator: + def __init__(self): + self.cancel_calls = 0 + + def cancel(self): + self.cancel_calls += 1 + + iterator = CancelOnlyIterator() + result_set = self._make_one(iterator) + + result_set._close() + result_set._close() + + self.assertEqual(iterator.cancel_calls, 1) + + def test_close_suppresses_iterator_close_error(self): + class FailingCloseIterator: + def close(self): + raise RuntimeError("close failure") + + result_set = self._make_one(FailingCloseIterator()) + result_set._close() + self.assertTrue(result_set._closed) + self.assertEqual(list(result_set), []) + + class FailingCancelIterator: + def cancel(self): + raise RuntimeError("cancel failure") + + cancel_result_set = self._make_one(FailingCancelIterator()) + cancel_result_set._close() + self.assertTrue(cancel_result_set._closed) + self.assertEqual(list(cancel_result_set), []) + + def test_close_clears_buffered_rows_and_stops_iteration(self): + partial = self._make_int_partial([1, 2, 3], with_metadata=True, last=False) + result_set = self._make_one(_MockCancellableIterator(partial)) + + collected = [] + for row in result_set: + collected.append(row) + result_set._close() + + self.assertEqual(collected, [[1]]) + self.assertEqual(result_set._rows, []) + self.assertEqual(list(result_set), []) + self.assertIsNotNone(result_set.metadata) + self.assertEqual(len(result_set.fields), 1) + self.assertEqual(result_set.decode_row(partial._pb.values[:1]), [1]) + + closed_before_consume = self._make_one(_MockCancellableIterator(partial)) + closed_before_consume._close() + self.assertIsNone(closed_before_consume.one_or_none()) + + def test_close_method_precedence(self): + calls = [] + + class DualCloseIterator: + async def aclose(self): + calls.append("aclose") + + def close(self): + calls.append("close") + + def cancel(self): + calls.append("cancel") + + result_set = self._make_one(DualCloseIterator()) + result_set._close() + self.assertEqual(calls, ["close"]) + + def test_stats_preserved_after_completion(self): + from google.cloud.spanner_v1.types import ResultSetStats + + partial = self._make_int_partial([1], with_metadata=True, last=True) + partial.stats = ResultSetStats(query_stats={"rows_returned": "1"}) + stats_result_set = self._make_one(_MockCancellableIterator(partial)) + self.assertEqual(list(stats_result_set), [[1]]) + self.assertEqual(stats_result_set.stats.query_stats["rows_returned"], "1") + + def test_iter_closes_on_normal_completion(self): + generator_closed = [] + partial = self._make_int_partial([1], with_metadata=True, last=True) + + def generator(): + try: + yield partial + finally: + generator_closed.append(True) + + result_set = self._make_one(generator()) + rows = list(result_set) + + self.assertEqual(rows, [[1]]) + self.assertEqual(generator_closed, [True]) + self.assertTrue(result_set._closed) + + def test_iter_closes_on_mid_stream_error(self): + from google.cloud.exceptions import NotFound + + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + + def generator(): + try: + yield first_partial + raise NotFound("row missing") + finally: + generator_closed.append(True) + + result_set = self._make_one(generator()) + collected_rows = [] + with self.assertRaisesRegex(NotFound, "row missing"): + for row in result_set: + collected_rows.append(row) + + self.assertEqual(collected_rows, [[1]]) + self.assertEqual(generator_closed, [True]) + self.assertTrue(result_set._closed) + + def test_iter_closes_on_early_break(self): + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + second_partial = self._make_int_partial([2], last=True) + + def generator(): + try: + yield first_partial + yield second_partial + finally: + generator_closed.append(True) + + result_set = self._make_one(generator()) + collected_rows = [] + for row in result_set: + collected_rows.append(row) + break + + self.assertEqual(collected_rows, [[1]]) + self.assertEqual(generator_closed, [True]) + self.assertTrue(result_set._closed) + + def test_one_and_one_or_none_close_across_outcomes(self): + from google.cloud.exceptions import NotFound + + empty_partial = self._make_int_partial([], with_metadata=True, last=True) + empty_iter = _MockCancellableIterator(empty_partial) + empty_result_set = self._make_one(empty_iter) + self.assertIsNone(empty_result_set.one_or_none()) + self.assertEqual(empty_iter.cancel_calls, 1) + + not_found_iter = _MockCancellableIterator(empty_partial) + not_found_result_set = self._make_one(not_found_iter) + with self.assertRaisesRegex(NotFound, "No rows matched the given query."): + not_found_result_set.one() + self.assertEqual(not_found_iter.cancel_calls, 1) + + one_partial = self._make_int_partial([7], with_metadata=True, last=True) + single_iter = _MockCancellableIterator(one_partial) + single_result_set = self._make_one(single_iter) + self.assertEqual(single_result_set.one(), [7]) + self.assertEqual(single_iter.cancel_calls, 1) + + multi_partial = self._make_int_partial([7, 8], with_metadata=True, last=True) + multi_iter = _MockCancellableIterator(multi_partial) + multi_result_set = self._make_one(multi_iter) + with self.assertRaisesRegex(ValueError, "Expected one result; got more."): + multi_result_set.one_or_none() + self.assertEqual(multi_iter.cancel_calls, 1) + + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + + def failing_generator(): + try: + yield first_partial + raise NotFound("one failed") + finally: + generator_closed.append(True) + + error_result_set = self._make_one(failing_generator()) + with self.assertRaisesRegex(NotFound, "one failed"): + error_result_set.one() + self.assertEqual(generator_closed, [True]) + + def test_to_dict_list_closes_on_completion_and_error(self): + from google.cloud.exceptions import NotFound + + partial = self._make_int_partial([1], with_metadata=True, last=True) + ok_iter = _MockCancellableIterator(partial) + ok_result_set = self._make_one(ok_iter) + self.assertEqual(ok_result_set.to_dict_list(), [{"id": 1}]) + self.assertEqual(ok_iter.cancel_calls, 1) + + generator_closed = [] + first_partial = self._make_int_partial([1], with_metadata=True, last=False) + + def failing_generator(): + try: + yield first_partial + raise NotFound("stream failed") + finally: + generator_closed.append(True) + + failing_result_set = self._make_one(failing_generator()) + with self.assertRaisesRegex(NotFound, "stream failed"): + failing_result_set.to_dict_list() + self.assertEqual(generator_closed, [True]) + self.assertTrue(failing_result_set._closed) + class _MockCancellableIterator(object): cancel_calls = 0 @@ -2760,6 +3039,9 @@ class _MockCancellableIterator(object): def __init__(self, *values): self.iter_values = iter(values) + def cancel(self): + self.cancel_calls += 1 + def next(self): return next(self.iter_values)