Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ def __init__(
self._field_decoders = None
self._lazy_decode = lazy_decode # Return protobuf values
self._done = False
self._closed = False

@property
def fields(self):
Expand Down Expand Up @@ -301,21 +302,49 @@ async def _consume_next(self):
if response_pb.last:
self._done = True

@CrossSync.convert
async def _close(self) -> None:
"""Close the underlying response iterator idempotently."""
if self._closed:
return
self._closed = True
self._rows = []
iterator = self._response_iterator
if iterator is None:
return
if CrossSync.is_async:
close_methods = ("aclose", "close", "cancel")
else:
close_methods = ("close", "cancel")
try:
for method_name in close_methods:
close_method = getattr(iterator, method_name, None)
if close_method is not None:
await CrossSync.run_if_async(close_method)
return
except Exception:
pass
Comment thread
olavloite marked this conversation as resolved.
Resolved

@CrossSync.convert(sync_name="__iter__")
async def __aiter__(self):
while True:
if not self._rows:
if self._done:
try:
while True:
if self._closed:
return
try:
await self._consume_next()
except StopAsyncIteration:
if not self._rows:
try:
await self._consume_next()
except StopAsyncIteration:
return
iter_rows, self._rows = self._rows, []
for row in iter_rows:
if self._closed:
return
yield row
if self._done:
return
iter_rows, self._rows = self._rows, []
for row in iter_rows:
yield row
if self._done:
return
finally:
await self._close()

def decode_row(self, row: []) -> []:
"""Decodes a row from protobuf values to Python objects. This function
Expand Down Expand Up @@ -375,17 +404,23 @@ async def one_or_none(self):
"stream consumption has already started."
)

while not self._done and len(self._rows) < 2:
try:
await self._consume_next()
except StopAsyncIteration:
break

if len(self._rows) == 1:
return self._rows.pop()
if not self._rows:
if self._closed:
return None
raise ValueError("Expected one result; got more.")

try:
while not self._done and len(self._rows) < 2:
try:
await self._consume_next()
except StopAsyncIteration:
break

if len(self._rows) == 1:
return self._rows.pop()
if not self._rows:
return None
raise ValueError("Expected one result; got more.")
finally:
await self._close()

@CrossSync.convert
async def to_dict_list(self):
Expand All @@ -399,12 +434,15 @@ async def to_dict_list(self):
"""
rows = []
column_names = None
async for row in self:
if column_names is None:
column_names = [
column.name for column in self._metadata.row_type.fields
]
rows.append(dict(zip(column_names, row)))
try:
async for row in self:
if column_names is None:
column_names = [
column.name for column in self._metadata.row_type.fields
]
rows.append(dict(zip(column_names, row)))
finally:
await self._close()
return rows


Expand Down
86 changes: 60 additions & 26 deletions packages/google-cloud-spanner/google/cloud/spanner_v1/streamed.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from google.protobuf.struct_pb2 import ListValue, Value

from google.cloud import exceptions
from google.cloud.aio._cross_sync import CrossSync
from google.cloud.spanner_v1._helpers import _get_type_decoder, _parse_nullable
from google.cloud.spanner_v1.types.result_set import PartialResultSet, ResultSetMetadata
from google.cloud.spanner_v1.types.type import TypeCode
Expand Down Expand Up @@ -54,6 +55,7 @@ def __init__(
self._field_decoders = None
self._lazy_decode = lazy_decode
self._done = False
self._closed = False

@property
def fields(self):
Expand Down Expand Up @@ -264,20 +266,44 @@ def _consume_next(self):
if response_pb.last:
self._done = True

def _close(self) -> None:
"""Close the underlying response iterator idempotently."""
if self._closed:
return
self._closed = True
self._rows = []
iterator = self._response_iterator
if iterator is None:
return
close_methods = ("close", "cancel")
try:
for method_name in close_methods:
close_method = getattr(iterator, method_name, None)
if close_method is not None:
CrossSync._Sync_Impl.run_if_async(close_method)
return
except Exception:
pass

def __iter__(self):
while True:
if not self._rows:
if self._done:
try:
while True:
if self._closed:
return
try:
self._consume_next()
except StopIteration:
if not self._rows:
try:
self._consume_next()
except StopIteration:
return
iter_rows, self._rows = (self._rows, [])
for row in iter_rows:
if self._closed:
return
yield row
if self._done:
return
iter_rows, self._rows = (self._rows, [])
for row in iter_rows:
yield row
if self._done:
return
finally:
self._close()

def decode_row(self, row: []) -> []:
"""Decodes a row from protobuf values to Python objects. This function
Expand Down Expand Up @@ -327,16 +353,21 @@ def one_or_none(self):
raise RuntimeError(
"Can not call `.one` or `.one_or_none` after stream consumption has already started."
)
while not self._done and len(self._rows) < 2:
try:
self._consume_next()
except StopIteration:
break
if len(self._rows) == 1:
return self._rows.pop()
if not self._rows:
if self._closed:
return None
raise ValueError("Expected one result; got more.")
try:
while not self._done and len(self._rows) < 2:
try:
self._consume_next()
except StopIteration:
break
if len(self._rows) == 1:
return self._rows.pop()
if not self._rows:
return None
raise ValueError("Expected one result; got more.")
finally:
self._close()

def to_dict_list(self):
"""Return the result of a query as a list of dictionaries.
Expand All @@ -348,12 +379,15 @@ def to_dict_list(self):
:returns: result rows as a list of dictionaries"""
rows = []
column_names = None
for row in self:
if column_names is None:
column_names = [
column.name for column in self._metadata.row_type.fields
]
rows.append(dict(zip(column_names, row)))
try:
for row in self:
if column_names is None:
column_names = [
column.name for column in self._metadata.row_type.fields
]
rows.append(dict(zip(column_names, row)))
finally:
self._close()
return rows


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -424,3 +424,63 @@ def test_operation_latency_streaming_read(self):
self.assertIn("operation_latencies", metrics)
op_lat_point = metrics["operation_latencies"][0]
self.assertEqual(op_lat_point.count, 1)

def test_operation_latency_recorded_on_streamed_result_set_close(self):
fields = [("col1", TypeCode.INT64)]
partials = _make_partial_result_sets(
fields,
[
{"values": ["1"], "resume_token": b"token-1", "last": False},
{"values": ["2"], "last": True},
],
)
add_execute_streaming_sql_results("select 1", partials)
reader = InMemoryMetricReader()
meter_provider = MeterProvider(metric_readers=[reader])

with (
mock.patch(
"google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider",
return_value=meter_provider,
),
mock.patch(
"google.cloud.spanner_v1.client.MeterProvider",
return_value=meter_provider,
),
mock.patch(
"google.cloud.spanner_v1.client._get_spanner_emulator_host",
return_value=None,
),
):
_, database = self._setup_client_and_database()
with database.snapshot() as snapshot:
results = snapshot.execute_sql("select 1")
rows = iter(results)
self.assertEqual(next(rows), [1])

# Stream is still mid-flight; operation metrics must not be recorded yet.
metrics_before = self._extract_metrics(
reader, method="Spanner.ExecuteStreamingSql"
)
self.assertNotIn("operation_latencies", metrics_before)
self.assertNotIn("operation_count", metrics_before)

# Closing the StreamedResultSet records completion metrics immediately
# even while `results` and `rows` remain alive in local scope.
results._close()
metrics_after = self._extract_metrics(
reader, method="Spanner.ExecuteStreamingSql"
)
self.assertIn("operation_latencies", metrics_after)
self.assertEqual(metrics_after["operation_latencies"][0].count, 1)
self.assertIn("operation_count", metrics_after)
self.assertEqual(metrics_after["operation_count"][0].value, 1)

# Idempotent: closing again does not double-record.
results._close()
rows.close()
metrics_final = self._extract_metrics(
reader, method="Spanner.ExecuteStreamingSql"
)
self.assertEqual(metrics_final["operation_latencies"][0].count, 1)
self.assertEqual(metrics_final["operation_count"][0].value, 1)
Original file line number Diff line number Diff line change
Expand Up @@ -2238,6 +2238,71 @@ async def test_partition_query_w_leader_enabled(self):
f"Expected {leader_header} in {metadata}",
)

async def test_streamed_result_set_records_metrics_completion_on_close(self):
from google.protobuf.struct_pb2 import Value

from google.cloud.spanner_v1.metrics.metrics_tracer_factory import (
MetricsTracerFactory,
)

class CancellableStream:
def __init__(self, items):
self._items = iter(items)
self.cancel = mock.Mock()

def __aiter__(self):
return self

async def __anext__(self):
try:
return next(self._items)
except StopIteration:
raise StopAsyncIteration

fields = [StructType.Field(name="col", type_=Type(code=TypeCode.INT64))]
metadata_pb = ResultSetMetadata(row_type=StructType(fields=fields))
first_partial = PartialResultSet(
metadata=metadata_pb, resume_token=b"tok-1", last=False
)
first_partial.values.append(Value(string_value="1"))
second_partial = PartialResultSet(last=True)
second_partial.values.append(Value(string_value="2"))

raw_stream = CancellableStream([first_partial, second_partial])
database = _Database()
database.spanner_api.execute_streaming_sql = mock.Mock(return_value=raw_stream)
session = _Session(database)
snapshot = self._make_snapshot(session, multi_use=False)

mock_tracer = mock.Mock()
with mock.patch.object(
MetricsTracerFactory, "create_metrics_tracer", return_value=mock_tracer
):
result_set = await snapshot.execute_sql(SQL_QUERY)
async_iter = result_set.__aiter__()
first_row = await async_iter.__anext__()

self.assertEqual(first_row, [1])
mock_tracer.record_operation_start.assert_called_once()
mock_tracer.record_operation_completion.assert_not_called()
raw_stream.cancel.assert_not_called()

await result_set._close()
mock_tracer.record_operation_completion.assert_called_once()
raw_stream.cancel.assert_called_once()
await async_iter.aclose()
mock_tracer.record_operation_completion.assert_called_once()

mock_tracer.reset_mock()
database.spanner_api.execute_streaming_sql.reset_mock()
unstarted_snapshot = self._make_snapshot(session, multi_use=False)
unstarted = await unstarted_snapshot.execute_sql(SQL_QUERY)
await unstarted._close()
self.assertEqual([row async for row in unstarted], [])
database.spanner_api.execute_streaming_sql.assert_not_called()
mock_tracer.record_operation_start.assert_not_called()
mock_tracer.record_operation_completion.assert_not_called()


class _Database(object):
def __init__(self):
Expand Down
Loading
Loading