Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
"""Wrapper for streaming results."""

__CROSS_SYNC_OUTPUT__ = "google.cloud.spanner_v1.streamed"
from typing import NoReturn

from google.protobuf.struct_pb2 import ListValue, Value

from google.cloud import exceptions
Expand Down Expand Up @@ -62,6 +64,7 @@ def __init__(
self._field_decoders = None
self._lazy_decode = lazy_decode # Return protobuf values
self._done = False
self._error = None

@property
def fields(self):
Expand Down Expand Up @@ -272,6 +275,25 @@ def _merge_values(self, values):
if remaining_values > full_values_count:
self._append_to_current_row(values[batch_end:])

def _set_terminal_error(self, message: str) -> NoReturn:
"""Record a sticky terminal InternalServerError and raise it."""
self._error = exceptions.InternalServerError(message)
self._done = True
raise self._error

def _validate_stream_termination(self) -> None:
"""Validate that the stream did not end mid-chunk or mid-row.

:raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the
stream ended with a pending chunked value or an incomplete row.
"""
if self._pending_chunk is not None:
self._set_terminal_error(
"Stream ended while waiting for continuation of chunked value."
)
if self._current_row:
self._set_terminal_error("Stream ended with an incomplete row.")

@CrossSync.convert
async def _consume_next(self):
"""Consume the next partial result set from the stream.
Expand All @@ -287,12 +309,20 @@ async def _consume_next(self):
if response_pb.HasField("stats"): # last response
self._stats = response.stats

if (response_pb.values or response_pb.chunked_value) and not self.fields:
self._set_terminal_error("Missing row_type in PartialResultSet metadata.")

if self._pending_chunk is not None or response_pb.chunked_value:
values = list(response_pb.values)
if self._pending_chunk is not None:
values[0] = self._merge_chunk(values[0])
if response_pb.chunked_value:
self._pending_chunk = values.pop()
if values:
if self._pending_chunk is not None:
values[0] = self._merge_chunk(values[0])
if response_pb.chunked_value:
self._pending_chunk = values.pop()
elif response_pb.chunked_value:
self._set_terminal_error(
"Received chunked_value=True with empty values."
)
else:
values = response_pb.values

Expand All @@ -303,18 +333,24 @@ async def _consume_next(self):

@CrossSync.convert(sync_name="__iter__")
async def __aiter__(self):
if self._error is not None:
raise self._error
while True:
if not self._rows:
if self._done:
self._validate_stream_termination()
return
try:
await self._consume_next()
except StopAsyncIteration:
self._done = True
self._validate_stream_termination()
return
iter_rows, self._rows = self._rows, []
for row in iter_rows:
yield row
if self._done:
self._validate_stream_termination()
return

def decode_row(self, row: []) -> []:
Expand Down Expand Up @@ -366,6 +402,8 @@ async def one_or_none(self):
:raises: :exc:`ValueError`: If there are multiple results.
:raises: :exc:`RuntimeError`: If consumption has already occurred,
in whole or in part.
:raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the
stream ended with a pending chunked value or an incomplete row.
Comment thread
olavloite marked this conversation as resolved.
Resolved
"""
# Sanity check: Has consumption of this query already started?
# If it has, then this is an exception.
Expand All @@ -379,8 +417,12 @@ async def one_or_none(self):
try:
await self._consume_next()
except StopAsyncIteration:
self._done = True
break

if self._done:
self._validate_stream_termination()

if len(self._rows) == 1:
return self._rows.pop()
if not self._rows:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

"""Wrapper for streaming results."""

from typing import NoReturn

from google.protobuf.struct_pb2 import ListValue, Value

from google.cloud import exceptions
Expand Down Expand Up @@ -54,6 +56,7 @@ def __init__(
self._field_decoders = None
self._lazy_decode = lazy_decode
self._done = False
self._error = None

@property
def fields(self):
Expand Down Expand Up @@ -242,6 +245,24 @@ def _merge_values(self, values):
if remaining_values > full_values_count:
self._append_to_current_row(values[batch_end:])

def _set_terminal_error(self, message: str) -> NoReturn:
"""Record a sticky terminal InternalServerError and raise it."""
self._error = exceptions.InternalServerError(message)
self._done = True
raise self._error

def _validate_stream_termination(self) -> None:
"""Validate that the stream did not end mid-chunk or mid-row.

:raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the
stream ended with a pending chunked value or an incomplete row."""
if self._pending_chunk is not None:
self._set_terminal_error(
"Stream ended while waiting for continuation of chunked value."
)
if self._current_row:
self._set_terminal_error("Stream ended with an incomplete row.")

def _consume_next(self):
"""Consume the next partial result set from the stream.

Expand All @@ -252,31 +273,44 @@ def _consume_next(self):
self._metadata = response_pb.metadata
if response_pb.HasField("stats"):
self._stats = response.stats
if (response_pb.values or response_pb.chunked_value) and (not self.fields):
self._set_terminal_error("Missing row_type in PartialResultSet metadata.")
if self._pending_chunk is not None or response_pb.chunked_value:
values = list(response_pb.values)
if self._pending_chunk is not None:
values[0] = self._merge_chunk(values[0])
if response_pb.chunked_value:
self._pending_chunk = values.pop()
if values:
if self._pending_chunk is not None:
values[0] = self._merge_chunk(values[0])
if response_pb.chunked_value:
self._pending_chunk = values.pop()
elif response_pb.chunked_value:
self._set_terminal_error(
"Received chunked_value=True with empty values."
)
else:
values = response_pb.values
self._merge_values(values)
if response_pb.last:
self._done = True

def __iter__(self):
if self._error is not None:
raise self._error
while True:
if not self._rows:
if self._done:
self._validate_stream_termination()
return
try:
self._consume_next()
except StopIteration:
self._done = True
self._validate_stream_termination()
return
iter_rows, self._rows = (self._rows, [])
for row in iter_rows:
yield row
if self._done:
self._validate_stream_termination()
return

def decode_row(self, row: []) -> []:
Expand Down Expand Up @@ -322,7 +356,9 @@ def one_or_none(self):

:raises: :exc:`ValueError`: If there are multiple results.
:raises: :exc:`RuntimeError`: If consumption has already occurred,
in whole or in part."""
in whole or in part.
:raises: :exc:`~google.cloud.exceptions.InternalServerError`: If the
stream ended with a pending chunked value or an incomplete row."""
Comment thread
olavloite marked this conversation as resolved.
Resolved
if self._metadata is not None:
raise RuntimeError(
"Can not call `.one` or `.one_or_none` after stream consumption has already started."
Expand All @@ -331,7 +367,10 @@ def one_or_none(self):
try:
self._consume_next()
except StopIteration:
self._done = True
break
if self._done:
self._validate_stream_termination()
if len(self._rows) == 1:
return self._rows.pop()
if not self._rows:
Expand Down
106 changes: 106 additions & 0 deletions packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py
Original file line number Diff line number Diff line change
Expand Up @@ -1071,6 +1071,29 @@ def test_dbapi_fetchall_mid_stream_aborted_issue_19079(self):
finally:
connection.close()

def test_streamed_result_set_invalid_termination_raises_internal_error(self):
for sql, fields, chunks, expected_message in _invalid_termination_scenarios():
add_execute_streaming_sql_results(
sql, _make_partial_result_sets(fields, chunks)
)
with self.database.snapshot(multi_use=True) as snapshot:
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
list(snapshot.execute_sql(sql))
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
snapshot.execute_sql(sql).one_or_none()
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
snapshot.execute_sql(sql).one()
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
snapshot.execute_sql(sql).to_dict_list()


class TestBasicsAsync(AsyncMockServerTestBase):
@asynccontextmanager
Expand Down Expand Up @@ -1199,6 +1222,89 @@ async def test_async_execute_streaming_sql_last_flag_trailing_error_drained(self
rows = [row async for row in results]
self.assertEqual([[10], [20]], rows)

async def test_async_streamed_result_set_invalid_termination_raises_internal_error(
self,
):
async with self._async_database() as database:
for sql, fields, chunks, expected_message in _invalid_termination_scenarios(
"async_"
):
add_execute_streaming_sql_results(
sql, _make_partial_result_sets(fields, chunks)
)
async with database.snapshot(multi_use=True) as snapshot:
results = await snapshot.execute_sql(sql)
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
_ = [row async for row in results]
one_or_none_results = await snapshot.execute_sql(sql)
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
await one_or_none_results.one_or_none()
one_results = await snapshot.execute_sql(sql)
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
await one_results.one()
dict_results = await snapshot.execute_sql(sql)
with self.assertRaisesRegex(
exceptions.InternalServerError, expected_message
):
await dict_results.to_dict_list()


def _invalid_termination_scenarios(prefix: str = ""):
fields = [("id", TypeCode.INT64), ("payload", TypeCode.STRING)]
return [
(
f"select id, payload from {prefix}invalid_chunk_eof",
fields,
[{"values": ["1", "partial "], "chunked_value": True, "last": False}],
"Stream ended while waiting for continuation of chunked value.",
),
(
f"select id, payload from {prefix}invalid_chunk_last",
fields,
[{"values": ["1", "partial "], "chunked_value": True, "last": True}],
"Stream ended while waiting for continuation of chunked value.",
),
(
f"select id, payload from {prefix}invalid_chunk_empty_last",
fields,
[
{"values": ["1", "partial "], "chunked_value": True, "last": False},
{"values": [], "chunked_value": False, "last": True},
],
"Stream ended while waiting for continuation of chunked value.",
),
(
f"select id, payload from {prefix}invalid_empty_values_chunked",
fields,
[{"values": [], "chunked_value": True, "last": False}],
"Received chunked_value=True with empty values.",
),
(
f"select id, payload from {prefix}invalid_row_eof",
fields,
[{"values": ["1", "ok", "2"], "chunked_value": False, "last": False}],
"Stream ended with an incomplete row.",
),
(
f"select id, payload from {prefix}invalid_row_last",
fields,
[{"values": ["1", "ok", "2"], "chunked_value": False, "last": True}],
"Stream ended with an incomplete row.",
),
(
f"select id, payload from {prefix}invalid_missing_row_type",
[],
[{"values": ["1"], "chunked_value": False, "last": True}],
"Missing row_type in PartialResultSet metadata.",
),
]


def _execute_query(transaction: Transaction, sql: str):
rows = transaction.execute_sql(sql, last_statement=True)
Expand Down
Loading
Loading