From 6134293f70de70106da099050a5e82ae17d65185 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 19 Nov 2026 05:41:32 +0100 Subject: [PATCH] fix(spanner): handle out-of-range read timestamps in result sets Change stream queries can return a transaction read timestamp that does not fit in a datetime. Checking the result set metadata by its truth value converts that timestamp through proto-plus and raises OverflowError, and Snapshot._update_for_transaction_pb and the DB-API cursor convert it again. Check for the transaction with HasField, and treat a read timestamp that does not fit in a datetime as no read timestamp. Refs #19127 --- .../google/cloud/spanner_dbapi/cursor.py | 8 ++--- .../cloud/spanner_v1/_async/snapshot.py | 14 +++++--- .../google/cloud/spanner_v1/_helpers.py | 22 +++++++++++++ .../google/cloud/spanner_v1/snapshot.py | 11 ++++--- .../tests/unit/_async/test_snapshot.py | 25 +++++++++++++++ .../tests/unit/spanner_dbapi/test_cursor.py | 18 +++++++++++ .../tests/unit/test__helpers.py | 30 +++++++++++++++++ .../tests/unit/test_snapshot.py | 32 +++++++++++++++++++ 8 files changed, 148 insertions(+), 12 deletions(-) diff --git a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/cursor.py b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/cursor.py index dc12d1b84609..c1cfb78d5bb6 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/cursor.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/cursor.py @@ -55,6 +55,7 @@ StreamedManyResultSets, ) from google.cloud.spanner_v1 import RequestOptions +from google.cloud.spanner_v1._helpers import _get_read_timestamp from google.cloud.spanner_v1.merged_result_set import MergedResultSet ColumnDetails = namedtuple("column_details", ["null_ok", "spanner_type"]) @@ -603,10 +604,9 @@ def _handle_DQL_with_snapshot(self, snapshot, sql, params): # Unfortunately, Spanner doesn't seem to send back # information about the number of rows available. self._row_count = None - if self._result_set.metadata.transaction.read_timestamp is not None: - snapshot._transaction_read_timestamp = ( - self._result_set.metadata.transaction.read_timestamp - ) + read_timestamp = _get_read_timestamp(self._result_set.metadata.transaction) + if read_timestamp is not None: + snapshot._transaction_read_timestamp = read_timestamp def _handle_DQL(self, sql, params): if self.connection.database is None: diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py index cc82d35de7cf..5a7e03c86e2b 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py @@ -33,6 +33,7 @@ from google.cloud.spanner_v1._helpers import ( _augment_error_with_request_id, _check_rst_stream_error, + _get_read_timestamp, _make_execute_sql_request, _make_value_pb, _merge_client_context, @@ -1047,8 +1048,12 @@ def _update_for_result_set_pb( self, result_set_pb: Union[ResultSet, PartialResultSet] ) -> None: """Updates the snapshot for the given result set.""" - if result_set_pb.metadata and result_set_pb.metadata.transaction: - self._update_for_transaction_pb(result_set_pb.metadata.transaction) + # Check for presence instead of truthiness: the truth value of a + # proto-plus message converts every field, including the read + # timestamp (see ``_get_read_timestamp``). + metadata = result_set_pb.metadata + if metadata is not None and metadata._pb.HasField("transaction"): + self._update_for_transaction_pb(metadata.transaction) def _update_for_transaction_pb(self, transaction_pb: Transaction) -> None: """Updates the snapshot for the given transaction.""" @@ -1148,5 +1153,6 @@ def _build_transaction_options_pb(self) -> TransactionOptions: def _update_for_transaction_pb(self, transaction_pb: Transaction) -> None: """Updates the snapshot for the given transaction.""" super(Snapshot, self)._update_for_transaction_pb(transaction_pb) - if transaction_pb.read_timestamp is not None: - self._transaction_read_timestamp = transaction_pb.read_timestamp + read_timestamp = _get_read_timestamp(transaction_pb) + if read_timestamp is not None: + self._transaction_read_timestamp = read_timestamp diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py index ba656f7ee70d..d7af1fe46349 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py @@ -817,6 +817,28 @@ def _parse_timestamp(value_pb): raise ValueError("Timestamp: {} does not match pattern".format(val)) from e +def _get_read_timestamp(transaction_pb): + """Return the read timestamp of a ``Transaction``, or ``None``. + + ``None`` is also returned when the timestamp is outside the range of + :class:`datetime.datetime`, which Spanner can return for change stream + queries. proto-plus raises ``OverflowError`` when converting it. + + :type transaction_pb: :class:`~google.cloud.spanner_v1.types.Transaction` + :param transaction_pb: the transaction returned by Spanner. + + :rtype: :class:`~google.api_core.datetime_helpers.DatetimeWithNanoseconds` + or None + :returns: the read timestamp, or ``None``. + """ + if not transaction_pb._pb.HasField("read_timestamp"): + return None + try: + return transaction_pb.read_timestamp + except OverflowError: + return None + + def _parse_proto(value_pb, column_info, field_name): bytes_value = base64.b64decode(value_pb.string_value) if column_info is not None and column_info.get(field_name) is not None: diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py index 3ba9b71014d1..41ec709bf26d 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py @@ -34,6 +34,7 @@ _augment_error_with_request_id, _check_rst_stream_error, _drain_stream, + _get_read_timestamp, _make_execute_sql_request, _make_value_pb, _merge_client_context, @@ -943,8 +944,9 @@ def _update_for_result_set_pb( self, result_set_pb: Union[ResultSet, PartialResultSet] ) -> None: """Updates the snapshot for the given result set.""" - if result_set_pb.metadata and result_set_pb.metadata.transaction: - self._update_for_transaction_pb(result_set_pb.metadata.transaction) + metadata = result_set_pb.metadata + if metadata is not None and metadata._pb.HasField("transaction"): + self._update_for_transaction_pb(metadata.transaction) def _update_for_transaction_pb(self, transaction_pb: Transaction) -> None: """Updates the snapshot for the given transaction.""" @@ -1029,5 +1031,6 @@ def _build_transaction_options_pb(self) -> TransactionOptions: def _update_for_transaction_pb(self, transaction_pb: Transaction) -> None: """Updates the snapshot for the given transaction.""" super(Snapshot, self)._update_for_transaction_pb(transaction_pb) - if transaction_pb.read_timestamp is not None: - self._transaction_read_timestamp = transaction_pb.read_timestamp + read_timestamp = _get_read_timestamp(transaction_pb) + if read_timestamp is not None: + self._transaction_read_timestamp = read_timestamp diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py index dd8ee834c37b..1bf9773565c4 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py @@ -261,6 +261,31 @@ def test__update_for_transaction_pb_multi_use(self): self.assertEqual(snapshot._transaction_read_timestamp, TIMESTAMP) self.assertTrue(snapshot._transaction_begin_event.is_set()) + def test__update_for_result_set_pb_w_read_timestamp(self): + snapshot = self._make_snapshot(_Session(), multi_use=False) + result_set_pb = PartialResultSet( + metadata={"transaction": {"read_timestamp": TIMESTAMP}} + ) + snapshot._update_for_result_set_pb(result_set_pb) + self.assertIsNone(snapshot._transaction_id) + self.assertEqual(snapshot._transaction_read_timestamp, TIMESTAMP) + + def test__update_for_result_set_pb_w_read_timestamp_out_of_range(self): + for multi_use in (False, True): + with self.subTest(multi_use=multi_use): + snapshot = self._make_snapshot(_Session(), multi_use=multi_use) + result_set_pb = PartialResultSet() + transaction_pb = result_set_pb._pb.metadata.transaction + if multi_use: + transaction_pb.id = TXN_ID + # 10000-01-01T00:00:00Z does not fit in a datetime. + transaction_pb.read_timestamp.seconds = 253402300800 + snapshot._update_for_result_set_pb(result_set_pb) + self.assertEqual( + snapshot._transaction_id, TXN_ID if multi_use else None + ) + self.assertIsNone(snapshot._transaction_read_timestamp) + async def test_restart_on_unavailable_precommit(self): from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable from tests._builders import build_precommit_token_pb diff --git a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_cursor.py b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_cursor.py index 8d7b33fe4a98..d059b343f27d 100644 --- a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_cursor.py +++ b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_cursor.py @@ -986,6 +986,24 @@ def test_handle_dql(self, MockedPeekIterator): self.assertEqual(cursor._itr, MockedPeekIterator()) self.assertEqual(cursor._row_count, None) + @mock.patch("google.cloud.spanner_dbapi.cursor.PeekIterator") + def test_handle_dql_w_read_timestamp_out_of_range(self, MockedPeekIterator): + from google.cloud.spanner_v1 import Transaction + + connection = self._make_connection(self.INSTANCE, mock.MagicMock()) + mock_snapshot = mock.MagicMock(_transaction_read_timestamp=None) + cursor = self._make_one(connection) + + transaction_pb = Transaction(id=b"txn-id") + # 10000-01-01T00:00:00Z does not fit in a datetime. + transaction_pb._pb.read_timestamp.seconds = 253402300800 + _result_set = mock.Mock() + _result_set.metadata.transaction = transaction_pb + mock_snapshot.execute_sql.return_value = _result_set + cursor._handle_DQL_with_snapshot(mock_snapshot, "sql", params=None) + self.assertEqual(cursor._result_set, _result_set) + self.assertIsNone(mock_snapshot._transaction_read_timestamp) + @mock.patch("google.cloud.spanner_dbapi.cursor.PeekIterator") def test_handle_dql_priority(self, MockedPeekIterator): from google.cloud.spanner_v1 import RequestOptions diff --git a/packages/google-cloud-spanner/tests/unit/test__helpers.py b/packages/google-cloud-spanner/tests/unit/test__helpers.py index 7a62f5dc6a2a..bdf10d6e57b4 100644 --- a/packages/google-cloud-spanner/tests/unit/test__helpers.py +++ b/packages/google-cloud-spanner/tests/unit/test__helpers.py @@ -291,6 +291,36 @@ def test_none_is_returned_unchanged(self): self.assertIsNone(self._callFUT(None, RequestOptions)) +class Test_get_read_timestamp(unittest.TestCase): + def _callFUT(self, *args, **kw): + from google.cloud.spanner_v1._helpers import _get_read_timestamp + + return _get_read_timestamp(*args, **kw) + + def test_not_set(self): + from google.cloud.spanner_v1 import Transaction + + self.assertIsNone(self._callFUT(Transaction(id=b"txn-id"))) + + def test_in_range(self): + from google.cloud.spanner_v1 import Transaction + + timestamp = datetime.datetime(2026, 11, 18, 21, 23, 35, tzinfo=timezone.utc) + transaction_pb = Transaction(read_timestamp=timestamp) + self.assertEqual(self._callFUT(transaction_pb), timestamp) + + def test_out_of_range(self): + from google.cloud.spanner_v1 import Transaction + + # 10000-01-01T00:00:00Z and the second before 0001-01-01T00:00:00Z + # do not fit in a datetime. + for seconds in (253402300800, -62135596801): + with self.subTest(seconds=seconds): + transaction_pb = Transaction() + transaction_pb._pb.read_timestamp.seconds = seconds + self.assertIsNone(self._callFUT(transaction_pb)) + + class Test_make_execute_sql_request(unittest.TestCase): SESSION_NAME = "projects/p/instances/i/databases/d/sessions/s" SQL = "SELECT * FROM Singers WHERE SingerId = @singer_id" diff --git a/packages/google-cloud-spanner/tests/unit/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/test_snapshot.py index b236ec2d8424..d3e5572b1ed2 100644 --- a/packages/google-cloud-spanner/tests/unit/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/test_snapshot.py @@ -3027,6 +3027,38 @@ def test__update_for_transaction_pb_multi_use(self): self.assertEqual(snapshot._transaction_read_timestamp, timestamp) self.assertTrue(snapshot._transaction_begin_event.is_set()) + def test__update_for_result_set_pb_w_read_timestamp(self): + from datetime import timezone + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + timestamp = datetime.now(timezone.utc) + snapshot = self._make_one(_Session(), multi_use=False) + result_set_pb = PartialResultSet( + metadata={"transaction": {"read_timestamp": timestamp}} + ) + snapshot._update_for_result_set_pb(result_set_pb) + self.assertIsNone(snapshot._transaction_id) + self.assertEqual(snapshot._transaction_read_timestamp, timestamp) + + def test__update_for_result_set_pb_w_read_timestamp_out_of_range(self): + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + for multi_use in (False, True): + with self.subTest(multi_use=multi_use): + snapshot = self._make_one(_Session(), multi_use=multi_use) + result_set_pb = PartialResultSet() + transaction_pb = result_set_pb._pb.metadata.transaction + if multi_use: + transaction_pb.id = TXN_ID + # 10000-01-01T00:00:00Z does not fit in a datetime. + transaction_pb.read_timestamp.seconds = 253402300800 + snapshot._update_for_result_set_pb(result_set_pb) + self.assertEqual( + snapshot._transaction_id, TXN_ID if multi_use else None + ) + self.assertIsNone(snapshot._transaction_read_timestamp) + def test_execute_sql_twice_single_use_fails(self): from google.cloud.spanner_v1 import PartialResultSet