From dc3c6378038ab23e56e842e9993e174ce98f271e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Wed, 18 Nov 2026 18:33:41 +0100 Subject: [PATCH] fix(spanner): add checksum iterator and harden DB-API retry handling Fixes transaction retry bugs in the DB-API driver and introduces the building blocks needed to safely retry queries that encounter mid-stream errors or partial result consumption: - Unexpected failures during replay now fail the retry cleanly instead of crashing when the original statement had succeeded. Retry attempts are also counted across the entire transaction rather than per retry call, and exhausting the limit re-raises the aborted error cleanly instead of crashing. - Replay error matching now requires both the underlying gRPC status code and error message to match, reading the raw gRPC status details when available so client-generated request IDs do not cause false mismatches. Non-gRPC errors continue to compare exact exception type and arguments. - Adds a checksum-tracking iterator wrapper that records row checksums, end-of-stream boundaries, and mid-stream errors so retries can verify that results did not change (modified rows, rows inserted past the end, or different errors) before resuming iteration. To keep this change self-contained, the new iterator wrapper is unit-tested alongside the existing retry path; wiring it into the DB-API cursor will follow in a separate PR. --- .../google/cloud/spanner_dbapi/checksum.py | 269 +++++- .../cloud/spanner_dbapi/transaction_helper.py | 163 +++- .../test_dbapi_retry_helper.py | 179 ++++ .../tests/unit/spanner_dbapi/test_checksum.py | 828 +++++++++++++++++- .../spanner_dbapi/test_transaction_helper.py | 743 +++++++++++++++- 5 files changed, 2109 insertions(+), 73 deletions(-) create mode 100644 packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py diff --git a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py index b2b3297db22d..cc7fd4f1594b 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py @@ -15,9 +15,21 @@ """API to calculate checksums of SQL statements results.""" import hashlib +import logging import pickle +from typing import Any, Callable, Iterator, Optional, Tuple -from google.cloud.spanner_dbapi.exceptions import RetryAborted +import grpc +from google.api_core.exceptions import Aborted, GoogleAPICallError + +from google.cloud.spanner_dbapi.exceptions import InterfaceError, RetryAborted + +_LOGGER = logging.getLogger(__name__) + +RETRY_ABORTED_ERROR = ( + "The transaction was aborted and could not be retried due to a concurrent " + "modification." +) class ResultsChecksum: @@ -62,8 +74,6 @@ def consume_result(self, result): def _compare_checksums(original, retried): - from google.cloud.spanner_dbapi.transaction_helper import RETRY_ABORTED_ERROR - """Compare the given checksums. Raise an error if the given checksums are not equal. @@ -78,3 +88,256 @@ def _compare_checksums(original, retried): """ if retried != original: raise RetryAborted(RETRY_ABORTED_ERROR) + + +def _extract_grpc_error_info( + source: Any, +) -> Tuple[Optional[grpc.StatusCode], Optional[str]]: + code_getter = getattr(source, "code", None) + if not callable(code_getter): + return None, None + code = code_getter() + if not isinstance(code, grpc.StatusCode): + return None, None + details_getter = getattr(source, "details", None) + details = details_getter() if callable(details_getter) else None + if details is None: + return code, None + return code, details if isinstance(details, str) else str(details) + + +def _get_grpc_error_info( + error: BaseException, +) -> Tuple[Optional[grpc.StatusCode], Optional[str]]: + """Extract (gRPC status code, message) from an error or its ``__cause__`` chain.""" + visited = set() + current: Optional[BaseException] = error + while current is not None and id(current) not in visited: + visited.add(id(current)) + if isinstance(current, GoogleAPICallError): + response = getattr(current, "response", None) + code, response_message = _extract_grpc_error_info(response) + if code is not None: + return ( + code, + response_message + if response_message is not None + else current.message, + ) + if isinstance(current.grpc_status_code, grpc.StatusCode): + return current.grpc_status_code, current.message + code, message = _extract_grpc_error_info(current) + if code is not None: + return code, message if message is not None else str(current) + current = current.__cause__ + return None, None + + +def _errors_equal_for_retry( + original: Optional[BaseException], + retried: Optional[BaseException], +) -> bool: + """Compare two exceptions for transaction retry equivalence. + + For gRPC errors (including through wrapped ``__cause__`` exceptions), + returns True only if both the non-OK gRPC status code and error message + match. For non-gRPC exceptions (or ``StatusCode.OK``), falls back to + comparing exception type and ``args``. + """ + if original is None or retried is None: + return False + if original is retried: + return True + original_code, original_message = _get_grpc_error_info(original) + retried_code, retried_message = _get_grpc_error_info(retried) + if original_code is not None or retried_code is not None: + if original_code != retried_code: + return False + if original_code != grpc.StatusCode.OK: + return original_message == retried_message + return type(original) is type(retried) and original.args == retried.args + + +def _close_iterator(iterator: Any) -> None: + """Close an iterator or generator if it exposes a callable ``close()``.""" + close_method = getattr(iterator, "close", None) + if callable(close_method): + try: + close_method() + except Exception as exception: + _LOGGER.warning("Failed to close iterator: %s", exception) + + +class ChecksumIterator: + """Iterator wrapper that tracks consumed query results for transaction retries. + + When a read-write transaction is aborted by Spanner, the transaction must be + replayed and can only continue safely if every query produces the exact same + results that the caller has already observed. This wrapper records a running + :class:`ResultsChecksum` of all yielded rows, the number of ``next()`` calls + made, and whether iteration ended at EOF or raised a non-``Aborted`` error, + allowing replay to detect concurrent modifications (changed rows, new rows + past EOF, or differing errors) before resuming iteration. + """ + + def __init__( + self, + iterator: Iterator[Any], + retry_callback: Optional[Callable[[], None]] = None, + ) -> None: + if iterator is None: + raise ValueError("iterator must not be None") + self._iterator: Iterator[Any] = iterator + self._retry_callback: Optional[Callable[[], None]] = retry_callback + self._next_count: int = 0 + self._checksum: ResultsChecksum = ResultsChecksum() + self._error: Optional[Exception] = None + self._error_index: Optional[int] = None + self._eof_reached: bool = False + self._closed: bool = False + + @property + def iterator(self) -> Iterator[Any]: + """Return the underlying iterator.""" + return self._iterator + + @property + def next_count(self) -> int: + """Return the number of completed ``next()`` calls.""" + return self._next_count + + @property + def checksum(self) -> ResultsChecksum: + """Return the running :class:`ResultsChecksum` of yielded rows.""" + return self._checksum + + @property + def error(self) -> Optional[Exception]: + """Return the recorded non-``Aborted`` error, if any.""" + return self._error + + @property + def error_index(self) -> Optional[int]: + """Return the 0-based ``next()`` index where ``error`` occurred.""" + return self._error_index + + @property + def eof_reached(self) -> bool: + """Return True if ``StopIteration`` has been observed.""" + return self._eof_reached + + @property + def is_closed(self) -> bool: + """Return True if this iterator has been closed.""" + return self._closed + + def close(self) -> None: + """Close this wrapper and its underlying iterator, if closeable.""" + if not self._closed: + self._closed = True + _close_iterator(self._iterator) + + def __iter__(self) -> "ChecksumIterator": + return self + + def __next__(self) -> Any: + if self._closed: + raise InterfaceError("ChecksumIterator is already closed.") + if self._eof_reached: + raise StopIteration + if self._error is not None: + raise self._error + + while True: + try: + row = next(self._iterator) + except Aborted: + if self._retry_callback is None: + raise + self._retry_callback() + if self._closed: + raise InterfaceError("ChecksumIterator is already closed.") + continue + except StopIteration: + self._eof_reached = True + self._next_count += 1 + raise + except Exception as exception: + self._error = exception + self._error_index = self._next_count + self._next_count += 1 + raise + + self._checksum.consume_result(row) + self._next_count += 1 + return row + + def retry(self, new_iterator: Iterator[Any]) -> None: + """Replay ``next_count`` calls on ``new_iterator`` and verify parity. + + Verifies that ``new_iterator`` produces the identical prefix checksum, + reproduces any recorded non-``Aborted`` error at ``error_index``, and + reaches ``StopIteration`` at the exact EOF position if EOF was reached. + On success, swaps ``self._iterator`` to ``new_iterator`` (or closes + ``new_iterator`` if this wrapper has already been closed). + + Raises: + Aborted: If ``new_iterator`` aborts during replay. + RetryAborted: If results, EOF position, or errors do not match. + """ + if new_iterator is None: + raise RetryAborted(RETRY_ABORTED_ERROR) + + try: + self._replay_and_verify_prefix(new_iterator) + self._verify_terminal_state(new_iterator) + except BaseException: + _close_iterator(new_iterator) + raise + + if self._closed: + _close_iterator(new_iterator) + else: + self._swap_iterator(new_iterator) + + def _replay_and_verify_prefix(self, new_iterator: Iterator[Any]) -> None: + if len(self._checksum) == 0: + return + new_checksum = ResultsChecksum() + for _ in range(len(self._checksum)): + try: + row = next(new_iterator) + except Aborted: + raise + except StopIteration: + raise RetryAborted(RETRY_ABORTED_ERROR) + except Exception as exception: + raise RetryAborted(RETRY_ABORTED_ERROR, exception) from exception + new_checksum.consume_result(row) + + _compare_checksums(self._checksum, new_checksum) + + def _verify_terminal_state(self, new_iterator: Iterator[Any]) -> None: + if not self._eof_reached and self._error is None: + return + try: + next(new_iterator) + except Aborted: + raise + except StopIteration: + if not self._eof_reached: + raise RetryAborted(RETRY_ABORTED_ERROR) + return + except Exception as exception: + if self._error is not None and _errors_equal_for_retry( + self._error, exception + ): + return + raise RetryAborted(RETRY_ABORTED_ERROR, exception) from exception + + raise RetryAborted(RETRY_ABORTED_ERROR) + + def _swap_iterator(self, new_iterator: Iterator[Any]) -> None: + old_iterator = self._iterator + self._iterator = new_iterator + _close_iterator(old_iterator) diff --git a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py index b9e9f816ca72..e830736f0f6b 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py @@ -11,28 +11,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +import sys import time from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any, Dict, List, Optional from google.api_core.exceptions import Aborted from google.cloud.spanner_dbapi.batch_dml_executor import BatchMode +from google.cloud.spanner_dbapi.checksum import ( + RETRY_ABORTED_ERROR, + ChecksumIterator, + ResultsChecksum, + _compare_checksums, + _errors_equal_for_retry, +) from google.cloud.spanner_dbapi.exceptions import RetryAborted from google.cloud.spanner_v1._helpers import _get_retry_delay +# TODO(#19143): Decouple TransactionRetryHelper from Cursor internals to remove +# the circular TYPE_CHECKING dependency. if TYPE_CHECKING: from google.cloud.spanner_dbapi import Connection, Cursor -from google.cloud.spanner_dbapi.checksum import ResultsChecksum, _compare_checksums - MAX_INTERNAL_RETRIES = 50 -RETRY_ABORTED_ERROR = "The transaction was aborted and could not be retried due to a concurrent modification." class TransactionRetryHelper: - def __init__(self, connection: "Connection"): + def __init__(self, connection: Connection): """Helper class used in retrying the transaction when aborted This will maintain all the statements executed on original transaction and replay them again in the retried transaction. @@ -50,6 +59,7 @@ def __init__(self, connection: "Connection"): # 1-1 map from original cursor object on which transaction ran to the # new cursor object used in the retry self._cursor_map: Dict[Cursor, Cursor] = {} + self._retry_attempts: int = 0 def _set_connection_for_retry(self): self._connection._spanner_transaction_started = False @@ -64,6 +74,7 @@ def reset(self): self._statement_result_details_list = [] self._last_statement_details_per_cursor = {} self._cursor_map = {} + self._retry_attempts = 0 def add_fetch_statement_for_retry( self, cursor, result_rows, exception, is_fetch_all @@ -163,7 +174,28 @@ def add_execute_statement_for_retry( self._last_statement_details_per_cursor[cursor] = last_statement_result_details self._statement_result_details_list.append(last_statement_result_details) - def retry_transaction(self, default_retry_delay=None): + def add_checksum_iterator_for_retry( + self, cursor: Cursor, checksum_iterator: ChecksumIterator + ) -> None: + """Register a ChecksumIterator for replay verification on abort. + + Args: + cursor (Cursor): Original Cursor object on which the query executed. + checksum_iterator (ChecksumIterator): ChecksumIterator wrapping the + query result iterator. + """ + if not self._connection._client_transaction_started: + return + statement_details = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._last_statement_details_per_cursor[cursor] = statement_details + self._statement_result_details_list.append(statement_details) + + def retry_transaction(self, default_retry_delay: Optional[float] = None) -> None: """Retry the aborted transaction. All the statements executed in the original transaction @@ -174,46 +206,74 @@ def retry_transaction(self, default_retry_delay=None): If results checksum of the retried statement is not equal to the checksum of the original one. """ - attempt = 0 while True: - attempt += 1 - if attempt > MAX_INTERNAL_RETRIES: - raise + self._check_retry_limit() + self._retry_attempts += 1 self._set_connection_for_retry() try: - for statement_result_details in self._statement_result_details_list: - if statement_result_details.cursor in self._cursor_map: - cursor = self._cursor_map.get(statement_result_details.cursor) - else: - cursor = self._connection.cursor() - cursor._in_retry_mode = True - cursor._timeout = getattr( - statement_result_details.cursor, "_timeout", None - ) - self._cursor_map[statement_result_details.cursor] = cursor - try: - _handle_statement(statement_result_details, cursor) - except Aborted: - raise - except RetryAborted: - raise - except Exception as ex: - if ( - type(statement_result_details.result_details) - is not type(ex) - or ex.args != statement_result_details.result_details.args - ): - raise RetryAborted(RETRY_ABORTED_ERROR, ex) + self._replay_statements() return - except Aborted as ex: - delay = _get_retry_delay( - ex.errors[0], attempt, default_retry_delay=default_retry_delay - ) - if delay: - time.sleep(delay) + except Aborted as exception: + if self._retry_attempts >= MAX_INTERNAL_RETRIES: + raise + self._sleep_before_retry(exception, default_retry_delay) + + def _check_retry_limit(self) -> None: + if self._retry_attempts < MAX_INTERNAL_RETRIES: + return + active_exception = sys.exc_info()[1] + if isinstance(active_exception, Aborted): + raise active_exception + raise Aborted("Internal transaction retry maximum exceeded.") + + def _sleep_before_retry( + self, exception: Aborted, default_retry_delay: Optional[float] + ) -> None: + cause = exception.errors[0] if exception.errors else exception + delay = _get_retry_delay( + cause, + self._retry_attempts, + default_retry_delay=default_retry_delay, + ) + if delay: + time.sleep(delay) + + def _replay_statements(self) -> None: + for statement_result_details in self._statement_result_details_list: + cursor = self._get_or_create_retry_cursor(statement_result_details.cursor) + _replay_statement(statement_result_details, cursor) + + def _get_or_create_retry_cursor(self, original_cursor: Cursor) -> Cursor: + cursor = self._cursor_map.get(original_cursor) + if cursor is not None: + return cursor + cursor = self._connection.cursor() + cursor._in_retry_mode = True + cursor._timeout = getattr(original_cursor, "_timeout", None) + self._cursor_map[original_cursor] = cursor + return cursor + + +def _replay_statement( + statement_result_details: StatementDetails, cursor: Cursor +) -> None: + try: + _handle_statement(statement_result_details, cursor) + except (Aborted, RetryAborted): + raise + except Exception as exception: + if ( + statement_result_details.result_type != ResultType.EXCEPTION + or not _errors_equal_for_retry( + statement_result_details.result_details, exception + ) + ): + raise RetryAborted(RETRY_ABORTED_ERROR, exception) from exception -def _handle_statement(statement_result_details, cursor): +def _handle_statement( + statement_result_details: StatementDetails, cursor: Cursor +) -> None: statement_type = statement_result_details.statement_type if _is_execute_type_statement(statement_type): if statement_type == CursorStatementType.EXECUTE: @@ -232,7 +292,22 @@ def _handle_statement(statement_result_details, cursor): and statement_result_details.result_details != cursor._batch_dml_rows_count ): raise RetryAborted(RETRY_ABORTED_ERROR) + elif statement_type == CursorStatementType.CHECKSUM_ITERATOR: + checksum_iterator = statement_result_details.result_details + checksum_iterator.retry(cursor._itr) + original_cursor = statement_result_details.cursor + if ( + original_cursor is not None + and not checksum_iterator.is_closed + and getattr(original_cursor, "_itr", None) is checksum_iterator + and hasattr(cursor, "_result_set") + ): + original_cursor._result_set = cursor._result_set + return else: + # Legacy FetchStatement path (FETCH_ALL / FETCH_MANY), kept until + # Cursor switches all fetch operations over to ChecksumIterator in + # a follow-up pull request. if statement_type == CursorStatementType.FETCH_ALL: res = cursor.fetchall() else: @@ -263,6 +338,7 @@ class CursorStatementType(Enum): FETCH_ONE = 3 FETCH_ALL = 4 FETCH_MANY = 5 + CHECKSUM_ITERATOR = 6 class ResultType(Enum): @@ -282,7 +358,7 @@ class ResultType(Enum): class StatementDetails: statement_type: CursorStatementType # The cursor object on which this statement was executed - cursor: "Cursor" + cursor: Cursor result_type: ResultType result_details: Any @@ -296,3 +372,8 @@ class ExecuteStatement(StatementDetails): @dataclass class FetchStatement(StatementDetails): size: int = None + + +@dataclass +class ChecksumStatement(StatementDetails): + pass diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py new file mode 100644 index 000000000000..39769adddb67 --- /dev/null +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py @@ -0,0 +1,179 @@ +# Copyright 2026 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end mockserver tests for DB-API transaction retry error handling.""" + +from unittest import mock + +from google.api_core.exceptions import Aborted +from google.rpc import code_pb2, status_pb2 +from grpc_status._common import code_to_grpc_status_code +from grpc_status.rpc_status import _Status + +from google.cloud.spanner_dbapi import Connection +from google.cloud.spanner_dbapi.checksum import RETRY_ABORTED_ERROR +from google.cloud.spanner_dbapi.exceptions import ProgrammingError, RetryAborted +from google.cloud.spanner_v1.testing.mock_spanner import SpannerServicer +from tests.mockserver_tests.mock_server_test_base import ( + MockServerTestBase, + aborted_status, + add_error, + add_update_count, +) + +_BATCH_INSERT_SQL = "insert into singers (id, name) values (1, 'Some Singer')" +_UPDATE_SQL = "update singers set name = 'Updated' where id = 1" + + +def _rpc_status(code: code_pb2.Code, message: str) -> _Status: + error = status_pb2.Status(code=code, message=message) + return _Status( + code=code_to_grpc_status_code(error.code), + details=error.message, + trailing_metadata=(("grpc-status-details-bin", error.SerializeToString()),), + ) + + +class TestDbapiRetryHelper(MockServerTestBase): + def setUp(self): + super().setUp() + add_update_count(_BATCH_INSERT_SQL, 1) + add_update_count(_UPDATE_SQL, 1) + + def tearDown(self): + self.spanner_service.mock_spanner._errors_list = {} + super().tearDown() + + def test_previously_succeeding_statement_fails_on_retry_raises_retry_aborted(self): + """Verifies that if a statement succeeds on the initial attempt, commit + is aborted, and replaying that statement during retry fails with a + non-Aborted error, commit raises RetryAborted. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + cursor.execute(_UPDATE_SQL) + self.assertEqual(cursor.rowcount, 1) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INTERNAL, "unexpected replay failure"), + ) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + connection.commit() + finally: + connection.close() + + def test_matching_grpc_error_with_different_request_ids_commits_on_retry(self): + """Verifies retry when the application catches a failed statement and + continues the transaction. When commit later aborts and the transaction + is replayed, the client attaches a new request_id to each RPC, so retry + must compare the underlying gRPC status code and message while ignoring + request_id. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Division by zero"), + ) + with self.assertRaises(ProgrammingError) as first_failure: + cursor.execute(_UPDATE_SQL) + first_request_id = getattr( + first_failure.exception.__cause__, "request_id", None + ) + self.assertIsNotNone(first_request_id) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Division by zero"), + ) + connection.commit() + + execute_sql_requests = [ + request + for request in self.spanner_service.requests + if type(request).__name__ == "ExecuteSqlRequest" + ] + commit_requests = [ + request + for request in self.spanner_service.requests + if type(request).__name__ == "CommitRequest" + ] + self.assertEqual(len(execute_sql_requests), 2) + self.assertEqual(len(commit_requests), 2) + finally: + connection.close() + + def test_different_grpc_message_on_retry_raises_retry_aborted(self): + """Verifies retry when the application catches a failed statement, commit + later aborts, and replaying that statement fails with the same gRPC + status code but a different error message. Because the error message + changed, commit must raise RetryAborted. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Division by zero"), + ) + with self.assertRaises(ProgrammingError): + cursor.execute(_UPDATE_SQL) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Out of range"), + ) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + connection.commit() + finally: + connection.close() + + def test_max_internal_retries_exceeded_raises_aborted(self): + """Verifies that when Aborted errors occur repeatedly within a single + transaction and exceed MAX_INTERNAL_RETRIES, the client stops retrying + and raises Aborted to the caller. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + cursor.execute(_UPDATE_SQL) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error(SpannerServicer.ExecuteStreamingSql.__name__, aborted_status()) + add_error(SpannerServicer.ExecuteStreamingSql.__name__, aborted_status()) + + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaises(Aborted): + connection.commit() + finally: + connection.close() diff --git a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py index ae57a21a8293..99c5f84c5b83 100644 --- a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py +++ b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py @@ -13,15 +13,39 @@ # limitations under the License. import unittest +from unittest import mock + +import grpc +from google.api_core.exceptions import ( + Aborted, + AlreadyExists, + DeadlineExceeded, + GoogleAPICallError, + InternalServerError, + InvalidArgument, + OutOfRange, + Unknown, + from_grpc_error, +) + +from google.cloud.spanner_dbapi.checksum import ( + RETRY_ABORTED_ERROR, + ChecksumIterator, + ResultsChecksum, + _compare_checksums, + _errors_equal_for_retry, +) +from google.cloud.spanner_dbapi.exceptions import ( + IntegrityError, + InterfaceError, + OperationalError, + RetryAborted, +) +from google.cloud.spanner_v1.exceptions import wrap_with_request_id class Test_compare_checksums(unittest.TestCase): def test_equal(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - original = ResultsChecksum() original.consume_result(5) @@ -31,27 +55,15 @@ def test_equal(self): self.assertIsNone(_compare_checksums(original, retried)) def test_less_results(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - from google.cloud.spanner_dbapi.exceptions import RetryAborted - original = ResultsChecksum() original.consume_result(5) retried = ResultsChecksum() - with self.assertRaises(RetryAborted): + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): _compare_checksums(original, retried) def test_more_results(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - from google.cloud.spanner_dbapi.exceptions import RetryAborted - original = ResultsChecksum() original.consume_result(5) @@ -59,21 +71,785 @@ def test_more_results(self): retried.consume_result(5) retried.consume_result(2) - with self.assertRaises(RetryAborted): + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): _compare_checksums(original, retried) def test_mismatch(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - from google.cloud.spanner_dbapi.exceptions import RetryAborted - original = ResultsChecksum() original.consume_result(5) retried = ResultsChecksum() retried.consume_result(2) - with self.assertRaises(RetryAborted): + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): _compare_checksums(original, retried) + + +class TestErrorsEqualForRetry(unittest.TestCase): + def test_none_inputs(self): + err = ValueError("boom") + self.assertFalse(_errors_equal_for_retry(None, None)) + self.assertFalse(_errors_equal_for_retry(err, None)) + self.assertFalse(_errors_equal_for_retry(None, err)) + + def test_same_instance(self): + err = ValueError("boom") + self.assertTrue(_errors_equal_for_retry(err, err)) + + def test_same_grpc_status_code_matching_and_different_messages(self): + self.assertTrue( + _errors_equal_for_retry( + InternalServerError("same"), + InternalServerError("same"), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + DeadlineExceeded("timeout"), + DeadlineExceeded("timeout"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + InternalServerError("req-1"), + InternalServerError("req-2"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + InvalidArgument("bad 1"), + InvalidArgument("bad 2"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + DeadlineExceeded("timeout 1"), + DeadlineExceeded("timeout 2"), + ) + ) + + def test_different_grpc_status_codes(self): + self.assertFalse( + _errors_equal_for_retry( + InvalidArgument("same"), + InternalServerError("same"), + ) + ) + + def test_wrapped_dbapi_cause_grpc_status_code(self): + first_cause = InternalServerError("same cause message") + first_error = OperationalError(["detail-1"]) + first_error.__cause__ = first_cause + + second_cause = InternalServerError("same cause message") + second_error = OperationalError(["detail-2"]) + second_error.__cause__ = second_cause + + self.assertTrue(_errors_equal_for_retry(first_error, second_error)) + + different_cause = InternalServerError("different cause message") + different_error = OperationalError(["detail-1"]) + different_error.__cause__ = different_cause + self.assertFalse(_errors_equal_for_retry(first_error, different_error)) + + third_cause = AlreadyExists("dup") + third_error = IntegrityError([]) + third_error.__cause__ = third_cause + + fourth_cause = OutOfRange("oob") + fourth_error = IntegrityError([]) + fourth_error.__cause__ = fourth_cause + + self.assertFalse(_errors_equal_for_retry(third_error, fourth_error)) + + error_without_cause = IntegrityError([]) + self.assertFalse(_errors_equal_for_retry(third_error, error_without_cause)) + self.assertFalse(_errors_equal_for_retry(error_without_cause, fourth_error)) + + def test_unknown_ok_and_rpc_details_matching(self): + self.assertFalse( + _errors_equal_for_retry( + Unknown("err-1"), + Unknown("err-2"), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + Unknown("same"), + Unknown("same"), + ) + ) + + class FakeRpcError(Exception): + def __init__(self, message, status_code): + super().__init__(message) + self._status_code = status_code + + def code(self): + return self._status_code + + class FakeRpcErrorWithDetails(FakeRpcError): + def __init__(self, message, status_code, details_text): + super().__init__(message, status_code) + self._details_text = details_text + + def details(self): + return self._details_text + + self.assertFalse( + _errors_equal_for_retry( + FakeRpcError("ok-1", grpc.StatusCode.OK), + FakeRpcError("ok-2", grpc.StatusCode.OK), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + FakeRpcError("unavail", grpc.StatusCode.UNAVAILABLE), + FakeRpcError("unavail", grpc.StatusCode.UNAVAILABLE), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + FakeRpcError("unavail-1", grpc.StatusCode.UNAVAILABLE), + FakeRpcError("unavail-2", grpc.StatusCode.UNAVAILABLE), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + FakeRpcErrorWithDetails( + "wrapper-1", grpc.StatusCode.UNAVAILABLE, "same details" + ), + FakeRpcErrorWithDetails( + "wrapper-2", grpc.StatusCode.UNAVAILABLE, "same details" + ), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + FakeRpcErrorWithDetails( + "same", grpc.StatusCode.UNAVAILABLE, "details-1" + ), + FakeRpcErrorWithDetails( + "same", grpc.StatusCode.UNAVAILABLE, "details-2" + ), + ) + ) + + def test_non_grpc_exceptions_fallback_to_type_and_args(self): + self.assertTrue( + _errors_equal_for_retry( + GoogleAPICallError("same"), + GoogleAPICallError("same"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + GoogleAPICallError("one"), + GoogleAPICallError("two"), + ) + ) + self.assertTrue(_errors_equal_for_retry(ValueError("a"), ValueError("a"))) + self.assertFalse(_errors_equal_for_retry(ValueError("a"), ValueError("b"))) + self.assertFalse(_errors_equal_for_retry(ValueError("a"), TypeError("a"))) + + def test_google_api_call_error_response_code_and_non_status_code(self): + class FakeResponse: + def __init__(self, status_code): + self._status_code = status_code + + def code(self): + return self._status_code + + class NonStatusCodeError(Exception): + def code(self): + return 500 + + self.assertTrue( + _errors_equal_for_retry( + GoogleAPICallError( + "msg", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + GoogleAPICallError( + "msg", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + GoogleAPICallError( + "msg-1", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + GoogleAPICallError( + "msg-2", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + GoogleAPICallError("msg-1", response=FakeResponse(503)), + GoogleAPICallError("msg-2", response=FakeResponse(503)), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + NonStatusCodeError("err-1"), + NonStatusCodeError("err-2"), + ) + ) + + def test_from_grpc_error_with_request_id_matches_raw_grpc_status(self): + class StubRpcError(grpc.RpcError): + def __init__(self, status_code, details_message): + super().__init__() + self._status_code = status_code + self._details_message = details_message + + def code(self): + return self._status_code + + def details(self): + return self._details_message + + def trailing_metadata(self): + return () + + error_1 = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.1", + ) + error_2 = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.2", + ) + error_diff_msg = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Out of range") + ), + "1.1.1.1.3", + ) + + self.assertTrue(_errors_equal_for_retry(error_1, error_2)) + self.assertFalse(_errors_equal_for_retry(error_1, error_diff_msg)) + + wrapped_1 = IntegrityError([]) + wrapped_1.__cause__ = error_1 + wrapped_2 = IntegrityError([]) + wrapped_2.__cause__ = error_2 + self.assertTrue(_errors_equal_for_retry(wrapped_1, wrapped_2)) + + def test_grpc_error_none_details_falls_back_to_error_message(self): + class NoneDetailsRpcError(grpc.RpcError): + def __init__(self, status_code, details_message, string_repr): + super().__init__() + self._status_code = status_code + self._details_message = details_message + self._string_repr = string_repr + + def code(self): + return self._status_code + + def details(self): + return self._details_message + + def __str__(self): + return self._string_repr + + first_api_error = InvalidArgument( + "Division by zero", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "unused" + ), + ) + second_api_error = InvalidArgument( + "Out of range", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "unused" + ), + ) + self.assertFalse(_errors_equal_for_retry(first_api_error, second_api_error)) + self.assertTrue( + _errors_equal_for_retry( + first_api_error, + InvalidArgument( + "Division by zero", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "unused" + ), + ), + ) + ) + + # Empty string details should still be used verbatim rather than replaced + # with GoogleAPICallError.message. + self.assertTrue( + _errors_equal_for_retry( + InvalidArgument( + "ignored-1", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, "", "unused" + ), + ), + InvalidArgument( + "ignored-2", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, "", "unused" + ), + ), + ) + ) + + first_rpc_error = NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "rpc detail A" + ) + second_rpc_error = NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "rpc detail B" + ) + self.assertFalse(_errors_equal_for_retry(first_rpc_error, second_rpc_error)) + self.assertTrue( + _errors_equal_for_retry( + first_rpc_error, + NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "rpc detail A" + ), + ) + ) + + def test_cyclic_cause_chain_terminates_cleanly(self): + first_error = OperationalError(["same"]) + second_error = OperationalError(["cause"]) + first_error.__cause__ = second_error + second_error.__cause__ = first_error + + matching_first = OperationalError(["same"]) + matching_second = OperationalError(["cause"]) + matching_first.__cause__ = matching_second + matching_second.__cause__ = matching_first + + different_error = OperationalError(["other"]) + different_error.__cause__ = different_error + + self.assertTrue(_errors_equal_for_retry(first_error, matching_first)) + self.assertFalse(_errors_equal_for_retry(first_error, different_error)) + + +def _iter_with_error(rows, error): + for row in rows: + yield row + raise error + + +class _CloseableIterator: + def __init__(self, rows): + self._iter = iter(rows) + self.close_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return next(self._iter) + + def close(self): + self.close_count += 1 + + +class TestChecksumIterator(unittest.TestCase): + def test_none_iterator_raises(self): + with self.assertRaisesRegex(ValueError, "iterator must not be None"): + ChecksumIterator(None) + + def test_iterate_all_rows_and_eof_sticky(self): + rows = [(1, "a"), (2, "b")] + checksum_iterator = ChecksumIterator(iter(rows)) + self.assertIs(iter(checksum_iterator), checksum_iterator) + + consumed = list(checksum_iterator) + self.assertEqual(consumed, rows) + self.assertEqual(checksum_iterator.next_count, 3) + self.assertTrue(checksum_iterator.eof_reached) + self.assertIsNone(checksum_iterator.error) + self.assertIsNone(checksum_iterator.error_index) + + # Calling next() again after EOF must not advance next_count. + with self.assertRaises(StopIteration): + next(checksum_iterator) + self.assertEqual(checksum_iterator.next_count, 3) + + def test_non_aborted_error_recorded_and_sticky(self): + rows = [(1,), (2,)] + original_error = InternalServerError("mid-stream boom") + checksum_iterator = ChecksumIterator(_iter_with_error(rows, original_error)) + + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(checksum_iterator) + + self.assertEqual(checksum_iterator.next_count, 3) + self.assertIs(checksum_iterator.error, original_error) + self.assertEqual(checksum_iterator.error_index, 2) + self.assertFalse(checksum_iterator.eof_reached) + self.assertEqual(len(checksum_iterator.checksum), 2) + + # Subsequent next() calls re-raise the same error without advancing next_count. + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(checksum_iterator) + self.assertEqual(checksum_iterator.next_count, 3) + + def test_aborted_without_callback_does_not_mutate_state(self): + checksum_iterator = ChecksumIterator(_iter_with_error([(1,)], Aborted("abort"))) + + self.assertEqual(next(checksum_iterator), (1,)) + with self.assertRaisesRegex(Aborted, "abort"): + next(checksum_iterator) + + self.assertEqual(checksum_iterator.next_count, 1) + self.assertIsNone(checksum_iterator.error) + self.assertIsNone(checksum_iterator.error_index) + self.assertFalse(checksum_iterator.eof_reached) + + def test_aborted_with_retry_callback_resumes(self): + retry_callback = mock.Mock() + + def on_retry(): + checksum_iterator.retry(iter([(1,), (2,), (3,)])) + + retry_callback.side_effect = on_retry + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort")), + retry_callback=retry_callback, + ) + + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,)]) + self.assertEqual(retry_callback.call_count, 1) + self.assertEqual(checksum_iterator.next_count, 4) + self.assertTrue(checksum_iterator.eof_reached) + + def test_aborted_with_failing_retry_callback_propagates_retry_aborted(self): + retry_callback = mock.Mock(side_effect=RetryAborted(RETRY_ABORTED_ERROR)) + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort")), + retry_callback=retry_callback, + ) + + self.assertEqual(next(checksum_iterator), (1,)) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + next(checksum_iterator) + + self.assertEqual(retry_callback.call_count, 1) + self.assertEqual(checksum_iterator.next_count, 1) + self.assertIsNone(checksum_iterator.error) + + def test_retry_zero_consumed(self): + old_iterator = iter([(1,), (2,)]) + new_iterator = iter([(1,), (2,), (3,)]) + checksum_iterator = ChecksumIterator(old_iterator) + + checksum_iterator.retry(new_iterator) + + self.assertIs(checksum_iterator.iterator, new_iterator) + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,)]) + + def test_retry_none_new_iterator_raises(self): + checksum_iterator = ChecksumIterator(iter([(1,)])) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(None) + + def test_retry_partial_consumption_success(self): + old_gen = (row for row in [(1,), (2,), (3,)]) + new_gen = (row for row in [(1,), (2,), (3,), (4,)]) + checksum_iterator = ChecksumIterator(old_gen) + + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + checksum_iterator.retry(new_gen) + + self.assertIs(checksum_iterator.iterator, new_gen) + # Old generator was closed upon swap. + self.assertIsNone(old_gen.gi_frame) + self.assertEqual(list(checksum_iterator), [(3,), (4,)]) + + def test_retry_partial_consumption_checksum_mismatch_keeps_old_iterator(self): + old_iterator = iter([(1,), (2,)]) + new_gen = (row for row in [(1,), (99,)]) + checksum_iterator = ChecksumIterator(old_iterator) + + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(new_gen) + + self.assertIs(checksum_iterator.iterator, old_iterator) + self.assertIsNone(new_gen.gi_frame) + + def test_retry_partial_consumption_unexpected_eof(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,), (3,)])) + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + def test_retry_partial_consumption_unexpected_error(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(_iter_with_error([(1,)], ValueError("unexpected"))) + + def test_retry_partial_consumption_aborted_propagates(self): + old_iterator = iter([(1,), (2,)]) + checksum_iterator = ChecksumIterator(old_iterator) + self.assertEqual(next(checksum_iterator), (1,)) + + with self.assertRaisesRegex(Aborted, "retry aborted"): + checksum_iterator.retry(_iter_with_error([], Aborted("retry aborted"))) + + self.assertIs(checksum_iterator.iterator, old_iterator) + + def test_retry_eof_verification(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(list(checksum_iterator), [(1,), (2,)]) + + # Exact match succeeds. + new_iterator = iter([(1,), (2,)]) + checksum_iterator.retry(new_iterator) + self.assertIs(checksum_iterator.iterator, new_iterator) + + # Concurrent insert past EOF fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (2,), (3,)])) + + # Concurrent delete (earlier EOF) fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + # Concurrent modification with same row count fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (99,)])) + + def test_retry_empty_result_set_eof_detects_concurrent_insert(self): + checksum_iterator = ChecksumIterator(iter([])) + self.assertEqual(list(checksum_iterator), []) + self.assertTrue(checksum_iterator.eof_reached) + self.assertEqual(checksum_iterator.next_count, 1) + + # Still empty succeeds. + checksum_iterator.retry(iter([])) + + # Newly inserted row fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + def test_retry_mid_stream_error_matching(self): + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,), (2,)], InternalServerError("mid-stream err")) + ) + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream err"): + next(checksum_iterator) + + # Same prefix and matching gRPC status code + message at error_index succeeds. + new_iterator = _iter_with_error( + [(1,), (2,)], InternalServerError("mid-stream err") + ) + checksum_iterator.retry(new_iterator) + self.assertIs(checksum_iterator.iterator, new_iterator) + + # Different prefix row fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,), (99,)], InternalServerError("mid-stream err")) + ) + + # Replay succeeds past error_index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (2,), (3,)])) + + # Replay reaches EOF at error_index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (2,)])) + + # Replay raises same status code with different message fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,), (2,)], InternalServerError("different msg")) + ) + + # Replay raises different error code at error_index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,), (2,)], InvalidArgument("mid-stream err")) + ) + + # Replay raises same error at earlier index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,)], InternalServerError("mid-stream err")) + ) + + def test_retry_error_on_first_row(self): + checksum_iterator = ChecksumIterator( + _iter_with_error([], InvalidArgument("bad arg")) + ) + with self.assertRaisesRegex(InvalidArgument, "bad arg"): + next(checksum_iterator) + + checksum_iterator.retry(_iter_with_error([], InvalidArgument("bad arg"))) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(_iter_with_error([], InvalidArgument("other arg"))) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + def test_retry_terminal_state_aborted_propagates(self): + eof_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(list(eof_iterator), [(1,)]) + + with self.assertRaisesRegex(Aborted, "abort at eof"): + eof_iterator.retry(_iter_with_error([(1,)], Aborted("abort at eof"))) + + error_iterator = ChecksumIterator( + _iter_with_error([(1,)], InternalServerError("req-1")) + ) + self.assertEqual(next(error_iterator), (1,)) + with self.assertRaisesRegex(InternalServerError, "req-1"): + next(error_iterator) + + with self.assertRaisesRegex(Aborted, "abort at err"): + error_iterator.retry(_iter_with_error([(1,)], Aborted("abort at err"))) + + def test_close_marks_closed_and_closes_closeable_iterator(self): + underlying = _CloseableIterator([(1,), (2,)]) + checksum_iterator = ChecksumIterator(underlying) + self.assertFalse(checksum_iterator.is_closed) + self.assertEqual(next(checksum_iterator), (1,)) + + checksum_iterator.close() + checksum_iterator.close() # Idempotent. + + self.assertTrue(checksum_iterator.is_closed) + self.assertEqual(underlying.close_count, 1) + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(checksum_iterator) + + def test_retry_when_closed_verifies_and_closes_new_iterator(self): + original = _CloseableIterator([(1,), (2,)]) + checksum_iterator = ChecksumIterator(original) + self.assertEqual(next(checksum_iterator), (1,)) + checksum_iterator.close() + + replacement = _CloseableIterator([(1,), (2,)]) + checksum_iterator.retry(replacement) + + self.assertTrue(checksum_iterator.is_closed) + self.assertIs(checksum_iterator.iterator, original) + self.assertEqual(replacement.close_count, 1) + + mismatched = _CloseableIterator([(99,), (2,)]) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(mismatched) + self.assertEqual(mismatched.close_count, 1) + + def test_close_iterator_exception_logged_and_does_not_mask_retry_error(self): + class FailingCloseIterator: + def __init__(self, rows): + self._iter = iter(rows) + + def __iter__(self): + return self + + def __next__(self): + return next(self._iter) + + def close(self): + raise RuntimeError("close failure") + + checksum_iterator = ChecksumIterator(FailingCloseIterator([(1,)])) + self.assertEqual(next(checksum_iterator), (1,)) + + with self.assertLogs( + "google.cloud.spanner_dbapi.checksum", level="WARNING" + ) as captured_logs: + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(FailingCloseIterator([(99,)])) + + self.assertIn( + "Failed to close iterator: close failure", captured_logs.output[0] + ) + + with self.assertLogs( + "google.cloud.spanner_dbapi.checksum", level="WARNING" + ) as close_logs: + checksum_iterator.close() + + self.assertIn("Failed to close iterator: close failure", close_logs.output[0]) + + def test_next_raises_if_closed_during_retry_callback(self): + def close_during_retry(): + checksum_iterator.close() + + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort")), + retry_callback=close_during_retry, + ) + + self.assertEqual(next(checksum_iterator), (1,)) + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(checksum_iterator) + + def test_close_after_eof_and_after_error_raises_interface_error_on_next(self): + eof_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(next(eof_iterator), (1,)) + with self.assertRaises(StopIteration): + next(eof_iterator) + eof_iterator.close() + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(eof_iterator) + + error_iterator = ChecksumIterator( + _iter_with_error([(1,)], InternalServerError("mid-stream boom")) + ) + self.assertEqual(next(error_iterator), (1,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(error_iterator) + error_iterator.close() + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(error_iterator) + + def test_next_multiple_aborted_retries_during_consumption(self): + attempts = [0] + + def retry_callback(): + attempts[0] += 1 + if attempts[0] == 1: + checksum_iterator.retry( + _iter_with_error([(1,), (2,), (3,)], Aborted("abort-2")) + ) + else: + checksum_iterator.retry(iter([(1,), (2,), (3,), (4,)])) + + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort-1")), + retry_callback=retry_callback, + ) + + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,), (4,)]) + self.assertEqual(attempts[0], 2) diff --git a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py index f425b2b32f90..e090ee628018 100644 --- a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py +++ b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py @@ -14,18 +14,44 @@ import unittest from unittest import mock -from google.api_core.exceptions import Aborted +import grpc +from google.api_core.exceptions import ( + Aborted, + InternalServerError, + Unknown, + from_grpc_error, +) -from google.cloud.spanner_dbapi.checksum import ResultsChecksum -from google.cloud.spanner_dbapi.exceptions import RetryAborted +from google.cloud.spanner_dbapi.checksum import ( + RETRY_ABORTED_ERROR, + ChecksumIterator, + ResultsChecksum, +) +from google.cloud.spanner_dbapi.exceptions import ( + OperationalError, + ProgrammingError, + RetryAborted, +) from google.cloud.spanner_dbapi.parsed_statement import ParsedStatement, StatementType from google.cloud.spanner_dbapi.transaction_helper import ( + ChecksumStatement, CursorStatementType, ExecuteStatement, FetchStatement, ResultType, TransactionRetryHelper, ) +from google.cloud.spanner_dbapi.utils import PeekIterator +from google.cloud.spanner_v1 import ( + PartialResultSet, + ResultSetMetadata, + StructType, + Type, + TypeCode, +) +from google.cloud.spanner_v1._helpers import _make_value_pb +from google.cloud.spanner_v1.exceptions import wrap_with_request_id +from google.cloud.spanner_v1.streamed import StreamedResultSet def _get_checksum(row): @@ -617,3 +643,714 @@ def test_add_fetch_statement_for_retry_when_last_statement_is_not_fetch(self): self._under_test._statement_result_details_list, [execute_statement, expected_fetch_statement], ) + + def test_add_checksum_iterator_for_retry_when_not_in_transaction(self): + self._under_test._connection._client_transaction_started = False + checksum_iterator = ChecksumIterator(iter([(1,)])) + + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + self.assertEqual(self._under_test._statement_result_details_list, []) + self.assertEqual(self._under_test._last_statement_details_per_cursor, {}) + + def test_add_checksum_iterator_for_retry_registers_statement(self): + self._under_test._connection._client_transaction_started = True + checksum_iterator = ChecksumIterator(iter([(1,)])) + + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + expected_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self.assertEqual( + self._under_test._last_statement_details_per_cursor, + {self._mock_cursor: expected_statement}, + ) + self.assertEqual( + self._under_test._statement_result_details_list, + [expected_statement], + ) + + def test_retry_transaction_checksum_iterator_success_and_result_set_swap(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,), (3,)])) + self.assertEqual(next(checksum_iterator), (1,)) + + original_result_set = mock.Mock() + retried_result_set = mock.Mock() + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = original_result_set + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,), (2,), (3,)]) + retried_cursor._result_set = retried_result_set + self._under_test._connection.cursor.return_value = retried_cursor + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.extend( + [execute_statement, checksum_statement] + ) + + self._under_test.retry_transaction() + + retried_cursor.execute.assert_called_once_with(SQL, ARGS) + self.assertIs(self._mock_cursor._result_set, retried_result_set) + self.assertEqual(list(checksum_iterator), [(2,), (3,)]) + + def test_retry_transaction_stale_checksum_iterator_does_not_clobber_result_set( + self, + ): + checksum_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(list(checksum_iterator), [(1,)]) + + current_result_set = mock.Mock() + self._mock_cursor._itr = mock.Mock() # Moved on to a newer statement. + self._mock_cursor._result_set = current_result_set + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,)]) + retried_cursor._result_set = mock.Mock() + self._under_test._connection.cursor.return_value = retried_cursor + + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.append(checksum_statement) + + self._under_test.retry_transaction() + + self.assertIs(self._mock_cursor._result_set, current_result_set) + + def test_retry_transaction_checksum_iterator_mismatch_raises_retry_aborted(self): + checksum_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(list(checksum_iterator), [(1,)]) + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,), (2,)]) # Concurrent insert past EOF. + self._under_test._connection.cursor.return_value = retried_cursor + + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.append(checksum_statement) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + def test_retry_transaction_checksum_iterator_aborted_retries_again(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(next(checksum_iterator), (1,)) + + def failing_then_succeeding(): + if retried_cursor.execute.call_count == 1: + raise Aborted("mid-stream abort") + yield (1,) + yield (2,) + + retried_cursor = mock.Mock() + retried_cursor._itr = None + + def assign_itr(*_args, **_kwargs): + retried_cursor._itr = failing_then_succeeding() + + retried_cursor.execute.side_effect = assign_itr + self._under_test._connection.cursor.return_value = retried_cursor + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.extend( + [execute_statement, checksum_statement] + ) + + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(retried_cursor.execute.call_count, 2) + self.assertEqual(list(checksum_iterator), [(2,)]) + + def test_retry_transaction_unexpected_exception_on_non_exception_statement(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + unexpected_error = InternalServerError("unexpected failure") + run_mock.side_effect = unexpected_error + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR) as context: + self._under_test.retry_transaction() + + self.assertIs(context.exception.__cause__, unexpected_error) + + def test_retry_transaction_unexpected_exception_on_fetch_checksum_statement(self): + fetch_statement = FetchStatement( + statement_type=CursorStatementType.FETCH_ALL, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=_get_checksum(("a",)), + ) + self._under_test._statement_result_details_list.append(fetch_statement) + run_mock = self._under_test._connection.cursor().fetchall = mock.Mock() + unexpected_error = InternalServerError("unexpected mid-stream failure") + run_mock.side_effect = unexpected_error + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR) as context: + self._under_test.retry_transaction() + + self.assertIs(context.exception.__cause__, unexpected_error) + + def test_retry_transaction_same_grpc_error_different_wrapper_details(self): + original_cause = InternalServerError("same error") + original_error = OperationalError(["detail-1"]) + original_error.__cause__ = original_cause + + retried_cause = InternalServerError("same error") + retried_error = OperationalError(["detail-2"]) + retried_error.__cause__ = retried_cause + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=original_error, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = retried_error + + self._under_test.retry_transaction() + + run_mock.assert_called_once_with(SQL, ARGS) + + def test_retry_transaction_same_grpc_code_different_message_raises(self): + original_cause = InternalServerError("first message") + original_error = OperationalError([]) + original_error.__cause__ = original_cause + + retried_cause = InternalServerError("second message") + retried_error = OperationalError([]) + retried_error.__cause__ = retried_cause + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=original_error, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = retried_error + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + def test_retry_transaction_unknown_grpc_error_different_args_raises(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=Unknown("first"), + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = Unknown("second") + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + def test_retry_transaction_aborted_without_errors_list_and_max_retries(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = Aborted("no errors list") + + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaisesRegex(Aborted, "no errors list"): + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(run_mock.call_count, 2) + + def test_retry_transaction_exception_statement_succeeds_raises_retry_aborted(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=InternalServerError("initial failure"), + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.return_value = None + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + run_mock.assert_called_once_with(SQL, ARGS) + + def test_retry_transaction_sleeps_with_retry_delay_on_aborted(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = [Aborted("transient"), None] + + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.time.sleep" + ) as sleep_mock: + self._under_test.retry_transaction(default_retry_delay=0.25) + + sleep_mock.assert_called_once_with(0.25) + self.assertEqual(run_mock.call_count, 2) + + def test_retry_transaction_multi_cursor_streamed_result_set_and_peek_iterator( + self, + ): + metadata = ResultSetMetadata( + row_type=StructType( + fields=[StructType.Field(name="id", type_=Type(code=TypeCode.INT64))] + ) + ) + + def make_streamed_peek_iterator(chunks, trailing_error=None): + def response_stream(): + for values in chunks: + partial_result_set = PartialResultSet(metadata=metadata) + for value in values: + partial_result_set.values.append(_make_value_pb(value)) + yield partial_result_set + if trailing_error is not None: + raise trailing_error + + result_set = StreamedResultSet(response_stream()) + return result_set, PeekIterator(result_set) + + # Cursor 1: DML statement + query consumed to EOF. + cursor1 = mock.Mock() + result_set_1, peek_iterator_1 = make_streamed_peek_iterator([[1], [2]]) + cursor1._result_set = result_set_1 + first_checksum_iterator = ChecksumIterator(peek_iterator_1) + cursor1._itr = first_checksum_iterator + self.assertEqual(list(first_checksum_iterator), [(1,), (2,)]) + + # Cursor 2: partially consumed query (1 of 3 rows consumed). + cursor2 = mock.Mock() + result_set_2, peek_iterator_2 = make_streamed_peek_iterator([[10], [20, 30]]) + cursor2._result_set = result_set_2 + second_checksum_iterator = ChecksumIterator(peek_iterator_2) + cursor2._itr = second_checksum_iterator + self.assertEqual(next(second_checksum_iterator), (10,)) + + # Cursor 3: query hitting mid-stream error on row 2. + cursor3 = mock.Mock() + result_set_3, peek_iterator_3 = make_streamed_peek_iterator( + [[100]], trailing_error=InternalServerError("mid-stream boom") + ) + cursor3._result_set = result_set_3 + third_checksum_iterator = ChecksumIterator(peek_iterator_3) + cursor3._itr = third_checksum_iterator + self.assertEqual(next(third_checksum_iterator), (100,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(third_checksum_iterator) + + # Prepare retried cursors corresponding 1-to-1 with original cursors. + retried_cursor1 = mock.Mock() + retried_cursor2 = mock.Mock() + retried_cursor3 = mock.Mock() + self._under_test._connection.cursor.side_effect = [ + retried_cursor1, + retried_cursor2, + retried_cursor3, + ] + + retried_result_set_1, first_retried_peek = make_streamed_peek_iterator( + [[1], [2]] + ) + retried_result_set_2, second_retried_peek = make_streamed_peek_iterator( + [[10], [20, 30]] + ) + retried_result_set_3, third_retried_peek = make_streamed_peek_iterator( + [[100]], trailing_error=InternalServerError("mid-stream boom") + ) + + def exec_cursor1(sql, _args): + if sql.startswith("UPDATE"): + retried_cursor1.rowcount = 4 + retried_cursor1._batch_dml_rows_count = None + else: + retried_cursor1._result_set = retried_result_set_1 + retried_cursor1._itr = first_retried_peek + + def exec_cursor2(_sql, _args): + retried_cursor2._result_set = retried_result_set_2 + retried_cursor2._itr = second_retried_peek + + def exec_cursor3(_sql, _args): + retried_cursor3._result_set = retried_result_set_3 + retried_cursor3._itr = third_retried_peek + + retried_cursor1.execute.side_effect = exec_cursor1 + retried_cursor2.execute.side_effect = exec_cursor2 + retried_cursor3.execute.side_effect = exec_cursor3 + + self._under_test._statement_result_details_list.extend( + [ + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor1, + sql="UPDATE T SET X = 1", + args=None, + result_type=ResultType.ROW_COUNT, + result_details=4, + ), + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor1, + sql="SELECT id FROM T1", + args=None, + result_type=ResultType.NONE, + result_details=None, + ), + ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor1, + result_type=ResultType.CHECKSUM, + result_details=first_checksum_iterator, + ), + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor2, + sql="SELECT id FROM T2", + args=None, + result_type=ResultType.NONE, + result_details=None, + ), + ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor2, + result_type=ResultType.CHECKSUM, + result_details=second_checksum_iterator, + ), + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor3, + sql="SELECT id FROM T3", + args=None, + result_type=ResultType.NONE, + result_details=None, + ), + ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor3, + result_type=ResultType.CHECKSUM, + result_details=third_checksum_iterator, + ), + ] + ) + + self._under_test.retry_transaction() + + self.assertIs(cursor1._result_set, retried_result_set_1) + self.assertIs(cursor2._result_set, retried_result_set_2) + self.assertIs(cursor3._result_set, retried_result_set_3) + # Cursor 2 seamlessly yields remaining rows from the retried StreamedResultSet. + self.assertEqual(list(second_checksum_iterator), [(20,), (30,)]) + + def test_retry_transaction_closed_checksum_iterator_does_not_overwrite_result_set( + self, + ): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(next(checksum_iterator), (1,)) + checksum_iterator.close() + + original_result_set = mock.Mock() + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = original_result_set + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,), (2,)]) + retried_cursor._result_set = mock.Mock() + self._under_test._connection.cursor.return_value = retried_cursor + + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.append(checksum_statement) + + self._under_test.retry_transaction() + + self.assertIs(self._mock_cursor._result_set, original_result_set) + + def test_checksum_iterator_retry_callback_invokes_retry_transaction_and_resumes( + self, + ): + self._under_test._connection._client_transaction_started = True + + def failing_stream(): + yield (1,) + raise Aborted("mid-stream abort") + + checksum_iterator = ChecksumIterator( + failing_stream(), + retry_callback=lambda: self._under_test.retry_transaction( + default_retry_delay=0 + ), + ) + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = mock.Mock() + self._mock_cursor._batch_dml_rows_count = None + self._mock_cursor._row_count = None + + retried_result_set = mock.Mock() + retried_cursor = mock.Mock() + retried_cursor.execute.side_effect = lambda _sql, _args: setattr( + retried_cursor, "_itr", iter([(1,), (2,), (3,)]) + ) + retried_cursor._result_set = retried_result_set + self._under_test._connection.cursor.return_value = retried_cursor + + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, SQL, ARGS, None, False + ) + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,)]) + retried_cursor.execute.assert_called_once_with(SQL, ARGS) + self.assertIs(self._mock_cursor._result_set, retried_result_set) + + def test_retry_transaction_with_request_id_augmented_grpc_errors(self): + class StubRpcError(grpc.RpcError): + def __init__(self, status_code, details_message): + super().__init__() + self._status_code = status_code + self._details_message = details_message + + def code(self): + return self._status_code + + def details(self): + return self._details_message + + def trailing_metadata(self): + return () + + original_cause = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.1", + ) + original_error = ProgrammingError(original_cause.details) + original_error.__cause__ = original_cause + + retried_cause = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.2", + ) + retried_error = ProgrammingError(retried_cause.details) + retried_error.__cause__ = retried_cause + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=original_error, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = retried_error + + self._under_test.retry_transaction() + + run_mock.assert_called_once_with(SQL, ARGS) + + def test_checksum_iterator_repeated_abort_on_row_k_respects_max_retries(self): + self._under_test._connection._client_transaction_started = True + + def stream_aborting_on_row_2(): + yield (1,) + raise Aborted("row 2 aborted") + + checksum_iterator = ChecksumIterator( + stream_aborting_on_row_2(), + retry_callback=lambda: self._under_test.retry_transaction( + default_retry_delay=0 + ), + ) + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = mock.Mock() + self._mock_cursor._batch_dml_rows_count = None + self._mock_cursor._row_count = None + + retried_cursor = mock.Mock() + retried_cursor.execute.side_effect = lambda _sql, _args: setattr( + retried_cursor, "_itr", stream_aborting_on_row_2() + ) + self._under_test._connection.cursor.return_value = retried_cursor + + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, SQL, ARGS, None, False + ) + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + self.assertEqual(next(checksum_iterator), (1,)) + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaisesRegex(Aborted, "row 2 aborted"): + next(checksum_iterator) + + self.assertEqual(retried_cursor.execute.call_count, 2) + + # Calling retry_transaction directly after exhaustion also raises without replaying. + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaisesRegex( + Aborted, "Internal transaction retry maximum exceeded." + ): + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(retried_cursor.execute.call_count, 2) + + # reset() clears _retry_attempts for the next transaction. + self._under_test.reset() + self.assertEqual(self._under_test._retry_attempts, 0) + + def test_retry_transaction_closes_attempt_1_swapped_iterator_on_second_attempt( + self, + ): + class CloseableRowIterator: + def __init__(self, rows): + self._iterator = iter(rows) + self.close_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return next(self._iterator) + + def close(self): + self.close_count += 1 + + self._under_test._connection._client_transaction_started = True + original_iterator = CloseableRowIterator([(1,), (2,)]) + attempt_1_iterator = CloseableRowIterator([(1,), (2,)]) + attempt_2_iterator = CloseableRowIterator([(1,), (2,)]) + + checksum_iterator = ChecksumIterator(original_iterator) + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._batch_dml_rows_count = None + self._mock_cursor._row_count = None + self.assertEqual(next(checksum_iterator), (1,)) + + retried_cursor = mock.Mock() + self._under_test._connection.cursor.return_value = retried_cursor + replay_iterators = iter([attempt_1_iterator, attempt_2_iterator]) + dml_attempt = [0] + + def execute_side_effect(sql, _args): + if sql.startswith("SELECT"): + retried_cursor._itr = next(replay_iterators) + else: + dml_attempt[0] += 1 + if dml_attempt[0] == 1: + raise Aborted("abort on second statement") + retried_cursor.rowcount = 1 + retried_cursor._batch_dml_rows_count = None + + retried_cursor.execute.side_effect = execute_side_effect + + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, "SELECT id FROM T", None, None, False + ) + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + self._mock_cursor._row_count = 1 + self._mock_cursor.rowcount = 1 + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, "UPDATE T SET x = 1", None, None, False + ) + + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(original_iterator.close_count, 1) + self.assertEqual(attempt_1_iterator.close_count, 1) + self.assertEqual(attempt_2_iterator.close_count, 0) + self.assertIs(checksum_iterator.iterator, attempt_2_iterator) + self.assertEqual(list(checksum_iterator), [(2,)])