diff --git a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py index b2b3297db22d..cc7fd4f1594b 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/checksum.py @@ -15,9 +15,21 @@ """API to calculate checksums of SQL statements results.""" import hashlib +import logging import pickle +from typing import Any, Callable, Iterator, Optional, Tuple -from google.cloud.spanner_dbapi.exceptions import RetryAborted +import grpc +from google.api_core.exceptions import Aborted, GoogleAPICallError + +from google.cloud.spanner_dbapi.exceptions import InterfaceError, RetryAborted + +_LOGGER = logging.getLogger(__name__) + +RETRY_ABORTED_ERROR = ( + "The transaction was aborted and could not be retried due to a concurrent " + "modification." +) class ResultsChecksum: @@ -62,8 +74,6 @@ def consume_result(self, result): def _compare_checksums(original, retried): - from google.cloud.spanner_dbapi.transaction_helper import RETRY_ABORTED_ERROR - """Compare the given checksums. Raise an error if the given checksums are not equal. @@ -78,3 +88,256 @@ def _compare_checksums(original, retried): """ if retried != original: raise RetryAborted(RETRY_ABORTED_ERROR) + + +def _extract_grpc_error_info( + source: Any, +) -> Tuple[Optional[grpc.StatusCode], Optional[str]]: + code_getter = getattr(source, "code", None) + if not callable(code_getter): + return None, None + code = code_getter() + if not isinstance(code, grpc.StatusCode): + return None, None + details_getter = getattr(source, "details", None) + details = details_getter() if callable(details_getter) else None + if details is None: + return code, None + return code, details if isinstance(details, str) else str(details) + + +def _get_grpc_error_info( + error: BaseException, +) -> Tuple[Optional[grpc.StatusCode], Optional[str]]: + """Extract (gRPC status code, message) from an error or its ``__cause__`` chain.""" + visited = set() + current: Optional[BaseException] = error + while current is not None and id(current) not in visited: + visited.add(id(current)) + if isinstance(current, GoogleAPICallError): + response = getattr(current, "response", None) + code, response_message = _extract_grpc_error_info(response) + if code is not None: + return ( + code, + response_message + if response_message is not None + else current.message, + ) + if isinstance(current.grpc_status_code, grpc.StatusCode): + return current.grpc_status_code, current.message + code, message = _extract_grpc_error_info(current) + if code is not None: + return code, message if message is not None else str(current) + current = current.__cause__ + return None, None + + +def _errors_equal_for_retry( + original: Optional[BaseException], + retried: Optional[BaseException], +) -> bool: + """Compare two exceptions for transaction retry equivalence. + + For gRPC errors (including through wrapped ``__cause__`` exceptions), + returns True only if both the non-OK gRPC status code and error message + match. For non-gRPC exceptions (or ``StatusCode.OK``), falls back to + comparing exception type and ``args``. + """ + if original is None or retried is None: + return False + if original is retried: + return True + original_code, original_message = _get_grpc_error_info(original) + retried_code, retried_message = _get_grpc_error_info(retried) + if original_code is not None or retried_code is not None: + if original_code != retried_code: + return False + if original_code != grpc.StatusCode.OK: + return original_message == retried_message + return type(original) is type(retried) and original.args == retried.args + + +def _close_iterator(iterator: Any) -> None: + """Close an iterator or generator if it exposes a callable ``close()``.""" + close_method = getattr(iterator, "close", None) + if callable(close_method): + try: + close_method() + except Exception as exception: + _LOGGER.warning("Failed to close iterator: %s", exception) + + +class ChecksumIterator: + """Iterator wrapper that tracks consumed query results for transaction retries. + + When a read-write transaction is aborted by Spanner, the transaction must be + replayed and can only continue safely if every query produces the exact same + results that the caller has already observed. This wrapper records a running + :class:`ResultsChecksum` of all yielded rows, the number of ``next()`` calls + made, and whether iteration ended at EOF or raised a non-``Aborted`` error, + allowing replay to detect concurrent modifications (changed rows, new rows + past EOF, or differing errors) before resuming iteration. + """ + + def __init__( + self, + iterator: Iterator[Any], + retry_callback: Optional[Callable[[], None]] = None, + ) -> None: + if iterator is None: + raise ValueError("iterator must not be None") + self._iterator: Iterator[Any] = iterator + self._retry_callback: Optional[Callable[[], None]] = retry_callback + self._next_count: int = 0 + self._checksum: ResultsChecksum = ResultsChecksum() + self._error: Optional[Exception] = None + self._error_index: Optional[int] = None + self._eof_reached: bool = False + self._closed: bool = False + + @property + def iterator(self) -> Iterator[Any]: + """Return the underlying iterator.""" + return self._iterator + + @property + def next_count(self) -> int: + """Return the number of completed ``next()`` calls.""" + return self._next_count + + @property + def checksum(self) -> ResultsChecksum: + """Return the running :class:`ResultsChecksum` of yielded rows.""" + return self._checksum + + @property + def error(self) -> Optional[Exception]: + """Return the recorded non-``Aborted`` error, if any.""" + return self._error + + @property + def error_index(self) -> Optional[int]: + """Return the 0-based ``next()`` index where ``error`` occurred.""" + return self._error_index + + @property + def eof_reached(self) -> bool: + """Return True if ``StopIteration`` has been observed.""" + return self._eof_reached + + @property + def is_closed(self) -> bool: + """Return True if this iterator has been closed.""" + return self._closed + + def close(self) -> None: + """Close this wrapper and its underlying iterator, if closeable.""" + if not self._closed: + self._closed = True + _close_iterator(self._iterator) + + def __iter__(self) -> "ChecksumIterator": + return self + + def __next__(self) -> Any: + if self._closed: + raise InterfaceError("ChecksumIterator is already closed.") + if self._eof_reached: + raise StopIteration + if self._error is not None: + raise self._error + + while True: + try: + row = next(self._iterator) + except Aborted: + if self._retry_callback is None: + raise + self._retry_callback() + if self._closed: + raise InterfaceError("ChecksumIterator is already closed.") + continue + except StopIteration: + self._eof_reached = True + self._next_count += 1 + raise + except Exception as exception: + self._error = exception + self._error_index = self._next_count + self._next_count += 1 + raise + + self._checksum.consume_result(row) + self._next_count += 1 + return row + + def retry(self, new_iterator: Iterator[Any]) -> None: + """Replay ``next_count`` calls on ``new_iterator`` and verify parity. + + Verifies that ``new_iterator`` produces the identical prefix checksum, + reproduces any recorded non-``Aborted`` error at ``error_index``, and + reaches ``StopIteration`` at the exact EOF position if EOF was reached. + On success, swaps ``self._iterator`` to ``new_iterator`` (or closes + ``new_iterator`` if this wrapper has already been closed). + + Raises: + Aborted: If ``new_iterator`` aborts during replay. + RetryAborted: If results, EOF position, or errors do not match. + """ + if new_iterator is None: + raise RetryAborted(RETRY_ABORTED_ERROR) + + try: + self._replay_and_verify_prefix(new_iterator) + self._verify_terminal_state(new_iterator) + except BaseException: + _close_iterator(new_iterator) + raise + + if self._closed: + _close_iterator(new_iterator) + else: + self._swap_iterator(new_iterator) + + def _replay_and_verify_prefix(self, new_iterator: Iterator[Any]) -> None: + if len(self._checksum) == 0: + return + new_checksum = ResultsChecksum() + for _ in range(len(self._checksum)): + try: + row = next(new_iterator) + except Aborted: + raise + except StopIteration: + raise RetryAborted(RETRY_ABORTED_ERROR) + except Exception as exception: + raise RetryAborted(RETRY_ABORTED_ERROR, exception) from exception + new_checksum.consume_result(row) + + _compare_checksums(self._checksum, new_checksum) + + def _verify_terminal_state(self, new_iterator: Iterator[Any]) -> None: + if not self._eof_reached and self._error is None: + return + try: + next(new_iterator) + except Aborted: + raise + except StopIteration: + if not self._eof_reached: + raise RetryAborted(RETRY_ABORTED_ERROR) + return + except Exception as exception: + if self._error is not None and _errors_equal_for_retry( + self._error, exception + ): + return + raise RetryAborted(RETRY_ABORTED_ERROR, exception) from exception + + raise RetryAborted(RETRY_ABORTED_ERROR) + + def _swap_iterator(self, new_iterator: Iterator[Any]) -> None: + old_iterator = self._iterator + self._iterator = new_iterator + _close_iterator(old_iterator) diff --git a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py index b9e9f816ca72..e830736f0f6b 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_dbapi/transaction_helper.py @@ -11,28 +11,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +import sys import time from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any, Dict, List, Optional from google.api_core.exceptions import Aborted from google.cloud.spanner_dbapi.batch_dml_executor import BatchMode +from google.cloud.spanner_dbapi.checksum import ( + RETRY_ABORTED_ERROR, + ChecksumIterator, + ResultsChecksum, + _compare_checksums, + _errors_equal_for_retry, +) from google.cloud.spanner_dbapi.exceptions import RetryAborted from google.cloud.spanner_v1._helpers import _get_retry_delay +# TODO(#19143): Decouple TransactionRetryHelper from Cursor internals to remove +# the circular TYPE_CHECKING dependency. if TYPE_CHECKING: from google.cloud.spanner_dbapi import Connection, Cursor -from google.cloud.spanner_dbapi.checksum import ResultsChecksum, _compare_checksums - MAX_INTERNAL_RETRIES = 50 -RETRY_ABORTED_ERROR = "The transaction was aborted and could not be retried due to a concurrent modification." class TransactionRetryHelper: - def __init__(self, connection: "Connection"): + def __init__(self, connection: Connection): """Helper class used in retrying the transaction when aborted This will maintain all the statements executed on original transaction and replay them again in the retried transaction. @@ -50,6 +59,7 @@ def __init__(self, connection: "Connection"): # 1-1 map from original cursor object on which transaction ran to the # new cursor object used in the retry self._cursor_map: Dict[Cursor, Cursor] = {} + self._retry_attempts: int = 0 def _set_connection_for_retry(self): self._connection._spanner_transaction_started = False @@ -64,6 +74,7 @@ def reset(self): self._statement_result_details_list = [] self._last_statement_details_per_cursor = {} self._cursor_map = {} + self._retry_attempts = 0 def add_fetch_statement_for_retry( self, cursor, result_rows, exception, is_fetch_all @@ -163,7 +174,28 @@ def add_execute_statement_for_retry( self._last_statement_details_per_cursor[cursor] = last_statement_result_details self._statement_result_details_list.append(last_statement_result_details) - def retry_transaction(self, default_retry_delay=None): + def add_checksum_iterator_for_retry( + self, cursor: Cursor, checksum_iterator: ChecksumIterator + ) -> None: + """Register a ChecksumIterator for replay verification on abort. + + Args: + cursor (Cursor): Original Cursor object on which the query executed. + checksum_iterator (ChecksumIterator): ChecksumIterator wrapping the + query result iterator. + """ + if not self._connection._client_transaction_started: + return + statement_details = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._last_statement_details_per_cursor[cursor] = statement_details + self._statement_result_details_list.append(statement_details) + + def retry_transaction(self, default_retry_delay: Optional[float] = None) -> None: """Retry the aborted transaction. All the statements executed in the original transaction @@ -174,46 +206,74 @@ def retry_transaction(self, default_retry_delay=None): If results checksum of the retried statement is not equal to the checksum of the original one. """ - attempt = 0 while True: - attempt += 1 - if attempt > MAX_INTERNAL_RETRIES: - raise + self._check_retry_limit() + self._retry_attempts += 1 self._set_connection_for_retry() try: - for statement_result_details in self._statement_result_details_list: - if statement_result_details.cursor in self._cursor_map: - cursor = self._cursor_map.get(statement_result_details.cursor) - else: - cursor = self._connection.cursor() - cursor._in_retry_mode = True - cursor._timeout = getattr( - statement_result_details.cursor, "_timeout", None - ) - self._cursor_map[statement_result_details.cursor] = cursor - try: - _handle_statement(statement_result_details, cursor) - except Aborted: - raise - except RetryAborted: - raise - except Exception as ex: - if ( - type(statement_result_details.result_details) - is not type(ex) - or ex.args != statement_result_details.result_details.args - ): - raise RetryAborted(RETRY_ABORTED_ERROR, ex) + self._replay_statements() return - except Aborted as ex: - delay = _get_retry_delay( - ex.errors[0], attempt, default_retry_delay=default_retry_delay - ) - if delay: - time.sleep(delay) + except Aborted as exception: + if self._retry_attempts >= MAX_INTERNAL_RETRIES: + raise + self._sleep_before_retry(exception, default_retry_delay) + + def _check_retry_limit(self) -> None: + if self._retry_attempts < MAX_INTERNAL_RETRIES: + return + active_exception = sys.exc_info()[1] + if isinstance(active_exception, Aborted): + raise active_exception + raise Aborted("Internal transaction retry maximum exceeded.") + + def _sleep_before_retry( + self, exception: Aborted, default_retry_delay: Optional[float] + ) -> None: + cause = exception.errors[0] if exception.errors else exception + delay = _get_retry_delay( + cause, + self._retry_attempts, + default_retry_delay=default_retry_delay, + ) + if delay: + time.sleep(delay) + + def _replay_statements(self) -> None: + for statement_result_details in self._statement_result_details_list: + cursor = self._get_or_create_retry_cursor(statement_result_details.cursor) + _replay_statement(statement_result_details, cursor) + + def _get_or_create_retry_cursor(self, original_cursor: Cursor) -> Cursor: + cursor = self._cursor_map.get(original_cursor) + if cursor is not None: + return cursor + cursor = self._connection.cursor() + cursor._in_retry_mode = True + cursor._timeout = getattr(original_cursor, "_timeout", None) + self._cursor_map[original_cursor] = cursor + return cursor + + +def _replay_statement( + statement_result_details: StatementDetails, cursor: Cursor +) -> None: + try: + _handle_statement(statement_result_details, cursor) + except (Aborted, RetryAborted): + raise + except Exception as exception: + if ( + statement_result_details.result_type != ResultType.EXCEPTION + or not _errors_equal_for_retry( + statement_result_details.result_details, exception + ) + ): + raise RetryAborted(RETRY_ABORTED_ERROR, exception) from exception -def _handle_statement(statement_result_details, cursor): +def _handle_statement( + statement_result_details: StatementDetails, cursor: Cursor +) -> None: statement_type = statement_result_details.statement_type if _is_execute_type_statement(statement_type): if statement_type == CursorStatementType.EXECUTE: @@ -232,7 +292,22 @@ def _handle_statement(statement_result_details, cursor): and statement_result_details.result_details != cursor._batch_dml_rows_count ): raise RetryAborted(RETRY_ABORTED_ERROR) + elif statement_type == CursorStatementType.CHECKSUM_ITERATOR: + checksum_iterator = statement_result_details.result_details + checksum_iterator.retry(cursor._itr) + original_cursor = statement_result_details.cursor + if ( + original_cursor is not None + and not checksum_iterator.is_closed + and getattr(original_cursor, "_itr", None) is checksum_iterator + and hasattr(cursor, "_result_set") + ): + original_cursor._result_set = cursor._result_set + return else: + # Legacy FetchStatement path (FETCH_ALL / FETCH_MANY), kept until + # Cursor switches all fetch operations over to ChecksumIterator in + # a follow-up pull request. if statement_type == CursorStatementType.FETCH_ALL: res = cursor.fetchall() else: @@ -263,6 +338,7 @@ class CursorStatementType(Enum): FETCH_ONE = 3 FETCH_ALL = 4 FETCH_MANY = 5 + CHECKSUM_ITERATOR = 6 class ResultType(Enum): @@ -282,7 +358,7 @@ class ResultType(Enum): class StatementDetails: statement_type: CursorStatementType # The cursor object on which this statement was executed - cursor: "Cursor" + cursor: Cursor result_type: ResultType result_details: Any @@ -296,3 +372,8 @@ class ExecuteStatement(StatementDetails): @dataclass class FetchStatement(StatementDetails): size: int = None + + +@dataclass +class ChecksumStatement(StatementDetails): + pass diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py new file mode 100644 index 000000000000..39769adddb67 --- /dev/null +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_dbapi_retry_helper.py @@ -0,0 +1,179 @@ +# Copyright 2026 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end mockserver tests for DB-API transaction retry error handling.""" + +from unittest import mock + +from google.api_core.exceptions import Aborted +from google.rpc import code_pb2, status_pb2 +from grpc_status._common import code_to_grpc_status_code +from grpc_status.rpc_status import _Status + +from google.cloud.spanner_dbapi import Connection +from google.cloud.spanner_dbapi.checksum import RETRY_ABORTED_ERROR +from google.cloud.spanner_dbapi.exceptions import ProgrammingError, RetryAborted +from google.cloud.spanner_v1.testing.mock_spanner import SpannerServicer +from tests.mockserver_tests.mock_server_test_base import ( + MockServerTestBase, + aborted_status, + add_error, + add_update_count, +) + +_BATCH_INSERT_SQL = "insert into singers (id, name) values (1, 'Some Singer')" +_UPDATE_SQL = "update singers set name = 'Updated' where id = 1" + + +def _rpc_status(code: code_pb2.Code, message: str) -> _Status: + error = status_pb2.Status(code=code, message=message) + return _Status( + code=code_to_grpc_status_code(error.code), + details=error.message, + trailing_metadata=(("grpc-status-details-bin", error.SerializeToString()),), + ) + + +class TestDbapiRetryHelper(MockServerTestBase): + def setUp(self): + super().setUp() + add_update_count(_BATCH_INSERT_SQL, 1) + add_update_count(_UPDATE_SQL, 1) + + def tearDown(self): + self.spanner_service.mock_spanner._errors_list = {} + super().tearDown() + + def test_previously_succeeding_statement_fails_on_retry_raises_retry_aborted(self): + """Verifies that if a statement succeeds on the initial attempt, commit + is aborted, and replaying that statement during retry fails with a + non-Aborted error, commit raises RetryAborted. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + cursor.execute(_UPDATE_SQL) + self.assertEqual(cursor.rowcount, 1) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INTERNAL, "unexpected replay failure"), + ) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + connection.commit() + finally: + connection.close() + + def test_matching_grpc_error_with_different_request_ids_commits_on_retry(self): + """Verifies retry when the application catches a failed statement and + continues the transaction. When commit later aborts and the transaction + is replayed, the client attaches a new request_id to each RPC, so retry + must compare the underlying gRPC status code and message while ignoring + request_id. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Division by zero"), + ) + with self.assertRaises(ProgrammingError) as first_failure: + cursor.execute(_UPDATE_SQL) + first_request_id = getattr( + first_failure.exception.__cause__, "request_id", None + ) + self.assertIsNotNone(first_request_id) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Division by zero"), + ) + connection.commit() + + execute_sql_requests = [ + request + for request in self.spanner_service.requests + if type(request).__name__ == "ExecuteSqlRequest" + ] + commit_requests = [ + request + for request in self.spanner_service.requests + if type(request).__name__ == "CommitRequest" + ] + self.assertEqual(len(execute_sql_requests), 2) + self.assertEqual(len(commit_requests), 2) + finally: + connection.close() + + def test_different_grpc_message_on_retry_raises_retry_aborted(self): + """Verifies retry when the application catches a failed statement, commit + later aborts, and replaying that statement fails with the same gRPC + status code but a different error message. Because the error message + changed, commit must raise RetryAborted. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Division by zero"), + ) + with self.assertRaises(ProgrammingError): + cursor.execute(_UPDATE_SQL) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error( + SpannerServicer.ExecuteStreamingSql.__name__, + _rpc_status(code_pb2.INVALID_ARGUMENT, "Out of range"), + ) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + connection.commit() + finally: + connection.close() + + def test_max_internal_retries_exceeded_raises_aborted(self): + """Verifies that when Aborted errors occur repeatedly within a single + transaction and exceed MAX_INTERNAL_RETRIES, the client stops retrying + and raises Aborted to the caller. + """ + connection = Connection(self.instance, self.database) + connection.autocommit = False + try: + with connection.cursor() as cursor: + cursor.executemany(_BATCH_INSERT_SQL, [()]) + cursor.execute(_UPDATE_SQL) + + add_error(SpannerServicer.Commit.__name__, aborted_status()) + add_error(SpannerServicer.ExecuteStreamingSql.__name__, aborted_status()) + add_error(SpannerServicer.ExecuteStreamingSql.__name__, aborted_status()) + + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaises(Aborted): + connection.commit() + finally: + connection.close() diff --git a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py index ae57a21a8293..99c5f84c5b83 100644 --- a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py +++ b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_checksum.py @@ -13,15 +13,39 @@ # limitations under the License. import unittest +from unittest import mock + +import grpc +from google.api_core.exceptions import ( + Aborted, + AlreadyExists, + DeadlineExceeded, + GoogleAPICallError, + InternalServerError, + InvalidArgument, + OutOfRange, + Unknown, + from_grpc_error, +) + +from google.cloud.spanner_dbapi.checksum import ( + RETRY_ABORTED_ERROR, + ChecksumIterator, + ResultsChecksum, + _compare_checksums, + _errors_equal_for_retry, +) +from google.cloud.spanner_dbapi.exceptions import ( + IntegrityError, + InterfaceError, + OperationalError, + RetryAborted, +) +from google.cloud.spanner_v1.exceptions import wrap_with_request_id class Test_compare_checksums(unittest.TestCase): def test_equal(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - original = ResultsChecksum() original.consume_result(5) @@ -31,27 +55,15 @@ def test_equal(self): self.assertIsNone(_compare_checksums(original, retried)) def test_less_results(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - from google.cloud.spanner_dbapi.exceptions import RetryAborted - original = ResultsChecksum() original.consume_result(5) retried = ResultsChecksum() - with self.assertRaises(RetryAborted): + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): _compare_checksums(original, retried) def test_more_results(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - from google.cloud.spanner_dbapi.exceptions import RetryAborted - original = ResultsChecksum() original.consume_result(5) @@ -59,21 +71,785 @@ def test_more_results(self): retried.consume_result(5) retried.consume_result(2) - with self.assertRaises(RetryAborted): + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): _compare_checksums(original, retried) def test_mismatch(self): - from google.cloud.spanner_dbapi.checksum import ( - ResultsChecksum, - _compare_checksums, - ) - from google.cloud.spanner_dbapi.exceptions import RetryAborted - original = ResultsChecksum() original.consume_result(5) retried = ResultsChecksum() retried.consume_result(2) - with self.assertRaises(RetryAborted): + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): _compare_checksums(original, retried) + + +class TestErrorsEqualForRetry(unittest.TestCase): + def test_none_inputs(self): + err = ValueError("boom") + self.assertFalse(_errors_equal_for_retry(None, None)) + self.assertFalse(_errors_equal_for_retry(err, None)) + self.assertFalse(_errors_equal_for_retry(None, err)) + + def test_same_instance(self): + err = ValueError("boom") + self.assertTrue(_errors_equal_for_retry(err, err)) + + def test_same_grpc_status_code_matching_and_different_messages(self): + self.assertTrue( + _errors_equal_for_retry( + InternalServerError("same"), + InternalServerError("same"), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + DeadlineExceeded("timeout"), + DeadlineExceeded("timeout"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + InternalServerError("req-1"), + InternalServerError("req-2"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + InvalidArgument("bad 1"), + InvalidArgument("bad 2"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + DeadlineExceeded("timeout 1"), + DeadlineExceeded("timeout 2"), + ) + ) + + def test_different_grpc_status_codes(self): + self.assertFalse( + _errors_equal_for_retry( + InvalidArgument("same"), + InternalServerError("same"), + ) + ) + + def test_wrapped_dbapi_cause_grpc_status_code(self): + first_cause = InternalServerError("same cause message") + first_error = OperationalError(["detail-1"]) + first_error.__cause__ = first_cause + + second_cause = InternalServerError("same cause message") + second_error = OperationalError(["detail-2"]) + second_error.__cause__ = second_cause + + self.assertTrue(_errors_equal_for_retry(first_error, second_error)) + + different_cause = InternalServerError("different cause message") + different_error = OperationalError(["detail-1"]) + different_error.__cause__ = different_cause + self.assertFalse(_errors_equal_for_retry(first_error, different_error)) + + third_cause = AlreadyExists("dup") + third_error = IntegrityError([]) + third_error.__cause__ = third_cause + + fourth_cause = OutOfRange("oob") + fourth_error = IntegrityError([]) + fourth_error.__cause__ = fourth_cause + + self.assertFalse(_errors_equal_for_retry(third_error, fourth_error)) + + error_without_cause = IntegrityError([]) + self.assertFalse(_errors_equal_for_retry(third_error, error_without_cause)) + self.assertFalse(_errors_equal_for_retry(error_without_cause, fourth_error)) + + def test_unknown_ok_and_rpc_details_matching(self): + self.assertFalse( + _errors_equal_for_retry( + Unknown("err-1"), + Unknown("err-2"), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + Unknown("same"), + Unknown("same"), + ) + ) + + class FakeRpcError(Exception): + def __init__(self, message, status_code): + super().__init__(message) + self._status_code = status_code + + def code(self): + return self._status_code + + class FakeRpcErrorWithDetails(FakeRpcError): + def __init__(self, message, status_code, details_text): + super().__init__(message, status_code) + self._details_text = details_text + + def details(self): + return self._details_text + + self.assertFalse( + _errors_equal_for_retry( + FakeRpcError("ok-1", grpc.StatusCode.OK), + FakeRpcError("ok-2", grpc.StatusCode.OK), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + FakeRpcError("unavail", grpc.StatusCode.UNAVAILABLE), + FakeRpcError("unavail", grpc.StatusCode.UNAVAILABLE), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + FakeRpcError("unavail-1", grpc.StatusCode.UNAVAILABLE), + FakeRpcError("unavail-2", grpc.StatusCode.UNAVAILABLE), + ) + ) + self.assertTrue( + _errors_equal_for_retry( + FakeRpcErrorWithDetails( + "wrapper-1", grpc.StatusCode.UNAVAILABLE, "same details" + ), + FakeRpcErrorWithDetails( + "wrapper-2", grpc.StatusCode.UNAVAILABLE, "same details" + ), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + FakeRpcErrorWithDetails( + "same", grpc.StatusCode.UNAVAILABLE, "details-1" + ), + FakeRpcErrorWithDetails( + "same", grpc.StatusCode.UNAVAILABLE, "details-2" + ), + ) + ) + + def test_non_grpc_exceptions_fallback_to_type_and_args(self): + self.assertTrue( + _errors_equal_for_retry( + GoogleAPICallError("same"), + GoogleAPICallError("same"), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + GoogleAPICallError("one"), + GoogleAPICallError("two"), + ) + ) + self.assertTrue(_errors_equal_for_retry(ValueError("a"), ValueError("a"))) + self.assertFalse(_errors_equal_for_retry(ValueError("a"), ValueError("b"))) + self.assertFalse(_errors_equal_for_retry(ValueError("a"), TypeError("a"))) + + def test_google_api_call_error_response_code_and_non_status_code(self): + class FakeResponse: + def __init__(self, status_code): + self._status_code = status_code + + def code(self): + return self._status_code + + class NonStatusCodeError(Exception): + def code(self): + return 500 + + self.assertTrue( + _errors_equal_for_retry( + GoogleAPICallError( + "msg", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + GoogleAPICallError( + "msg", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + GoogleAPICallError( + "msg-1", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + GoogleAPICallError( + "msg-2", response=FakeResponse(grpc.StatusCode.UNAVAILABLE) + ), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + GoogleAPICallError("msg-1", response=FakeResponse(503)), + GoogleAPICallError("msg-2", response=FakeResponse(503)), + ) + ) + self.assertFalse( + _errors_equal_for_retry( + NonStatusCodeError("err-1"), + NonStatusCodeError("err-2"), + ) + ) + + def test_from_grpc_error_with_request_id_matches_raw_grpc_status(self): + class StubRpcError(grpc.RpcError): + def __init__(self, status_code, details_message): + super().__init__() + self._status_code = status_code + self._details_message = details_message + + def code(self): + return self._status_code + + def details(self): + return self._details_message + + def trailing_metadata(self): + return () + + error_1 = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.1", + ) + error_2 = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.2", + ) + error_diff_msg = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Out of range") + ), + "1.1.1.1.3", + ) + + self.assertTrue(_errors_equal_for_retry(error_1, error_2)) + self.assertFalse(_errors_equal_for_retry(error_1, error_diff_msg)) + + wrapped_1 = IntegrityError([]) + wrapped_1.__cause__ = error_1 + wrapped_2 = IntegrityError([]) + wrapped_2.__cause__ = error_2 + self.assertTrue(_errors_equal_for_retry(wrapped_1, wrapped_2)) + + def test_grpc_error_none_details_falls_back_to_error_message(self): + class NoneDetailsRpcError(grpc.RpcError): + def __init__(self, status_code, details_message, string_repr): + super().__init__() + self._status_code = status_code + self._details_message = details_message + self._string_repr = string_repr + + def code(self): + return self._status_code + + def details(self): + return self._details_message + + def __str__(self): + return self._string_repr + + first_api_error = InvalidArgument( + "Division by zero", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "unused" + ), + ) + second_api_error = InvalidArgument( + "Out of range", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "unused" + ), + ) + self.assertFalse(_errors_equal_for_retry(first_api_error, second_api_error)) + self.assertTrue( + _errors_equal_for_retry( + first_api_error, + InvalidArgument( + "Division by zero", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "unused" + ), + ), + ) + ) + + # Empty string details should still be used verbatim rather than replaced + # with GoogleAPICallError.message. + self.assertTrue( + _errors_equal_for_retry( + InvalidArgument( + "ignored-1", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, "", "unused" + ), + ), + InvalidArgument( + "ignored-2", + response=NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, "", "unused" + ), + ), + ) + ) + + first_rpc_error = NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "rpc detail A" + ) + second_rpc_error = NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "rpc detail B" + ) + self.assertFalse(_errors_equal_for_retry(first_rpc_error, second_rpc_error)) + self.assertTrue( + _errors_equal_for_retry( + first_rpc_error, + NoneDetailsRpcError( + grpc.StatusCode.INVALID_ARGUMENT, None, "rpc detail A" + ), + ) + ) + + def test_cyclic_cause_chain_terminates_cleanly(self): + first_error = OperationalError(["same"]) + second_error = OperationalError(["cause"]) + first_error.__cause__ = second_error + second_error.__cause__ = first_error + + matching_first = OperationalError(["same"]) + matching_second = OperationalError(["cause"]) + matching_first.__cause__ = matching_second + matching_second.__cause__ = matching_first + + different_error = OperationalError(["other"]) + different_error.__cause__ = different_error + + self.assertTrue(_errors_equal_for_retry(first_error, matching_first)) + self.assertFalse(_errors_equal_for_retry(first_error, different_error)) + + +def _iter_with_error(rows, error): + for row in rows: + yield row + raise error + + +class _CloseableIterator: + def __init__(self, rows): + self._iter = iter(rows) + self.close_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return next(self._iter) + + def close(self): + self.close_count += 1 + + +class TestChecksumIterator(unittest.TestCase): + def test_none_iterator_raises(self): + with self.assertRaisesRegex(ValueError, "iterator must not be None"): + ChecksumIterator(None) + + def test_iterate_all_rows_and_eof_sticky(self): + rows = [(1, "a"), (2, "b")] + checksum_iterator = ChecksumIterator(iter(rows)) + self.assertIs(iter(checksum_iterator), checksum_iterator) + + consumed = list(checksum_iterator) + self.assertEqual(consumed, rows) + self.assertEqual(checksum_iterator.next_count, 3) + self.assertTrue(checksum_iterator.eof_reached) + self.assertIsNone(checksum_iterator.error) + self.assertIsNone(checksum_iterator.error_index) + + # Calling next() again after EOF must not advance next_count. + with self.assertRaises(StopIteration): + next(checksum_iterator) + self.assertEqual(checksum_iterator.next_count, 3) + + def test_non_aborted_error_recorded_and_sticky(self): + rows = [(1,), (2,)] + original_error = InternalServerError("mid-stream boom") + checksum_iterator = ChecksumIterator(_iter_with_error(rows, original_error)) + + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(checksum_iterator) + + self.assertEqual(checksum_iterator.next_count, 3) + self.assertIs(checksum_iterator.error, original_error) + self.assertEqual(checksum_iterator.error_index, 2) + self.assertFalse(checksum_iterator.eof_reached) + self.assertEqual(len(checksum_iterator.checksum), 2) + + # Subsequent next() calls re-raise the same error without advancing next_count. + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(checksum_iterator) + self.assertEqual(checksum_iterator.next_count, 3) + + def test_aborted_without_callback_does_not_mutate_state(self): + checksum_iterator = ChecksumIterator(_iter_with_error([(1,)], Aborted("abort"))) + + self.assertEqual(next(checksum_iterator), (1,)) + with self.assertRaisesRegex(Aborted, "abort"): + next(checksum_iterator) + + self.assertEqual(checksum_iterator.next_count, 1) + self.assertIsNone(checksum_iterator.error) + self.assertIsNone(checksum_iterator.error_index) + self.assertFalse(checksum_iterator.eof_reached) + + def test_aborted_with_retry_callback_resumes(self): + retry_callback = mock.Mock() + + def on_retry(): + checksum_iterator.retry(iter([(1,), (2,), (3,)])) + + retry_callback.side_effect = on_retry + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort")), + retry_callback=retry_callback, + ) + + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,)]) + self.assertEqual(retry_callback.call_count, 1) + self.assertEqual(checksum_iterator.next_count, 4) + self.assertTrue(checksum_iterator.eof_reached) + + def test_aborted_with_failing_retry_callback_propagates_retry_aborted(self): + retry_callback = mock.Mock(side_effect=RetryAborted(RETRY_ABORTED_ERROR)) + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort")), + retry_callback=retry_callback, + ) + + self.assertEqual(next(checksum_iterator), (1,)) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + next(checksum_iterator) + + self.assertEqual(retry_callback.call_count, 1) + self.assertEqual(checksum_iterator.next_count, 1) + self.assertIsNone(checksum_iterator.error) + + def test_retry_zero_consumed(self): + old_iterator = iter([(1,), (2,)]) + new_iterator = iter([(1,), (2,), (3,)]) + checksum_iterator = ChecksumIterator(old_iterator) + + checksum_iterator.retry(new_iterator) + + self.assertIs(checksum_iterator.iterator, new_iterator) + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,)]) + + def test_retry_none_new_iterator_raises(self): + checksum_iterator = ChecksumIterator(iter([(1,)])) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(None) + + def test_retry_partial_consumption_success(self): + old_gen = (row for row in [(1,), (2,), (3,)]) + new_gen = (row for row in [(1,), (2,), (3,), (4,)]) + checksum_iterator = ChecksumIterator(old_gen) + + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + checksum_iterator.retry(new_gen) + + self.assertIs(checksum_iterator.iterator, new_gen) + # Old generator was closed upon swap. + self.assertIsNone(old_gen.gi_frame) + self.assertEqual(list(checksum_iterator), [(3,), (4,)]) + + def test_retry_partial_consumption_checksum_mismatch_keeps_old_iterator(self): + old_iterator = iter([(1,), (2,)]) + new_gen = (row for row in [(1,), (99,)]) + checksum_iterator = ChecksumIterator(old_iterator) + + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(new_gen) + + self.assertIs(checksum_iterator.iterator, old_iterator) + self.assertIsNone(new_gen.gi_frame) + + def test_retry_partial_consumption_unexpected_eof(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,), (3,)])) + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + def test_retry_partial_consumption_unexpected_error(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(_iter_with_error([(1,)], ValueError("unexpected"))) + + def test_retry_partial_consumption_aborted_propagates(self): + old_iterator = iter([(1,), (2,)]) + checksum_iterator = ChecksumIterator(old_iterator) + self.assertEqual(next(checksum_iterator), (1,)) + + with self.assertRaisesRegex(Aborted, "retry aborted"): + checksum_iterator.retry(_iter_with_error([], Aborted("retry aborted"))) + + self.assertIs(checksum_iterator.iterator, old_iterator) + + def test_retry_eof_verification(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(list(checksum_iterator), [(1,), (2,)]) + + # Exact match succeeds. + new_iterator = iter([(1,), (2,)]) + checksum_iterator.retry(new_iterator) + self.assertIs(checksum_iterator.iterator, new_iterator) + + # Concurrent insert past EOF fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (2,), (3,)])) + + # Concurrent delete (earlier EOF) fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + # Concurrent modification with same row count fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (99,)])) + + def test_retry_empty_result_set_eof_detects_concurrent_insert(self): + checksum_iterator = ChecksumIterator(iter([])) + self.assertEqual(list(checksum_iterator), []) + self.assertTrue(checksum_iterator.eof_reached) + self.assertEqual(checksum_iterator.next_count, 1) + + # Still empty succeeds. + checksum_iterator.retry(iter([])) + + # Newly inserted row fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + def test_retry_mid_stream_error_matching(self): + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,), (2,)], InternalServerError("mid-stream err")) + ) + self.assertEqual(next(checksum_iterator), (1,)) + self.assertEqual(next(checksum_iterator), (2,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream err"): + next(checksum_iterator) + + # Same prefix and matching gRPC status code + message at error_index succeeds. + new_iterator = _iter_with_error( + [(1,), (2,)], InternalServerError("mid-stream err") + ) + checksum_iterator.retry(new_iterator) + self.assertIs(checksum_iterator.iterator, new_iterator) + + # Different prefix row fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,), (99,)], InternalServerError("mid-stream err")) + ) + + # Replay succeeds past error_index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (2,), (3,)])) + + # Replay reaches EOF at error_index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,), (2,)])) + + # Replay raises same status code with different message fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,), (2,)], InternalServerError("different msg")) + ) + + # Replay raises different error code at error_index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,), (2,)], InvalidArgument("mid-stream err")) + ) + + # Replay raises same error at earlier index fails. + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry( + _iter_with_error([(1,)], InternalServerError("mid-stream err")) + ) + + def test_retry_error_on_first_row(self): + checksum_iterator = ChecksumIterator( + _iter_with_error([], InvalidArgument("bad arg")) + ) + with self.assertRaisesRegex(InvalidArgument, "bad arg"): + next(checksum_iterator) + + checksum_iterator.retry(_iter_with_error([], InvalidArgument("bad arg"))) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(_iter_with_error([], InvalidArgument("other arg"))) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(iter([(1,)])) + + def test_retry_terminal_state_aborted_propagates(self): + eof_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(list(eof_iterator), [(1,)]) + + with self.assertRaisesRegex(Aborted, "abort at eof"): + eof_iterator.retry(_iter_with_error([(1,)], Aborted("abort at eof"))) + + error_iterator = ChecksumIterator( + _iter_with_error([(1,)], InternalServerError("req-1")) + ) + self.assertEqual(next(error_iterator), (1,)) + with self.assertRaisesRegex(InternalServerError, "req-1"): + next(error_iterator) + + with self.assertRaisesRegex(Aborted, "abort at err"): + error_iterator.retry(_iter_with_error([(1,)], Aborted("abort at err"))) + + def test_close_marks_closed_and_closes_closeable_iterator(self): + underlying = _CloseableIterator([(1,), (2,)]) + checksum_iterator = ChecksumIterator(underlying) + self.assertFalse(checksum_iterator.is_closed) + self.assertEqual(next(checksum_iterator), (1,)) + + checksum_iterator.close() + checksum_iterator.close() # Idempotent. + + self.assertTrue(checksum_iterator.is_closed) + self.assertEqual(underlying.close_count, 1) + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(checksum_iterator) + + def test_retry_when_closed_verifies_and_closes_new_iterator(self): + original = _CloseableIterator([(1,), (2,)]) + checksum_iterator = ChecksumIterator(original) + self.assertEqual(next(checksum_iterator), (1,)) + checksum_iterator.close() + + replacement = _CloseableIterator([(1,), (2,)]) + checksum_iterator.retry(replacement) + + self.assertTrue(checksum_iterator.is_closed) + self.assertIs(checksum_iterator.iterator, original) + self.assertEqual(replacement.close_count, 1) + + mismatched = _CloseableIterator([(99,), (2,)]) + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(mismatched) + self.assertEqual(mismatched.close_count, 1) + + def test_close_iterator_exception_logged_and_does_not_mask_retry_error(self): + class FailingCloseIterator: + def __init__(self, rows): + self._iter = iter(rows) + + def __iter__(self): + return self + + def __next__(self): + return next(self._iter) + + def close(self): + raise RuntimeError("close failure") + + checksum_iterator = ChecksumIterator(FailingCloseIterator([(1,)])) + self.assertEqual(next(checksum_iterator), (1,)) + + with self.assertLogs( + "google.cloud.spanner_dbapi.checksum", level="WARNING" + ) as captured_logs: + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + checksum_iterator.retry(FailingCloseIterator([(99,)])) + + self.assertIn( + "Failed to close iterator: close failure", captured_logs.output[0] + ) + + with self.assertLogs( + "google.cloud.spanner_dbapi.checksum", level="WARNING" + ) as close_logs: + checksum_iterator.close() + + self.assertIn("Failed to close iterator: close failure", close_logs.output[0]) + + def test_next_raises_if_closed_during_retry_callback(self): + def close_during_retry(): + checksum_iterator.close() + + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort")), + retry_callback=close_during_retry, + ) + + self.assertEqual(next(checksum_iterator), (1,)) + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(checksum_iterator) + + def test_close_after_eof_and_after_error_raises_interface_error_on_next(self): + eof_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(next(eof_iterator), (1,)) + with self.assertRaises(StopIteration): + next(eof_iterator) + eof_iterator.close() + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(eof_iterator) + + error_iterator = ChecksumIterator( + _iter_with_error([(1,)], InternalServerError("mid-stream boom")) + ) + self.assertEqual(next(error_iterator), (1,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(error_iterator) + error_iterator.close() + with self.assertRaisesRegex( + InterfaceError, "ChecksumIterator is already closed." + ): + next(error_iterator) + + def test_next_multiple_aborted_retries_during_consumption(self): + attempts = [0] + + def retry_callback(): + attempts[0] += 1 + if attempts[0] == 1: + checksum_iterator.retry( + _iter_with_error([(1,), (2,), (3,)], Aborted("abort-2")) + ) + else: + checksum_iterator.retry(iter([(1,), (2,), (3,), (4,)])) + + checksum_iterator = ChecksumIterator( + _iter_with_error([(1,)], Aborted("abort-1")), + retry_callback=retry_callback, + ) + + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,), (4,)]) + self.assertEqual(attempts[0], 2) diff --git a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py index f425b2b32f90..e090ee628018 100644 --- a/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py +++ b/packages/google-cloud-spanner/tests/unit/spanner_dbapi/test_transaction_helper.py @@ -14,18 +14,44 @@ import unittest from unittest import mock -from google.api_core.exceptions import Aborted +import grpc +from google.api_core.exceptions import ( + Aborted, + InternalServerError, + Unknown, + from_grpc_error, +) -from google.cloud.spanner_dbapi.checksum import ResultsChecksum -from google.cloud.spanner_dbapi.exceptions import RetryAborted +from google.cloud.spanner_dbapi.checksum import ( + RETRY_ABORTED_ERROR, + ChecksumIterator, + ResultsChecksum, +) +from google.cloud.spanner_dbapi.exceptions import ( + OperationalError, + ProgrammingError, + RetryAborted, +) from google.cloud.spanner_dbapi.parsed_statement import ParsedStatement, StatementType from google.cloud.spanner_dbapi.transaction_helper import ( + ChecksumStatement, CursorStatementType, ExecuteStatement, FetchStatement, ResultType, TransactionRetryHelper, ) +from google.cloud.spanner_dbapi.utils import PeekIterator +from google.cloud.spanner_v1 import ( + PartialResultSet, + ResultSetMetadata, + StructType, + Type, + TypeCode, +) +from google.cloud.spanner_v1._helpers import _make_value_pb +from google.cloud.spanner_v1.exceptions import wrap_with_request_id +from google.cloud.spanner_v1.streamed import StreamedResultSet def _get_checksum(row): @@ -617,3 +643,714 @@ def test_add_fetch_statement_for_retry_when_last_statement_is_not_fetch(self): self._under_test._statement_result_details_list, [execute_statement, expected_fetch_statement], ) + + def test_add_checksum_iterator_for_retry_when_not_in_transaction(self): + self._under_test._connection._client_transaction_started = False + checksum_iterator = ChecksumIterator(iter([(1,)])) + + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + self.assertEqual(self._under_test._statement_result_details_list, []) + self.assertEqual(self._under_test._last_statement_details_per_cursor, {}) + + def test_add_checksum_iterator_for_retry_registers_statement(self): + self._under_test._connection._client_transaction_started = True + checksum_iterator = ChecksumIterator(iter([(1,)])) + + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + expected_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self.assertEqual( + self._under_test._last_statement_details_per_cursor, + {self._mock_cursor: expected_statement}, + ) + self.assertEqual( + self._under_test._statement_result_details_list, + [expected_statement], + ) + + def test_retry_transaction_checksum_iterator_success_and_result_set_swap(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,), (3,)])) + self.assertEqual(next(checksum_iterator), (1,)) + + original_result_set = mock.Mock() + retried_result_set = mock.Mock() + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = original_result_set + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,), (2,), (3,)]) + retried_cursor._result_set = retried_result_set + self._under_test._connection.cursor.return_value = retried_cursor + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.extend( + [execute_statement, checksum_statement] + ) + + self._under_test.retry_transaction() + + retried_cursor.execute.assert_called_once_with(SQL, ARGS) + self.assertIs(self._mock_cursor._result_set, retried_result_set) + self.assertEqual(list(checksum_iterator), [(2,), (3,)]) + + def test_retry_transaction_stale_checksum_iterator_does_not_clobber_result_set( + self, + ): + checksum_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(list(checksum_iterator), [(1,)]) + + current_result_set = mock.Mock() + self._mock_cursor._itr = mock.Mock() # Moved on to a newer statement. + self._mock_cursor._result_set = current_result_set + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,)]) + retried_cursor._result_set = mock.Mock() + self._under_test._connection.cursor.return_value = retried_cursor + + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.append(checksum_statement) + + self._under_test.retry_transaction() + + self.assertIs(self._mock_cursor._result_set, current_result_set) + + def test_retry_transaction_checksum_iterator_mismatch_raises_retry_aborted(self): + checksum_iterator = ChecksumIterator(iter([(1,)])) + self.assertEqual(list(checksum_iterator), [(1,)]) + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,), (2,)]) # Concurrent insert past EOF. + self._under_test._connection.cursor.return_value = retried_cursor + + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.append(checksum_statement) + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + def test_retry_transaction_checksum_iterator_aborted_retries_again(self): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(next(checksum_iterator), (1,)) + + def failing_then_succeeding(): + if retried_cursor.execute.call_count == 1: + raise Aborted("mid-stream abort") + yield (1,) + yield (2,) + + retried_cursor = mock.Mock() + retried_cursor._itr = None + + def assign_itr(*_args, **_kwargs): + retried_cursor._itr = failing_then_succeeding() + + retried_cursor.execute.side_effect = assign_itr + self._under_test._connection.cursor.return_value = retried_cursor + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.extend( + [execute_statement, checksum_statement] + ) + + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(retried_cursor.execute.call_count, 2) + self.assertEqual(list(checksum_iterator), [(2,)]) + + def test_retry_transaction_unexpected_exception_on_non_exception_statement(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + unexpected_error = InternalServerError("unexpected failure") + run_mock.side_effect = unexpected_error + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR) as context: + self._under_test.retry_transaction() + + self.assertIs(context.exception.__cause__, unexpected_error) + + def test_retry_transaction_unexpected_exception_on_fetch_checksum_statement(self): + fetch_statement = FetchStatement( + statement_type=CursorStatementType.FETCH_ALL, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=_get_checksum(("a",)), + ) + self._under_test._statement_result_details_list.append(fetch_statement) + run_mock = self._under_test._connection.cursor().fetchall = mock.Mock() + unexpected_error = InternalServerError("unexpected mid-stream failure") + run_mock.side_effect = unexpected_error + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR) as context: + self._under_test.retry_transaction() + + self.assertIs(context.exception.__cause__, unexpected_error) + + def test_retry_transaction_same_grpc_error_different_wrapper_details(self): + original_cause = InternalServerError("same error") + original_error = OperationalError(["detail-1"]) + original_error.__cause__ = original_cause + + retried_cause = InternalServerError("same error") + retried_error = OperationalError(["detail-2"]) + retried_error.__cause__ = retried_cause + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=original_error, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = retried_error + + self._under_test.retry_transaction() + + run_mock.assert_called_once_with(SQL, ARGS) + + def test_retry_transaction_same_grpc_code_different_message_raises(self): + original_cause = InternalServerError("first message") + original_error = OperationalError([]) + original_error.__cause__ = original_cause + + retried_cause = InternalServerError("second message") + retried_error = OperationalError([]) + retried_error.__cause__ = retried_cause + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=original_error, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = retried_error + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + def test_retry_transaction_unknown_grpc_error_different_args_raises(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=Unknown("first"), + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = Unknown("second") + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + def test_retry_transaction_aborted_without_errors_list_and_max_retries(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = Aborted("no errors list") + + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaisesRegex(Aborted, "no errors list"): + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(run_mock.call_count, 2) + + def test_retry_transaction_exception_statement_succeeds_raises_retry_aborted(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=InternalServerError("initial failure"), + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.return_value = None + + with self.assertRaisesRegex(RetryAborted, RETRY_ABORTED_ERROR): + self._under_test.retry_transaction() + + run_mock.assert_called_once_with(SQL, ARGS) + + def test_retry_transaction_sleeps_with_retry_delay_on_aborted(self): + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.NONE, + result_details=None, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = [Aborted("transient"), None] + + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.time.sleep" + ) as sleep_mock: + self._under_test.retry_transaction(default_retry_delay=0.25) + + sleep_mock.assert_called_once_with(0.25) + self.assertEqual(run_mock.call_count, 2) + + def test_retry_transaction_multi_cursor_streamed_result_set_and_peek_iterator( + self, + ): + metadata = ResultSetMetadata( + row_type=StructType( + fields=[StructType.Field(name="id", type_=Type(code=TypeCode.INT64))] + ) + ) + + def make_streamed_peek_iterator(chunks, trailing_error=None): + def response_stream(): + for values in chunks: + partial_result_set = PartialResultSet(metadata=metadata) + for value in values: + partial_result_set.values.append(_make_value_pb(value)) + yield partial_result_set + if trailing_error is not None: + raise trailing_error + + result_set = StreamedResultSet(response_stream()) + return result_set, PeekIterator(result_set) + + # Cursor 1: DML statement + query consumed to EOF. + cursor1 = mock.Mock() + result_set_1, peek_iterator_1 = make_streamed_peek_iterator([[1], [2]]) + cursor1._result_set = result_set_1 + first_checksum_iterator = ChecksumIterator(peek_iterator_1) + cursor1._itr = first_checksum_iterator + self.assertEqual(list(first_checksum_iterator), [(1,), (2,)]) + + # Cursor 2: partially consumed query (1 of 3 rows consumed). + cursor2 = mock.Mock() + result_set_2, peek_iterator_2 = make_streamed_peek_iterator([[10], [20, 30]]) + cursor2._result_set = result_set_2 + second_checksum_iterator = ChecksumIterator(peek_iterator_2) + cursor2._itr = second_checksum_iterator + self.assertEqual(next(second_checksum_iterator), (10,)) + + # Cursor 3: query hitting mid-stream error on row 2. + cursor3 = mock.Mock() + result_set_3, peek_iterator_3 = make_streamed_peek_iterator( + [[100]], trailing_error=InternalServerError("mid-stream boom") + ) + cursor3._result_set = result_set_3 + third_checksum_iterator = ChecksumIterator(peek_iterator_3) + cursor3._itr = third_checksum_iterator + self.assertEqual(next(third_checksum_iterator), (100,)) + with self.assertRaisesRegex(InternalServerError, "mid-stream boom"): + next(third_checksum_iterator) + + # Prepare retried cursors corresponding 1-to-1 with original cursors. + retried_cursor1 = mock.Mock() + retried_cursor2 = mock.Mock() + retried_cursor3 = mock.Mock() + self._under_test._connection.cursor.side_effect = [ + retried_cursor1, + retried_cursor2, + retried_cursor3, + ] + + retried_result_set_1, first_retried_peek = make_streamed_peek_iterator( + [[1], [2]] + ) + retried_result_set_2, second_retried_peek = make_streamed_peek_iterator( + [[10], [20, 30]] + ) + retried_result_set_3, third_retried_peek = make_streamed_peek_iterator( + [[100]], trailing_error=InternalServerError("mid-stream boom") + ) + + def exec_cursor1(sql, _args): + if sql.startswith("UPDATE"): + retried_cursor1.rowcount = 4 + retried_cursor1._batch_dml_rows_count = None + else: + retried_cursor1._result_set = retried_result_set_1 + retried_cursor1._itr = first_retried_peek + + def exec_cursor2(_sql, _args): + retried_cursor2._result_set = retried_result_set_2 + retried_cursor2._itr = second_retried_peek + + def exec_cursor3(_sql, _args): + retried_cursor3._result_set = retried_result_set_3 + retried_cursor3._itr = third_retried_peek + + retried_cursor1.execute.side_effect = exec_cursor1 + retried_cursor2.execute.side_effect = exec_cursor2 + retried_cursor3.execute.side_effect = exec_cursor3 + + self._under_test._statement_result_details_list.extend( + [ + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor1, + sql="UPDATE T SET X = 1", + args=None, + result_type=ResultType.ROW_COUNT, + result_details=4, + ), + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor1, + sql="SELECT id FROM T1", + args=None, + result_type=ResultType.NONE, + result_details=None, + ), + ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor1, + result_type=ResultType.CHECKSUM, + result_details=first_checksum_iterator, + ), + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor2, + sql="SELECT id FROM T2", + args=None, + result_type=ResultType.NONE, + result_details=None, + ), + ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor2, + result_type=ResultType.CHECKSUM, + result_details=second_checksum_iterator, + ), + ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=cursor3, + sql="SELECT id FROM T3", + args=None, + result_type=ResultType.NONE, + result_details=None, + ), + ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=cursor3, + result_type=ResultType.CHECKSUM, + result_details=third_checksum_iterator, + ), + ] + ) + + self._under_test.retry_transaction() + + self.assertIs(cursor1._result_set, retried_result_set_1) + self.assertIs(cursor2._result_set, retried_result_set_2) + self.assertIs(cursor3._result_set, retried_result_set_3) + # Cursor 2 seamlessly yields remaining rows from the retried StreamedResultSet. + self.assertEqual(list(second_checksum_iterator), [(20,), (30,)]) + + def test_retry_transaction_closed_checksum_iterator_does_not_overwrite_result_set( + self, + ): + checksum_iterator = ChecksumIterator(iter([(1,), (2,)])) + self.assertEqual(next(checksum_iterator), (1,)) + checksum_iterator.close() + + original_result_set = mock.Mock() + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = original_result_set + + retried_cursor = mock.Mock() + retried_cursor._itr = iter([(1,), (2,)]) + retried_cursor._result_set = mock.Mock() + self._under_test._connection.cursor.return_value = retried_cursor + + checksum_statement = ChecksumStatement( + statement_type=CursorStatementType.CHECKSUM_ITERATOR, + cursor=self._mock_cursor, + result_type=ResultType.CHECKSUM, + result_details=checksum_iterator, + ) + self._under_test._statement_result_details_list.append(checksum_statement) + + self._under_test.retry_transaction() + + self.assertIs(self._mock_cursor._result_set, original_result_set) + + def test_checksum_iterator_retry_callback_invokes_retry_transaction_and_resumes( + self, + ): + self._under_test._connection._client_transaction_started = True + + def failing_stream(): + yield (1,) + raise Aborted("mid-stream abort") + + checksum_iterator = ChecksumIterator( + failing_stream(), + retry_callback=lambda: self._under_test.retry_transaction( + default_retry_delay=0 + ), + ) + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = mock.Mock() + self._mock_cursor._batch_dml_rows_count = None + self._mock_cursor._row_count = None + + retried_result_set = mock.Mock() + retried_cursor = mock.Mock() + retried_cursor.execute.side_effect = lambda _sql, _args: setattr( + retried_cursor, "_itr", iter([(1,), (2,), (3,)]) + ) + retried_cursor._result_set = retried_result_set + self._under_test._connection.cursor.return_value = retried_cursor + + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, SQL, ARGS, None, False + ) + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + self.assertEqual(list(checksum_iterator), [(1,), (2,), (3,)]) + retried_cursor.execute.assert_called_once_with(SQL, ARGS) + self.assertIs(self._mock_cursor._result_set, retried_result_set) + + def test_retry_transaction_with_request_id_augmented_grpc_errors(self): + class StubRpcError(grpc.RpcError): + def __init__(self, status_code, details_message): + super().__init__() + self._status_code = status_code + self._details_message = details_message + + def code(self): + return self._status_code + + def details(self): + return self._details_message + + def trailing_metadata(self): + return () + + original_cause = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.1", + ) + original_error = ProgrammingError(original_cause.details) + original_error.__cause__ = original_cause + + retried_cause = wrap_with_request_id( + from_grpc_error( + StubRpcError(grpc.StatusCode.INVALID_ARGUMENT, "Division by zero") + ), + "1.1.1.1.2", + ) + retried_error = ProgrammingError(retried_cause.details) + retried_error.__cause__ = retried_cause + + execute_statement = ExecuteStatement( + statement_type=CursorStatementType.EXECUTE, + cursor=self._mock_cursor, + sql=SQL, + args=ARGS, + result_type=ResultType.EXCEPTION, + result_details=original_error, + ) + self._under_test._statement_result_details_list.append(execute_statement) + run_mock = self._under_test._connection.cursor().execute = mock.Mock() + run_mock.side_effect = retried_error + + self._under_test.retry_transaction() + + run_mock.assert_called_once_with(SQL, ARGS) + + def test_checksum_iterator_repeated_abort_on_row_k_respects_max_retries(self): + self._under_test._connection._client_transaction_started = True + + def stream_aborting_on_row_2(): + yield (1,) + raise Aborted("row 2 aborted") + + checksum_iterator = ChecksumIterator( + stream_aborting_on_row_2(), + retry_callback=lambda: self._under_test.retry_transaction( + default_retry_delay=0 + ), + ) + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._result_set = mock.Mock() + self._mock_cursor._batch_dml_rows_count = None + self._mock_cursor._row_count = None + + retried_cursor = mock.Mock() + retried_cursor.execute.side_effect = lambda _sql, _args: setattr( + retried_cursor, "_itr", stream_aborting_on_row_2() + ) + self._under_test._connection.cursor.return_value = retried_cursor + + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, SQL, ARGS, None, False + ) + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + + self.assertEqual(next(checksum_iterator), (1,)) + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaisesRegex(Aborted, "row 2 aborted"): + next(checksum_iterator) + + self.assertEqual(retried_cursor.execute.call_count, 2) + + # Calling retry_transaction directly after exhaustion also raises without replaying. + with mock.patch( + "google.cloud.spanner_dbapi.transaction_helper.MAX_INTERNAL_RETRIES", 2 + ): + with self.assertRaisesRegex( + Aborted, "Internal transaction retry maximum exceeded." + ): + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(retried_cursor.execute.call_count, 2) + + # reset() clears _retry_attempts for the next transaction. + self._under_test.reset() + self.assertEqual(self._under_test._retry_attempts, 0) + + def test_retry_transaction_closes_attempt_1_swapped_iterator_on_second_attempt( + self, + ): + class CloseableRowIterator: + def __init__(self, rows): + self._iterator = iter(rows) + self.close_count = 0 + + def __iter__(self): + return self + + def __next__(self): + return next(self._iterator) + + def close(self): + self.close_count += 1 + + self._under_test._connection._client_transaction_started = True + original_iterator = CloseableRowIterator([(1,), (2,)]) + attempt_1_iterator = CloseableRowIterator([(1,), (2,)]) + attempt_2_iterator = CloseableRowIterator([(1,), (2,)]) + + checksum_iterator = ChecksumIterator(original_iterator) + self._mock_cursor._itr = checksum_iterator + self._mock_cursor._batch_dml_rows_count = None + self._mock_cursor._row_count = None + self.assertEqual(next(checksum_iterator), (1,)) + + retried_cursor = mock.Mock() + self._under_test._connection.cursor.return_value = retried_cursor + replay_iterators = iter([attempt_1_iterator, attempt_2_iterator]) + dml_attempt = [0] + + def execute_side_effect(sql, _args): + if sql.startswith("SELECT"): + retried_cursor._itr = next(replay_iterators) + else: + dml_attempt[0] += 1 + if dml_attempt[0] == 1: + raise Aborted("abort on second statement") + retried_cursor.rowcount = 1 + retried_cursor._batch_dml_rows_count = None + + retried_cursor.execute.side_effect = execute_side_effect + + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, "SELECT id FROM T", None, None, False + ) + self._under_test.add_checksum_iterator_for_retry( + self._mock_cursor, checksum_iterator + ) + self._mock_cursor._row_count = 1 + self._mock_cursor.rowcount = 1 + self._under_test.add_execute_statement_for_retry( + self._mock_cursor, "UPDATE T SET x = 1", None, None, False + ) + + self._under_test.retry_transaction(default_retry_delay=0) + + self.assertEqual(original_iterator.close_count, 1) + self.assertEqual(attempt_1_iterator.close_count, 1) + self.assertEqual(attempt_2_iterator.close_count, 0) + self.assertIs(checksum_iterator.iterator, attempt_2_iterator) + self.assertEqual(list(checksum_iterator), [(2,)])