From d59d784d4ca11b4625371c2ecbbb15f063cb1b09 Mon Sep 17 00:00:00 2001 From: Antonio Aranda <102337110+arandito@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:08:05 -0500 Subject: [PATCH] Retry known service errors and 5xx responses --- ...ture-8c2599dc3fb64afe86dfa1b8dc1c97b4.json | 4 + .../smithy_aws_core/_private/query/errors.py | 34 +- .../src/smithy_aws_core/protocols.py | 57 ++- .../integration/test_error_code_retries.py | 171 ++++++++ .../smithy-aws-core/tests/unit/test_query.py | 9 +- .../tests/unit/test_retry_classification.py | 367 ++++++++++++++++++ ...gfix-8f8ccd7f8fc44adab0ddc66b395562a0.json | 4 + ...ture-d0e7906d9d90444d8b7d30b164b1da96.json | 4 + .../smithy-http/src/smithy_http/protocols.py | 132 +++++-- .../tests/unit/test_retry_classification.py | 340 ++++++++++++++++ scripts/api-snapshot/api-rex.json | 3 + uv.lock | 2 +- 12 files changed, 1043 insertions(+), 84 deletions(-) create mode 100644 packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-8c2599dc3fb64afe86dfa1b8dc1c97b4.json create mode 100644 packages/smithy-aws-core/tests/integration/test_error_code_retries.py create mode 100644 packages/smithy-aws-core/tests/unit/test_retry_classification.py create mode 100644 packages/smithy-http/.changes/next-release/smithy-http-bugfix-8f8ccd7f8fc44adab0ddc66b395562a0.json create mode 100644 packages/smithy-http/.changes/next-release/smithy-http-feature-d0e7906d9d90444d8b7d30b164b1da96.json create mode 100644 packages/smithy-http/tests/unit/test_retry_classification.py diff --git a/packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-8c2599dc3fb64afe86dfa1b8dc1c97b4.json b/packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-8c2599dc3fb64afe86dfa1b8dc1c97b4.json new file mode 100644 index 000000000..d54b07193 --- /dev/null +++ b/packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-8c2599dc3fb64afe86dfa1b8dc1c97b4.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "AWS protocols now retry errors with a known throttling or transient error code, such as EC2's `RequestLimitExceeded`, and errors with a 500, 502, 503 or 504 status code." +} diff --git a/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py b/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py index accc66fc3..c211855cd 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py @@ -10,12 +10,11 @@ MissingDependencyError, ModeledError, ) -from smithy_core.interfaces import TypedProperties from smithy_core.schemas import APIOperation from smithy_core.shapes import ShapeID from ...traits import AwsQueryErrorTrait -from ._xml import find_child, local_name, parse_xml_root +from ._xml import find_child, local_name try: from smithy_xml import XMLCodec @@ -30,18 +29,16 @@ def create_aws_query_error( *, - body: bytes | Element | None, + root: Element | None, + code: str | None, operation: APIOperation[Any, Any], error_registry: TypeRegistry, default_namespace: str, wrapper_elements: tuple[str, ...], status: int, - context: TypedProperties, retry_after: float | None = None, ) -> CallError: """Create a modeled or generic CallError from an awsQuery error response.""" - root = parse_xml_root(body) if isinstance(body, bytes) else body - code = _parse_aws_query_error_code(root, wrapper_elements) if code is not None: shape_id = _resolve_aws_query_error_shape_id( code=code, @@ -70,28 +67,14 @@ def create_aws_query_error( if code is not None: message += f", code: {code}" - is_timeout = status == 408 - is_throttle = status == 429 - fault = "client" if status < 500 else "server" - return CallError( message=message, - fault=fault, - is_throttling_error=is_throttle, - is_timeout_error=is_timeout, - is_retry_safe=is_throttle or is_timeout or None, + fault="client" if status < 500 else "server", retry_after=retry_after, ) -def _assert_xml() -> None: - if not _HAS_XML: - raise MissingDependencyError( - "Attempted to use XML codec, but smithy-xml is not installed." - ) - - -def _parse_aws_query_error_code( +def parse_aws_query_error_code( element: Element | None, wrapper_elements: tuple[str, ...] ) -> str | None: """Parse the ``Code`` field from a wrapped awsQuery error response.""" @@ -111,6 +94,13 @@ def _parse_aws_query_error_code( return code_element.text if code_element is not None else None +def _assert_xml() -> None: + if not _HAS_XML: + raise MissingDependencyError( + "Attempted to use XML codec, but smithy-xml is not installed." + ) + + def _resolve_aws_query_error_shape_id( *, code: str, diff --git a/packages/smithy-aws-core/src/smithy_aws_core/protocols.py b/packages/smithy-aws-core/src/smithy_aws_core/protocols.py index d30136930..0053570e7 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/protocols.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/protocols.py @@ -44,7 +44,7 @@ from smithy_http.schema_extensions import HTTP_BINDING_SCHEMA_EXTENSION from ._private.query._xml import parse_xml_root -from ._private.query.errors import create_aws_query_error +from ._private.query.errors import create_aws_query_error, parse_aws_query_error_code from ._private.query.metadata import parse_aws_query_request_id from ._private.query.serializers import Ec2QueryShapeSerializer, QueryShapeSerializer from ._private.restxml import parse_rest_xml_error @@ -285,7 +285,7 @@ def _create_error( modeled_error.args = (error_info.message,) if retry_after is not None: modeled_error.retry_after = retry_after - return modeled_error + return self._classify_error_for_retry(modeled_error, response, code) message = ( f"Unknown error for operation {operation.schema.id} " @@ -298,16 +298,12 @@ def _create_error( if response.reason is not None: message += f" - reason: {response.reason}" - is_timeout = response.status == 408 - is_throttle = response.status == 429 - return CallError( + error = CallError( message=message, fault="client" if response.status < 500 else "server", - is_throttling_error=is_throttle, - is_timeout_error=is_timeout, - is_retry_safe=is_throttle or is_timeout or None, retry_after=retry_after, ) + return self._classify_error_for_retry(error, response, code) class _AWSJSONClientProtocol(_AWSResponseMetadataMixin, RpcClientProtocol): @@ -451,6 +447,7 @@ def _create_error( error_id = self._resolve_error_id(operation=operation, error_id=error_id) retry_after = parse_retry_after(response) + code = error_id.name if error_id is not None else None if ( error_id is None @@ -459,18 +456,23 @@ def _create_error( ): deserializer = self.payload_codec.create_deserializer(response_body) document = deserializer.read_document(schema=DOCUMENT) - document_error_id = document.discriminator - if document_error_id not in error_registry: - document_error_id = self._resolve_error_id( - operation=operation, error_id=document_error_id - ) + if (document_error_id := self._document_error_id(document)) is not None: + code = document_error_id.name + if document_error_id not in error_registry: + document_error_id = self._resolve_error_id( + operation=operation, error_id=document_error_id + ) - if document_error_id in error_registry: - error_id = document_error_id + if document_error_id in error_registry: + error_id = document_error_id if error_id is None or error_id not in error_registry: error_id = self._resolve_query_error_id(operation, response) or error_id + # A query-compatible service's awsQuery code takes precedence. + if (query_error := self._query_error(response)) is not None: + code = query_error[0] + if error_id is not None and error_id in error_registry: error_shape = error_registry.get(error_id) @@ -486,7 +488,7 @@ def _create_error( modeled_error = error_shape.deserialize(deserializer) if retry_after is not None: modeled_error.retry_after = retry_after - return modeled_error + return self._classify_error_for_retry(modeled_error, response, code) message = ( f"Unknown error for operation {operation.schema.id} " @@ -497,20 +499,12 @@ def _create_error( if response.reason is not None: message += f" - reason: {response.reason}" - is_timeout = response.status == 408 - is_throttle = response.status == 429 fault = self._query_error_fault(response) or ( "client" if response.status < 500 else "server" ) - return CallError( - message=message, - fault=fault, - is_throttling_error=is_throttle, - is_timeout_error=is_timeout, - is_retry_safe=is_throttle or is_timeout or None, - retry_after=retry_after, - ) + error = CallError(message=message, fault=fault, retry_after=retry_after) + return self._classify_error_for_retry(error, response, code) def _matches_content_type(self, response: HTTPResponse) -> bool: if "content-type" not in response.fields: @@ -687,16 +681,19 @@ def _create_error( context: TypedProperties, root: Element | None, ) -> CallError: - return create_aws_query_error( - body=root, + wrapper_elements = self._error_wrapper_elements() + code = parse_aws_query_error_code(root, wrapper_elements) + error = create_aws_query_error( + root=root, + code=code, operation=operation, error_registry=error_registry, default_namespace=self._default_namespace, - wrapper_elements=self._error_wrapper_elements(), + wrapper_elements=wrapper_elements, status=response.status, - context=context, retry_after=parse_retry_after(response), ) + return self._classify_error_for_retry(error, response, code) def _create_serializer( self, diff --git a/packages/smithy-aws-core/tests/integration/test_error_code_retries.py b/packages/smithy-aws-core/tests/integration/test_error_code_retries.py new file mode 100644 index 000000000..72eab212d --- /dev/null +++ b/packages/smithy-aws-core/tests/integration/test_error_code_retries.py @@ -0,0 +1,171 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Retries of known service errors through the generated ec2Query client. + +EC2 models none of its errors, so its throttling errors are only known by code. +""" + +import inspect +import os +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("ec2query") + +from ec2query.aio.client import AsyncEC2ProtocolClient +from ec2query.aio.config import AsyncEC2ProtocolConfig +from ec2query.client import EC2ProtocolClient +from ec2query.config import EC2ProtocolConfig +from smithy_core.exceptions import CallError +from smithy_core.retries import ( + ExponentialRetryBackoffStrategy, + StandardRetryQuota, + StandardRetryStrategy, +) +from smithy_http import HTTPResponse, tuples_to_fields + + +class _Transport: + """Answers each request with the next canned response, then with a success.""" + + TIMEOUT_EXCEPTIONS = () + + def __init__(self, responses: list[tuple[int, str | None]]) -> None: + self.attempts = 0 + self._responses = responses + + def send(self, request: Any, *, request_config: Any = None) -> HTTPResponse: + self.attempts += 1 + if not self._responses: + return HTTPResponse(status=200, fields=tuples_to_fields([]), body=b"") + status, code = self._responses.pop(0) + body = ( + f"{code}" + "request-id" + if code is not None + else "" + ) + return HTTPResponse( + status=status, + fields=tuples_to_fields([("content-type", "text/xml")]), + body=body.encode(), + ) + + +class _AsyncTransport(_Transport): + async def send( # pyright: ignore[reportIncompatibleMethodOverride] + self, request: Any, *, request_config: Any = None + ) -> HTTPResponse: + return super().send(request, request_config=request_config) + + +@pytest.fixture(params=[True, False], ids=["sync", "async"]) +def sync(request: pytest.FixtureRequest) -> bool: + return request.param + + +@pytest.fixture(autouse=True) +def _clean_environment(monkeypatch: pytest.MonkeyPatch) -> None: + # Keep the host's AWS settings out of the client's config. + for name in list(os.environ): + if name.startswith("AWS_"): + monkeypatch.delenv(name) + + +async def _call( + sync: bool, + tmp_path: Path, + responses: list[tuple[int, str | None]], + *, + max_attempts: int = 3, +) -> tuple[_Transport, StandardRetryQuota, Exception | None]: + """Make a call, returning its transport, retry quota and error, if it failed.""" + quota = StandardRetryQuota() + no_backoff = ExponentialRetryBackoffStrategy(backoff_scale_value=0) + missing = str(tmp_path / "missing") + config: dict[str, Any] = { + "config_file_path": missing, + "credentials_file_path": missing, + "region": "us-west-2", + "endpoint_uri": "https://example.com", + "aws_access_key_id": "AKID", + "aws_secret_access_key": "secret", + "retry_strategy": StandardRetryStrategy( + backoff_strategy=no_backoff, + throttling_backoff_strategy=no_backoff, + max_attempts=max_attempts, + retry_quota=quota, + ), + } + if sync: + transport = _Transport(responses) + client: Any = EC2ProtocolClient( + config=EC2ProtocolConfig.resolve(transport=transport, **config) + ) + else: + transport = _AsyncTransport(responses) + client = AsyncEC2ProtocolClient( + config=await AsyncEC2ProtocolConfig.resolve(transport=transport, **config) + ) + + try: + result = client.empty_input_and_empty_output() + if inspect.isawaitable(result): + await result + except Exception as e: + return transport, quota, e + return transport, quota, None + + +@pytest.mark.parametrize( + ("status", "code"), [(503, "RequestLimitExceeded"), (502, "EC2ThrottledException")] +) +async def test_throttling_error_is_retried_as_a_throttling_error( + sync: bool, tmp_path: Path, status: int, code: str +) -> None: + transport, quota, error = await _call( + sync, tmp_path, [(status, code)] * 2, max_attempts=2 + ) + + assert isinstance(error, CallError) + assert error.is_throttling_error is True + assert transport.attempts == 2 + # The retry was charged as a throttling error. + assert quota.available_capacity == 500 - StandardRetryQuota.THROTTLING_RETRY_COST + + +@pytest.mark.parametrize( + ("status", "code"), [(500, "InternalError"), (503, "Unavailable"), (504, None)] +) +async def test_transient_error_is_retried( + sync: bool, tmp_path: Path, status: int, code: str | None +) -> None: + transport, quota, error = await _call( + sync, tmp_path, [(status, code)] * 2, max_attempts=2 + ) + + assert isinstance(error, CallError) + assert error.is_throttling_error is False + assert transport.attempts == 2 + assert quota.available_capacity == 500 - StandardRetryQuota.RETRY_COST + + +async def test_retried_error_can_succeed(sync: bool, tmp_path: Path) -> None: + transport, _, error = await _call(sync, tmp_path, [(400, "RequestLimitExceeded")]) + + assert error is None + assert transport.attempts == 2 + + +@pytest.mark.parametrize( + ("status", "code"), [(400, "InvalidInstanceID.NotFound"), (501, None)] +) +async def test_other_errors_are_not_retried( + sync: bool, tmp_path: Path, status: int, code: str | None +) -> None: + transport, _, error = await _call(sync, tmp_path, [(status, code)]) + + assert isinstance(error, CallError) + assert transport.attempts == 1 diff --git a/packages/smithy-aws-core/tests/unit/test_query.py b/packages/smithy-aws-core/tests/unit/test_query.py index f89e48f43..d62de4dd7 100644 --- a/packages/smithy-aws-core/tests/unit/test_query.py +++ b/packages/smithy-aws-core/tests/unit/test_query.py @@ -15,7 +15,6 @@ from smithy_core.serializers import ShapeSerializer from smithy_core.shapes import ShapeID, ShapeType from smithy_core.traits import XMLFlattenedTrait, XMLNameTrait -from smithy_core.types import TypedProperties def test_query_list_serialization() -> None: @@ -300,13 +299,13 @@ def _error_test_operation() -> APIOperation[Any, Any]: def test_aws_query_error_sets_retry_after_on_generic_error() -> None: error = create_aws_query_error( - body=b"", + root=None, + code=None, operation=_error_test_operation(), error_registry=TypeRegistry({}), default_namespace="com.example", wrapper_elements=("ErrorResponse", "Error"), status=503, - context=TypedProperties(), retry_after=1.5, ) assert error.retry_after == 1.5 @@ -314,13 +313,13 @@ def test_aws_query_error_sets_retry_after_on_generic_error() -> None: def test_aws_query_error_retry_after_none_by_default() -> None: error = create_aws_query_error( - body=b"", + root=None, + code=None, operation=_error_test_operation(), error_registry=TypeRegistry({}), default_namespace="com.example", wrapper_elements=("ErrorResponse", "Error"), status=503, - context=TypedProperties(), ) assert error.retry_after is None diff --git a/packages/smithy-aws-core/tests/unit/test_retry_classification.py b/packages/smithy-aws-core/tests/unit/test_retry_classification.py new file mode 100644 index 000000000..b764cd774 --- /dev/null +++ b/packages/smithy-aws-core/tests/unit/test_retry_classification.py @@ -0,0 +1,367 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""How the AWS protocols classify error responses for retries.""" + +import json +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, ClassVar, Self, cast +from unittest.mock import Mock + +import pytest +from smithy_aws_core.protocols import ( + AwsJson10ClientProtocol, + AwsJson11ClientProtocol, + AwsQueryClientProtocol, + Ec2QueryClientProtocol, + RestJsonClientProtocol, + RestXmlClientProtocol, +) +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import TypeRegistry +from smithy_core.exceptions import CallError, ModeledError +from smithy_core.interfaces import ProtocolSettings +from smithy_core.prelude import STRING +from smithy_core.program import run_program +from smithy_core.schemas import APIOperation, Schema +from smithy_core.shapes import ShapeID, ShapeType +from smithy_core.traits import Trait +from smithy_core.types import TypedProperties +from smithy_core.utils import read_streaming_blob +from smithy_http import HTTPRequest, HTTPResponse, tuples_to_fields + +_THROTTLING_ERROR_CODES = [ + "Throttling", + "ThrottlingException", + "ThrottledException", + "RequestThrottledException", + "TooManyRequestsException", + "ProvisionedThroughputExceededException", + "TransactionInProgressException", + "RequestLimitExceeded", + "BandwidthLimitExceeded", + "LimitExceededException", + "RequestThrottled", + "SlowDown", + "PriorRequestNotComplete", + "EC2ThrottledException", +] + +_TRANSIENT_ERROR_CODES = ["RequestTimeout", "InternalError", "RequestTimeoutException"] + + +def _json_body(code: str) -> bytes: + return json.dumps({"__type": code}).encode() + + +@dataclass(frozen=True) +class _Case: + protocol: Callable[[ProtocolSettings], Any] + fields: tuple[tuple[str, str], ...] + body: Callable[[str], bytes] + + +_CASES = [ + pytest.param( + _Case( + RestJsonClientProtocol, (("content-type", "application/json"),), _json_body + ), + id="restJson1", + ), + pytest.param( + _Case( + RestXmlClientProtocol, + (("content-type", "application/xml"),), + lambda code: ( + f"{code}" + ).encode(), + ), + id="restXml", + ), + pytest.param( + _Case( + AwsJson10ClientProtocol, + (("content-type", "application/x-amz-json-1.0"),), + _json_body, + ), + id="awsJson1_0", + ), + pytest.param( + _Case( + AwsJson11ClientProtocol, + (("content-type", "application/x-amz-json-1.1"),), + _json_body, + ), + id="awsJson1_1", + ), + pytest.param( + _Case( + AwsQueryClientProtocol, + (("content-type", "text/xml"),), + lambda code: ( + f"{code}" + ).encode(), + ), + id="awsQuery", + ), + pytest.param( + _Case( + Ec2QueryClientProtocol, + (("content-type", "text/xml"),), + lambda code: ( + f"{code}" + "" + ).encode(), + ), + id="ec2Query", + ), +] + +_AWS_JSON_CASES = [p for p in _CASES if p.id in ("awsJson1_0", "awsJson1_1")] +# Deserializing a modeled error with HTTP bindings needs a real operation schema. +_UNBOUND_CASES = [p for p in _CASES if p.id not in ("restJson1", "restXml")] + + +def _settings(*, query_compatible: bool = False) -> ProtocolSettings: + traits = ( + [Trait.new(id=ShapeID("aws.protocols#awsQueryCompatible"))] + if query_compatible + else [] + ) + return ProtocolSettings( + namespace="com.test", + service_target="Service", + version="2020-01-08", + service_schema=Schema.collection( + id=ShapeID("com.test#Service"), shape_type=ShapeType.SERVICE, traits=traits + ), + ) + + +def _error_schema(name: str) -> Schema: + return Schema.collection( + id=ShapeID(f"com.test#{name}"), + traits=[Trait.new(id=ShapeID("smithy.api#error"), value="server")], + members={"message": {"target": STRING}}, + ) + + +class _MessageError(ModeledError): + schema: ClassVar[Schema] + + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: + kwargs: dict[str, Any] = {} + + def _consumer(member: Schema, de: ShapeDeserializer) -> None: + kwargs["message"] = de.read_string(member) + + deserializer.read_struct(cls.schema, consumer=_consumer) + return cls(**kwargs) + + +@dataclass(kw_only=True) +class _UndecidedError(_MessageError): + schema = _error_schema("UndecidedError") + + +@dataclass(kw_only=True) +class _RetryableError(_MessageError): + schema = _error_schema("RetryableError") + is_retry_safe: bool | None = True + + +@dataclass(kw_only=True) +class _NotRetryableError(_MessageError): + schema = _error_schema("NotRetryableError") + is_retry_safe: bool | None = False + + +@dataclass(kw_only=True) +class _ThrottlingException(_MessageError): + schema = _error_schema("ThrottlingException") + + +_MODELED_ERRORS: list[type[_MessageError]] = [ + _UndecidedError, + _RetryableError, + _NotRetryableError, +] + + +def _operation(modeled_errors: list[type[_MessageError]]) -> APIOperation[Any, Any]: + operation = Mock(spec=APIOperation) + operation.schema = Schema( + id=ShapeID("com.test#Operation"), shape_type=ShapeType.OPERATION + ) + operation.output_schema = Schema.collection(id=ShapeID("com.test#Output")) + operation.error_schemas = [error.schema for error in modeled_errors] + operation.input_stream_member = None + operation.output_stream_member = None + return cast("APIOperation[Any, Any]", operation) + + +def _raise( + case: _Case, + status: int, + body: bytes, + *, + fields: tuple[tuple[str, str], ...] = (), + query_compatible: bool = False, + modeled_errors: list[type[_MessageError]] = _MODELED_ERRORS, +) -> CallError: + protocol = case.protocol(_settings(query_compatible=query_compatible)) + context = TypedProperties() + response = HTTPResponse( + status=status, + fields=tuples_to_fields([*case.fields, *fields]), + body=body, + ) + with pytest.raises(CallError) as exc_info: + run_program( + protocol.deserialize_response_program( + operation=_operation(modeled_errors), + request=cast(HTTPRequest, Mock()), + response=response, + error_registry=TypeRegistry( + {error.schema.id: error for error in modeled_errors} + ), + context=context, + read_body=read_streaming_blob, + ) + ) + return exc_info.value + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("status", [500, 502, 503, 504]) +def test_unmodeled_transient_status_is_retry_safe(case: _Case, status: int) -> None: + error = _raise(case, status, case.body("SomeError")) + assert type(error) is CallError + assert error.is_retry_safe is True + assert error.is_throttling_error is False + assert error.fault == "server" + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("status", [500, 503]) +def test_transient_status_without_a_body_is_retry_safe( + case: _Case, status: int +) -> None: + error = _raise(case, status, b"") + assert error.is_retry_safe is True + + +@pytest.mark.parametrize( + "case", [p for p in _CASES if p.id in ("restJson1", "awsJson1_0", "awsJson1_1")] +) +def test_json_body_without_an_error_type_is_an_unmodeled_error(case: _Case) -> None: + # Such as a load balancer's response. + error = _raise(case, 503, b'{"message": "Service Unavailable"}') + assert type(error) is CallError + assert error.is_retry_safe is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("status", [400, 403, 501, 505]) +def test_other_status_leaves_retry_safety_undecided(case: _Case, status: int) -> None: + error = _raise(case, status, case.body("SomeError")) + assert error.is_retry_safe is None + + +@pytest.mark.parametrize("case", _CASES) +def test_unmodeled_429_is_a_throttling_error(case: _Case) -> None: + error = _raise(case, 429, case.body("SomeError")) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +def test_unmodeled_408_is_a_timeout(case: _Case) -> None: + error = _raise(case, 408, case.body("SomeError")) + assert error.is_retry_safe is True + assert error.is_timeout_error is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("code", _THROTTLING_ERROR_CODES) +def test_throttling_error_code_is_a_throttling_error(case: _Case, code: str) -> None: + error = _raise(case, 400, case.body(code)) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("code", _TRANSIENT_ERROR_CODES) +def test_transient_error_code_is_retry_safe(case: _Case, code: str) -> None: + error = _raise(case, 400, case.body(code)) + assert error.is_retry_safe is True + assert error.is_throttling_error is False + + +@pytest.mark.parametrize("case", _CASES) +def test_throttling_error_code_takes_precedence_over_a_transient_status( + case: _Case, +) -> None: + error = _raise(case, 503, case.body("SlowDown")) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +def test_service_specific_error_code_is_not_retry_safe(case: _Case) -> None: + # The SDK retry behavior retries it for STS clients only. + error = _raise(case, 400, case.body("IDPCommunicationError")) + assert error.is_retry_safe is None + + +@pytest.mark.parametrize("case", _UNBOUND_CASES) +def test_modeled_error_with_a_throttling_error_code_is_a_throttling_error( + case: _Case, +) -> None: + error = _raise( + case, + 400, + case.body("ThrottlingException"), + modeled_errors=[_ThrottlingException], + ) + assert type(error) is _ThrottlingException + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _AWS_JSON_CASES) +def test_query_error_code_of_a_query_compatible_service_is_classified( + case: _Case, +) -> None: + error = _raise( + case, + 400, + case.body("SomeError"), + fields=(("x-amzn-query-error", "Throttling;Sender"),), + query_compatible=True, + ) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _UNBOUND_CASES) +@pytest.mark.parametrize( + ("error_type", "status", "expected"), + [ + (_UndecidedError, 503, True), + (_UndecidedError, 500, True), + (_UndecidedError, 429, None), + (_UndecidedError, 408, None), + (_UndecidedError, 400, None), + (_RetryableError, 400, True), + (_NotRetryableError, 503, False), + ], +) +def test_modeled_error_retry_safety( + case: _Case, error_type: type[_MessageError], status: int, expected: bool | None +) -> None: + error = _raise(case, status, case.body(error_type.schema.id.name)) + assert type(error) is error_type + assert error.is_retry_safe is expected + assert error.is_throttling_error is False diff --git a/packages/smithy-http/.changes/next-release/smithy-http-bugfix-8f8ccd7f8fc44adab0ddc66b395562a0.json b/packages/smithy-http/.changes/next-release/smithy-http-bugfix-8f8ccd7f8fc44adab0ddc66b395562a0.json new file mode 100644 index 000000000..608cc5a21 --- /dev/null +++ b/packages/smithy-http/.changes/next-release/smithy-http-bugfix-8f8ccd7f8fc44adab0ddc66b395562a0.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "JSON error bodies with no error type are now unmodeled errors instead of parse failures." +} \ No newline at end of file diff --git a/packages/smithy-http/.changes/next-release/smithy-http-feature-d0e7906d9d90444d8b7d30b164b1da96.json b/packages/smithy-http/.changes/next-release/smithy-http-feature-d0e7906d9d90444d8b7d30b164b1da96.json new file mode 100644 index 000000000..2853b5c6b --- /dev/null +++ b/packages/smithy-http/.changes/next-release/smithy-http-feature-d0e7906d9d90444d8b7d30b164b1da96.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Protocols now retry errors with a known throttling or transient error code, such as `ThrottlingException`, and errors with a 500, 502, 503 or 504 status code. RPCv2 protocols now also retry unmodeled 408 and 429 errors, as the other protocols do." +} diff --git a/packages/smithy-http/src/smithy_http/protocols.py b/packages/smithy-http/src/smithy_http/protocols.py index 4ce1b4248..b728736ce 100644 --- a/packages/smithy-http/src/smithy_http/protocols.py +++ b/packages/smithy-http/src/smithy_http/protocols.py @@ -9,9 +9,10 @@ from smithy_core import URI as _URI from smithy_core.codecs import Codec from smithy_core.deserializers import DeserializeableShape -from smithy_core.documents import TypeRegistry +from smithy_core.documents import Document, TypeRegistry from smithy_core.exceptions import ( CallError, + DiscriminatorError, ExpectationNotMetError, MissingDependencyError, ModeledError, @@ -82,6 +83,31 @@ # ClientOptionalTrait class. _CLIENT_OPTIONAL = ShapeID("smithy.api#clientOptional") +# Known throttling and transient error codes +_THROTTLING_ERROR_CODES = frozenset( + { + "Throttling", + "ThrottlingException", + "ThrottledException", + "RequestThrottledException", + "TooManyRequestsException", + "ProvisionedThroughputExceededException", + "TransactionInProgressException", + "RequestLimitExceeded", + "BandwidthLimitExceeded", + "LimitExceededException", + "RequestThrottled", + "SlowDown", + "PriorRequestNotComplete", + "EC2ThrottledException", + } +) +_TRANSIENT_ERROR_CODES = frozenset( + {"RequestTimeout", "InternalError", "RequestTimeoutException"} +) +# Known transient status codes +_TRANSIENT_STATUS_CODES = frozenset({500, 502, 503, 504}) + try: from smithy_cbor import CBORCodec from smithy_cbor import loads as _cbor_loads @@ -159,6 +185,34 @@ def extract_response_metadata( """ return ResponseMetadata(http_status_code=response.status) + def _document_error_id(self, document: Document) -> ShapeID | None: + """The error shape ID in the document's discriminator, if it has one. + + A body without one, such as a load balancer's, is an unmodeled error. + """ + try: + return document.discriminator + except DiscriminatorError: + return None + + def _classify_error_for_retry[E: CallError]( + self, error: E, response: HTTPResponse, code: str | None + ) -> E: + """Mark the error as retryable if it's a throttling, timeout or transient error.""" + if error.is_retry_safe is False: + return error + status = response.status + modeled = isinstance(error, ModeledError) + if _is_throttling_error(status, code, modeled=modeled): + error.is_retry_safe = True + error.is_throttling_error = True + elif _is_timeout_error(status, modeled=modeled): + error.is_retry_safe = True + error.is_timeout_error = True + elif _is_transient_error(status, code): + error.is_retry_safe = True + return error + class HttpBindingClientProtocol(HttpClientProtocol): """An HTTP-based protocol that uses HTTP binding traits.""" @@ -303,23 +357,27 @@ def _create_error( ) retry_after = self._retry_after(response) + code = error_id.name if error_id is not None else None - if error_id is None and self._matches_content_type(response): + # An empty body has no error type to read. + is_empty = isinstance(response_body, bytes | bytearray) and not response_body + if error_id is None and not is_empty and self._matches_content_type(response): if isinstance(response_body, bytearray): response_body = bytes(response_body) deserializer = self.payload_codec.create_deserializer(source=response_body) document = deserializer.read_document(schema=DOCUMENT) - document_error_id = document.discriminator - if document_error_id not in error_registry: - document_error_id = self._resolve_error_id( - operation=operation, - error_id=document_error_id, - ) - - if document_error_id in error_registry: - error_id = document_error_id - if isinstance(response_body, SeekableBytesReader): - response_body.seek(0) + if (document_error_id := self._document_error_id(document)) is not None: + code = document_error_id.name + if document_error_id not in error_registry: + document_error_id = self._resolve_error_id( + operation=operation, + error_id=document_error_id, + ) + + if document_error_id in error_registry: + error_id = document_error_id + if isinstance(response_body, SeekableBytesReader): + response_body.seek(0) if error_id is not None and error_id in error_registry: error_shape = error_registry.get(error_id) @@ -339,7 +397,7 @@ def _create_error( modeled_error = error_shape.deserialize(deserializer) if retry_after is not None: modeled_error.retry_after = retry_after - return modeled_error + return self._classify_error_for_retry(modeled_error, response, code) message = ( f"Unknown error for operation {operation.schema.id} " @@ -350,18 +408,12 @@ def _create_error( if response.reason is not None: message += f" - reason: {response.status}" - is_timeout = response.status == 408 - is_throttle = response.status == 429 - fault = "client" if response.status < 500 else "server" - - return CallError( + error = CallError( message=message, - fault=fault, - is_throttling_error=is_throttle, - is_timeout_error=is_timeout, - is_retry_safe=is_throttle or is_timeout or None, + fault="client" if response.status < 500 else "server", retry_after=retry_after, ) + return self._classify_error_for_retry(error, response, code) def _retry_after(self, response: HTTPResponse) -> float | None: """The retry delay in seconds requested by the server, if the response carries one.""" @@ -717,7 +769,7 @@ def deserialize_response_program[ ) if actual_protocol != self._smithy_protocol: yield from self._discard_body(operation, response, read_body) - raise CallError( + error = CallError( message=( f"Malformed response for operation {operation.schema.id}: " f"expected Smithy-Protocol header {self._smithy_protocol!r}, " @@ -725,6 +777,8 @@ def deserialize_response_program[ ), fault="client" if response.status < 500 else "server", ) + # The body isn't read, so only the status code classifies the error. + raise self._classify_error_for_retry(error, response, code=None) return ( yield from super().deserialize_response_program( operation=operation, @@ -814,6 +868,7 @@ def _create_error( if error_id is None or error_id not in error_registry: error_id = self._resolve_query_error_id(operation, response) or error_id + code = self._rpc_error_code(response, raw_type) if error_id is not None and error_id in error_registry: error_shape = error_registry.get(error_id) if not issubclass(error_shape, ModeledError): @@ -821,7 +876,8 @@ def _create_error( f"Modeled errors must derive from 'ModeledError', got {error_shape}" ) deserializer = self._codec.create_deserializer(body or self._empty_document) - return error_shape.deserialize(deserializer) + modeled_error = error_shape.deserialize(deserializer) + return self._classify_error_for_retry(modeled_error, response, code) message = ( f"Unknown error for operation {operation.schema.id} " @@ -831,11 +887,20 @@ def _create_error( message += f" - id: {error_id}" elif isinstance(raw_type, str): message += f" - id: {raw_type}" - return CallError( + error = CallError( message=message, fault=self._query_error_fault(response) or ("client" if response.status < 500 else "server"), ) + return self._classify_error_for_retry(error, response, code) + + def _rpc_error_code(self, response: HTTPResponse, raw_type: object) -> str | None: + """The error code, preferring a query-compatible service's awsQuery code.""" + if (query_error := self._query_error(response)) is not None: + return query_error[0] + if isinstance(raw_type, str): + return raw_type.split(":", 1)[0].rpartition("#")[2] or None + return None @staticmethod def _normalize_error_id( @@ -922,6 +987,21 @@ def _assert_json() -> None: ) +def _is_throttling_error(status: int, code: str | None, *, modeled: bool) -> bool: + """Whether the error has a known throttling error code or an unmodeled 429.""" + return code in _THROTTLING_ERROR_CODES or (status == 429 and not modeled) + + +def _is_timeout_error(status: int, *, modeled: bool) -> bool: + """Whether the error is an unmodeled 408.""" + return status == 408 and not modeled + + +def _is_transient_error(status: int, code: str | None) -> bool: + """Whether the error has a 500, 502, 503 or 504 status or a known transient code.""" + return status in _TRANSIENT_STATUS_CODES or code in _TRANSIENT_ERROR_CODES + + def _close(stream: Any) -> Any: """Effect: close a stream, returning the awaitable if its ``close`` is async. diff --git a/packages/smithy-http/tests/unit/test_retry_classification.py b/packages/smithy-http/tests/unit/test_retry_classification.py new file mode 100644 index 000000000..27341a7bc --- /dev/null +++ b/packages/smithy-http/tests/unit/test_retry_classification.py @@ -0,0 +1,340 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""How the RPCv2 protocols classify error responses for retries.""" + +import json +from collections.abc import Callable +from dataclasses import dataclass, replace +from typing import Any, ClassVar, Self, cast +from unittest.mock import Mock + +import pytest +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import TypeRegistry +from smithy_core.exceptions import CallError, ModeledError +from smithy_core.interfaces import ProtocolSettings +from smithy_core.prelude import STRING +from smithy_core.program import run_program +from smithy_core.schemas import APIOperation, Schema +from smithy_core.shapes import ShapeID, ShapeType +from smithy_core.traits import Trait +from smithy_core.types import TypedProperties +from smithy_core.utils import read_streaming_blob +from smithy_http import HTTPRequest, HTTPResponse, tuples_to_fields +from smithy_http.protocols import RpcV2CborClientProtocol, RpcV2JsonClientProtocol + +_THROTTLING_ERROR_CODES = [ + "Throttling", + "ThrottlingException", + "ThrottledException", + "RequestThrottledException", + "TooManyRequestsException", + "ProvisionedThroughputExceededException", + "TransactionInProgressException", + "RequestLimitExceeded", + "BandwidthLimitExceeded", + "LimitExceededException", + "RequestThrottled", + "SlowDown", + "PriorRequestNotComplete", + "EC2ThrottledException", +] + +_TRANSIENT_ERROR_CODES = ["RequestTimeout", "InternalError", "RequestTimeoutException"] + + +def _cbor_body(code: str) -> bytes: + # {"__type": "com.test#"}, with a one-byte string length. + value = f"com.test#{code}".encode() + assert len(value) < 256 + return b"\xa1\x66__type\x78" + bytes([len(value)]) + value + + +def _json_body(code: str) -> bytes: + return json.dumps({"__type": f"com.test#{code}"}).encode() + + +@dataclass(frozen=True) +class _Case: + protocol: Callable[[ProtocolSettings], Any] + fields: tuple[tuple[str, str], ...] + body: Callable[[str], bytes] + + +_CASES = [ + pytest.param( + _Case( + RpcV2CborClientProtocol, + (("smithy-protocol", "rpc-v2-cbor"), ("content-type", "application/cbor")), + _cbor_body, + ), + id="rpcv2Cbor", + ), + pytest.param( + _Case( + RpcV2JsonClientProtocol, + (("smithy-protocol", "rpc-v2-json"), ("content-type", "application/json")), + _json_body, + ), + id="rpcv2Json", + ), +] + + +def _settings(*, query_compatible: bool = False) -> ProtocolSettings: + traits = ( + [Trait.new(id=ShapeID("aws.protocols#awsQueryCompatible"))] + if query_compatible + else [] + ) + return ProtocolSettings( + namespace="com.test", + service_target="Service", + version="2020-01-08", + service_schema=Schema.collection( + id=ShapeID("com.test#Service"), shape_type=ShapeType.SERVICE, traits=traits + ), + ) + + +def _error_schema(name: str) -> Schema: + return Schema.collection( + id=ShapeID(f"com.test#{name}"), + traits=[Trait.new(id=ShapeID("smithy.api#error"), value="server")], + members={"message": {"target": STRING}}, + ) + + +class _MessageError(ModeledError): + schema: ClassVar[Schema] + + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: + kwargs: dict[str, Any] = {} + + def _consumer(member: Schema, de: ShapeDeserializer) -> None: + kwargs["message"] = de.read_string(member) + + deserializer.read_struct(cls.schema, consumer=_consumer) + return cls(**kwargs) + + +@dataclass(kw_only=True) +class _UndecidedError(_MessageError): + schema = _error_schema("UndecidedError") + + +@dataclass(kw_only=True) +class _RetryableError(_MessageError): + schema = _error_schema("RetryableError") + is_retry_safe: bool | None = True + + +@dataclass(kw_only=True) +class _NotRetryableError(_MessageError): + schema = _error_schema("NotRetryableError") + is_retry_safe: bool | None = False + + +@dataclass(kw_only=True) +class _ThrottlingException(_MessageError): + schema = _error_schema("ThrottlingException") + + +_MODELED_ERRORS: list[type[_MessageError]] = [ + _UndecidedError, + _RetryableError, + _NotRetryableError, +] + + +def _operation(modeled_errors: list[type[_MessageError]]) -> APIOperation[Any, Any]: + operation = Mock(spec=APIOperation) + operation.schema = Schema( + id=ShapeID("com.test#Operation"), shape_type=ShapeType.OPERATION + ) + operation.output_schema = Schema.collection(id=ShapeID("com.test#Output")) + operation.error_schemas = [error.schema for error in modeled_errors] + operation.input_stream_member = None + operation.output_stream_member = None + return cast("APIOperation[Any, Any]", operation) + + +def _raise( + case: _Case, + status: int, + body: bytes, + *, + fields: tuple[tuple[str, str], ...] = (), + query_compatible: bool = False, + modeled_errors: list[type[_MessageError]] = _MODELED_ERRORS, +) -> CallError: + protocol = case.protocol(_settings(query_compatible=query_compatible)) + response = HTTPResponse( + status=status, + fields=tuples_to_fields([*case.fields, *fields]), + body=body, + ) + with pytest.raises(CallError) as exc_info: + run_program( + protocol.deserialize_response_program( + operation=_operation(modeled_errors), + request=cast(HTTPRequest, Mock()), + response=response, + error_registry=TypeRegistry( + {error.schema.id: error for error in modeled_errors} + ), + context=TypedProperties(), + read_body=read_streaming_blob, + ) + ) + return exc_info.value + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("status", [500, 502, 503, 504]) +def test_unmodeled_transient_status_is_retry_safe(case: _Case, status: int) -> None: + error = _raise(case, status, case.body("SomeError")) + assert type(error) is CallError + assert error.is_retry_safe is True + assert error.is_throttling_error is False + assert error.fault == "server" + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("status", [500, 503]) +def test_transient_status_without_a_body_is_retry_safe( + case: _Case, status: int +) -> None: + error = _raise(case, status, b"") + assert error.is_retry_safe is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("status", [400, 403, 501, 505]) +def test_other_status_leaves_retry_safety_undecided(case: _Case, status: int) -> None: + error = _raise(case, status, case.body("SomeError")) + assert error.is_retry_safe is None + + +@pytest.mark.parametrize("case", _CASES) +def test_unmodeled_429_is_a_throttling_error(case: _Case) -> None: + error = _raise(case, 429, case.body("SomeError")) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +def test_unmodeled_408_is_a_timeout(case: _Case) -> None: + error = _raise(case, 408, case.body("SomeError")) + assert error.is_retry_safe is True + assert error.is_timeout_error is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("code", _THROTTLING_ERROR_CODES) +def test_throttling_error_code_is_a_throttling_error(case: _Case, code: str) -> None: + error = _raise(case, 400, case.body(code)) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize("code", _TRANSIENT_ERROR_CODES) +def test_transient_error_code_is_retry_safe(case: _Case, code: str) -> None: + error = _raise(case, 400, case.body(code)) + assert error.is_retry_safe is True + assert error.is_throttling_error is False + + +@pytest.mark.parametrize("case", _CASES) +def test_throttling_error_code_takes_precedence_over_a_transient_status( + case: _Case, +) -> None: + error = _raise(case, 503, case.body("SlowDown")) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +def test_service_specific_error_code_is_not_retry_safe(case: _Case) -> None: + # The SDK retry behavior retries it for STS clients only. + error = _raise(case, 400, case.body("IDPCommunicationError")) + assert error.is_retry_safe is None + + +@pytest.mark.parametrize("case", _CASES) +def test_modeled_error_with_a_throttling_error_code_is_a_throttling_error( + case: _Case, +) -> None: + error = _raise( + case, + 400, + case.body("ThrottlingException"), + modeled_errors=[_ThrottlingException], + ) + assert type(error) is _ThrottlingException + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +def test_query_error_code_of_a_query_compatible_service_is_classified( + case: _Case, +) -> None: + error = _raise( + case, + 400, + case.body("SomeError"), + fields=(("x-amzn-query-error", "Throttling;Sender"),), + query_compatible=True, + ) + assert error.is_retry_safe is True + assert error.is_throttling_error is True + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize( + ("status", "retry_safe", "throttling", "timeout"), + [ + (500, True, False, False), + (503, True, False, False), + (429, True, True, False), + (408, True, False, True), + (400, None, False, False), + (200, None, False, False), + ], +) +def test_malformed_response_is_classified_by_status( + case: _Case, status: int, retry_safe: bool | None, throttling: bool, timeout: bool +) -> None: + # Such as a load balancer's response, which has no Smithy-Protocol header. + fields = tuple(f for f in case.fields if f[0] != "smithy-protocol") + malformed = replace(case, fields=fields) + error = _raise(malformed, status, case.body("NotRetryableError")) + assert type(error) is CallError + assert error.is_retry_safe is retry_safe + assert error.is_throttling_error is throttling + assert error.is_timeout_error is timeout + + +@pytest.mark.parametrize("case", _CASES) +@pytest.mark.parametrize( + ("error_type", "status", "expected"), + [ + (_UndecidedError, 503, True), + (_UndecidedError, 500, True), + (_UndecidedError, 429, None), + (_UndecidedError, 408, None), + (_UndecidedError, 400, None), + (_RetryableError, 400, True), + (_NotRetryableError, 503, False), + ], +) +def test_modeled_error_retry_safety( + case: _Case, error_type: type[_MessageError], status: int, expected: bool | None +) -> None: + error = _raise(case, status, case.body(error_type.schema.id.name)) + assert type(error) is error_type + assert error.is_retry_safe is expected + assert error.is_throttling_error is False diff --git a/scripts/api-snapshot/api-rex.json b/scripts/api-snapshot/api-rex.json index dc6c176a3..fe92a8951 100644 --- a/scripts/api-snapshot/api-rex.json +++ b/scripts/api-snapshot/api-rex.json @@ -586,6 +586,7 @@ "URI": "rex-protocol", "XMLCodec": "rex-class", "create_aws_query_error": "rex-function", + "parse_aws_query_error_code": "rex-function", "parse_aws_query_request_id": "rex-function", "parse_document_discriminator": "rex-function", "parse_error_code": "rex-function", @@ -1181,6 +1182,8 @@ "DOCUMENT": "rex-Schema", "DefaultTrait": "rex-class", "DeserializeableShape": "rex-protocol", + "DiscriminatorError": "rex-exception", + "Document": "rex-class", "Endpoint": "rex-protocol", "ExpectationNotMetError": "rex-exception", "HTTPBindingSerializer": "rex-class", diff --git a/uv.lock b/uv.lock index 59c1a388f..6b94175fa 100644 --- a/uv.lock +++ b/uv.lock @@ -667,7 +667,7 @@ wheels = [ [[package]] name = "smithy-aws-checksums" -version = "0.1.0" +version = "0.1.1" source = { editable = "packages/smithy-aws-checksums" } [[package]]