Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type": "feature",
"description": "AWS protocols now retry errors with a known throttling or transient error code, such as EC2's `RequestLimitExceeded`, and errors with a 500, 502, 503 or 504 status code."
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,11 @@
MissingDependencyError,
ModeledError,
)
from smithy_core.interfaces import TypedProperties
from smithy_core.schemas import APIOperation
from smithy_core.shapes import ShapeID

from ...traits import AwsQueryErrorTrait
from ._xml import find_child, local_name, parse_xml_root
from ._xml import find_child, local_name

try:
from smithy_xml import XMLCodec
Expand All @@ -30,18 +29,16 @@

def create_aws_query_error(
*,
body: bytes | Element | None,
root: Element | None,
code: str | None,
operation: APIOperation[Any, Any],
error_registry: TypeRegistry,
default_namespace: str,
wrapper_elements: tuple[str, ...],
status: int,
context: TypedProperties,
retry_after: float | None = None,
) -> CallError:
"""Create a modeled or generic CallError from an awsQuery error response."""
root = parse_xml_root(body) if isinstance(body, bytes) else body
code = _parse_aws_query_error_code(root, wrapper_elements)
if code is not None:
shape_id = _resolve_aws_query_error_shape_id(
code=code,
Expand Down Expand Up @@ -70,28 +67,14 @@ def create_aws_query_error(
if code is not None:
message += f", code: {code}"

is_timeout = status == 408
is_throttle = status == 429
fault = "client" if status < 500 else "server"

return CallError(
message=message,
fault=fault,
is_throttling_error=is_throttle,
is_timeout_error=is_timeout,
is_retry_safe=is_throttle or is_timeout or None,
fault="client" if status < 500 else "server",
retry_after=retry_after,
)


def _assert_xml() -> None:
if not _HAS_XML:
raise MissingDependencyError(
"Attempted to use XML codec, but smithy-xml is not installed."
)


def _parse_aws_query_error_code(
def parse_aws_query_error_code(
element: Element | None, wrapper_elements: tuple[str, ...]
) -> str | None:
"""Parse the ``Code`` field from a wrapped awsQuery error response."""
Expand All @@ -111,6 +94,13 @@ def _parse_aws_query_error_code(
return code_element.text if code_element is not None else None


def _assert_xml() -> None:
if not _HAS_XML:
raise MissingDependencyError(
"Attempted to use XML codec, but smithy-xml is not installed."
)


def _resolve_aws_query_error_shape_id(
*,
code: str,
Expand Down
57 changes: 27 additions & 30 deletions packages/smithy-aws-core/src/smithy_aws_core/protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
from smithy_http.schema_extensions import HTTP_BINDING_SCHEMA_EXTENSION

from ._private.query._xml import parse_xml_root
from ._private.query.errors import create_aws_query_error
from ._private.query.errors import create_aws_query_error, parse_aws_query_error_code
from ._private.query.metadata import parse_aws_query_request_id
from ._private.query.serializers import Ec2QueryShapeSerializer, QueryShapeSerializer
from ._private.restxml import parse_rest_xml_error
Expand Down Expand Up @@ -285,7 +285,7 @@ def _create_error(
modeled_error.args = (error_info.message,)
if retry_after is not None:
modeled_error.retry_after = retry_after
return modeled_error
return self._classify_error_for_retry(modeled_error, response, code)

message = (
f"Unknown error for operation {operation.schema.id} "
Expand All @@ -298,16 +298,12 @@ def _create_error(
if response.reason is not None:
message += f" - reason: {response.reason}"

is_timeout = response.status == 408
is_throttle = response.status == 429
return CallError(
error = CallError(
message=message,
fault="client" if response.status < 500 else "server",
is_throttling_error=is_throttle,
is_timeout_error=is_timeout,
is_retry_safe=is_throttle or is_timeout or None,
retry_after=retry_after,
)
return self._classify_error_for_retry(error, response, code)


class _AWSJSONClientProtocol(_AWSResponseMetadataMixin, RpcClientProtocol):
Expand Down Expand Up @@ -451,6 +447,7 @@ def _create_error(
error_id = self._resolve_error_id(operation=operation, error_id=error_id)

retry_after = parse_retry_after(response)
code = error_id.name if error_id is not None else None

if (
error_id is None
Expand All @@ -459,18 +456,23 @@ def _create_error(
):
deserializer = self.payload_codec.create_deserializer(response_body)
document = deserializer.read_document(schema=DOCUMENT)
document_error_id = document.discriminator
if document_error_id not in error_registry:
document_error_id = self._resolve_error_id(
operation=operation, error_id=document_error_id
)
if (document_error_id := self._document_error_id(document)) is not None:
code = document_error_id.name
if document_error_id not in error_registry:
document_error_id = self._resolve_error_id(
operation=operation, error_id=document_error_id
)

if document_error_id in error_registry:
error_id = document_error_id
if document_error_id in error_registry:
error_id = document_error_id

if error_id is None or error_id not in error_registry:
error_id = self._resolve_query_error_id(operation, response) or error_id

# A query-compatible service's awsQuery code takes precedence.
if (query_error := self._query_error(response)) is not None:
code = query_error[0]

if error_id is not None and error_id in error_registry:
error_shape = error_registry.get(error_id)

Expand All @@ -486,7 +488,7 @@ def _create_error(
modeled_error = error_shape.deserialize(deserializer)
if retry_after is not None:
modeled_error.retry_after = retry_after
return modeled_error
return self._classify_error_for_retry(modeled_error, response, code)

message = (
f"Unknown error for operation {operation.schema.id} "
Expand All @@ -497,20 +499,12 @@ def _create_error(
if response.reason is not None:
message += f" - reason: {response.reason}"

is_timeout = response.status == 408
is_throttle = response.status == 429
fault = self._query_error_fault(response) or (
"client" if response.status < 500 else "server"
)

return CallError(
message=message,
fault=fault,
is_throttling_error=is_throttle,
is_timeout_error=is_timeout,
is_retry_safe=is_throttle or is_timeout or None,
retry_after=retry_after,
)
error = CallError(message=message, fault=fault, retry_after=retry_after)
return self._classify_error_for_retry(error, response, code)

def _matches_content_type(self, response: HTTPResponse) -> bool:
if "content-type" not in response.fields:
Expand Down Expand Up @@ -687,16 +681,19 @@ def _create_error(
context: TypedProperties,
root: Element | None,
) -> CallError:
return create_aws_query_error(
body=root,
wrapper_elements = self._error_wrapper_elements()
code = parse_aws_query_error_code(root, wrapper_elements)
error = create_aws_query_error(
root=root,
code=code,
operation=operation,
error_registry=error_registry,
default_namespace=self._default_namespace,
wrapper_elements=self._error_wrapper_elements(),
wrapper_elements=wrapper_elements,
status=response.status,
context=context,
retry_after=parse_retry_after(response),
)
return self._classify_error_for_retry(error, response, code)

def _create_serializer(
self,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""Retries of known service errors through the generated ec2Query client.

EC2 models none of its errors, so its throttling errors are only known by code.
"""

import inspect
import os
from pathlib import Path
from typing import Any

import pytest

pytest.importorskip("ec2query")

from ec2query.aio.client import AsyncEC2ProtocolClient
from ec2query.aio.config import AsyncEC2ProtocolConfig
from ec2query.client import EC2ProtocolClient
from ec2query.config import EC2ProtocolConfig
from smithy_core.exceptions import CallError
from smithy_core.retries import (
ExponentialRetryBackoffStrategy,
StandardRetryQuota,
StandardRetryStrategy,
)
from smithy_http import HTTPResponse, tuples_to_fields


class _Transport:
"""Answers each request with the next canned response, then with a success."""

TIMEOUT_EXCEPTIONS = ()

def __init__(self, responses: list[tuple[int, str | None]]) -> None:
self.attempts = 0
self._responses = responses

def send(self, request: Any, *, request_config: Any = None) -> HTTPResponse:
self.attempts += 1
if not self._responses:
return HTTPResponse(status=200, fields=tuples_to_fields([]), body=b"")
status, code = self._responses.pop(0)
body = (
f"<Response><Errors><Error><Code>{code}</Code></Error></Errors>"
"<RequestID>request-id</RequestID></Response>"
if code is not None
else ""
)
return HTTPResponse(
status=status,
fields=tuples_to_fields([("content-type", "text/xml")]),
body=body.encode(),
)


class _AsyncTransport(_Transport):
async def send( # pyright: ignore[reportIncompatibleMethodOverride]
self, request: Any, *, request_config: Any = None
) -> HTTPResponse:
return super().send(request, request_config=request_config)


@pytest.fixture(params=[True, False], ids=["sync", "async"])
def sync(request: pytest.FixtureRequest) -> bool:
return request.param


@pytest.fixture(autouse=True)
def _clean_environment(monkeypatch: pytest.MonkeyPatch) -> None:
# Keep the host's AWS settings out of the client's config.
for name in list(os.environ):
if name.startswith("AWS_"):
monkeypatch.delenv(name)


async def _call(
sync: bool,
tmp_path: Path,
responses: list[tuple[int, str | None]],
*,
max_attempts: int = 3,
) -> tuple[_Transport, StandardRetryQuota, Exception | None]:
"""Make a call, returning its transport, retry quota and error, if it failed."""
quota = StandardRetryQuota()
no_backoff = ExponentialRetryBackoffStrategy(backoff_scale_value=0)
missing = str(tmp_path / "missing")
config: dict[str, Any] = {
"config_file_path": missing,
"credentials_file_path": missing,
"region": "us-west-2",
"endpoint_uri": "https://example.com",
"aws_access_key_id": "AKID",
"aws_secret_access_key": "secret",
"retry_strategy": StandardRetryStrategy(
backoff_strategy=no_backoff,
throttling_backoff_strategy=no_backoff,
max_attempts=max_attempts,
retry_quota=quota,
),
}
if sync:
transport = _Transport(responses)
client: Any = EC2ProtocolClient(
config=EC2ProtocolConfig.resolve(transport=transport, **config)
)
else:
transport = _AsyncTransport(responses)
client = AsyncEC2ProtocolClient(
config=await AsyncEC2ProtocolConfig.resolve(transport=transport, **config)
)

try:
result = client.empty_input_and_empty_output()
if inspect.isawaitable(result):
await result
except Exception as e:
return transport, quota, e
return transport, quota, None


@pytest.mark.parametrize(
("status", "code"), [(503, "RequestLimitExceeded"), (502, "EC2ThrottledException")]
)
async def test_throttling_error_is_retried_as_a_throttling_error(
sync: bool, tmp_path: Path, status: int, code: str
) -> None:
transport, quota, error = await _call(
sync, tmp_path, [(status, code)] * 2, max_attempts=2
)

assert isinstance(error, CallError)
assert error.is_throttling_error is True
assert transport.attempts == 2
# The retry was charged as a throttling error.
assert quota.available_capacity == 500 - StandardRetryQuota.THROTTLING_RETRY_COST


@pytest.mark.parametrize(
("status", "code"), [(500, "InternalError"), (503, "Unavailable"), (504, None)]
)
async def test_transient_error_is_retried(
sync: bool, tmp_path: Path, status: int, code: str | None
) -> None:
transport, quota, error = await _call(
sync, tmp_path, [(status, code)] * 2, max_attempts=2
)

assert isinstance(error, CallError)
assert error.is_throttling_error is False
assert transport.attempts == 2
assert quota.available_capacity == 500 - StandardRetryQuota.RETRY_COST


async def test_retried_error_can_succeed(sync: bool, tmp_path: Path) -> None:
transport, _, error = await _call(sync, tmp_path, [(400, "RequestLimitExceeded")])

assert error is None
assert transport.attempts == 2


@pytest.mark.parametrize(
("status", "code"), [(400, "InvalidInstanceID.NotFound"), (501, None)]
)
async def test_other_errors_are_not_retried(
sync: bool, tmp_path: Path, status: int, code: str | None
) -> None:
transport, _, error = await _call(sync, tmp_path, [(status, code)])

assert isinstance(error, CallError)
assert transport.attempts == 1
Loading
Loading