diff --git a/codegen-output-py/features-client/.smithy-python-manifest.json b/codegen-output-py/features-client/.smithy-python-manifest.json index 1eef86129..302189c50 100644 --- a/codegen-output-py/features-client/.smithy-python-manifest.json +++ b/codegen-output-py/features-client/.smithy-python-manifest.json @@ -4,19 +4,19 @@ "pyproject.toml": "3bf9e8ba4f2d5620cb43a34b5a05445485d96bbc71b79d568cfe10e92d3deef6", "src/features_client/__init__.py": "32592f5725712e19c306d0c817040e64c67203ae420e3fac2a05301915b742aa", "src/features_client/_private/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "src/features_client/_private/endpoints.py": "f5bec5c0c17f9ec3fab86ea81c1472b1771118e6064f52cbb58cf85d89e09b20", + "src/features_client/_private/endpoints.py": "90e1bc94c3ca6338b495b94870e7b1e433c1c2da064e336a85ad94616953916a", "src/features_client/_private/schemas.py": "7b2007610a0ec1e352198c77230810edaa6a2ec41cc4d3034b58fd36b278b613", "src/features_client/aio/__init__.py": "2d6c204a7bbceeed5b175fe0df922722d341068027b29287cfdcbb9cb29403af", "src/features_client/aio/client.py": "910b8914637d5e09218664a96792ca6727c941a52d4e678ba4c31fdbc81e360e", - "src/features_client/aio/config.py": "c341294cd16f4940fceb60c8513e29b5b81636a8ba4b999951e09c23efa9e093", + "src/features_client/aio/config.py": "96baec406f19b8641e301fa89088e83a3cdde57a8ae14a270fd16ffe25e70214", "src/features_client/aio/user_agent.py": "05cbb5932a96cb83d55ece172f9b9b5b8aeb625782140647e177d5d5cbd18dd5", "src/features_client/client.py": "fb0acd7beabe576c39ba2f1f1067366d5ba63635ad66f4849280c546fe3d059d", - "src/features_client/config.py": "9ac6470542449161d0aa85f94eec11623e79cf5fbc5db6749b8ce462940e8fdb", + "src/features_client/config.py": "0380227b86fc8409c15e381894be6396fdaa00cdd9a76c7da968537f855adb68", "src/features_client/models.py": "b326674510de3a4f6dd77a4d172cf4dfce08a6ed24f01fa162e99c31dfa13567", "src/features_client/models.pyi": "26858d43ae4e5d88d0ef0db95bd332d12de4784e8b8bf9b0a0a71d54cec5b93a", "src/features_client/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "src/features_client/user_agent.py": "6d7e1477fa0123330b26d750070dcb49afe76bd14baa8bbe4b66a3d28e7ad75f", - "tests/test_endpoints.py": "dd98319dec82644ab5f58d0793e0c2dcf877f358b60b96dcb26f2367c8c37f21" + "tests/test_endpoints.py": "395f805a2889159097487f70f8ee3d6bfb83403ec370dc97e56e5e9304ba93a9" }, "version": 1 } diff --git a/codegen-output-py/features-client/src/features_client/_private/endpoints.py b/codegen-output-py/features-client/src/features_client/_private/endpoints.py index 1cca3f69f..bcc9185e3 100644 --- a/codegen-output-py/features-client/src/features_client/_private/endpoints.py +++ b/codegen-output-py/features-client/src/features_client/_private/endpoints.py @@ -7,29 +7,33 @@ parameters=( ("Endpoint", "string", False, None, "SDK::Endpoint", None), ("Region", "string", False, None, "AWS::Region", None), + ("UseFIPS", "boolean", True, False, "AWS::UseFIPS", None), ("Stage", "string", False, None, None, "stage"), ("ForcePathStyle", "boolean", True, False, None, "force_path_style"), ("Name", "string", False, None, None, None), ("Kind", "string", False, None, None, None), ), - functions=("booleanEquals", "isSet", "isValidHostLabel"), + functions=("aws.partition", "booleanEquals", "isSet", "isValidHostLabel"), conditions=( ((2, "isSet", (0, "Endpoint")), None), # 0: isSet(Endpoint) - ((2, "isSet", (0, "Region")), None), # 1: isSet(Region) - ((2, "booleanEquals", (0, "ForcePathStyle"), True), None), # 2: booleanEquals(ForcePathStyle, true) - ((2, "isSet", (0, "Kind")), None), # 3: isSet(Kind) - ((2, "isSet", (0, "Stage")), None), # 4: isSet(Stage) - ((2, "isSet", (0, "Name")), None), # 5: isSet(Name) - ((2, "isValidHostLabel", (0, "Name"), False), None), # 6: isValidHostLabel(Name, false) + ((2, "booleanEquals", (0, "ForcePathStyle"), True), None), # 1: booleanEquals(ForcePathStyle, true) + ((2, "isSet", (0, "Kind")), None), # 2: isSet(Kind) + ((2, "isSet", (0, "Stage")), None), # 3: isSet(Stage) + ((2, "isSet", (0, "Region")), None), # 4: isSet(Region) + ((2, "aws.partition", (0, "Region")), "PartitionResult"), # 5: PartitionResult = aws.partition(Region) + ((2, "booleanEquals", (0, "UseFIPS"), True), None), # 6: booleanEquals(UseFIPS, true) + ((2, "isSet", (0, "Name")), None), # 7: isSet(Name) + ((2, "isValidHostLabel", (0, "Name"), False), None), # 8: isValidHostLabel(Name, false) ), results=( None, # 0: no match (6, (0, "Endpoint"), None, ()), # 1 - (6, "https://{Region}.example.com", None, ()), # 2 - (6, "https://path.example.com", None, ()), # 3 - (6, "https://{Name}.{Stage}.example.com", None, (("x-kind", ("{Kind}",)),)), # 4 - (6, "https://{Stage}.example.com", (4, ("kind",), "{Kind}"), ()), # 5 - (5, "No stage"), # 6 + (6, "https://path.example.com", None, ()), # 2 + (6, "https://{Name}.{Stage}.example.com", None, (("x-kind", ("{Kind}",)),)), # 3 + (6, "https://{Stage}.example.com", (4, ("kind",), "{Kind}"), ()), # 4 + (6, "https://fieldnotes-fips.{Region}.{PartitionResult#dnsSuffix}", None, ()), # 5 + (6, "https://fieldnotes.{Region}.{PartitionResult#dnsSuffix}", (4, ("authSchemes",), (3, (4, ("name", "signingName", "signingRegion"), "sigv4", "fieldnotes", "{Region}"))), ()), # 6 + (5, "No region or stage"), # 7 ), root=2, # (condition, high, low); 1 and -1 are terminals, a negative reference is a @@ -38,11 +42,13 @@ -1, 1, -1, 0, 100_000_001, 3, 1, 100_000_002, 4, - 2, 100_000_003, 5, - 3, 6, 100_000_006, - 4, 7, 100_000_006, - 5, 8, 100_000_005, - 6, 100_000_004, 100_000_005, + 2, 5, 6, + 3, 9, 6, + 4, 7, 100_000_007, + 5, 8, 100_000_007, + 6, 100_000_005, 100_000_006, + 7, 10, 100_000_004, + 8, 100_000_003, 100_000_004, ), ) # fmt: on diff --git a/codegen-output-py/features-client/src/features_client/aio/config.py b/codegen-output-py/features-client/src/features_client/aio/config.py index dc1060b2d..8548e60d0 100644 --- a/codegen-output-py/features-client/src/features_client/aio/config.py +++ b/codegen-output-py/features-client/src/features_client/aio/config.py @@ -8,8 +8,12 @@ AsyncAwsConfigOverrides, ) from smithy_aws_core.aio.config.filesystem import AsyncFileSystem -from smithy_aws_core.config.resolvers import EndpointUriResolver +from smithy_aws_core.config.resolvers import ( + EndpointUriResolver, + resolve_use_fips_endpoint_program, +) from smithy_aws_core.config.types import FieldSpec +from smithy_aws_core.endpoints import AWS_RULES_EXTENSION from smithy_core.aio.interfaces import AsyncEndpointResolver from smithy_core.aio.interfaces.auth import AsyncAuthScheme from smithy_core.aio.interfaces.identity import AsyncIdentityResolver @@ -57,6 +61,7 @@ class _AsyncFieldNotesConfigOverrides(AsyncAwsConfigOverrides, total=False): ) stage: str | None force_path_style: bool | None + use_fips_endpoint: bool | None @dataclass(kw_only=True, repr=False, init=False) @@ -94,13 +99,18 @@ class AsyncFieldNotesConfig(AsyncAwsConfig): force_path_style: bool | None = None "Forces path-style addressing." + use_fips_endpoint: bool | None = None + "Whether to use a FIPS endpoint." + _FIELDS: ClassVar[dict[str, FieldSpec]] = { **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("field_notes").program ), "endpoint_resolver": FieldSpec( - default_factory=lambda: BddEndpointResolver(BDD) + default_factory=lambda: BddEndpointResolver( + BDD, extensions=[AWS_RULES_EXTENSION] + ) ), "protocol": FieldSpec( default_factory=lambda: AsyncRpcV2CborClientProtocol(_PROTOCOL_SETTINGS), @@ -123,6 +133,9 @@ class AsyncFieldNotesConfig(AsyncAwsConfig): ), "stage": FieldSpec(default=None), "force_path_style": FieldSpec(default=None), + "use_fips_endpoint": FieldSpec( + default=None, resolver=resolve_use_fips_endpoint_program + ), "transport": FieldSpec(default_factory=lambda: AIOHTTPClient()), } diff --git a/codegen-output-py/features-client/src/features_client/config.py b/codegen-output-py/features-client/src/features_client/config.py index 91612c864..b663696d5 100644 --- a/codegen-output-py/features-client/src/features_client/config.py +++ b/codegen-output-py/features-client/src/features_client/config.py @@ -5,8 +5,12 @@ from smithy_aws_core.auth.sigv4 import SigV4AuthScheme from smithy_aws_core.config.aws_config import AwsConfig, AwsConfigOverrides from smithy_aws_core.config.filesystem import FileSystem -from smithy_aws_core.config.resolvers import EndpointUriResolver +from smithy_aws_core.config.resolvers import ( + EndpointUriResolver, + resolve_use_fips_endpoint_program, +) from smithy_aws_core.config.types import FieldSpec +from smithy_aws_core.endpoints import AWS_RULES_EXTENSION from smithy_core.auth import DefaultAuthResolver from smithy_core.endpoints import BddEndpointResolver from smithy_core.interceptors import Interceptor @@ -52,6 +56,7 @@ class _FieldNotesConfigOverrides(AwsConfigOverrides, total=False): ) stage: str | None force_path_style: bool | None + use_fips_endpoint: bool | None @dataclass(kw_only=True, repr=False, init=False) @@ -89,13 +94,18 @@ class FieldNotesConfig(AwsConfig): force_path_style: bool | None = None "Forces path-style addressing." + use_fips_endpoint: bool | None = None + "Whether to use a FIPS endpoint." + _FIELDS: ClassVar[dict[str, FieldSpec]] = { **AwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("field_notes").program ), "endpoint_resolver": FieldSpec( - default_factory=lambda: BddEndpointResolver(BDD) + default_factory=lambda: BddEndpointResolver( + BDD, extensions=[AWS_RULES_EXTENSION] + ) ), "protocol": FieldSpec( default_factory=lambda: RpcV2CborClientProtocol(_PROTOCOL_SETTINGS), @@ -118,6 +128,9 @@ class FieldNotesConfig(AwsConfig): ), "stage": FieldSpec(default=None), "force_path_style": FieldSpec(default=None), + "use_fips_endpoint": FieldSpec( + default=None, resolver=resolve_use_fips_endpoint_program + ), "transport": FieldSpec(default_factory=lambda: Urllib3HTTPClient()), } diff --git a/codegen-output-py/features-client/tests/test_endpoints.py b/codegen-output-py/features-client/tests/test_endpoints.py index 9ebbb3540..3519c8ec6 100644 --- a/codegen-output-py/features-client/tests/test_endpoints.py +++ b/codegen-output-py/features-client/tests/test_endpoints.py @@ -4,6 +4,9 @@ from features_client.aio.client import AsyncFieldNotesClient from features_client.aio.config import AsyncFieldNotesConfig from pytest import mark, param +from smithy_aws_core.aio.identity.static import AsyncStaticCredentialsResolver +from smithy_aws_core.endpoints import AWS_RULES_EXTENSION +from smithy_aws_core.identity.components import AWSCredentialsIdentity from smithy_core.aio.utils import close_async from smithy_core.endpoints import BddEndpointResolver from smithy_core.interceptors import Interceptor @@ -18,8 +21,14 @@ _TEST_CONFIG: dict[str, Any] = { "region": "us-east-1", - "aws_access_key_id": "AKIDTEST", - "aws_secret_access_key": "secret", + "endpoint_uri": None, + "aws_credentials_identity_resolver": AsyncStaticCredentialsResolver( + AWSCredentialsIdentity( + access_key_id="test-access-key-id", + secret_access_key="test-secret-access-key", + ) + ), + "use_fips_endpoint": None, } "Config fields every test sets. A case's own fields replace them." @@ -30,6 +39,10 @@ async def _config(**fields: Any) -> AsyncFieldNotesConfig: ) +_RULES_EXTENSIONS = [AWS_RULES_EXTENSION] +"The extensions of the default resolver, which build the auth schemes a case expects." + + # Every rules test shares one resolver, so its caches see changing parameters. _resolver: BddEndpointResolver | None = None @@ -99,25 +112,58 @@ async def _client( ), param( {"Region": "us-west-2"}, - {"endpoint": {"url": "https://us-west-2.example.com"}}, + { + "endpoint": { + "url": "https://fieldnotes.us-west-2.amazonaws.com", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "us-west-2", + } + ] + }, + } + }, id="Region from the built-in", ), param( - {}, - {"error": "No stage"}, - id="Missing stage", + {"Region": "cn-north-1"}, + { + "endpoint": { + "url": "https://fieldnotes.cn-north-1.amazonaws.com.cn", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "cn-north-1", + } + ] + }, + } + }, + id="Partition from the region", ), param( - {"Kind": "put"}, - {"error": "No stage"}, - id="Missing stage (2)", + {"Region": "us-east-1", "UseFIPS": True}, + {"endpoint": {"url": "https://fieldnotes-fips.us-east-1.amazonaws.com"}}, + id="FIPS from the built-in", + ), + param( + {}, + {"error": "No region or stage"}, + id="No region or stage", ), ] @mark.parametrize(("params", "expect"), RULES_CASES) async def test_endpoint_rules(params: dict[str, Any], expect: dict[str, Any]) -> None: - check_rules_case(await _endpoint_resolver(), params, expect) + check_rules_case( + await _endpoint_resolver(), params, expect, extensions=_RULES_EXTENSIONS + ) CLIENT_CASES = [ @@ -172,19 +218,51 @@ async def test_endpoint_rules(params: dict[str, Any], expect: dict[str, Any]) -> id="Custom endpoint", ), param( + {"region": "us-west-2"}, + AsyncFieldNotesClient.put_note, {}, - AsyncFieldNotesClient.ping, - {}, - {"endpoint": {"url": "https://us-west-2.example.com"}}, + { + "endpoint": { + "url": "https://fieldnotes.us-west-2.amazonaws.com", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "us-west-2", + } + ] + }, + } + }, id="Region from the built-in", - marks=mark.skip(reason="No config field sets the built-in AWS::Region"), ), param( + {"region": "cn-north-1"}, + AsyncFieldNotesClient.ping, {}, + { + "endpoint": { + "url": "https://fieldnotes.cn-north-1.amazonaws.com.cn", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "cn-north-1", + } + ] + }, + } + }, + id="Partition from the region", + ), + param( + {"region": "us-east-1", "use_fips_endpoint": True}, AsyncFieldNotesClient.ping, {}, - {"error": "No stage"}, - id="Missing stage", + {"endpoint": {"url": "https://fieldnotes-fips.us-east-1.amazonaws.com"}}, + id="FIPS from the built-in", ), ] @@ -197,5 +275,9 @@ async def test_endpoint_client( expect: dict[str, Any], ) -> None: await check_client_case_async( - lambda capture: _client(config, capture), operation, input, expect + lambda capture: _client(config, capture), + operation, + input, + expect, + extensions=_RULES_EXTENSIONS, ) diff --git a/designs/codegen/client.md b/designs/codegen/client.md index 797aa0dfe..f091ae9c7 100644 --- a/designs/codegen/client.md +++ b/designs/codegen/client.md @@ -220,8 +220,10 @@ BDD compiled from the default ruleset of smithy-typescript, which reads ruleset into a BDD is Smithy's `CompileBdd` transform. The generated client calls rules functions from the `smithy_core.endpoints` -standard library only. Rules that call any other function fail generation with -a diagnostic naming the functions. +standard library and from the rules extensions integrations add. The default +resolver is `BddEndpointResolver(BDD, extensions=[...])` when there are +extensions. Rules that call any other function fail generation with a diagnostic +naming the functions. Each `smithy.rules#clientContextParams` entry becomes a config field typed `bool | None` or `str | None` and defaulting to None. Its name is the snake-case form @@ -305,7 +307,9 @@ defaults to no change: | `write_config` | The config class, written in place of the core one | | `new_config` | The callable that creates a default config | | `write_client_methods` | Extra client methods, such as `_post_setup` | +| `rules_extensions` | Rules extensions the default endpoint resolver uses | | `endpoint_test_config` | Config fields every generated endpoint test sets | +| `endpoint_test_builtin` | The config field that sets a built-in in an endpoint test | | `client_plugins` | Plugins every client applies, before the core ones | | `write_files` | Extra modules | @@ -353,6 +357,26 @@ not take an `AwsConfig` field name. S3 signing properties come later with S3 support. +`AwsEndpointsIntegration` matches an AWS service, and any service whose rules +call `aws.*` functions or read `AWS::*` built-ins. It follows the Java +generator's `AwsEndpointRulesIntegration`: + +* The default endpoint resolver uses `AWS_RULES_EXTENSION`, which adds + `aws.partition`, `aws.parseArn`, and `aws.isVirtualHostableS3Bucket`, binds + the AWS built-ins to config fields, and builds endpoint auth schemes. +* An AWS config adds a field for each built-in its rules declare: + `use_fips_endpoint` for `AWS::UseFIPS`, `use_dualstack_endpoint` for + `AWS::UseDualStack`, and `account_id_endpoint_mode` for + `AWS::Auth::AccountIdEndpointMode`. Each resolves from the environment and + profile. `AWS::Region` reads the base `region` field. +* Generated endpoint tests set those fields, `region`, and an account ID in + the test credentials from a case's built-ins. Every test also sets a region, + no `endpoint_uri`, static test credentials, and the built-in fields' + defaults, so nothing is read from the environment. + +S3's bucket and S3 Control's account ID host prefix customizations come with +S3 support. + ## Clients and Configuration Service classes are thin subclasses of `Client[Config]` and @@ -416,8 +440,8 @@ standard library. Models that require other auth schemes or event or data streams fail generation with a diagnostic naming the operation and feature. Authentication and endpoint extension points are provider-neutral. AWS config -resolution, the AWS user agent, and SigV4 are AWS integrations. AWS endpoint -functions and AWS protocols are not supported yet. +resolution, the AWS user agent, SigV4, and AWS endpoint functions are AWS +integrations. AWS protocols are not supported yet. ## Validation @@ -436,7 +460,7 @@ model assembled by Smithy with its ruleset compiled by `CompileBdd`: | Feature | Fixture | | --- | --- | -| Endpoint rules | Client context fields, `SDK::Endpoint`, an input context parameter, and static context parameters | +| Endpoint rules | Client context fields, `SDK::Endpoint`, `AWS::Region`, `AWS::UseFIPS`, `aws.partition`, an endpoint auth scheme, an input context parameter, and static context parameters | | Endpoint tests | Rules and client cases, run by the snapshot tests | | SigV4 | The service's default auth | | API key auth | Operations with `@auth([httpApiKeyAuth])`, one also `@optionalAuth` | diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-9e87816034df415ab72ebc61fa2348b4.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-9e87816034df415ab72ebc61fa2348b4.json new file mode 100644 index 000000000..942816b34 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-9e87816034df415ab72ebc61fa2348b4.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Resolve AWS endpoints with the AWS rules extension, so rules can call aws.partition and other AWS functions. AWS configs get fields for the AWS built-ins their rules declare, and generated endpoint tests set AWS built-ins." +} \ No newline at end of file diff --git a/packages/smithy-python/src/smithy_python/_client_emitter.py b/packages/smithy-python/src/smithy_python/_client_emitter.py index 1c20ce97b..0b15296c5 100644 --- a/packages/smithy-python/src/smithy_python/_client_emitter.py +++ b/packages/smithy-python/src/smithy_python/_client_emitter.py @@ -73,7 +73,15 @@ def plan_endpoints( ) for integration in integrations: reserved.update(integration.config_fields(service)) - return _endpoints.plan_endpoints(service, reserved=reserved) + return _endpoints.plan_endpoints( + service, + reserved=reserved, + extensions=[ + extension + for integration in integrations + for extension in integration.rules_extensions(service) + ], + ) def write_client( @@ -151,6 +159,15 @@ def write_client( for integration in integrations for name, value in integration.endpoint_test_config(service).items() }, + lambda builtin, value: next( + ( + entry + for integration in integrations + if (entry := integration.endpoint_test_builtin(service, builtin, value)) + is not None + ), + None, + ), ) aio_init = context.files.python_module( f"src/{directory}/aio/__init__.py", f"{import_name}.aio" @@ -197,7 +214,6 @@ def _typing(name: str) -> TypeReference: _URI: Final = _core("URI", "interfaces") _AUTH_SCHEME_RESOLVER: Final = _core("AuthSchemeResolver", "interfaces.auth") _DEFAULT_AUTH_RESOLVER: Final = _core("DefaultAuthResolver", "auth") -_BDD_ENDPOINT_RESOLVER: Final = _core("BddEndpointResolver", "endpoints") _RETRY_STRATEGY: Final = _core("RetryStrategy", "interfaces.retries") _RETRY_STRATEGY_OPTIONS: Final = _core("RetryStrategyOptions", "retries") _SHAPE_ID: Final = _core("ShapeID", "shapes") @@ -494,10 +510,7 @@ def _write_core_config(writer: PythonWriter, config: ConfigPlan) -> None: writer.line("}") writer.line( "self.endpoint_resolver = endpoint_resolver or ", - _BDD_ENDPOINT_RESOLVER, - "(", - bdd, - ")", + *endpoints.default_resolver(bdd), ) writer.line("self.endpoint_uri = endpoint_uri") writer.line("self.interceptors = interceptors or []") diff --git a/packages/smithy-python/src/smithy_python/_endpoint_tests.py b/packages/smithy-python/src/smithy_python/_endpoint_tests.py index ed69217bd..b866de5e5 100644 --- a/packages/smithy-python/src/smithy_python/_endpoint_tests.py +++ b/packages/smithy-python/src/smithy_python/_endpoint_tests.py @@ -14,7 +14,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final from . import _endpoints @@ -36,7 +36,8 @@ def write_endpoint_tests( client: TypeReference, config: TypeReference, new_config: tuple[str | TypeReference, ...] | None = None, - test_config: Mapping[str, str] | None = None, + test_config: Mapping[str, tuple[Part, ...]] | None = None, + builtin_entry: BuiltinEntry | None = None, ) -> None: """Register ``tests/test_endpoints.py`` when the service has endpoint tests. @@ -44,8 +45,10 @@ def write_endpoint_tests( :param config: The async config class. :param new_config: The callable that creates an async config from keyword fields. Defaults to the config class. - :param test_config: Config fields every test sets, as Python literals by - field name. A case's own fields replace them. + :param test_config: Config fields every test sets, as expressions by field + name. A case's own fields replace them. + :param builtin_entry: The config field and expression that set a built-in to + a case's value, or None when no config field sets it. :raises CodegenError: If a case names an operation the service doesn't contain, expects neither an endpoint nor an error, or gives an operation an input member it doesn't have. @@ -72,9 +75,17 @@ def write_endpoint_tests( config, new_config or (config,), test_config or {}, + builtin_entry or (lambda builtin, value: None), ).write(service, tuple(_object(case, "test case") for case in cases)) +type Part = str | TypeReference +"""A piece of a generated line.""" + +type BuiltinEntry = Callable[[str, _model.Node], tuple[str, tuple[Part, ...]] | None] +"""Finds the config field and expression that set a built-in in a test.""" + + def literal(node: _model.Node) -> str: """Render a JSON node as a one-line Python literal of lists and dicts.""" if node is None: @@ -146,11 +157,16 @@ def __init__( client: TypeReference, config: TypeReference, new_config: tuple[str | TypeReference, ...], - test_config: Mapping[str, str], + test_config: Mapping[str, tuple[Part, ...]], + builtin_entry: BuiltinEntry, ) -> None: self._writer = writer self._new_config = new_config self._test_config = test_config + self._builtin_entry = builtin_entry + self._extensions = tuple( + extension.reference for extension in endpoints.extensions + ) self._plan = plan self._client = client self._config = config @@ -185,7 +201,7 @@ def _write_factories(self) -> None: w.line("_TEST_CONFIG: dict[str, ", any_, "] = {") with w.indent(): for name, value in self._test_config.items(): - w.line(_endpoints.literal(name), ": ", value, ",") + w.line(_endpoints.literal(name), ": ", *value, ",") w.line("}") w.line( _endpoints.literal( @@ -203,6 +219,16 @@ def _write_factories(self) -> None: _RETRY, "(max_attempts=1))", ) + if self._extensions: + w.line() + w.line() + w.line("_RULES_EXTENSIONS = [", *_join(self._extensions, ", "), "]") + w.line( + _endpoints.literal( + "The extensions of the default resolver, which build the auth" + " schemes a case expects." + ) + ) w.line() w.line() w.line( @@ -288,7 +314,12 @@ def _write_rules_tests(self, cases: tuple[Mapping[str, _model.Node], ...]) -> No w.line("params: dict[str, ", _ANY, "], expect: dict[str, ", _ANY, "]") w.line(") -> None:") with w.indent(): - w.line(_CHECK_RULES, "(await _endpoint_resolver(), params, expect)") + w.line( + _CHECK_RULES, + "(await _endpoint_resolver(), params, expect", + self._extensions_argument(), + ")", + ) def _write_client_tests(self, cases: tuple[Mapping[str, _model.Node], ...]) -> None: inputs = [ @@ -337,7 +368,8 @@ def _write_client_tests(self, cases: tuple[Mapping[str, _model.Node], ...]) -> N w.line("await ", _CHECK_CLIENT, "(") with w.indent(): w.line( - "lambda capture: _client(config, capture), operation, input, expect" + "lambda capture: _client(config, capture), operation, input, expect", + self._extensions_argument(), ) w.line(")") @@ -358,7 +390,7 @@ def _write_client_case( ) w.line(_PYTEST_PARAM, "(") with w.indent(): - w.line("{", ", ".join(entries), "},") + w.line("{", *_join(entries, ", "), "},") w.line(self._client, ".", operation.method_name, ",") w.line("{", ", ".join(arguments), "},") w.line(_expectation(case), ",") @@ -371,10 +403,13 @@ def _write_client_case( # The config entries an operation input's built-in and client parameters set. A # parameter the client has no config field for adds the reason the case is # skipped. + def _extensions_argument(self) -> str: + return ", extensions=_RULES_EXTENSIONS" if self._extensions else "" + def _config_entries( self, operation_input: Mapping[str, _model.Node], skips: list[str] - ) -> list[str]: - entries: list[str] = [] + ) -> list[tuple[Part, ...]]: + entries: list[tuple[Part, ...]] = [] builtins = _object(operation_input.get("builtInParams", {}), "builtInParams") for builtin, value in builtins.items(): parameter = self._builtins.get(builtin) @@ -385,7 +420,9 @@ def _config_entries( if field is None and builtin == "SDK::Endpoint": field = "endpoint_uri" if field is not None: - entries.append(f"{_endpoints.literal(field)}: {literal(value)}") + entries.append((f"{_endpoints.literal(field)}: {literal(value)}",)) + elif (entry := self._builtin_entry(builtin, value)) is not None: + entries.append((_endpoints.literal(entry[0]), ": ", *entry[1])) elif parameter is not None and parameter.get("default") != value: skips.append(f"No config field sets the built-in {builtin}") clients = _object(operation_input.get("clientParams", {}), "clientParams") @@ -394,7 +431,7 @@ def _config_entries( if field is None: skips.append(f"{name} is not a client context parameter") else: - entries.append(f"{_endpoints.literal(field)}: {literal(value)}") + entries.append((f"{_endpoints.literal(field)}: {literal(value)}",)) return entries def _arguments( @@ -472,6 +509,20 @@ def _operation(self, name: str) -> OperationPlan: ) +def _join( + items: tuple[TypeReference, ...] | list[tuple[Part, ...]], separator: str +) -> tuple[Part, ...]: + parts: list[Part] = [] + for index, item in enumerate(items): + if index: + parts.append(separator) + if isinstance(item, tuple): + parts.extend(item) + else: + parts.append(item) + return tuple(parts) + + # The case's expect object as the trait models it. def _expectation(case: Mapping[str, _model.Node]) -> str: expect = _object(case.get("expect"), "expect") diff --git a/packages/smithy-python/src/smithy_python/_endpoints.py b/packages/smithy-python/src/smithy_python/_endpoints.py index ee71da60a..e6f58513c 100644 --- a/packages/smithy-python/src/smithy_python/_endpoints.py +++ b/packages/smithy-python/src/smithy_python/_endpoints.py @@ -15,7 +15,7 @@ import binascii import re import struct -from collections.abc import Collection, Mapping +from collections.abc import Collection, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final @@ -60,6 +60,17 @@ class ClientContextField: documentation: str +@dataclass(frozen=True, slots=True) +class RulesExtension: + """A runtime ``smithy_core.endpoints.RulesExtension`` a client's resolver uses.""" + + reference: TypeReference + """The extension constant.""" + + functions: frozenset[str] + """The rules functions it adds to the standard library.""" + + @dataclass(frozen=True, slots=True) class EndpointPlan: """A service's lowered endpoint rules and the config fields they read.""" @@ -69,10 +80,30 @@ class EndpointPlan: bdd: EndpointBdd client_context_fields: tuple[ClientContextField, ...] + extensions: tuple[RulesExtension, ...] = () + """The rules extensions the default endpoint resolver uses, in order.""" + + def default_resolver(self, bdd: TypeReference) -> tuple[str | TypeReference, ...]: + """The default endpoint resolver expression. + + :param bdd: The ``BDD`` constant of the generated endpoints module. + """ + if not self.extensions: + return (_BDD_RESOLVER, "(", bdd, ")") + parts: list[str | TypeReference] = [_BDD_RESOLVER, "(", bdd, ", extensions=["] + for index, extension in enumerate(self.extensions): + if index: + parts.append(", ") + parts.append(extension.reference) + parts.append("])") + return tuple(parts) def plan_endpoints( - service: _model.ServiceShape, *, reserved: Collection[str] = () + service: _model.ServiceShape, + *, + reserved: Collection[str] = (), + extensions: Sequence[RulesExtension] = (), ) -> EndpointPlan: """Lower a service's endpoint rules. @@ -81,9 +112,11 @@ def plan_endpoints( :param service: The service. :param reserved: Config attribute names a client context field must not take. + :param extensions: Rules extensions whose functions the rules may also call. :raises CodegenError: If the service has only an endpointRuleSet trait, its - endpointBdd trait is invalid, its rules call functions outside the rules - standard library, or a client context field is unsupported or collides. + endpointBdd trait is invalid, its rules call functions neither the rules + standard library nor an extension provides, or a client context field is + unsupported or collides. """ if service.has_trait(_ENDPOINT_BDD): trait = _object(service.get_trait(_ENDPOINT_BDD), "the trait") @@ -97,13 +130,21 @@ def plan_endpoints( trait = MANUAL_ENDPOINT_BDD fields = client_context_fields(service, reserved=reserved) bdd = lower(trait, {field.parameter: field.attribute for field in fields}) - unsupported = [name for name in bdd.functions if name not in STANDARD_FUNCTIONS] + provided = STANDARD_FUNCTIONS.union( + *(extension.functions for extension in extensions) + ) + unsupported = [name for name in bdd.functions if name not in provided] if unsupported: raise _exceptions.CodegenError( f"Service {service.id} endpoint rules call {', '.join(unsupported)}, " "which client generation does not support yet" ) - return EndpointPlan(trait=trait, bdd=bdd, client_context_fields=fields) + return EndpointPlan( + trait=trait, + bdd=bdd, + client_context_fields=fields, + extensions=tuple(extensions), + ) def client_context_fields( @@ -272,6 +313,11 @@ def literal(value: Value) -> str: _ENDPOINT_RULE_SET: Final = _model.ShapeId("smithy.rules", "endpointRuleSet") _CLIENT_CONTEXT_PARAMS: Final = _model.ShapeId("smithy.rules", "clientContextParams") _FIELD_TYPES: Final = {"boolean": "bool", "string": "str"} +_BDD_RESOLVER: Final = TypeReference( + "BddEndpointResolver", + "smithy_core.endpoints", + dependencies=(_dependencies.SMITHY_CORE,), +) _ENDPOINT_BDD_TYPE: Final = TypeReference( "EndpointBdd", "smithy_core.endpoints", diff --git a/packages/smithy-python/src/smithy_python/aws/__init__.py b/packages/smithy-python/src/smithy_python/aws/__init__.py index 08237e447..21e9ef176 100644 --- a/packages/smithy-python/src/smithy_python/aws/__init__.py +++ b/packages/smithy-python/src/smithy_python/aws/__init__.py @@ -5,7 +5,8 @@ A service with the ``aws.api#service`` trait gets clients named for its SDK ID and configs that resolve from the environment and shared config files. Its clients also send the AWS user agent. A service with the ``aws.auth#sigv4`` trait also gets SigV4 -auth and the default AWS credentials chain. +auth and the default AWS credentials chain. AWS services, and services whose rules +use AWS functions or built-ins, resolve endpoints with the AWS rules extension. """ from __future__ import annotations @@ -14,14 +15,20 @@ from ._config import AwsConfigIntegration from ._dependencies import SMITHY_AWS_CORE +from ._endpoints import AwsEndpointsIntegration from ._sigv4 import AwsSigV4Integration __all__ = [ "AWS_INTEGRATIONS", "SMITHY_AWS_CORE", "AwsConfigIntegration", + "AwsEndpointsIntegration", "AwsSigV4Integration", ] -AWS_INTEGRATIONS: Final = (AwsConfigIntegration(), AwsSigV4Integration()) +AWS_INTEGRATIONS: Final = ( + AwsConfigIntegration(), + AwsSigV4Integration(), + AwsEndpointsIntegration(), +) """The integrations AWS services get, in order.""" diff --git a/packages/smithy-python/src/smithy_python/aws/_config.py b/packages/smithy-python/src/smithy_python/aws/_config.py index a11737941..7624c72cc 100644 --- a/packages/smithy-python/src/smithy_python/aws/_config.py +++ b/packages/smithy-python/src/smithy_python/aws/_config.py @@ -14,6 +14,7 @@ from ..integrations import ClientIntegration, ClientMode, ClientNames, ConfigPlan from ..symbols import TypeReference from ._dependencies import SMITHY_AWS_CORE +from ._endpoints import BuiltinField, builtin_fields, static_credentials if TYPE_CHECKING: from ..clients import ClientPlan @@ -49,22 +50,28 @@ def client_names( ) def config_fields(self, service: _model.ServiceShape) -> Collection[str]: - del service - return _BASE_FIELDS + return _BASE_FIELDS | {field.name for _, field in builtin_fields(service)} def write_config(self, writer: PythonWriter, config: ConfigPlan) -> bool: _write_config(writer, config) return True - def endpoint_test_config(self, service: _model.ServiceShape) -> Mapping[str, str]: - del service - # AwsConfig requires a region. Static credentials keep the tests from - # reaching the credentials chain. - return { - "region": _quote("us-east-1"), - "aws_access_key_id": _quote("AKIDTEST"), - "aws_secret_access_key": _quote("secret"), + def endpoint_test_config( + self, service: _model.ServiceShape + ) -> Mapping[str, tuple[Part, ...]]: + # AwsConfig requires a region. Explicit values keep endpoint_uri, the + # credentials, and the built-in fields from being read from the + # environment. + config: dict[str, tuple[Part, ...]] = { + "region": (_quote("us-east-1"),), + "endpoint_uri": ("None",), + "aws_credentials_identity_resolver": static_credentials(), } + for _definition, field in builtin_fields(service): + config[field.name] = ( + "None" if field.default is None else _quote(field.default), + ) + return config def new_config( self, mode: ClientMode, config: TypeReference @@ -188,13 +195,7 @@ def _write_config(writer: PythonWriter, config: ConfigPlan) -> None: "endpoint_resolver", (*endpoint_resolver, " | None"), "Resolves the endpoint of each operation.", - ( - "default_factory=lambda: ", - _BDD_ENDPOINT_RESOLVER, - "(", - config.bdd, - ")", - ), + ("default_factory=lambda: ", *endpoints.default_resolver(config.bdd)), ), ( "protocol", @@ -239,6 +240,10 @@ def _write_config(writer: PythonWriter, config: ConfigPlan) -> None: ) for field in endpoints.client_context_fields ), + *( + _builtin_field(definition, field) + for definition, field in builtin_fields(service) + ), ] overrides = f"_{mode.config}Overrides" writer.line("class ", overrides, "(", overrides_base, ", total=False):") @@ -338,6 +343,33 @@ def _write_config(writer: PythonWriter, config: ConfigPlan) -> None: writer.line(")") +def _builtin_field( + definition: Mapping[str, _model.Node], field: BuiltinField +) -> tuple[str, tuple[Part, ...], str, tuple[Part, ...]]: + """A built-in's config field, typed and documented by its rules parameter.""" + kind = definition.get("type") + python_type = ( + "bool" if isinstance(kind, str) and kind.lower() == "boolean" else "str" + ) + documentation = definition.get("documentation") + docs = ( + documentation + if isinstance(documentation, str) + else f"The {definition.get('builtIn')} endpoint parameter." + ) + if field.default is not None: + docs += f" Defaults to {field.default}." + spec: list[Part] = [ + "default=", + "None" if field.default is None else _quote(field.default), + ", resolver=", + field.resolver, + ] + if field.validator is not None: + spec.extend((", validator=", field.validator)) + return (field.name, (python_type, " | None"), docs, tuple(spec)) + + def _default_auth_schemes(config: ConfigPlan) -> tuple[Part, ...]: parts: list[Part] = ["{"] for scheme in config.auth_schemes: @@ -405,4 +437,3 @@ def _join(references: tuple[TypeReference, ...]) -> tuple[Part, ...]: _SHAPE_ID: Final = _core("ShapeID", "shapes") _AUTH_SCHEME_RESOLVER: Final = _core("AuthSchemeResolver", "interfaces.auth") _DEFAULT_AUTH_RESOLVER: Final = _core("DefaultAuthResolver", "auth") -_BDD_ENDPOINT_RESOLVER: Final = _core("BddEndpointResolver", "endpoints") diff --git a/packages/smithy-python/src/smithy_python/aws/_endpoints.py b/packages/smithy-python/src/smithy_python/aws/_endpoints.py new file mode 100644 index 000000000..c47834eda --- /dev/null +++ b/packages/smithy-python/src/smithy_python/aws/_endpoints.py @@ -0,0 +1,192 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The AWS endpoint rules extension and the config fields its built-ins read.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final + +from .. import model as _model +from .. import models as _models +from .._endpoints import RulesExtension +from ..integrations import ClientIntegration +from ..symbols import TypeReference +from ._dependencies import SMITHY_AWS_CORE + +if TYPE_CHECKING: + from ..integrations import Part + +__all__ = [ + "AWS_RULES_EXTENSION", + "BUILTIN_FIELDS", + "AwsEndpointsIntegration", + "BuiltinField", + "builtin_fields", + "static_credentials", +] + + +@dataclass(frozen=True, slots=True) +class BuiltinField: + """A config field the AWS rules extension binds a built-in to.""" + + name: str + default: str | None + """The field's default, or None.""" + + resolver: TypeReference + """The program that resolves the field from the environment and profile.""" + + validator: TypeReference | None = None + + +class AwsEndpointsIntegration(ClientIntegration): + """The AWS rules extension, for AWS services and rules that use AWS parts. + + The default endpoint resolver gets ``AWS_RULES_EXTENSION``, which adds the + ``aws.*`` functions and binds the ``AWS::*`` built-ins to config fields. In + generated endpoint tests of AWS services, those built-ins set their fields. + """ + + def matches(self, service: _model.ServiceShape) -> bool: + return service.has_trait(_SERVICE) or _uses_aws_rules(service) + + def rules_extensions( + self, service: _model.ServiceShape + ) -> tuple[RulesExtension, ...]: + del service + return (AWS_RULES_EXTENSION,) + + def endpoint_test_builtin( + self, service: _model.ServiceShape, builtin: str, value: _model.Node + ) -> tuple[str, tuple[Part, ...]] | None: + # Only AWS configs have fields for the AWS built-ins. + if not service.has_trait(_SERVICE): + return None + if builtin == _REGION: + return ("region", (_literal(value),)) + if (field := BUILTIN_FIELDS.get(builtin)) is not None: + return (field.name, (_literal(value),)) + # The account ID is read from the identity, so the test credentials + # carry it. + if builtin == _ACCOUNT_ID and isinstance(value, str): + return ("aws_credentials_identity_resolver", static_credentials(value)) + return None + + +def builtin_fields( + service: _model.ServiceShape, +) -> tuple[tuple[Mapping[str, _model.Node], BuiltinField], ...]: + """The config field of each built-in the service's rules declare. + + :returns: Each rules parameter definition with its field, in trait order. + """ + trait = service.get_trait(_ENDPOINT_BDD) + parameters = trait.get("parameters") if isinstance(trait, Mapping) else None + if not isinstance(parameters, Mapping): + return () + fields: list[tuple[Mapping[str, _model.Node], BuiltinField]] = [] + for definition in parameters.values(): + if not isinstance(definition, Mapping): + continue + builtin = definition.get("builtIn") + if isinstance(builtin, str) and (field := BUILTIN_FIELDS.get(builtin)): + fields.append((definition, field)) + return tuple(fields) + + +def static_credentials(account_id: str | None = None) -> tuple[Part, ...]: + """An async static credentials resolver with test credentials.""" + arguments = ( + 'access_key_id="test-access-key-id", secret_access_key="test-secret-access-key"' + ) + if account_id is not None: + arguments += f", account_id={_quote(account_id)}" + return (_STATIC_CREDENTIALS, "(", _CREDENTIALS_IDENTITY, f"({arguments}))") + + +def _aws(name: str, module: str) -> TypeReference: + return TypeReference( + name, f"smithy_aws_core.{module}", dependencies=(SMITHY_AWS_CORE,) + ) + + +def _quote(value: str) -> str: + return _models._quote(value) # pyright: ignore[reportPrivateUsage] + + +def _literal(value: _model.Node) -> str: + if isinstance(value, bool): + return "True" if value else "False" + if isinstance(value, str): + return _quote(value) + return "None" if value is None else repr(value) + + +def _uses_aws_rules(service: _model.ServiceShape) -> bool: + trait = service.get_trait(_ENDPOINT_BDD) + if not isinstance(trait, Mapping): + return False + parameters = trait.get("parameters") + if isinstance(parameters, Mapping): + for definition in parameters.values(): + builtin = ( + definition.get("builtIn") if isinstance(definition, Mapping) else None + ) + if isinstance(builtin, str) and builtin.startswith("AWS::"): + return True + return _calls_aws_function(trait.get("conditions")) + + +def _calls_aws_function(node: _model.Node) -> bool: + if isinstance(node, Mapping): + function = node.get("fn") + if isinstance(function, str) and function.startswith("aws."): + return True + return any(_calls_aws_function(value) for value in node.values()) + if isinstance(node, tuple): + return any(_calls_aws_function(item) for item in node) + return False + + +_SERVICE: Final = _model.ShapeId("aws.api", "service") +_ENDPOINT_BDD: Final = _model.ShapeId("smithy.rules", "endpointBdd") +_REGION: Final = "AWS::Region" +_ACCOUNT_ID: Final = "AWS::Auth::AccountId" +_STATIC_CREDENTIALS: Final = _aws( + "AsyncStaticCredentialsResolver", "aio.identity.static" +) +_CREDENTIALS_IDENTITY: Final = _aws("AWSCredentialsIdentity", "identity.components") + +AWS_RULES_EXTENSION: Final = RulesExtension( + reference=_aws("AWS_RULES_EXTENSION", "endpoints"), + functions=frozenset( + {"aws.partition", "aws.parseArn", "aws.isVirtualHostableS3Bucket"} + ), +) +"""``smithy_aws_core.endpoints.AWS_RULES_EXTENSION``.""" + +# Keep in sync with AWS_RULES_EXTENSION in smithy_aws_core.endpoints, and with +# AwsEndpointRulesIntegration.BUILT_IN_FIELDS in the Java generator. +BUILTIN_FIELDS: Final[Mapping[str, BuiltinField]] = { + "AWS::UseFIPS": BuiltinField( + "use_fips_endpoint", + None, + _aws("resolve_use_fips_endpoint_program", "config.resolvers"), + ), + "AWS::UseDualStack": BuiltinField( + "use_dualstack_endpoint", + None, + _aws("resolve_use_dualstack_endpoint_program", "config.resolvers"), + ), + # The SDKs and Tools Reference Guide default. + "AWS::Auth::AccountIdEndpointMode": BuiltinField( + "account_id_endpoint_mode", + "preferred", + _aws("resolve_account_id_endpoint_mode_program", "config.resolvers"), + _aws("validate_account_id_endpoint_mode", "config.validators"), + ), +} +"""The config fields AWS configs add for built-ins, besides ``region``.""" diff --git a/packages/smithy-python/src/smithy_python/integrations.py b/packages/smithy-python/src/smithy_python/integrations.py index be27c1ef7..d34f82076 100644 --- a/packages/smithy-python/src/smithy_python/integrations.py +++ b/packages/smithy-python/src/smithy_python/integrations.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from ._endpoints import RulesExtension from .symbols import TypeReference if TYPE_CHECKING: @@ -23,7 +24,14 @@ from .generation import ClientSettings, GenerationContext from .writer import PythonWriter -__all__ = ["ClientIntegration", "ClientMode", "ClientNames", "ConfigPlan", "Part"] +__all__ = [ + "ClientIntegration", + "ClientMode", + "ClientNames", + "ConfigPlan", + "Part", + "RulesExtension", +] type Part = str | TypeReference """A piece of a generated line.""" @@ -130,8 +138,20 @@ def new_config( del mode, config return None - def endpoint_test_config(self, service: _model.ServiceShape) -> Mapping[str, str]: - """Config fields every generated endpoint test sets, as Python literals. + def rules_extensions( + self, service: _model.ServiceShape + ) -> tuple[RulesExtension, ...]: + """Rules extensions the default endpoint resolver uses. + + Their functions join the standard library the rules may call. + """ + del service + return () + + def endpoint_test_config( + self, service: _model.ServiceShape + ) -> Mapping[str, tuple[Part, ...]]: + """Config fields every generated endpoint test sets, as expressions. A config that resolves from the environment, for example, may need a region and credentials. A test case's own fields replace these. @@ -139,6 +159,18 @@ def endpoint_test_config(self, service: _model.ServiceShape) -> Mapping[str, str del service return {} + def endpoint_test_builtin( + self, service: _model.ServiceShape, builtin: str, value: _model.Node + ) -> tuple[str, tuple[Part, ...]] | None: + """The config field and expression that set a built-in in an endpoint test. + + None if the integration doesn't set the built-in. A test case that sets a + built-in no integration sets, to a value other than its default, is + skipped. + """ + del service, builtin, value + return None + def write_client_methods( self, writer: PythonWriter, diff --git a/packages/smithy-python/tests/unit/test_client_generation.py b/packages/smithy-python/tests/unit/test_client_generation.py index a26c5e37e..8fedf99c3 100644 --- a/packages/smithy-python/tests/unit/test_client_generation.py +++ b/packages/smithy-python/tests/unit/test_client_generation.py @@ -752,3 +752,61 @@ def test_sigv4_trait_needs_a_signing_name(tmp_path: Path) -> None: shapes["example#Greeter"]["traits"]["aws.auth#sigv4"] = {} with pytest.raises(CodegenError, match="needs a name"): _aws_generate(tmp_path, "greeter_no_signing_name", shapes) + + +def _add_aws_rules(shapes: dict[str, Any]) -> None: + nodes = (-1, 1, -1, 0, 100_000_001, 100_000_002) + shapes["example#Greeter"]["traits"]["smithy.rules#endpointBdd"] = { + "version": "1.1", + "parameters": { + "Region": {"type": "string", "builtIn": "AWS::Region"}, + "UseFIPS": { + "type": "boolean", + "builtIn": "AWS::UseFIPS", + "required": True, + "default": False, + "documentation": "Whether to use FIPS.", + }, + }, + "conditions": [ + {"fn": "aws.partition", "argv": [{"ref": "Region"}], "assign": "p"} + ], + "results": [ + { + "conditions": [], + "type": "endpoint", + "endpoint": {"url": "https://{Region}.{p#dnsSuffix}"}, + }, + {"conditions": [], "type": "error", "error": "No region"}, + ], + "root": 2, + "nodeCount": 2, + "nodes": base64.b64encode(struct.pack(">6i", *nodes)).decode(), + } + + +def test_aws_rules_use_the_aws_rules_extension(tmp_path: Path) -> None: + shapes = _shapes() + _add_aws_rules(shapes) + shapes["example#Greeter"]["traits"]["aws.api#service"] = {"sdkId": "Greeter"} + _aws_generate(tmp_path, "greeter_aws_rules", shapes) + config = (tmp_path / "src/greeter_aws_rules/aio/config.py").read_text() + assert "BddEndpointResolver(BDD, extensions=[AWS_RULES_EXTENSION])" in config + assert "use_fips_endpoint: bool | None = None" in config + assert "resolver=resolve_use_fips_endpoint_program" in config + + +def test_non_aws_services_with_aws_rules_use_the_extension(tmp_path: Path) -> None: + shapes = _shapes() + _add_aws_rules(shapes) + _aws_generate(tmp_path, "greeter_plain_aws_rules", shapes) + config = (tmp_path / "src/greeter_plain_aws_rules/config.py").read_text() + assert "class Config:" in config + assert "BddEndpointResolver(BDD, extensions=[AWS_RULES_EXTENSION])" in config + + +def test_aws_rule_functions_need_the_aws_integrations(tmp_path: Path) -> None: + shapes = _shapes() + _add_aws_rules(shapes) + with pytest.raises(CodegenError, match=r"rules call aws\.partition"): + _generate(tmp_path, "greeter_no_aws_rules", shapes) diff --git a/scripts/api-snapshot/api-rex.json b/scripts/api-snapshot/api-rex.json index 36c86e043..812cab85c 100644 --- a/scripts/api-snapshot/api-rex.json +++ b/scripts/api-snapshot/api-rex.json @@ -1330,6 +1330,7 @@ }, "smithy_python.aws": { "AwsConfigIntegration": "rex-class", + "AwsEndpointsIntegration": "rex-class", "AwsSigV4Integration": "rex-class", "SMITHY_AWS_CORE": "rex-Dependency" }, @@ -1345,6 +1346,7 @@ }, "smithy_python.integrations": { "Part": "rex-type_alias", + "RulesExtension": "rex-class", "TypeReference": "rex-class" }, "smithy_python.model": { diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 866a2df67..065966b3d 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -104,7 +104,11 @@ def _format_output(output: Path) -> None: for command in ("check", "format"): args = [sys.executable, "-m", "ruff", command] if command == "check": - args.append("--fix") + # The staged tree is outside the repository, where the config's + # tests ignore doesn't reach, so apply it to the staged tests too. + args.extend( + ("--fix", "--extend-per-file-ignores", f"{output}/**/tests/*:S") + ) subprocess.run( # noqa: S603 [*args, str(output), "--config", str(config), "--no-cache"], check=True, diff --git a/scripts/codegen-output-py/models/features.json b/scripts/codegen-output-py/models/features.json index 798b6823e..c7fa9f5c9 100644 --- a/scripts/codegen-output-py/models/features.json +++ b/scripts/codegen-output-py/models/features.json @@ -76,6 +76,13 @@ "documentation": "The region.", "type": "string" }, + "UseFIPS": { + "builtIn": "AWS::UseFIPS", + "required": true, + "default": false, + "documentation": "Whether to use a FIPS endpoint.", + "type": "boolean" + }, "Stage": { "required": false, "documentation": "The deployment stage.", @@ -107,14 +114,6 @@ } ] }, - { - "fn": "isSet", - "argv": [ - { - "ref": "Region" - } - ] - }, { "fn": "booleanEquals", "argv": [ @@ -140,6 +139,32 @@ } ] }, + { + "fn": "isSet", + "argv": [ + { + "ref": "Region" + } + ] + }, + { + "fn": "aws.partition", + "argv": [ + { + "ref": "Region" + } + ], + "assign": "PartitionResult" + }, + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + }, { "fn": "isSet", "argv": [ @@ -170,15 +195,6 @@ }, "type": "endpoint" }, - { - "conditions": [], - "endpoint": { - "url": "https://{Region}.example.com", - "properties": {}, - "headers": {} - }, - "type": "endpoint" - }, { "conditions": [], "endpoint": { @@ -214,13 +230,39 @@ }, { "conditions": [], - "error": "No stage", + "endpoint": { + "url": "https://fieldnotes-fips.{Region}.{PartitionResult#dnsSuffix}", + "properties": {}, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "endpoint": { + "url": "https://fieldnotes.{Region}.{PartitionResult#dnsSuffix}", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "{Region}" + } + ] + }, + "headers": {} + }, + "type": "endpoint" + }, + { + "conditions": [], + "error": "No region or stage", "type": "error" } ], "root": 2, - "nodeCount": 8, - "nodes": "/////wAAAAH/////AAAAAAX14QEAAAADAAAAAQX14QIAAAAEAAAAAgX14QMAAAAFAAAAAwAAAAYF9eEGAAAABAAAAAcF9eEGAAAABQAAAAgF9eEFAAAABgX14QQF9eEF" + "nodeCount": 10, + "nodes": "/////wAAAAH/////AAAAAAX14QEAAAADAAAAAQX14QIAAAAEAAAAAgAAAAUAAAAGAAAAAwAAAAkAAAAGAAAABAAAAAcF9eEHAAAABQAAAAgF9eEHAAAABgX14QUF9eEGAAAABwAAAAoF9eEEAAAACAX14QMF9eEE" }, "smithy.rules#endpointRuleSet": { "version": "1.0", @@ -235,6 +277,13 @@ "builtIn": "AWS::Region", "documentation": "The region." }, + "UseFIPS": { + "type": "boolean", + "builtIn": "AWS::UseFIPS", + "required": true, + "default": false, + "documentation": "Whether to use a FIPS endpoint." + }, "Stage": { "type": "string", "documentation": "The deployment stage." @@ -273,22 +322,6 @@ }, "type": "endpoint" }, - { - "conditions": [ - { - "fn": "isSet", - "argv": [ - { - "ref": "Region" - } - ] - } - ], - "endpoint": { - "url": "https://{Region}.example.com" - }, - "type": "endpoint" - }, { "conditions": [ { @@ -379,9 +412,66 @@ }, "type": "endpoint" }, + { + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "Region" + } + ] + }, + { + "fn": "aws.partition", + "argv": [ + { + "ref": "Region" + } + ], + "assign": "PartitionResult" + } + ], + "rules": [ + { + "conditions": [ + { + "fn": "booleanEquals", + "argv": [ + { + "ref": "UseFIPS" + }, + true + ] + } + ], + "endpoint": { + "url": "https://fieldnotes-fips.{Region}.{PartitionResult#dnsSuffix}" + }, + "type": "endpoint" + }, + { + "conditions": [], + "endpoint": { + "url": "https://fieldnotes.{Region}.{PartitionResult#dnsSuffix}", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "{Region}" + } + ] + } + }, + "type": "endpoint" + } + ], + "type": "tree" + }, { "conditions": [], - "error": "No stage", + "error": "No region or stage", "type": "error" } ] @@ -524,12 +614,21 @@ }, "expect": { "endpoint": { - "url": "https://us-west-2.example.com" + "url": "https://fieldnotes.us-west-2.amazonaws.com", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "us-west-2" + } + ] + } } }, "operationInputs": [ { - "operationName": "Ping", + "operationName": "PutNote", "builtInParams": { "AWS::Region": "us-west-2" } @@ -537,24 +636,59 @@ ] }, { - "documentation": "Missing stage", - "params": {}, + "documentation": "Partition from the region", + "params": { + "Region": "cn-north-1" + }, "expect": { - "error": "No stage" + "endpoint": { + "url": "https://fieldnotes.cn-north-1.amazonaws.com.cn", + "properties": { + "authSchemes": [ + { + "name": "sigv4", + "signingName": "fieldnotes", + "signingRegion": "cn-north-1" + } + ] + } + } }, "operationInputs": [ { - "operationName": "Ping" + "operationName": "Ping", + "builtInParams": { + "AWS::Region": "cn-north-1" + } } ] }, { - "documentation": "Missing stage", + "documentation": "FIPS from the built-in", "params": { - "Kind": "put" + "Region": "us-east-1", + "UseFIPS": true }, "expect": { - "error": "No stage" + "endpoint": { + "url": "https://fieldnotes-fips.us-east-1.amazonaws.com" + } + }, + "operationInputs": [ + { + "operationName": "Ping", + "builtInParams": { + "AWS::Region": "us-east-1", + "AWS::UseFIPS": true + } + } + ] + }, + { + "documentation": "No region or stage", + "params": {}, + "expect": { + "error": "No region or stage" } } ] diff --git a/scripts/codegen-output-py/models/features.smithy b/scripts/codegen-output-py/models/features.smithy index 14d5b06d5..d437bde30 100644 --- a/scripts/codegen-output-py/models/features.smithy +++ b/scripts/codegen-output-py/models/features.smithy @@ -28,6 +28,7 @@ use smithy.rules#staticContextParams parameters: { Endpoint: { type: "string", builtIn: "SDK::Endpoint", documentation: "A custom endpoint." } Region: { type: "string", builtIn: "AWS::Region", documentation: "The region." } + UseFIPS: { type: "boolean", builtIn: "AWS::UseFIPS", required: true, default: false, documentation: "Whether to use a FIPS endpoint." } Stage: { type: "string", documentation: "The deployment stage." } ForcePathStyle: { type: "boolean", required: true, default: false, documentation: "Forces path-style addressing." } Name: { type: "string", documentation: "The thing name." } @@ -39,11 +40,6 @@ use smithy.rules#staticContextParams endpoint: { url: { ref: "Endpoint" } } type: "endpoint" } - { - conditions: [{ fn: "isSet", argv: [{ ref: "Region" }] }] - endpoint: { url: "https://{Region}.example.com" } - type: "endpoint" - } { conditions: [{ fn: "booleanEquals", argv: [{ ref: "ForcePathStyle" }, true] }] endpoint: { url: "https://path.example.com" } @@ -70,7 +66,31 @@ use smithy.rules#staticContextParams endpoint: { url: "https://{Stage}.example.com", properties: { kind: "{Kind}" } } type: "endpoint" } - { conditions: [], error: "No stage", type: "error" } + { + conditions: [ + { fn: "isSet", argv: [{ ref: "Region" }] } + { fn: "aws.partition", argv: [{ ref: "Region" }], assign: "PartitionResult" } + ] + rules: [ + { + conditions: [{ fn: "booleanEquals", argv: [{ ref: "UseFIPS" }, true] }] + endpoint: { url: "https://fieldnotes-fips.{Region}.{PartitionResult#dnsSuffix}" } + type: "endpoint" + } + { + conditions: [] + endpoint: { + url: "https://fieldnotes.{Region}.{PartitionResult#dnsSuffix}" + properties: { + authSchemes: [{ name: "sigv4", signingName: "fieldnotes", signingRegion: "{Region}" }] + } + } + type: "endpoint" + } + ] + type: "tree" + } + { conditions: [], error: "No region or stage", type: "error" } ] }) @endpointTests({ @@ -146,24 +166,49 @@ use smithy.rules#staticContextParams documentation: "Region from the built-in" params: { Region: "us-west-2" } expect: { - endpoint: { url: "https://us-west-2.example.com" } + endpoint: { + url: "https://fieldnotes.us-west-2.amazonaws.com" + properties: { + authSchemes: [{ name: "sigv4", signingName: "fieldnotes", signingRegion: "us-west-2" }] + } + } + } + operationInputs: [ + { operationName: "PutNote", builtInParams: { "AWS::Region": "us-west-2" } } + ] + } + { + documentation: "Partition from the region" + params: { Region: "cn-north-1" } + expect: { + endpoint: { + url: "https://fieldnotes.cn-north-1.amazonaws.com.cn" + properties: { + authSchemes: [{ name: "sigv4", signingName: "fieldnotes", signingRegion: "cn-north-1" }] + } + } } operationInputs: [ - { operationName: "Ping", builtInParams: { "AWS::Region": "us-west-2" } } + { operationName: "Ping", builtInParams: { "AWS::Region": "cn-north-1" } } ] } { - documentation: "Missing stage" - params: {} - expect: { error: "No stage" } + documentation: "FIPS from the built-in" + params: { Region: "us-east-1", UseFIPS: true } + expect: { + endpoint: { url: "https://fieldnotes-fips.us-east-1.amazonaws.com" } + } operationInputs: [ - { operationName: "Ping" } + { + operationName: "Ping" + builtInParams: { "AWS::Region": "us-east-1", "AWS::UseFIPS": true } + } ] } { - documentation: "Missing stage" - params: { Kind: "put" } - expect: { error: "No stage" } + documentation: "No region or stage" + params: {} + expect: { error: "No region or stage" } } ] }) diff --git a/scripts/codegen-output-py/tests/test_client_snapshots.py b/scripts/codegen-output-py/tests/test_client_snapshots.py index aa2aeb0cd..d677b05f5 100644 --- a/scripts/codegen-output-py/tests/test_client_snapshots.py +++ b/scripts/codegen-output-py/tests/test_client_snapshots.py @@ -175,7 +175,8 @@ def test_generated_endpoint_tests_pass(aws_env: pytest.MonkeyPatch) -> None: check=False, ) assert result.returncode == 0, result.stdout + result.stderr - assert "14 passed, 1 skipped" in result.stdout + assert "17 passed" in result.stdout + assert "skipped" not in result.stdout @pytest.fixture(scope="module") diff --git a/scripts/verifications/import-cost.json b/scripts/verifications/import-cost.json index ef63cd343..97585f4cc 100644 --- a/scripts/verifications/import-cost.json +++ b/scripts/verifications/import-cost.json @@ -16,7 +16,7 @@ "smithy_json": 21, "smithy_json (all public modules)": 21, "smithy_python": 1, - "smithy_python (all public modules)": 32, + "smithy_python (all public modules)": 33, "smithy_test": 44, "smithy_test (all public modules)": 60, "smithy_xml": 20,