Add support for endpoint discovery in core (#1247)

jderusse · web-flow · commit 279d9d782ed9 · 2022-06-09T00:15:33.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## NOT RELEASED
 
+### Added
+
+- Added support for endpoint discovery
+
 ## 1.15.0
 
 ### Added
diff --git a/src/AbstractApi.php b/src/AbstractApi.php
@@ -9,8 +9,10 @@
 use AsyncAws\Core\Credentials\CacheProvider;
 use AsyncAws\Core\Credentials\ChainProvider;
 use AsyncAws\Core\Credentials\CredentialProvider;
+use AsyncAws\Core\EndpointDiscovery\EndpointCache;
 use AsyncAws\Core\Exception\InvalidArgument;
 use AsyncAws\Core\Exception\LogicException;
+use AsyncAws\Core\Exception\RuntimeException;
 use AsyncAws\Core\HttpClient\AwsRetryStrategy;
 use AsyncAws\Core\Signer\Signer;
 use AsyncAws\Core\Signer\SignerV4;
@@ -59,6 +61,11 @@ abstract class AbstractApi
      */
     private $awsErrorFactory;
 
+    /**
+     * @var EndpointCache
+     */
+    private $endpointCache;
+
     /**
      * @param Configuration|array $configuration
      */
@@ -72,6 +79,7 @@ public function __construct($configuration = [], ?CredentialProvider $credential
 
         $this->logger = $logger ?? new NullLogger();
         $this->awsErrorFactory = $this->getAwsErrorFactory();
+        $this->endpointCache = new EndpointCache();
         if (!isset($httpClient)) {
             $httpClient = HttpClient::create();
             if (class_exists(RetryableHttpClient::class)) {
@@ -132,7 +140,7 @@ protected function getSignatureScopeName(): string
 
     final protected function getResponse(Request $request, ?RequestContext $context = null): Response
     {
-        $request->setEndpoint($this->getEndpoint($request->getUri(), $request->getQuery(), $context ? $context->getRegion() : null));
+        $request->setEndpoint($this->getDiscoveredEndpoint($request->getUri(), $request->getQuery(), $context ? $context->getRegion() : null, $context ? $context->usesEndpointDiscovery() : false, $context ? $context->requiresEndpointDiscovery() : false));
 
         if (null !== $credentials = $this->credentialProvider->getCredentials($this->configuration)) {
             $this->getSigner($context ? $context->getRegion() : null)->sign($request, $credentials, $context ?? new RequestContext());
@@ -166,7 +174,7 @@ final protected function getResponse(Request $request, ?RequestContext $context
             ]);
         }
 
-        return new Response($response, $this->httpClient, $this->logger, $this->awsErrorFactory, $debug, $context ? $context->getExceptionMapping() : []);
+        return new Response($response, $this->httpClient, $this->logger, $this->awsErrorFactory, $this->endpointCache, $request, $debug, $context ? $context->getExceptionMapping() : []);
     }
 
     /**
@@ -255,6 +263,49 @@ protected function getEndpoint(string $uri, array $query, ?string $region): stri
         return $endpoint . (false === strpos($endpoint, '?') ? '?' : '&') . http_build_query($query);
     }
 
+    protected function discoverEndpoints(?string $region): array
+    {
+        throw new LogicException(sprintf('The Client "%s" must implement the "%s" method.', \get_class($this), 'discoverEndpoints'));
+    }
+
+    private function getDiscoveredEndpoint(string $uri, array $query, ?string $region, bool $usesEndpointDiscovery, bool $requiresEndpointDiscovery)
+    {
+        if (!$this->configuration->isDefault('endpoint')) {
+            return $this->getEndpoint($uri, $query, $region);
+        }
+
+        $usesEndpointDiscovery = $requiresEndpointDiscovery || ($usesEndpointDiscovery && filter_var($this->configuration->get(Configuration::OPTION_ENDPOINT_DISCOVERY_ENABLED), \FILTER_VALIDATE_BOOLEAN));
+        if (!$usesEndpointDiscovery) {
+            return $this->getEndpoint($uri, $query, $region);
+        }
+
+        // 1. use an active endpoints
+        if (null === $endpoint = $this->endpointCache->getActiveEndpoint($region)) {
+            $previous = null;
+
+            try {
+                // 2. call API to fetch new endpoints
+                $endpoints = $this->discoverEndpoints($region);
+                $this->endpointCache->addEndpoints($region, $endpoints);
+
+                // 3. use active endpoints that has just been injected
+                $endpoint = $this->endpointCache->getActiveEndpoint($region);
+            } catch (\Exception $previous) {
+            }
+
+            // 4. if endpoint is still null, fallback to expired endpoint
+            if (null === $endpoint && null === $endpoint = $this->endpointCache->getExpiredEndpoint($region)) {
+                if ($requiresEndpointDiscovery) {
+                    throw new RuntimeException(sprintf('The Client "%s" failed to fetch the endpoint.', \get_class($this)), 0, $previous);
+                }
+
+                return $this->getEndpoint($uri, $query, $region);
+            }
+        }
+
+        return $endpoint;
+    }
+
     /**
      * @param ?string $region region provided by the user in the `@region` parameter of the Input
      */
diff --git a/src/Configuration.php b/src/Configuration.php
@@ -30,6 +30,7 @@ final class Configuration
     public const OPTION_WEB_IDENTITY_TOKEN_FILE = 'webIdentityTokenFile';
     public const OPTION_ROLE_SESSION_NAME = 'roleSessionName';
     public const OPTION_CONTAINER_CREDENTIALS_RELATIVE_URI = 'containerCredentialsRelativeUri';
+    public const OPTION_ENDPOINT_DISCOVERY_ENABLED = 'endpointDiscoveryEnabled';
 
     // S3 specific option
     public const OPTION_PATH_STYLE_ENDPOINT = 'pathStyleEndpoint';
@@ -49,6 +50,7 @@ final class Configuration
         self::OPTION_WEB_IDENTITY_TOKEN_FILE => true,
         self::OPTION_ROLE_SESSION_NAME => true,
         self::OPTION_CONTAINER_CREDENTIALS_RELATIVE_URI => true,
+        self::OPTION_ENDPOINT_DISCOVERY_ENABLED => true,
         self::OPTION_PATH_STYLE_ENDPOINT => true,
         self::OPTION_SEND_CHUNKED_BODY => true,
     ];
@@ -70,6 +72,7 @@ final class Configuration
             self::OPTION_ROLE_SESSION_NAME => 'AWS_ROLE_SESSION_NAME',
         ],
         [self::OPTION_CONTAINER_CREDENTIALS_RELATIVE_URI => 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'],
+        [self::OPTION_ENDPOINT_DISCOVERY_ENABLED => ['AWS_ENDPOINT_DISCOVERY_ENABLED', 'AWS_ENABLE_ENDPOINT_DISCOVERY']],
     ];
 
     private const DEFAULT_OPTIONS = [
@@ -82,6 +85,7 @@ final class Configuration
         self::OPTION_ENDPOINT => 'https://%service%.%region%.amazonaws.com',
         self::OPTION_PATH_STYLE_ENDPOINT => 'false',
         self::OPTION_SEND_CHUNKED_BODY => 'false',
+        self::OPTION_ENDPOINT_DISCOVERY_ENABLED => 'false',
     ];
 
     private $data = [];
diff --git a/src/EndpointDiscovery/EndpointCache.php b/src/EndpointDiscovery/EndpointCache.php
@@ -0,0 +1,118 @@
+<?php
+
+namespace AsyncAws\Core\EndpointDiscovery;
+
+use AsyncAws\Core\Exception\LogicException;
+
+/**
+ * @author Jérémy Derussé <jeremy@derusse.com>
+ *
+ * @internal
+ */
+class EndpointCache
+{
+    private $endpoints = [];
+
+    private $expired = [];
+
+    public function addEndpoints(?string $region, array $endpoints): void
+    {
+        $now = time();
+
+        if (null === $region) {
+            $region = '';
+        }
+        if (!isset($this->endpoints[$region])) {
+            $this->endpoints[$region] = [];
+        }
+
+        /** @var EndpointInterface $endpoint */
+        foreach ($endpoints as $endpoint) {
+            $this->endpoints[$region][$this->sanitizeEndpoint($endpoint->getAddress())] = $now + ($endpoint->getCachePeriodInMinutes() * 60);
+        }
+        arsort($this->endpoints[$region]);
+    }
+
+    public function removeEndpoint(string $endpoint): void
+    {
+        $endpoint = $this->sanitizeEndpoint($endpoint);
+        foreach ($this->endpoints as &$endpoints) {
+            unset($endpoints[$endpoint]);
+        }
+        unset($endpoints);
+        foreach ($this->expired as &$endpoints) {
+            unset($endpoints[$endpoint]);
+        }
+
+        unset($endpoints);
+    }
+
+    public function getActiveEndpoint(?string $region): ?string
+    {
+        if (null === $region) {
+            $region = '';
+        }
+        $now = time();
+
+        foreach ($this->endpoints[$region] ?? [] as $endpoint => $expiresAt) {
+            if ($expiresAt < $now) {
+                $this->expired[$region] = \array_slice($this->expired[$region] ?? [], -100); // keep only the last 100 items
+                unset($this->endpoints[$region][$endpoint]);
+                $this->expired[$region][$endpoint] = $expiresAt;
+
+                continue;
+            }
+
+            return $endpoint;
+        }
+
+        return null;
+    }
+
+    public function getExpiredEndpoint(?string $region): ?string
+    {
+        if (null === $region) {
+            $region = '';
+        }
+        if (empty($this->expired[$region])) {
+            return null;
+        }
+
+        return array_key_last($this->expired[$region]);
+    }
+
+    private function sanitizeEndpoint(string $address): string
+    {
+        $parsed = parse_url($address);
+
+        // parse_url() will correctly parse full URIs with schemes
+        if (isset($parsed['host'])) {
+            return rtrim(sprintf(
+                '%s://%s/%s',
+                $parsed['scheme'] ?? 'https',
+                $parsed['host'],
+                ltrim($parsed['path'] ?? '/', '/')
+            ), '/');
+        }
+
+        // parse_url() will put host & path in 'path' if scheme is not provided
+        if (isset($parsed['path'])) {
+            $split = explode('/', $parsed['path'], 2);
+            $parsed['host'] = $split[0];
+            if (isset($split[1])) {
+                $parsed['path'] = $split[1];
+            } else {
+                $parsed['path'] = '';
+            }
+
+            return rtrim(sprintf(
+                '%s://%s/%s',
+                $parsed['scheme'] ?? 'https',
+                $parsed['host'],
+                ltrim($parsed['path'], '/')
+            ), '/');
+        }
+
+        throw new LogicException(sprintf('The supplied endpoint "%s" is invalid.', $address));
+    }
+}
diff --git a/src/EndpointDiscovery/EndpointInterface.php b/src/EndpointDiscovery/EndpointInterface.php
@@ -0,0 +1,10 @@
+<?php
+
+namespace AsyncAws\Core\EndpointDiscovery;
+
+interface EndpointInterface
+{
+    public function getAddress(): string;
+
+    public function getCachePeriodInMinutes(): int;
+}
diff --git a/src/RequestContext.php b/src/RequestContext.php
@@ -19,13 +19,25 @@ class RequestContext
         'expirationDate' => true,
         'currentDate' => true,
         'exceptionMapping' => true,
+        'usesEndpointDiscovery' => true,
+        'requiresEndpointDiscovery' => true,
     ];
 
     /**
      * @var string|null
      */
     private $operation;
 
+    /**
+     * @var bool
+     */
+    private $usesEndpointDiscovery = false;
+
+    /**
+     * @var bool
+     */
+    private $requiresEndpointDiscovery = false;
+
     /**
      * @var string|null
      */
@@ -53,6 +65,8 @@ class RequestContext
      *  expirationDate?: null|\DateTimeImmutable
      *  currentDate?: null|\DateTimeImmutable
      *  exceptionMapping?: string[]
+     *  usesEndpointDiscovery?: bool
+     *  requiresEndpointDiscovery?: bool
      * }
      */
     public function __construct(array $options = [])
@@ -90,4 +104,14 @@ public function getExceptionMapping(): array
     {
         return $this->exceptionMapping;
     }
+
+    public function usesEndpointDiscovery(): bool
+    {
+        return $this->usesEndpointDiscovery;
+    }
+
+    public function requiresEndpointDiscovery(): bool
+    {
+        return $this->requiresEndpointDiscovery;
+    }
 }
diff --git a/src/Response.php b/src/Response.php
@@ -6,6 +6,7 @@
 
 use AsyncAws\Core\AwsError\AwsErrorFactoryInterface;
 use AsyncAws\Core\AwsError\ChainAwsErrorFactory;
+use AsyncAws\Core\EndpointDiscovery\EndpointCache;
 use AsyncAws\Core\Exception\Exception;
 use AsyncAws\Core\Exception\Http\ClientException;
 use AsyncAws\Core\Exception\Http\HttpException;
@@ -79,6 +80,16 @@ class Response
      */
     private $awsErrorFactory;
 
+    /**
+     * @var ?EndpointCache
+     */
+    private $endpointCache;
+
+    /**
+     * @var ?Request
+     */
+    private $request;
+
     /**
      * @var bool
      */
@@ -89,12 +100,14 @@ class Response
      */
     private $exceptionMapping;
 
-    public function __construct(ResponseInterface $response, HttpClientInterface $httpClient, LoggerInterface $logger, AwsErrorFactoryInterface $awsErrorFactory = null, bool $debug = false, array $exceptionMapping = [])
+    public function __construct(ResponseInterface $response, HttpClientInterface $httpClient, LoggerInterface $logger, AwsErrorFactoryInterface $awsErrorFactory = null, EndpointCache $endpointCache = null, Request $request = null, bool $debug = false, array $exceptionMapping = [])
     {
         $this->httpResponse = $response;
         $this->httpClient = $httpClient;
         $this->logger = $logger;
         $this->awsErrorFactory = $awsErrorFactory ?? new ChainAwsErrorFactory();
+        $this->endpointCache = $endpointCache;
+        $this->request = $request;
         $this->debug = $debug;
         $this->exceptionMapping = $exceptionMapping;
     }
@@ -385,6 +398,9 @@ private function defineResolveStatus(): void
         if (300 <= $statusCode) {
             try {
                 $awsError = $this->awsErrorFactory->createFromResponse($this->httpResponse);
+                if ($this->request && $this->endpointCache && (400 === $statusCode || 'InvalidEndpointException' === $awsError->getCode())) {
+                    $this->endpointCache->removeEndpoint($this->request->getEndpoint());
+                }
             } catch (UnparsableResponse $e) {
                 $awsError = null;
             }
diff --git a/tests/Unit/AbstractApiTest.php b/tests/Unit/AbstractApiTest.php
diff --git a/tests/Unit/EndpointDiscovery/EndpointCacheTest.php b/tests/Unit/EndpointDiscovery/EndpointCacheTest.php
