diff --git a/README.md b/README.md index ce6ace3..da4473d 100644 --- a/README.md +++ b/README.md @@ -1387,6 +1387,8 @@ response = await fga_client.execute_api_request( If a network request fails with a 429 or 5xx error from the server, the SDK will automatically retry the request up to 3 times with a minimum wait time of 100 milliseconds between each attempt. +The SDK honors `Retry-After` delay seconds or HTTP dates up to 30 minutes. Unparseable or out-of-range values fall back to exponential backoff. In the sync client, urllib3's own `Retry-After` handling is disabled unless you set `Configuration.retries`. + To customize this behavior, create a `RetryParams` object and assign values to the `max_retry` and `min_wait_in_ms` constructor parameters. `max_retry` determines the maximum number of retries (up to 15), while `min_wait_in_ms` sets the minimum wait time between retries in milliseconds. Apply your custom retry values by passing the object to the `ClientConfiguration` constructor's `retry_params` parameter. diff --git a/openfga_sdk/api_client.py b/openfga_sdk/api_client.py index 4c521bc..223b838 100644 --- a/openfga_sdk/api_client.py +++ b/openfga_sdk/api_client.py @@ -8,6 +8,7 @@ import time import urllib +from email.utils import parsedate_to_datetime from multiprocessing.pool import ThreadPool from dateutil.parser import parse # type: ignore[import-untyped] @@ -19,6 +20,7 @@ from openfga_sdk.configuration import Configuration from openfga_sdk.constants import ( MAX_BACKOFF_TIME_IN_SEC, + RETRY_AFTER_HEADER_NAME, RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC, ) from openfga_sdk.exceptions import ( @@ -423,23 +425,24 @@ async def __call_api( return (return_data, response_data.status, response_data.headers) def _parse_retry_after_header(self, headers) -> int: - retry_after_header = headers.get("retry-after") + retry_after_header = headers.get(RETRY_AFTER_HEADER_NAME.lower()) if not retry_after_header: raise ValueError("Retry-After header is not present") + # RFC 9110: Retry-After is either delay-seconds or an HTTP-date try: - parsed_http_date = self.__deserialize_datetime(retry_after_header).replace( - tzinfo=datetime.timezone.utc - ) - now = datetime.datetime.now(datetime.timezone.utc) - wait_time_in_sec = (parsed_http_date - now).total_seconds() - except ApiException: wait_time_in_sec = int(retry_after_header) + except ValueError: + try: + retry_at = parsedate_to_datetime(retry_after_header) + except ValueError as e: + raise ValueError("Retry-After header is invalid") from e + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=datetime.timezone.utc) + now = datetime.datetime.now(datetime.timezone.utc) + wait_time_in_sec = (retry_at - now).total_seconds() - if ( - wait_time_in_sec > RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC - or wait_time_in_sec < 1 - ): + if not 1 <= wait_time_in_sec <= RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC: raise ValueError("Retry-After header is invalid") return math.ceil(wait_time_in_sec) diff --git a/openfga_sdk/configuration.py b/openfga_sdk/configuration.py index 19756c4..02f8e5e 100644 --- a/openfga_sdk/configuration.py +++ b/openfga_sdk/configuration.py @@ -304,7 +304,8 @@ def __init__( """Safe chars for path_param """ self.retries = None - """Adding retries to override urllib3 default value 3 + """urllib3 Retry for the sync client. Defaults to 3 retries with + Retry-After handling disabled; the SDK retries 429/5xx itself. """ # Enable client side validation self.client_side_validation = True diff --git a/openfga_sdk/exceptions.py b/openfga_sdk/exceptions.py index 0523b23..ca3afac 100644 --- a/openfga_sdk/exceptions.py +++ b/openfga_sdk/exceptions.py @@ -10,6 +10,9 @@ NOTE: This file was auto generated by OpenAPI Generator (https://openapi-generator.tech). DO NOT EDIT. """ +from openfga_sdk.constants import RETRY_AFTER_HEADER_NAME + + # Specific FGA header to be parsed X_RATELIMIT_LIMIT = "x-ratelimit-limit" X_RATELIMIT_REMAINING = "x_ratelimit_remaining" @@ -17,7 +20,7 @@ FGA_REQUEST_ID = "fga-request-id" FGA_QUERY_DURATION_MS = "fga-query-duration-ms" OPENFGA_AUTHORIZATION_MODEL_ID = "openfga_authorization_model_id" -RETRY_AFTER = "retry-after" +RETRY_AFTER = RETRY_AFTER_HEADER_NAME.lower() RESPONSE_HEADERS_TO_KEEP = [ X_RATELIMIT_LIMIT, X_RATELIMIT_REMAINING, diff --git a/openfga_sdk/sync/api_client.py b/openfga_sdk/sync/api_client.py index af0e67e..9a7ae3e 100644 --- a/openfga_sdk/sync/api_client.py +++ b/openfga_sdk/sync/api_client.py @@ -7,6 +7,7 @@ import time import urllib +from email.utils import parsedate_to_datetime from multiprocessing.pool import ThreadPool from dateutil.parser import parse # type: ignore[import-untyped] @@ -17,6 +18,7 @@ from openfga_sdk.configuration import Configuration from openfga_sdk.constants import ( MAX_BACKOFF_TIME_IN_SEC, + RETRY_AFTER_HEADER_NAME, RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC, ) from openfga_sdk.exceptions import ( @@ -424,23 +426,24 @@ def __call_api( return (return_data, response_data.status, response_data.headers) def _parse_retry_after_header(self, headers) -> int: - retry_after_header = headers.get("retry-after") + retry_after_header = headers.get(RETRY_AFTER_HEADER_NAME.lower()) if not retry_after_header: raise ValueError("Retry-After header is not present") + # RFC 9110: Retry-After is either delay-seconds or an HTTP-date try: - parsed_http_date = self.__deserialize_datetime(retry_after_header).replace( - tzinfo=datetime.timezone.utc - ) - now = datetime.datetime.now(datetime.timezone.utc) - wait_time_in_sec = (parsed_http_date - now).total_seconds() - except ApiException: wait_time_in_sec = int(retry_after_header) + except ValueError: + try: + retry_at = parsedate_to_datetime(retry_after_header) + except ValueError as e: + raise ValueError("Retry-After header is invalid") from e + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=datetime.timezone.utc) + now = datetime.datetime.now(datetime.timezone.utc) + wait_time_in_sec = (retry_at - now).total_seconds() - if ( - wait_time_in_sec > RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC - or wait_time_in_sec < 1 - ): + if not 1 <= wait_time_in_sec <= RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC: raise ValueError("Retry-After header is invalid") return math.ceil(wait_time_in_sec) diff --git a/openfga_sdk/sync/rest.py b/openfga_sdk/sync/rest.py index 35a8fde..dd2f098 100644 --- a/openfga_sdk/sync/rest.py +++ b/openfga_sdk/sync/rest.py @@ -167,6 +167,12 @@ def __init__( if hasattr(configuration, "retries") and configuration.retries is not None: addition_pool_args["retries"] = configuration.retries + else: + # The SDK retries 429/5xx itself; keep urllib3 from also retrying + # idempotent requests and sleeping for an uncapped Retry-After. + addition_pool_args["retries"] = urllib3.Retry( + total=3, respect_retry_after_header=False + ) if ( hasattr(configuration, "socket_options") diff --git a/test/api/open_fga_api_test.py b/test/api/open_fga_api_test.py index a3b9c67..cb31290 100644 --- a/test/api/open_fga_api_test.py +++ b/test/api/open_fga_api_test.py @@ -1456,7 +1456,7 @@ async def test_429_error_retry_configured_unparseable_retry_after( ): """ Test to ensure 429 errors are handled properly. - For this case, retry is configured and the Retry-After header is provided as an HTTP date. + For this case, the Retry-After header is invalid. """ response_body = '{"allowed": true, "resolution": "1234"}' error_response_body = """ @@ -1465,12 +1465,12 @@ async def test_429_error_retry_configured_unparseable_retry_after( "message": "Rate Limit exceeded" } """ - retry_after_in_sec = 5 - five_seconds_from_now = f"{retry_after_in_sec}s" + min_wait_in_ms = 10 + max_wait_in_sec = 1 mock_http_response = http_mock_response( body=error_response_body, status=429, - headers={"Retry-After": five_seconds_from_now}, + headers={"Retry-After": "not-a-date"}, ) mock_request.side_effect = [ RateLimitExceededError(http_resp=mock_http_response), @@ -1478,7 +1478,9 @@ async def test_429_error_retry_configured_unparseable_retry_after( ] retry = openfga_sdk.configuration.RetryParams( - max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + max_retry=1, + min_wait_in_ms=min_wait_in_ms, + max_wait_in_sec=max_wait_in_sec, ) configuration = self.configuration configuration.store_id = store_id @@ -1499,7 +1501,10 @@ async def test_429_error_retry_configured_unparseable_retry_after( self.assertTrue(api_response.allowed) mock_request.assert_called() self.assertEqual(mock_request.call_count, 2) - self.assertNotEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + # an invalid header falls back to exponential backoff + self.assertTrue( + min_wait_in_ms / 1000 <= mock_sleep.call_args[0][0] <= max_wait_in_sec + ) @patch("asyncio.sleep") @patch.object(rest.RESTClientObject, "request") @@ -1566,7 +1571,7 @@ async def test_429_error_retry_configured_with_delay_seconds( ): """ Test to ensure 429 errors are handled properly. - For this case, retry is configured and the Retry-After header is provided as an HTTP date. + For this case, the Retry-After header is provided as delay seconds. """ response_body = '{"allowed": true, "resolution": "1234"}' error_response_body = """ @@ -1575,11 +1580,11 @@ async def test_429_error_retry_configured_with_delay_seconds( "message": "Rate Limit exceeded" } """ - retry_after_in_sec = 10 + retry_after_in_sec = 60 mock_http_response = http_mock_response( body=error_response_body, status=429, - headers={"Retry-After": retry_after_in_sec}, + headers={"Retry-After": str(retry_after_in_sec)}, ) mock_request.side_effect = [ RateLimitExceededError(http_resp=mock_http_response), diff --git a/test/retry_after_test.py b/test/retry_after_test.py new file mode 100644 index 0000000..450047f --- /dev/null +++ b/test/retry_after_test.py @@ -0,0 +1,154 @@ +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from unittest.mock import AsyncMock, patch + +import pytest +import pytest_asyncio + +from openfga_sdk.api_client import ApiClient as AsyncApiClient +from openfga_sdk.configuration import Configuration, RetryParams +from openfga_sdk.sync.api_client import ApiClient as SyncApiClient + + +BACKOFF = 0.125 +IMF_FIXDATE = "%a, %d %b %Y %H:%M:%S GMT" + + +def http_date(**delta): + return (datetime.now(timezone.utc) + timedelta(**delta)).strftime(IMF_FIXDATE) + + +# urllib3 retries idempotent methods like GET on its own unless the SDK stops it. +http_methods = pytest.mark.parametrize("method", ["GET", "POST"]) + +retry_after_cases = pytest.mark.parametrize( + "status,retry_after,expected_wait", + [ + (429, "0", BACKOFF), + (429, "1", 1), + (429, "10", 10), + (429, "31", 31), + (429, "60", 60), + (429, "1800", 1800), + (429, "-1", BACKOFF), + (429, "1801", BACKOFF), + (429, "not-a-date", BACKOFF), + (503, "60", 60), + ], +) + + +@pytest.fixture +def retry_server(status, retry_after): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def _respond(self): + requests.append(self.path) + self.send_response(status if len(requests) == 1 else 200) + self.send_header("Content-Type", "application/json") + self.send_header("Retry-After", retry_after) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"{}") + + do_GET = do_POST = _respond + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", requests + finally: + server.shutdown() + thread.join() + + +@http_methods +@retry_after_cases +@pytest.mark.asyncio +async def test_async_retry_after_over_http(retry_server, method, expected_wait): + url, requests = retry_server + configuration = Configuration( + api_url=url, retry_params=RetryParams(max_retry=1, max_wait_in_sec=1) + ) + with ( + patch("openfga_sdk.api_client.random_time", return_value=BACKOFF), + patch("asyncio.sleep", new_callable=AsyncMock) as sleep, + ): + async with AsyncApiClient(configuration) as client: + result = await client.call_api( + "/retry", method, response_types_map={}, _return_http_data_only=True + ) + + assert result is None + assert requests == ["/retry", "/retry"] + sleep.assert_awaited_once_with(expected_wait) + + +@http_methods +@retry_after_cases +def test_sync_retry_after_over_http(retry_server, method, expected_wait): + url, requests = retry_server + configuration = Configuration( + api_url=url, retry_params=RetryParams(max_retry=1, max_wait_in_sec=1) + ) + with ( + patch("openfga_sdk.sync.api_client.random_time", return_value=BACKOFF), + patch("time.sleep") as sleep, + SyncApiClient(configuration) as client, + ): + result = client.call_api( + "/retry", method, response_types_map={}, _return_http_data_only=True + ) + + assert result is None + assert requests == ["/retry", "/retry"] + sleep.assert_called_once_with(expected_wait) + + +@pytest_asyncio.fixture(params=["async", "sync"]) +async def parse_retry_after(request): + configuration = Configuration(api_url="http://localhost:8080") + if request.param == "async": + async with AsyncApiClient(configuration) as client: + yield lambda value: client._parse_retry_after_header({"retry-after": value}) + else: + with SyncApiClient(configuration) as client: + yield lambda value: client._parse_retry_after_header({"retry-after": value}) + + +# HTTP-date formats a recipient must accept per RFC 9110 section 5.6.7. +@pytest.mark.parametrize( + "http_date_format", + [IMF_FIXDATE, "%A, %d-%b-%y %H:%M:%S GMT", "%a %b %d %H:%M:%S %Y"], + ids=["imf-fixdate", "rfc850", "asctime"], +) +@pytest.mark.asyncio +async def test_parse_retry_after_http_date(parse_retry_after, http_date_format): + retry_at = datetime.now(timezone.utc) + timedelta(seconds=60) + + wait = parse_retry_after(retry_at.strftime(http_date_format)) + + assert 59 <= wait <= 60 + + +@pytest.mark.parametrize( + "value", + [ + "1.5", + "5s", + pytest.param(http_date(seconds=-5), id="date-in-past"), + pytest.param(http_date(hours=1), id="date-above-max"), + ], +) +@pytest.mark.asyncio +async def test_parse_retry_after_rejects_invalid(parse_retry_after, value): + with pytest.raises(ValueError): + parse_retry_after(value) diff --git a/test/sync/open_fga_api_test.py b/test/sync/open_fga_api_test.py index 5193163..b3cf80c 100644 --- a/test/sync/open_fga_api_test.py +++ b/test/sync/open_fga_api_test.py @@ -1519,7 +1519,7 @@ async def test_429_error_retry_configured_unparseable_retry_after( ): """ Test to ensure 429 errors are handled properly. - For this case, retry is configured and the Retry-After header is provided as an HTTP date. + For this case, the Retry-After header is invalid. """ response_body = '{"allowed": true, "resolution": "1234"}' error_response_body = """ @@ -1528,12 +1528,12 @@ async def test_429_error_retry_configured_unparseable_retry_after( "message": "Rate Limit exceeded" } """ - retry_after_in_sec = 5 - five_seconds_from_now = f"{retry_after_in_sec}s" + min_wait_in_ms = 10 + max_wait_in_sec = 1 mock_http_response = http_mock_response( body=error_response_body, status=429, - headers={"Retry-After": five_seconds_from_now}, + headers={"Retry-After": "not-a-date"}, ) mock_request.side_effect = [ RateLimitExceededError(http_resp=mock_http_response), @@ -1541,7 +1541,9 @@ async def test_429_error_retry_configured_unparseable_retry_after( ] retry = openfga_sdk.configuration.RetryParams( - max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1 + max_retry=1, + min_wait_in_ms=min_wait_in_ms, + max_wait_in_sec=max_wait_in_sec, ) configuration = self.configuration configuration.store_id = store_id @@ -1562,7 +1564,10 @@ async def test_429_error_retry_configured_unparseable_retry_after( self.assertTrue(api_response.allowed) mock_request.assert_called() self.assertEqual(mock_request.call_count, 2) - self.assertNotEqual(mock_sleep.call_args[0][0], retry_after_in_sec) + # an invalid header falls back to exponential backoff + self.assertTrue( + min_wait_in_ms / 1000 <= mock_sleep.call_args[0][0] <= max_wait_in_sec + ) @patch("time.sleep") @patch.object(rest.RESTClientObject, "request") @@ -1629,7 +1634,7 @@ async def test_429_error_retry_configured_with_delay_seconds( ): """ Test to ensure 429 errors are handled properly. - For this case, retry is configured and the Retry-After header is provided as an HTTP date. + For this case, the Retry-After header is provided as delay seconds. """ response_body = '{"allowed": true, "resolution": "1234"}' error_response_body = """ @@ -1638,11 +1643,11 @@ async def test_429_error_retry_configured_with_delay_seconds( "message": "Rate Limit exceeded" } """ - retry_after_in_sec = 10 + retry_after_in_sec = 60 mock_http_response = http_mock_response( body=error_response_body, status=429, - headers={"Retry-After": retry_after_in_sec}, + headers={"Retry-After": str(retry_after_in_sec)}, ) mock_request.side_effect = [ RateLimitExceededError(http_resp=mock_http_response),