Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1387,6 +1387,8 @@ response = await fga_client.execute_api_request(

If a network request fails with a 429 or 5xx error from the server, the SDK will automatically retry the request up to 3 times with a minimum wait time of 100 milliseconds between each attempt.

The SDK honors `Retry-After` delay seconds or HTTP dates up to 30 minutes. Unparseable or out-of-range values fall back to exponential backoff. In the sync client, urllib3's own `Retry-After` handling is disabled unless you set `Configuration.retries`.

To customize this behavior, create a `RetryParams` object and assign values to the `max_retry` and `min_wait_in_ms` constructor parameters. `max_retry` determines the maximum number of retries (up to 15), while `min_wait_in_ms` sets the minimum wait time between retries in milliseconds.

Apply your custom retry values by passing the object to the `ClientConfiguration` constructor's `retry_params` parameter.
Expand Down
25 changes: 14 additions & 11 deletions openfga_sdk/api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import time
import urllib

from email.utils import parsedate_to_datetime
from multiprocessing.pool import ThreadPool

from dateutil.parser import parse # type: ignore[import-untyped]
Expand All @@ -19,6 +20,7 @@
from openfga_sdk.configuration import Configuration
from openfga_sdk.constants import (
MAX_BACKOFF_TIME_IN_SEC,
RETRY_AFTER_HEADER_NAME,
RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC,
)
from openfga_sdk.exceptions import (
Expand Down Expand Up @@ -423,23 +425,24 @@ async def __call_api(
return (return_data, response_data.status, response_data.headers)

def _parse_retry_after_header(self, headers) -> int:
retry_after_header = headers.get("retry-after")
retry_after_header = headers.get(RETRY_AFTER_HEADER_NAME.lower())
if not retry_after_header:
raise ValueError("Retry-After header is not present")

# RFC 9110: Retry-After is either delay-seconds or an HTTP-date
try:
parsed_http_date = self.__deserialize_datetime(retry_after_header).replace(
tzinfo=datetime.timezone.utc
)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (parsed_http_date - now).total_seconds()
except ApiException:
wait_time_in_sec = int(retry_after_header)
except ValueError:
try:
retry_at = parsedate_to_datetime(retry_after_header)
except ValueError as e:
raise ValueError("Retry-After header is invalid") from e
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=datetime.timezone.utc)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (retry_at - now).total_seconds()

if (
wait_time_in_sec > RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC
or wait_time_in_sec < 1
):
if not 1 <= wait_time_in_sec <= RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC:
raise ValueError("Retry-After header is invalid")

return math.ceil(wait_time_in_sec)
Expand Down
3 changes: 2 additions & 1 deletion openfga_sdk/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@ def __init__(
"""Safe chars for path_param
"""
self.retries = None
"""Adding retries to override urllib3 default value 3
"""urllib3 Retry for the sync client. Defaults to 3 retries with
Retry-After handling disabled; the SDK retries 429/5xx itself.
"""
# Enable client side validation
self.client_side_validation = True
Expand Down
5 changes: 4 additions & 1 deletion openfga_sdk/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@
NOTE: This file was auto generated by OpenAPI Generator (https://openapi-generator.tech). DO NOT EDIT.
"""

from openfga_sdk.constants import RETRY_AFTER_HEADER_NAME


# Specific FGA header to be parsed
X_RATELIMIT_LIMIT = "x-ratelimit-limit"
X_RATELIMIT_REMAINING = "x_ratelimit_remaining"
X_RATELIMIT_RESET = "x_ratelimit_reset"
FGA_REQUEST_ID = "fga-request-id"
FGA_QUERY_DURATION_MS = "fga-query-duration-ms"
OPENFGA_AUTHORIZATION_MODEL_ID = "openfga_authorization_model_id"
RETRY_AFTER = "retry-after"
RETRY_AFTER = RETRY_AFTER_HEADER_NAME.lower()
RESPONSE_HEADERS_TO_KEEP = [
X_RATELIMIT_LIMIT,
X_RATELIMIT_REMAINING,
Expand Down
25 changes: 14 additions & 11 deletions openfga_sdk/sync/api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import time
import urllib

from email.utils import parsedate_to_datetime
from multiprocessing.pool import ThreadPool

from dateutil.parser import parse # type: ignore[import-untyped]
Expand All @@ -17,6 +18,7 @@
from openfga_sdk.configuration import Configuration
from openfga_sdk.constants import (
MAX_BACKOFF_TIME_IN_SEC,
RETRY_AFTER_HEADER_NAME,
RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC,
)
from openfga_sdk.exceptions import (
Expand Down Expand Up @@ -424,23 +426,24 @@ def __call_api(
return (return_data, response_data.status, response_data.headers)

def _parse_retry_after_header(self, headers) -> int:
retry_after_header = headers.get("retry-after")
retry_after_header = headers.get(RETRY_AFTER_HEADER_NAME.lower())
if not retry_after_header:
raise ValueError("Retry-After header is not present")

# RFC 9110: Retry-After is either delay-seconds or an HTTP-date
try:
parsed_http_date = self.__deserialize_datetime(retry_after_header).replace(
tzinfo=datetime.timezone.utc
)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (parsed_http_date - now).total_seconds()
except ApiException:
wait_time_in_sec = int(retry_after_header)
except ValueError:
try:
retry_at = parsedate_to_datetime(retry_after_header)
except ValueError as e:
raise ValueError("Retry-After header is invalid") from e
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=datetime.timezone.utc)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (retry_at - now).total_seconds()

if (
wait_time_in_sec > RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC
or wait_time_in_sec < 1
):
if not 1 <= wait_time_in_sec <= RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC:
raise ValueError("Retry-After header is invalid")

return math.ceil(wait_time_in_sec)
Expand Down
6 changes: 6 additions & 0 deletions openfga_sdk/sync/rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ def __init__(

if hasattr(configuration, "retries") and configuration.retries is not None:
addition_pool_args["retries"] = configuration.retries
else:
# The SDK retries 429/5xx itself; keep urllib3 from also retrying
# idempotent requests and sleeping for an uncapped Retry-After.
addition_pool_args["retries"] = urllib3.Retry(
total=3, respect_retry_after_header=False
)

if (
hasattr(configuration, "socket_options")
Expand Down
23 changes: 14 additions & 9 deletions test/api/open_fga_api_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1456,7 +1456,7 @@ async def test_429_error_retry_configured_unparseable_retry_after(
):
"""
Test to ensure 429 errors are handled properly.
For this case, retry is configured and the Retry-After header is provided as an HTTP date.
For this case, the Retry-After header is invalid.
"""
response_body = '{"allowed": true, "resolution": "1234"}'
error_response_body = """
Expand All @@ -1465,20 +1465,22 @@ async def test_429_error_retry_configured_unparseable_retry_after(
"message": "Rate Limit exceeded"
}
"""
retry_after_in_sec = 5
five_seconds_from_now = f"{retry_after_in_sec}s"
min_wait_in_ms = 10
max_wait_in_sec = 1
mock_http_response = http_mock_response(
body=error_response_body,
status=429,
headers={"Retry-After": five_seconds_from_now},
headers={"Retry-After": "not-a-date"},
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
)
mock_request.side_effect = [
RateLimitExceededError(http_resp=mock_http_response),
mock_response(response_body, 200),
]

retry = openfga_sdk.configuration.RetryParams(
max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1
max_retry=1,
min_wait_in_ms=min_wait_in_ms,
max_wait_in_sec=max_wait_in_sec,
)
configuration = self.configuration
configuration.store_id = store_id
Expand All @@ -1499,7 +1501,10 @@ async def test_429_error_retry_configured_unparseable_retry_after(
self.assertTrue(api_response.allowed)
mock_request.assert_called()
self.assertEqual(mock_request.call_count, 2)
self.assertNotEqual(mock_sleep.call_args[0][0], retry_after_in_sec)
# an invalid header falls back to exponential backoff
self.assertTrue(
min_wait_in_ms / 1000 <= mock_sleep.call_args[0][0] <= max_wait_in_sec
)

@patch("asyncio.sleep")
@patch.object(rest.RESTClientObject, "request")
Expand Down Expand Up @@ -1566,7 +1571,7 @@ async def test_429_error_retry_configured_with_delay_seconds(
):
"""
Test to ensure 429 errors are handled properly.
For this case, retry is configured and the Retry-After header is provided as an HTTP date.
For this case, the Retry-After header is provided as delay seconds.
"""
response_body = '{"allowed": true, "resolution": "1234"}'
error_response_body = """
Expand All @@ -1575,11 +1580,11 @@ async def test_429_error_retry_configured_with_delay_seconds(
"message": "Rate Limit exceeded"
}
"""
retry_after_in_sec = 10
retry_after_in_sec = 60
mock_http_response = http_mock_response(
body=error_response_body,
status=429,
headers={"Retry-After": retry_after_in_sec},
headers={"Retry-After": str(retry_after_in_sec)},
)
mock_request.side_effect = [
RateLimitExceededError(http_resp=mock_http_response),
Expand Down
154 changes: 154 additions & 0 deletions test/retry_after_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from threading import Thread
from unittest.mock import AsyncMock, patch

import pytest
import pytest_asyncio

from openfga_sdk.api_client import ApiClient as AsyncApiClient
from openfga_sdk.configuration import Configuration, RetryParams
from openfga_sdk.sync.api_client import ApiClient as SyncApiClient


BACKOFF = 0.125
IMF_FIXDATE = "%a, %d %b %Y %H:%M:%S GMT"


def http_date(**delta):
return (datetime.now(timezone.utc) + timedelta(**delta)).strftime(IMF_FIXDATE)


# urllib3 retries idempotent methods like GET on its own unless the SDK stops it.
http_methods = pytest.mark.parametrize("method", ["GET", "POST"])

retry_after_cases = pytest.mark.parametrize(
"status,retry_after,expected_wait",
[
(429, "0", BACKOFF),
(429, "1", 1),
(429, "10", 10),
(429, "31", 31),
(429, "60", 60),
(429, "1800", 1800),
(429, "-1", BACKOFF),
(429, "1801", BACKOFF),
(429, "not-a-date", BACKOFF),
(503, "60", 60),
],
)


@pytest.fixture
def retry_server(status, retry_after):
requests = []

class Handler(BaseHTTPRequestHandler):
def log_message(self, format, *args):
pass

def _respond(self):
requests.append(self.path)
self.send_response(status if len(requests) == 1 else 200)
self.send_header("Content-Type", "application/json")
self.send_header("Retry-After", retry_after)
self.send_header("Content-Length", "2")
self.end_headers()
self.wfile.write(b"{}")

do_GET = do_POST = _respond

with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server:
thread = Thread(
target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}", requests
finally:
server.shutdown()
thread.join()


@http_methods
@retry_after_cases
@pytest.mark.asyncio
async def test_async_retry_after_over_http(retry_server, method, expected_wait):
url, requests = retry_server
configuration = Configuration(
api_url=url, retry_params=RetryParams(max_retry=1, max_wait_in_sec=1)
)
with (
patch("openfga_sdk.api_client.random_time", return_value=BACKOFF),
patch("asyncio.sleep", new_callable=AsyncMock) as sleep,
):
async with AsyncApiClient(configuration) as client:
result = await client.call_api(
"/retry", method, response_types_map={}, _return_http_data_only=True
)

assert result is None
assert requests == ["/retry", "/retry"]
sleep.assert_awaited_once_with(expected_wait)


@http_methods
@retry_after_cases
def test_sync_retry_after_over_http(retry_server, method, expected_wait):
url, requests = retry_server
configuration = Configuration(
api_url=url, retry_params=RetryParams(max_retry=1, max_wait_in_sec=1)
)
with (
patch("openfga_sdk.sync.api_client.random_time", return_value=BACKOFF),
patch("time.sleep") as sleep,
SyncApiClient(configuration) as client,
):
result = client.call_api(
"/retry", method, response_types_map={}, _return_http_data_only=True
)

assert result is None
assert requests == ["/retry", "/retry"]
sleep.assert_called_once_with(expected_wait)


@pytest_asyncio.fixture(params=["async", "sync"])
async def parse_retry_after(request):
configuration = Configuration(api_url="http://localhost:8080")
if request.param == "async":
async with AsyncApiClient(configuration) as client:
yield lambda value: client._parse_retry_after_header({"retry-after": value})
else:
with SyncApiClient(configuration) as client:
yield lambda value: client._parse_retry_after_header({"retry-after": value})


# HTTP-date formats a recipient must accept per RFC 9110 section 5.6.7.
@pytest.mark.parametrize(
"http_date_format",
[IMF_FIXDATE, "%A, %d-%b-%y %H:%M:%S GMT", "%a %b %d %H:%M:%S %Y"],
ids=["imf-fixdate", "rfc850", "asctime"],
)
@pytest.mark.asyncio
async def test_parse_retry_after_http_date(parse_retry_after, http_date_format):
retry_at = datetime.now(timezone.utc) + timedelta(seconds=60)

wait = parse_retry_after(retry_at.strftime(http_date_format))

assert 59 <= wait <= 60


@pytest.mark.parametrize(
"value",
[
"1.5",
"5s",
pytest.param(http_date(seconds=-5), id="date-in-past"),
pytest.param(http_date(hours=1), id="date-above-max"),
],
)
@pytest.mark.asyncio
async def test_parse_retry_after_rejects_invalid(parse_retry_after, value):
with pytest.raises(ValueError):
parse_retry_after(value)
Loading
Loading