From 84621470bdbbe69e8490df7793b11da076731db7 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:11:26 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(gateway):=20429=20=E5=90=8C=20key=20?= =?UTF-8?q?=E4=BB=8D=E9=87=8D=E8=AF=95=E4=B8=89=E6=AC=A1=EF=BC=8C=E9=97=B4?= =?UTF-8?q?=E9=9A=94=E6=94=B9=E4=B8=BA=E6=8C=87=E6=95=B0=E9=80=80=E9=81=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #817 --- .../src/windup_framework/gateway/chat.py | 14 ++++------ .../src/windup_framework/gateway/image.py | 14 ++++------ .../src/windup_framework/gateway/policy.py | 11 ++++++++ .../src/windup_framework/gateway/video.py | 15 ++++------ backend/tests/test_gateway_image.py | 28 +++++++++++++++++++ backend/tests/test_gateway_policy.py | 16 ++++++++++- 6 files changed, 72 insertions(+), 26 deletions(-) diff --git a/backend/packages/framework/src/windup_framework/gateway/chat.py b/backend/packages/framework/src/windup_framework/gateway/chat.py index 580769617..7f7d831c7 100644 --- a/backend/packages/framework/src/windup_framework/gateway/chat.py +++ b/backend/packages/framework/src/windup_framework/gateway/chat.py @@ -17,7 +17,7 @@ from windup_framework.gateway.budget import AttemptBudget from windup_framework.gateway.circuit import CircuitBreaker from windup_framework.gateway.context import current_call_context -from windup_framework.gateway.policy import decide +from windup_framework.gateway.policy import decide, rate_limit_wait_s from windup_framework.gateway.routes import ( GatewayRoute, config_for_route, @@ -30,8 +30,6 @@ from windup_framework.gateway.types import Family, NextStep, Scene _CIRCUIT = CircuitBreaker() -_DEFAULT_RETRY_AFTER_S = 2.0 -_SLEEP_CAP_S = 30.0 _ERROR_MESSAGE_LIMIT = 2_000 @@ -462,12 +460,12 @@ def fail(http_status: int | None) -> None: fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: - wait = ( - result.retry_after_s - if result.retry_after_s is not None - else _DEFAULT_RETRY_AFTER_S + time.sleep( + rate_limit_wait_s( + retry_count=retry_count, + retry_after_s=result.retry_after_s, + ) ) - time.sleep(min(wait, _SLEEP_CAP_S)) retry_count += 1 continue if step is NextStep.FALLBACK: diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py index 767621929..e216b8dc0 100644 --- a/backend/packages/framework/src/windup_framework/gateway/image.py +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -10,7 +10,7 @@ from windup_framework.gateway.budget import AttemptBudget from windup_framework.gateway.circuit import CircuitBreaker from windup_framework.gateway.context import current_call_context -from windup_framework.gateway.policy import decide +from windup_framework.gateway.policy import decide, rate_limit_wait_s from windup_framework.gateway.registry import ModelRegistry, RegistryError from windup_framework.gateway.routes import ( GatewayRoute, @@ -31,8 +31,6 @@ from windup_framework.gateway.types import NextStep, Scene _CIRCUIT = CircuitBreaker() -_DEFAULT_RETRY_AFTER_S = 2.0 -_SLEEP_CAP_S = 30.0 def _utc_now() -> str: @@ -354,12 +352,12 @@ def fail(http_status: int | None) -> None: fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: - wait = ( - result.retry_after_s - if result.retry_after_s is not None - else _DEFAULT_RETRY_AFTER_S + time.sleep( + rate_limit_wait_s( + retry_count=retry_count, + retry_after_s=result.retry_after_s, + ) ) - time.sleep(min(wait, _SLEEP_CAP_S)) retry_count += 1 if error_type is ModelErrorType.UNREACHED: resend_spent = 1 diff --git a/backend/packages/framework/src/windup_framework/gateway/policy.py b/backend/packages/framework/src/windup_framework/gateway/policy.py index c19970e4f..9d60f2877 100644 --- a/backend/packages/framework/src/windup_framework/gateway/policy.py +++ b/backend/packages/framework/src/windup_framework/gateway/policy.py @@ -3,6 +3,17 @@ from windup_common.enums.model import ModelErrorType from windup_framework.gateway.types import NextStep +RATE_LIMIT_BACKOFF_BASE_S = 2.0 +RATE_LIMIT_SLEEP_CAP_S = 30.0 + + +def rate_limit_wait_s(*, retry_count: int, retry_after_s: float | None) -> float: + """429 同 key 重试等待：2s、4s 指数退避，Retry-After 作下限，封顶 30s。""" + wait = RATE_LIMIT_BACKOFF_BASE_S * (2 ** retry_count) + if retry_after_s is not None: + wait = max(wait, retry_after_s) + return min(wait, RATE_LIMIT_SLEEP_CAP_S) + def decide( *, diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py index f3f50c35f..314fe1384 100644 --- a/backend/packages/framework/src/windup_framework/gateway/video.py +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -11,7 +11,7 @@ from windup_framework.gateway.budget import AttemptBudget from windup_framework.gateway.context import current_call_context from windup_framework.gateway.image import _CIRCUIT -from windup_framework.gateway.policy import decide +from windup_framework.gateway.policy import decide, rate_limit_wait_s from windup_framework.gateway.registry import ModelRegistry, RegistryError from windup_framework.gateway.routes import ( GatewayRoute, @@ -31,9 +31,6 @@ ) from windup_framework.gateway.types import AdapterResult, NextStep, Scene -_DEFAULT_RETRY_AFTER_S = 2.0 -_SLEEP_CAP_S = 30.0 - @dataclass(frozen=True) class SubmittedVideoJob: @@ -396,12 +393,12 @@ def fail(http_status: int | None) -> None: fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: - wait = ( - result.retry_after_s - if result.retry_after_s is not None - else _DEFAULT_RETRY_AFTER_S + time.sleep( + rate_limit_wait_s( + retry_count=retry_count, + retry_after_s=result.retry_after_s, + ) ) - time.sleep(min(wait, _SLEEP_CAP_S)) retry_count += 1 if error_type is ModelErrorType.UNREACHED: resend_spent = 1 diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py index c61f9ed56..e645838b7 100644 --- a/backend/tests/test_gateway_image.py +++ b/backend/tests/test_gateway_image.py @@ -113,6 +113,34 @@ def test_429_does_not_switch_model_when_only_one_key(monkeypatch): assert "gemini-2.5-flash-image-alt" not in ad.calls +def test_429_same_key_retries_use_exponential_backoff(monkeypatch): + slept: list[float] = [] + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", slept.append) + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + ad = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate, rate]}) + gw = _make_gw(ad) + with pytest.raises(RuntimeError, match="429"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image"] * 3 + assert slept == [2.0, 4.0] + + +def test_429_backoff_waits_at_least_retry_after(monkeypatch): + slept: list[float] = [] + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", slept.append) + rate = AdapterResult( + ok=False, + error_type=ModelErrorType.RATE_LIMIT, + http_status=429, + retry_after_s=10.0, + ) + ad = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate, rate]}) + gw = _make_gw(ad) + with pytest.raises(RuntimeError, match="429"): + gw.gen_image("p", []) + assert slept == [10.0, 10.0] + + def test_429_switches_key_on_same_base_url_before_model(monkeypatch, caplog): monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) caplog.set_level(logging.INFO, logger="windup.gateway") diff --git a/backend/tests/test_gateway_policy.py b/backend/tests/test_gateway_policy.py index 2bf248d1d..3c74dab4f 100644 --- a/backend/tests/test_gateway_policy.py +++ b/backend/tests/test_gateway_policy.py @@ -3,7 +3,7 @@ from windup_common.enums.model import ModelErrorType from windup_framework.gateway.circuit import CircuitBreaker -from windup_framework.gateway.policy import decide +from windup_framework.gateway.policy import decide, rate_limit_wait_s from windup_framework.gateway.types import NextStep @@ -18,6 +18,20 @@ def test_429_retries_twice_then_fallback_key(): assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=2, has_job_id=False) is NextStep.FALLBACK_KEY +def test_429_backoff_is_exponential(): + assert rate_limit_wait_s(retry_count=0, retry_after_s=None) == 2.0 + assert rate_limit_wait_s(retry_count=1, retry_after_s=None) == 4.0 + + +def test_429_backoff_uses_retry_after_as_floor(): + assert rate_limit_wait_s(retry_count=0, retry_after_s=10.0) == 10.0 + assert rate_limit_wait_s(retry_count=1, retry_after_s=3.0) == 4.0 + + +def test_429_backoff_caps_at_30s(): + assert rate_limit_wait_s(retry_count=0, retry_after_s=100.0) == 30.0 + + def test_520_never_retries(): assert decide(error_type=ModelErrorType.MAYBE_BILLED, retry_count=0, has_job_id=False) is NextStep.FAIL From 37083281e887b7b060b7be033a6816c988c04d9d Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:41:33 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(gateway):=20429=20=E5=90=8C=20key=20?= =?UTF-8?q?=E5=8F=AA=E5=86=8D=E8=AF=95=E4=B8=80=E6=AC=A1=EF=BC=8C=E6=8D=A2?= =?UTF-8?q?=20key=20=E5=89=8D=2016s=20=E9=80=80=E9=81=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/windup_framework/gateway/chat.py | 14 ++++- .../src/windup_framework/gateway/image.py | 14 ++++- .../src/windup_framework/gateway/policy.py | 8 +-- .../src/windup_framework/gateway/video.py | 14 ++++- backend/tests/test_gateway_chat.py | 4 +- backend/tests/test_gateway_image.py | 63 +++++++++++++++---- backend/tests/test_gateway_policy.py | 15 +++-- backend/tests/test_gateway_video.py | 10 +-- 8 files changed, 109 insertions(+), 33 deletions(-) diff --git a/backend/packages/framework/src/windup_framework/gateway/chat.py b/backend/packages/framework/src/windup_framework/gateway/chat.py index 7f7d831c7..84cb97a19 100644 --- a/backend/packages/framework/src/windup_framework/gateway/chat.py +++ b/backend/packages/framework/src/windup_framework/gateway/chat.py @@ -453,10 +453,22 @@ def fail(http_status: int | None) -> None: break if step is NextStep.FALLBACK_KEY: if has_next_route: + nxt = self._routes[route_index + 1] + time.sleep( + rate_limit_wait_s( + retry_count=retry_count, + retry_after_s=result.retry_after_s, + ) + ) fallback_used = True - route_reason_override = "key_rate_limit" + if nxt.base_url_id != route.base_url_id: + self._circuit.open("base_url:" + route.base_url_id) + route_reason_override = "base_url_unreached" + else: + route_reason_override = "key_rate_limit" switch_to_next_route = True break + self._circuit.open("aggregator") fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py index e216b8dc0..21cd574dc 100644 --- a/backend/packages/framework/src/windup_framework/gateway/image.py +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -345,10 +345,22 @@ def fail(http_status: int | None) -> None: break if step is NextStep.FALLBACK_KEY: if has_next_route: + nxt = routes[route_index + 1] + time.sleep( + rate_limit_wait_s( + retry_count=retry_count, + retry_after_s=result.retry_after_s, + ) + ) fallback_used = True - route_reason_override = "key_rate_limit" + if nxt.base_url_id != route.base_url_id: + self._circuit.open("base_url:" + route.base_url_id) + route_reason_override = "base_url_unreached" + else: + route_reason_override = "key_rate_limit" switch_to_next_route = True break + self._circuit.open("aggregator") fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: diff --git a/backend/packages/framework/src/windup_framework/gateway/policy.py b/backend/packages/framework/src/windup_framework/gateway/policy.py index 9d60f2877..f385d8474 100644 --- a/backend/packages/framework/src/windup_framework/gateway/policy.py +++ b/backend/packages/framework/src/windup_framework/gateway/policy.py @@ -3,12 +3,12 @@ from windup_common.enums.model import ModelErrorType from windup_framework.gateway.types import NextStep -RATE_LIMIT_BACKOFF_BASE_S = 2.0 -RATE_LIMIT_SLEEP_CAP_S = 30.0 +RATE_LIMIT_BACKOFF_BASE_S = 8.0 +RATE_LIMIT_SLEEP_CAP_S = 60.0 def rate_limit_wait_s(*, retry_count: int, retry_after_s: float | None) -> float: - """429 同 key 重试等待：2s、4s 指数退避，Retry-After 作下限，封顶 30s。""" + """429 等待：同 key 再试 8s，换 key 16s。Retry-After 作下限，封顶 60s。""" wait = RATE_LIMIT_BACKOFF_BASE_S * (2 ** retry_count) if retry_after_s is not None: wait = max(wait, retry_after_s) @@ -39,7 +39,7 @@ def decide( return NextStep.FAIL if error_type is ModelErrorType.UNREACHED: return NextStep.OPEN_AGGREGATOR - if error_type is ModelErrorType.RATE_LIMIT and retry_count < 2: + if error_type is ModelErrorType.RATE_LIMIT and retry_count == 0: return NextStep.RETRY_SAME if error_type is ModelErrorType.RATE_LIMIT: return NextStep.FALLBACK_KEY diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py index 314fe1384..30e0a582e 100644 --- a/backend/packages/framework/src/windup_framework/gateway/video.py +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -386,10 +386,22 @@ def fail(http_status: int | None) -> None: if bound_job_id is not None: fail(last_http_status) if has_next_route: + nxt = routes[route_index + 1] + time.sleep( + rate_limit_wait_s( + retry_count=retry_count, + retry_after_s=result.retry_after_s, + ) + ) fallback_used = True - route_reason_override = "key_rate_limit" + if nxt.base_url_id != route.base_url_id: + self._circuit.open("base_url:" + route.base_url_id) + route_reason_override = "base_url_unreached" + else: + route_reason_override = "key_rate_limit" switch_to_next_route = True break + self._circuit.open("aggregator") fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: diff --git a/backend/tests/test_gateway_chat.py b/backend/tests/test_gateway_chat.py index ee95a6c79..765a02556 100644 --- a/backend/tests/test_gateway_chat.py +++ b/backend/tests/test_gateway_chat.py @@ -73,7 +73,7 @@ def test_chat_gateway_switches_key_after_429(monkeypatch, caplog): monkeypatch.setattr("windup_framework.gateway.chat.time.sleep", lambda _: None) caplog.set_level(logging.INFO, logger="windup.gateway") rate = ChatAdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) - key_a = FakeChatAdapter({"gpt-4o-mini": [rate, rate, rate]}) + key_a = FakeChatAdapter({"gpt-4o-mini": [rate, rate]}) key_b = FakeChatAdapter({"gpt-4o-mini": [OK]}) cfg = AIProviderSettings( model="gpt-4o-mini", @@ -90,7 +90,7 @@ def test_chat_gateway_switches_key_after_429(monkeypatch, caplog): ) assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" - assert key_a.calls == ["gpt-4o-mini"] * 3 + assert key_a.calls == ["gpt-4o-mini"] * 2 assert key_b.calls == ["gpt-4o-mini"] records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py index e645838b7..69ef4a658 100644 --- a/backend/tests/test_gateway_image.py +++ b/backend/tests/test_gateway_image.py @@ -103,26 +103,28 @@ def test_429_does_not_switch_model_when_only_one_key(monkeypatch): monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) ad = FakeImageAdapter({ - "gemini-2.5-flash-image": [rate, rate, rate], + "gemini-2.5-flash-image": [rate, rate], "gemini-2.5-flash-image-alt": [PNG], }) gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") with pytest.raises(RuntimeError, match="429"): gw.gen_image("p", []) - assert ad.calls == ["gemini-2.5-flash-image"] * 3 + assert ad.calls == ["gemini-2.5-flash-image"] * 2 assert "gemini-2.5-flash-image-alt" not in ad.calls -def test_429_same_key_retries_use_exponential_backoff(monkeypatch): +def test_429_same_key_retries_once_then_opens_aggregator(monkeypatch): slept: list[float] = [] monkeypatch.setattr("windup_framework.gateway.image.time.sleep", slept.append) rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) - ad = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate, rate]}) - gw = _make_gw(ad) + ad = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate]}) + br = CircuitBreaker() + gw = _make_gw(ad, circuit=br) with pytest.raises(RuntimeError, match="429"): gw.gen_image("p", []) - assert ad.calls == ["gemini-2.5-flash-image"] * 3 - assert slept == [2.0, 4.0] + assert ad.calls == ["gemini-2.5-flash-image"] * 2 + assert slept == [8.0] + assert br.is_open("aggregator") def test_429_backoff_waits_at_least_retry_after(monkeypatch): @@ -134,19 +136,20 @@ def test_429_backoff_waits_at_least_retry_after(monkeypatch): http_status=429, retry_after_s=10.0, ) - ad = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate, rate]}) + ad = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate]}) gw = _make_gw(ad) with pytest.raises(RuntimeError, match="429"): gw.gen_image("p", []) - assert slept == [10.0, 10.0] + assert slept == [10.0] def test_429_switches_key_on_same_base_url_before_model(monkeypatch, caplog): - monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) + slept: list[float] = [] + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", slept.append) caplog.set_level(logging.INFO, logger="windup.gateway") rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) key_a = FakeImageAdapter({ - "gemini-2.5-flash-image": [rate, rate, rate], + "gemini-2.5-flash-image": [rate, rate], "gemini-2.5-flash-image-alt": [PNG], }) key_b = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) @@ -167,9 +170,10 @@ def test_429_switches_key_on_same_base_url_before_model(monkeypatch, caplog): ) assert gw.gen_image("p", []).startswith(b"\x89PNG") - assert key_a.calls == ["gemini-2.5-flash-image"] * 3 + assert key_a.calls == ["gemini-2.5-flash-image"] * 2 assert key_b.calls == ["gemini-2.5-flash-image"] assert "gemini-2.5-flash-image-alt" not in key_a.calls + assert slept == [8.0, 16.0] records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] @@ -181,6 +185,41 @@ def test_429_switches_key_on_same_base_url_before_model(monkeypatch, caplog): assert line["api_key_id"].endswith("key1") +def test_429_exhausted_keys_switches_backup_entry(monkeypatch, caplog): + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) + caplog.set_level(logging.INFO, logger="windup.gateway") + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeImageAdapter({"gemini-2.5-flash-image": [rate, rate]}) + backup = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="key-c", + ) + br = CircuitBreaker() + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + key_a, + br, + cfg, + route_adapters={"primary.key0": key_a, "backup.key0": backup}, + ) + + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert key_a.calls == ["gemini-2.5-flash-image"] * 2 + assert backup.calls == ["gemini-2.5-flash-image"] + assert br.is_open("base_url:primary") + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + line = success[-1] + assert line["route_reason"] == "base_url_unreached" + assert line["base_url_id"] == "backup" + + def test_522_skips_remaining_keys_on_same_url(caplog): caplog.set_level(logging.INFO, logger="windup.gateway") key_a = FakeImageAdapter({ diff --git a/backend/tests/test_gateway_policy.py b/backend/tests/test_gateway_policy.py index 3c74dab4f..078dda141 100644 --- a/backend/tests/test_gateway_policy.py +++ b/backend/tests/test_gateway_policy.py @@ -12,24 +12,23 @@ def test_522_retries_once_then_opens_aggregator(): assert decide(error_type=ModelErrorType.UNREACHED, retry_count=1, has_job_id=False) is NextStep.OPEN_AGGREGATOR -def test_429_retries_twice_then_fallback_key(): +def test_429_retries_once_then_fallback_key(): assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME - assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=1, has_job_id=False) is NextStep.RETRY_SAME - assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=2, has_job_id=False) is NextStep.FALLBACK_KEY + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=1, has_job_id=False) is NextStep.FALLBACK_KEY def test_429_backoff_is_exponential(): - assert rate_limit_wait_s(retry_count=0, retry_after_s=None) == 2.0 - assert rate_limit_wait_s(retry_count=1, retry_after_s=None) == 4.0 + assert rate_limit_wait_s(retry_count=0, retry_after_s=None) == 8.0 + assert rate_limit_wait_s(retry_count=1, retry_after_s=None) == 16.0 def test_429_backoff_uses_retry_after_as_floor(): assert rate_limit_wait_s(retry_count=0, retry_after_s=10.0) == 10.0 - assert rate_limit_wait_s(retry_count=1, retry_after_s=3.0) == 4.0 + assert rate_limit_wait_s(retry_count=1, retry_after_s=3.0) == 16.0 -def test_429_backoff_caps_at_30s(): - assert rate_limit_wait_s(retry_count=0, retry_after_s=100.0) == 30.0 +def test_429_backoff_caps_at_60s(): + assert rate_limit_wait_s(retry_count=0, retry_after_s=100.0) == 60.0 def test_520_never_retries(): diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py index 43ca558e2..e81f045b1 100644 --- a/backend/tests/test_gateway_video.py +++ b/backend/tests/test_gateway_video.py @@ -107,7 +107,7 @@ def test_submit_429_switches_key_on_same_base_url(monkeypatch): rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) key_a = FakeVideoAdapter( submits={ - "kling-v2-5-turbo": [rate, rate, rate], + "kling-v2-5-turbo": [rate, rate], "kling-v2-6": [AdapterResult(ok=True, job_id="wrong", maybe_billed=True)], }, follows={}, @@ -136,7 +136,7 @@ def test_submit_429_switches_key_on_same_base_url(monkeypatch): ) assert gw.i2v(b"frame", "walk").startswith(b"\x00\x00\x00\x18ftyp") - assert key_a.submit_models == ["kling-v2-5-turbo"] * 3 + assert key_a.submit_models == ["kling-v2-5-turbo"] * 2 assert key_b.submit_models == ["kling-v2-5-turbo"] assert "kling-v2-6" not in key_a.submit_models @@ -172,7 +172,8 @@ def test_timeout_does_not_submit_fallback(): "error_type", [ModelErrorType.RATE_LIMIT, ModelErrorType.INVALID_RESPONSE], ) -def test_follow_fallback_without_upstream_fail_does_not_open_second_job(error_type): +def test_follow_fallback_without_upstream_fail_does_not_open_second_job(error_type, monkeypatch): + monkeypatch.setattr("windup_framework.gateway.video.time.sleep", lambda _: None) follow_result = AdapterResult( ok=False, error_type=error_type, @@ -190,7 +191,8 @@ def test_follow_fallback_without_upstream_fail_does_not_open_second_job(error_ty with pytest.raises(RuntimeError, match=error_type.value): _video_gw(ad).i2v(b"frame", "walk") assert ad.submit_models == ["kling-v2-5-turbo"] - assert ad.followed == ["j1", "j1", "j1"] + follows = 2 if error_type is ModelErrorType.RATE_LIMIT else 3 + assert ad.followed == ["j1"] * follows def test_success_trace_has_phase_timings(caplog):