🤖 fix: prevent retry spam by waiting until stream-end to reset retry state (#548)

## Problem

Auth errors and other stream failures caused unlimited retries with no
backoff due to a stream lifecycle timing bug. The retry counter was
reset on `stream-start` (when stream initiates) but auth errors happen
**after** stream-start on the backend, preventing exponential backoff
from progressing.

## Solution

**Two-part fix:**

1. **Move retry reset from `stream-start` to `stream-end`** - Only reset
attempt counter when stream completes successfully
2. **Increment counter on `stream-error` event** - Ensures backoff
progresses even for errors that happen after stream initiation

Created `src/utils/messages/retryState.ts` with utility functions for
state transitions:
- `createManualRetryState()` - preserves attempt counter for manual
retries
- `createFailedRetryState()` - increments attempt counter on failure
- `createFreshRetryState()` - resets on successful completion

## Testing

Verified exponential backoff progression with invalid API key:
- Attempt 0 → 1s delay
- Attempt 1 → 2s delay  
- Attempt 2 → 4s delay
- Continues up to 60s cap

All tests pass (997 pass, 1 skip, 0 fail).

## Cleanup

Final commit cleaned up debug logging (65 lines removed) while
preserving `window.__CMUX_FORCE_ALL_RETRYABLE` debug flag for testing.

_Generated with `cmux`_

Author: ammar-agent

src/components/Messages/ChatBarrier/RetryBarrier.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect, useCallback, useMemo } from "react";
+import React, { useState, useEffect, useMemo } from "react";
 import { usePersistedState, updatePersistedState } from "@/hooks/usePersistedState";
 import { getRetryStateKey, getAutoRetryKey } from "@/constants/storage";
 import { CUSTOM_EVENTS, createCustomEvent } from "@/constants/events";
@@ -7,15 +7,13 @@ import type { RetryState } from "@/hooks/useResumeManager";
 import { useWorkspaceState } from "@/stores/WorkspaceStore";
 import { isEligibleForAutoRetry, isNonRetryableSendError } from "@/utils/messages/retryEligibility";
 import { formatSendMessageError } from "@/utils/errors/formatSendError";
+import { createManualRetryState, calculateBackoffDelay } from "@/utils/messages/retryState";
 
 interface RetryBarrierProps {
   workspaceId: string;
   className?: string;
 }
 
-const INITIAL_DELAY = 1000; // 1 second
-const MAX_DELAY = 60000; // 60 seconds (cap for exponential backoff)
-
 const defaultRetryState: RetryState = {
   attempt: 0,
   retryStartTime: Date.now(),
@@ -45,7 +43,9 @@ export const RetryBarrier: React.FC<RetryBarrierProps> = ({ workspaceId, classNa
   // Compute effective autoRetry state: user preference AND error is retryable
   // This ensures UI shows "Retry" button (not "Retrying...") for non-retryable errors
   const effectiveAutoRetry = useMemo(() => {
-    if (!autoRetry || !workspaceState) return false;
+    if (!autoRetry || !workspaceState) {
+      return false;
+    }
 
     // Check if current state is eligible for auto-retry
     const messagesEligible = isEligibleForAutoRetry(
@@ -54,6 +54,7 @@ export const RetryBarrier: React.FC<RetryBarrierProps> = ({ workspaceId, classNa
     );
 
     // Also check RetryState for SendMessageErrors (from resumeStream failures)
+    // Note: isNonRetryableSendError already respects window.__CMUX_FORCE_ALL_RETRYABLE
     if (lastError && isNonRetryableSendError(lastError)) {
       return false; // Non-retryable SendMessageError
     }
@@ -64,37 +65,31 @@ export const RetryBarrier: React.FC<RetryBarrierProps> = ({ workspaceId, classNa
   // Local state for UI
   const [countdown, setCountdown] = useState(0);
 
-  // Calculate delay with exponential backoff (same as useResumeManager)
-  const getDelay = useCallback((attemptNum: number) => {
-    const exponentialDelay = INITIAL_DELAY * Math.pow(2, attemptNum);
-    return Math.min(exponentialDelay, MAX_DELAY);
-  }, []);
-
   // Update countdown display (pure display logic, no side effects)
   // useResumeManager handles the actual retry logic
   useEffect(() => {
     if (!autoRetry) return;
 
     const interval = setInterval(() => {
-      const delay = getDelay(attempt);
+      const delay = calculateBackoffDelay(attempt);
       const nextRetryTime = retryStartTime + delay;
       const timeUntilRetry = Math.max(0, nextRetryTime - Date.now());
 
       setCountdown(Math.ceil(timeUntilRetry / 1000));
     }, 100);
 
     return () => clearInterval(interval);
-  }, [autoRetry, attempt, retryStartTime, getDelay]);
+  }, [autoRetry, attempt, retryStartTime]);
 
   // Manual retry handler (user-initiated, immediate)
   // Emits event to useResumeManager instead of calling resumeStream directly
   // This keeps all retry logic centralized in one place
   const handleManualRetry = () => {
     setAutoRetry(true); // Re-enable auto-retry for next failure
 
-    // Clear retry state to make workspace immediately eligible for resume
-    // Use updatePersistedState to ensure listener-enabled hooks receive the update
-    updatePersistedState(getRetryStateKey(workspaceId), null);
+    // Create manual retry state: immediate retry BUT preserves attempt counter
+    // This prevents infinite retry loops without backoff if the retry fails
+    updatePersistedState(getRetryStateKey(workspaceId), createManualRetryState(attempt));
 
     // Emit event to useResumeManager - it will handle the actual resume
     // Pass isManual flag to bypass eligibility checks (user explicitly wants to retry)

src/hooks/useResumeManager.ts

@@ -7,16 +7,18 @@ import { readPersistedState, updatePersistedState } from "./usePersistedState";
 import { isEligibleForAutoRetry, isNonRetryableSendError } from "@/utils/messages/retryEligibility";
 import { applyCompactionOverrides } from "@/utils/messages/compactionOptions";
 import type { SendMessageError } from "@/types/errors";
+import {
+  createFailedRetryState,
+  calculateBackoffDelay,
+  INITIAL_DELAY,
+} from "@/utils/messages/retryState";
 
 export interface RetryState {
   attempt: number;
   retryStartTime: number;
   lastError?: SendMessageError;
 }
 
-const INITIAL_DELAY = 1000; // 1 second
-const MAX_DELAY = 60000; // 60 seconds
-
 /**
  * Centralized auto-resume manager for interrupted streams
  *
@@ -122,7 +124,7 @@ export function useResumeManager() {
 
     // 5. Check exponential backoff timer
     const { attempt, retryStartTime } = retryState;
-    const delay = Math.min(INITIAL_DELAY * Math.pow(2, attempt), MAX_DELAY);
+    const delay = calculateBackoffDelay(attempt);
     const timeSinceLastRetry = Date.now() - retryStartTime;
 
     if (timeSinceLastRetry < delay) return false; // Not time yet
@@ -151,6 +153,9 @@ export function useResumeManager() {
     });
 
     const { attempt } = retryState;
+    console.debug(
+      `[retry] ${workspaceId} attemptResume: current attempt=${attempt}, isManual=${isManual}`
+    );
 
     try {
       // Start with workspace defaults
@@ -171,27 +176,25 @@ export function useResumeManager() {
 
       if (!result.success) {
         // Store error in retry state so RetryBarrier can display it
-        const newState: RetryState = {
-          attempt: attempt + 1,
-          retryStartTime: Date.now(),
-          lastError: result.error,
-        };
+        const newState = createFailedRetryState(attempt, result.error);
+        console.debug(
+          `[retry] ${workspaceId} resumeStream failed: attempt ${attempt} → ${newState.attempt}`
+        );
         updatePersistedState(getRetryStateKey(workspaceId), newState);
-      } else {
-        // Success - clear retry state entirely
-        // If stream fails again, we'll start fresh (immediately eligible)
-        updatePersistedState(getRetryStateKey(workspaceId), null);
       }
+      // Note: Don't clear retry state on success - stream-end event will handle that
+      // resumeStream success just means "stream initiated", not "stream completed"
+      // Clearing here causes backoff reset bug when stream starts then immediately fails
     } catch (error) {
       // Store error in retry state for display
-      const newState: RetryState = {
-        attempt: attempt + 1,
-        retryStartTime: Date.now(),
-        lastError: {
-          type: "unknown",
-          raw: error instanceof Error ? error.message : "Failed to resume stream",
-        },
+      const errorData: SendMessageError = {
+        type: "unknown",
+        raw: error instanceof Error ? error.message : "Failed to resume stream",
       };
+      const newState = createFailedRetryState(attempt, errorData);
+      console.debug(
+        `[retry] ${workspaceId} resumeStream exception: attempt ${attempt} → ${newState.attempt}`
+      );
       updatePersistedState(getRetryStateKey(workspaceId), newState);
     } finally {
       // Always clear retrying flag

src/stores/WorkspaceStore.ts

@@ -18,6 +18,7 @@ import type { TokenConsumer } from "@/types/chatStats";
 import type { LanguageModelV2Usage } from "@ai-sdk/provider";
 import { getCancelledCompactionKey } from "@/constants/storage";
 import { isCompactingStream, findCompactionRequestMessage } from "@/utils/compaction/handler";
+import { createFreshRetryState } from "@/utils/messages/retryState";
 
 export interface WorkspaceState {
   name: string; // User-facing workspace name (e.g., "feature-branch")
@@ -123,10 +124,8 @@ export class WorkspaceStore {
       if (this.onModelUsed) {
         this.onModelUsed((data as { model: string }).model);
       }
-      updatePersistedState(getRetryStateKey(workspaceId), {
-        attempt: 0,
-        retryStartTime: Date.now(),
-      });
+      // Don't reset retry state here - stream might still fail after starting
+      // Retry state will be reset on stream-end (successful completion)
       this.states.bump(workspaceId);
     },
     "stream-delta": (workspaceId, aggregator, data) => {
@@ -141,6 +140,9 @@ export class WorkspaceStore {
         return;
       }
 
+      // Reset retry state on successful stream completion
+      updatePersistedState(getRetryStateKey(workspaceId), createFreshRetryState());
+
       this.states.bump(workspaceId);
       this.checkAndBumpRecencyIfChanged();
       this.finalizeUsageStats(workspaceId, (data as { metadata?: never }).metadata);
@@ -920,6 +922,24 @@ export class WorkspaceStore {
     // Handle non-buffered special events first
     if (isStreamError(data)) {
       aggregator.handleStreamError(data);
+
+      // Increment retry attempt counter when stream fails
+      // This handles auth errors that happen AFTER stream-start
+      updatePersistedState(
+        getRetryStateKey(workspaceId),
+        (prev) => {
+          const newAttempt = prev.attempt + 1;
+          console.debug(
+            `[retry] ${workspaceId} stream-error: incrementing attempt ${prev.attempt} → ${newAttempt}`
+          );
+          return {
+            attempt: newAttempt,
+            retryStartTime: Date.now(),
+          };
+        },
+        { attempt: 0, retryStartTime: Date.now() }
+      );
+
       this.states.bump(workspaceId);
       this.dispatchResumeCheck(workspaceId);
       return;

src/utils/messages/retryEligibility.ts

@@ -1,6 +1,30 @@
 import type { DisplayedMessage } from "@/types/message";
 import type { StreamErrorType, SendMessageError } from "@/types/errors";
 
+/**
+ * Debug flag to force all errors to be retryable
+ * Set in browser console: window.__CMUX_FORCE_ALL_RETRYABLE = true
+ *
+ * Useful for testing retry/backoff logic without needing to simulate
+ * specific network conditions or rate limits.
+ *
+ * Note: If you set this flag after an error occurs, you may need to
+ * trigger a manual retry first (click "Retry" button) to clear the
+ * stored non-retryable error state.
+ */
+declare global {
+  interface Window {
+    __CMUX_FORCE_ALL_RETRYABLE?: boolean;
+  }
+}
+
+/**
+ * Check if the debug flag to force all errors to be retryable is enabled
+ */
+function isForceAllRetryableEnabled(): boolean {
+  return typeof window !== "undefined" && window.__CMUX_FORCE_ALL_RETRYABLE === true;
+}
+
 /**
  * Error types that should NOT be auto-retried because they require user action
  * These errors won't resolve on their own - the user must fix the underlying issue
@@ -17,6 +41,11 @@ const NON_RETRYABLE_STREAM_ERRORS: StreamErrorType[] = [
  * Check if a SendMessageError (from resumeStream failures) is non-retryable
  */
 export function isNonRetryableSendError(error: SendMessageError): boolean {
+  // Debug flag: force all errors to be retryable
+  if (isForceAllRetryableEnabled()) {
+    return false;
+  }
+
   switch (error.type) {
     case "api_key_not_found": // Missing API key - user must configure
     case "provider_not_supported": // Unsupported provider - user must switch
@@ -91,6 +120,10 @@ export function isEligibleForAutoRetry(
   // (but manual retry is still available via hasInterruptedStream)
   const lastMessage = messages[messages.length - 1];
   if (lastMessage.type === "stream-error") {
+    // Debug flag: force all errors to be retryable
+    if (isForceAllRetryableEnabled()) {
+      return true;
+    }
     return !NON_RETRYABLE_STREAM_ERRORS.includes(lastMessage.errorType);
   }
 

src/utils/messages/retryState.test.ts

@@ -0,0 +1,99 @@
+import { describe, it, expect } from "bun:test";
+import {
+  createFreshRetryState,
+  createManualRetryState,
+  createFailedRetryState,
+  calculateBackoffDelay,
+  INITIAL_DELAY,
+} from "./retryState";
+
+describe("retryState utilities", () => {
+  describe("calculateBackoffDelay", () => {
+    it("returns exponential backoff: 1s → 2s → 4s → 8s...", () => {
+      expect(calculateBackoffDelay(0)).toBe(1000);
+      expect(calculateBackoffDelay(1)).toBe(2000);
+      expect(calculateBackoffDelay(2)).toBe(4000);
+      expect(calculateBackoffDelay(3)).toBe(8000);
+    });
+
+    it("caps at 60 seconds for large attempts", () => {
+      expect(calculateBackoffDelay(6)).toBe(60000);
+      expect(calculateBackoffDelay(10)).toBe(60000);
+    });
+  });
+
+  describe("createFreshRetryState", () => {
+    it("creates a state with attempt 0 and no error", () => {
+      const state = createFreshRetryState();
+      expect(state.attempt).toBe(0);
+      expect(state.lastError).toBeUndefined();
+      expect(state.retryStartTime).toBeLessThanOrEqual(Date.now());
+    });
+  });
+
+  describe("createManualRetryState", () => {
+    it("preserves attempt counter (critical for backoff)", () => {
+      const currentAttempt = 3;
+      const state = createManualRetryState(currentAttempt);
+
+      // CRITICAL: Manual retry must preserve attempt counter
+      // This ensures exponential backoff continues if the retry fails
+      expect(state.attempt).toBe(currentAttempt);
+    });
+
+    it("makes retry immediately eligible by backdating retryStartTime", () => {
+      const state = createManualRetryState(0);
+      const expectedTime = Date.now() - INITIAL_DELAY;
+      expect(state.retryStartTime).toBeLessThanOrEqual(expectedTime);
+    });
+
+    it("clears any previous error", () => {
+      const state = createManualRetryState(2);
+      expect(state.lastError).toBeUndefined();
+    });
+
+    it("prevents no-backoff bug: preserves attempt counter for continued backoff", () => {
+      // Bug scenario: After 3 failed attempts, manual retry should preserve counter
+      // so next failure waits 2^3=8s, not reset to 2^0=1s
+      const state = createManualRetryState(3);
+      expect(state.attempt).toBe(3); // NOT reset to 0
+    });
+  });
+
+  describe("createFailedRetryState", () => {
+    it("increments attempt counter and stores error", () => {
+      const error = { type: "unknown" as const, raw: "Test error" };
+      const state = createFailedRetryState(2, error);
+
+      expect(state.attempt).toBe(3);
+      expect(state.lastError).toEqual(error);
+      expect(state.retryStartTime).toBeLessThanOrEqual(Date.now());
+    });
+  });
+
+  describe("backoff progression scenario", () => {
+    it("maintains exponential backoff through manual retries", () => {
+      // 3 auto-retry failures → manual retry → preserves attempt counter
+      let state = createFailedRetryState(0, { type: "unknown" as const, raw: "Error" });
+      state = createFailedRetryState(state.attempt, { type: "unknown" as const, raw: "Error" });
+      state = createFailedRetryState(state.attempt, { type: "unknown" as const, raw: "Error" });
+      expect(state.attempt).toBe(3);
+
+      state = createManualRetryState(state.attempt);
+      expect(state.attempt).toBe(3); // NOT reset to 0
+
+      state = createFailedRetryState(state.attempt, { type: "unknown" as const, raw: "Error" });
+      expect(state.attempt).toBe(4); // Continues progression
+    });
+
+    it("resets backoff on successful stream start", () => {
+      let state = createFailedRetryState(0, { type: "unknown" as const, raw: "Error" });
+      state = createFailedRetryState(state.attempt, { type: "unknown" as const, raw: "Error" });
+      expect(state.attempt).toBe(2);
+
+      state = createFreshRetryState();
+      expect(state.attempt).toBe(0);
+      expect(state.lastError).toBeUndefined();
+    });
+  });
+});