From efafeb7196fbd16e903425a5d4c9d9b2c4fe0e01 Mon Sep 17 00:00:00 2001
From: alban bertolini <albanb@forestadmin.com>
Date: Mon, 10 Aug 2026 15:31:49 +0200
Subject: [PATCH] test(ai-proxy): skip models the provider cannot serve

The tool-support check counted a provider 5xx as "this model does not
support tool calls", so an OpenAI outage failed the suite on unrelated PRs.
It happened twice in a row on gpt-3.5-turbo-16k while main and other
branches were green, with AIProviderUnavailableError (HTTP 500).

A 5xx carries no signal about tool support and cannot be answered by the
denylist this test feeds, so those models are now reported as skipped
instead. A real capability failure still fails, and the run is asserted to
have verified at least one model so a total outage cannot pass green.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../ai-proxy/test/llm.integration.test.ts     | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/packages/ai-proxy/test/llm.integration.test.ts b/packages/ai-proxy/test/llm.integration.test.ts
index 5bc7f22626..28a0cc1143 100644
--- a/packages/ai-proxy/test/llm.integration.test.ts
+++ b/packages/ai-proxy/test/llm.integration.test.ts
@@ -412,6 +412,7 @@ providers.forEach(
 
         it('all models should support tool calls', async () => {
           const results: { model: string; success: boolean; error?: string }[] = [];
+          const unavailable: { model: string; error: string }[] = [];
 
           for (const model of modelsToTest) {
             const modelRouter = new Router({
@@ -464,12 +465,32 @@ providers.forEach(
                 throw new Error(`Infrastructure error testing model ${model}: ${errorMessage}`);
               }
 
+              // A provider 5xx says nothing about whether the model supports tool calls, so it
+              // cannot be answered by the denylist this test feeds. Counting it as a failure makes
+              // the suite fail on the provider's uptime instead of on our own contract.
+              if (errorMessage.includes('AIProviderUnavailableError')) {
+                unavailable.push({ model, error: errorMessage });
+
+                // eslint-disable-next-line no-continue
+                continue;
+              }
+
               results.push({ model, success: false, error: errorMessage });
             }
           }
 
           const failures = results.filter(r => !r.success);
 
+          if (unavailable.length > 0) {
+            const unavailableModelNames = unavailable.map(f => f.model).join(', ');
+            // eslint-disable-next-line no-console
+            console.warn(
+              `\n⚠️ ${unavailable.length} ${label} model(s) skipped, provider unavailable: ` +
+                `${unavailableModelNames}\n`,
+              unavailable,
+            );
+          }
+
           if (failures.length > 0) {
             const failedModelNames = failures.map(f => f.model).join(', ');
             // eslint-disable-next-line no-console
@@ -480,6 +501,10 @@ providers.forEach(
           }
 
           expect(failures).toEqual([]);
+
+          // Guards against the skip above hiding a total outage, which would otherwise let the
+          // suite pass green while having verified nothing.
+          expect(results.length).toBeGreaterThan(0);
         }, 300000); // 5 minutes for all models
       });
     });