From cffd7378115eb19662cfc0164450ab19c01df803 Mon Sep 17 00:00:00 2001 From: Brian Genisio Date: Thu, 3 Sep 2026 06:46:50 -0400 Subject: [PATCH 1/2] Select the LLM from session.config.json model refs. Co-authored-by: Cursor --- .env.example | 12 +--- README.md | 10 ++-- lib/llm/anthropic.js | 11 ++-- lib/llm/gemini.js | 11 ++-- lib/llm/model-ref.js | 83 +++++++++++++++++++++++++++ lib/llm/openai.js | 11 ++-- lib/llm/provider.js | 34 +++++------- lib/session-config.js | 73 +++++++++++++++++++++++- server.js | 53 +++++++++++++----- session.config.example.json | 6 ++ tests/llm-gemini.test.js | 16 ++---- tests/llm-model-ref.test.js | 85 ++++++++++++++++++++++++++++ tests/llm-openai.test.js | 16 ++---- tests/llm-provider.test.js | 22 ++++---- tests/server.test.js | 105 +++++++++++++++++++++++++++++++---- tests/session-config.test.js | 76 ++++++++++++++++++++++++- 16 files changed, 516 insertions(+), 108 deletions(-) create mode 100644 lib/llm/model-ref.js create mode 100644 tests/llm-model-ref.test.js diff --git a/.env.example b/.env.example index 94f031c..73f10dc 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,14 @@ -# LLM provider: anthropic (default), openai, or gemini -LLM_PROVIDER=anthropic - -# Anthropic (LLM_PROVIDER=anthropic) +# Anthropic (session.config.json model: anthropic/…) ANTHROPIC_BASE_URL=https://api.anthropic.com ANTHROPIC_API_KEY= -# ANTHROPIC_MODEL=claude-sonnet-4-6 -# OpenAI (LLM_PROVIDER=openai) +# OpenAI (session.config.json model: openai/…) # OPENAI_BASE_URL=https://api.openai.com/v1 # OPENAI_API_KEY= -# OPENAI_MODEL=gpt-4o -# Gemini (LLM_PROVIDER=gemini) +# Gemini (session.config.json model: google/…) # GOOGLE_BASE_URL=https://generativelanguage.googleapis.com # GOOGLE_API_KEY= -# GOOGLE_MODEL=gemini-2.5-flash # Optional: HTTP port (default 3000) # PORT=3000 diff --git a/README.md b/README.md index 4f53173..fd41517 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,16 @@ cp .env.example .env cp session.config.example.json session.config.json ``` -Fill in `.env` for the provider you want: +Fill in `.env` with the API key (and optional `*_BASE_URL`) for the provider you want. Choose the model in `session.config.json` as `provider/model-id`: -- Anthropic (default): `ANTHROPIC_API_KEY`, optional `ANTHROPIC_BASE_URL` / `ANTHROPIC_MODEL` -- OpenAI: set `LLM_PROVIDER=openai` and `OPENAI_API_KEY`, optional `OPENAI_BASE_URL` / `OPENAI_MODEL` -- Gemini: set `LLM_PROVIDER=gemini` and `GOOGLE_API_KEY`, optional `GOOGLE_BASE_URL` / `GOOGLE_MODEL` +- `anthropic/claude-sonnet-4-6` — needs `ANTHROPIC_API_KEY`, optional `ANTHROPIC_BASE_URL` +- `openai/gpt-5.6-luna` — needs `OPENAI_API_KEY`, optional `OPENAI_BASE_URL` +- `google/gemini-3.6-flash` — needs `GOOGLE_API_KEY`, optional `GOOGLE_BASE_URL` (`gemini/…` also routes to Gemini) `session.config.json` is separate from `.env`. It is local (not checked in) and holds **session defaults**, not secrets: +- `model` (optional) — `provider/model-id` (default `anthropic/claude-sonnet-4-6`); must be listed in `allowedModels` +- `allowedModels` (optional) — picker list of `provider/model-id` refs (defaults to Anthropic, OpenAI, and Gemini examples above) - `defaults` (optional) — `minRuns`, `maxRuns`, `minCases`, `maxCases` (each 1–5) - `initialSession` (optional) — `promptA`, `promptB`, and `cases` (`input` / `expectedAnswer`) diff --git a/lib/llm/anthropic.js b/lib/llm/anthropic.js index 69f49b7..6bc649a 100644 --- a/lib/llm/anthropic.js +++ b/lib/llm/anthropic.js @@ -1,9 +1,9 @@ /** * Anthropic Messages API provider. * - * Configured from ANTHROPIC_API_KEY and optional ANTHROPIC_BASE_URL / - * ANTHROPIC_MODEL. Model ids may still use the Octavus-style - * `anthropic/claude-…` prefix; it is stripped before the API call. + * Configured from ANTHROPIC_API_KEY and optional ANTHROPIC_BASE_URL. + * Model ids may still use an `anthropic/` prefix; it is stripped before + * the API call. */ import Anthropic from '@anthropic-ai/sdk'; @@ -36,9 +36,10 @@ export function extractMessageText(message) { /** * @param {NodeJS.ProcessEnv} [env] + * @param {string} [modelId] * @returns {import('./types.js').LlmProvider} */ -export function createAnthropicProvider(env = process.env) { +export function createAnthropicProvider(env = process.env, modelId) { const apiKey = env.ANTHROPIC_API_KEY; if (!apiKey) { const err = new Error('ANTHROPIC_API_KEY is not configured'); @@ -47,7 +48,7 @@ export function createAnthropicProvider(env = process.env) { } const baseURL = optionalBaseUrl(env.ANTHROPIC_BASE_URL, 'ANTHROPIC_BASE_URL'); - const model = normalizeAnthropicModelId(env.ANTHROPIC_MODEL) || DEFAULT_ANTHROPIC_MODEL; + const model = normalizeAnthropicModelId(modelId) || DEFAULT_ANTHROPIC_MODEL; const client = new Anthropic({ apiKey, diff --git a/lib/llm/gemini.js b/lib/llm/gemini.js index eb04430..91ca62c 100644 --- a/lib/llm/gemini.js +++ b/lib/llm/gemini.js @@ -1,9 +1,9 @@ /** * Google Gemini generateContent provider. * - * Configured from GOOGLE_API_KEY and optional GOOGLE_BASE_URL / - * GOOGLE_MODEL. Model ids may still use a `google/` or `gemini/` - * prefix; it is stripped before the API call. + * Configured from GOOGLE_API_KEY and optional GOOGLE_BASE_URL. + * Model ids may still use a `google/` or `gemini/` prefix; it is + * stripped before the API call. */ import { GoogleGenAI } from '@google/genai'; @@ -51,9 +51,10 @@ export function toGeminiContents(messages = []) { /** * @param {NodeJS.ProcessEnv} [env] + * @param {string} [modelId] * @returns {import('./types.js').LlmProvider} */ -export function createGeminiProvider(env = process.env) { +export function createGeminiProvider(env = process.env, modelId) { const apiKey = env.GOOGLE_API_KEY; if (!apiKey) { const err = new Error('GOOGLE_API_KEY is not configured'); @@ -62,7 +63,7 @@ export function createGeminiProvider(env = process.env) { } const baseURL = optionalBaseUrl(env.GOOGLE_BASE_URL, 'GOOGLE_BASE_URL'); - const model = normalizeGeminiModelId(env.GOOGLE_MODEL) || DEFAULT_GEMINI_MODEL; + const model = normalizeGeminiModelId(modelId) || DEFAULT_GEMINI_MODEL; const client = new GoogleGenAI({ apiKey, diff --git a/lib/llm/model-ref.js b/lib/llm/model-ref.js new file mode 100644 index 0000000..3325f06 --- /dev/null +++ b/lib/llm/model-ref.js @@ -0,0 +1,83 @@ +/** + * Parse a session model ref (`provider/model-id`) into a routed provider. + * + * Browser-safe — no SDK imports. Prefix aliases: + * anthropic/… → anthropic + * openai/… → openai + * google/… → gemini + * gemini/… → gemini + */ + +export const DEFAULT_MODEL_REF = 'anthropic/claude-sonnet-4-6'; + +/** @type {Record} */ +const PROVIDERS = { + anthropic: { name: 'anthropic', apiKeyName: 'ANTHROPIC_API_KEY' }, + openai: { name: 'openai', apiKeyName: 'OPENAI_API_KEY' }, + google: { name: 'gemini', apiKeyName: 'GOOGLE_API_KEY' }, + gemini: { name: 'gemini', apiKeyName: 'GOOGLE_API_KEY' }, +}; + +/** + * @typedef {object} ParsedModelRef + * @property {string} provider + * @property {string} modelId + * @property {string} prefix + * @property {string} raw + * @property {string} apiKeyName + */ + +/** + * @param {string} raw + * @param {string} [code] + * @returns {Error} + */ +function modelRefError(raw, code = 'LLM_INVALID_MODEL') { + const err = new Error( + code === 'LLM_UNSUPPORTED_PROVIDER' + ? `Unsupported model provider "${raw}"` + : `Model must use provider/model-id format (e.g. openai/gpt-5.6-luna), got "${raw}"`, + ); + err.code = code; + return err; +} + +/** + * @param {unknown} [modelRef] + * @returns {ParsedModelRef} + */ +export function parseModelRef(modelRef) { + const raw = String(modelRef ?? '').trim() || DEFAULT_MODEL_REF; + const slash = raw.indexOf('/'); + if (slash <= 0 || slash === raw.length - 1) { + throw modelRefError(raw); + } + + const prefix = raw.slice(0, slash).trim().toLowerCase(); + const modelId = raw.slice(slash + 1).trim(); + if (!prefix || !modelId) { + throw modelRefError(raw); + } + + const provider = PROVIDERS[prefix]; + if (!provider) { + throw modelRefError(prefix, 'LLM_UNSUPPORTED_PROVIDER'); + } + + return { + provider: provider.name, + modelId, + prefix, + raw, + apiKeyName: provider.apiKeyName, + }; +} + +/** + * Env var that must be set for the provider in a model ref. + * @param {unknown} [modelRef] + * @returns {string} + */ +export function requiredApiKeyName(modelRef = DEFAULT_MODEL_REF) { + return parseModelRef(modelRef).apiKeyName; +} diff --git a/lib/llm/openai.js b/lib/llm/openai.js index f96d7f4..0c30580 100644 --- a/lib/llm/openai.js +++ b/lib/llm/openai.js @@ -1,9 +1,9 @@ /** * OpenAI Chat Completions API provider. * - * Configured from OPENAI_API_KEY and optional OPENAI_BASE_URL / - * OPENAI_MODEL. Model ids may still use an `openai/gpt-…` prefix; - * it is stripped before the API call. + * Configured from OPENAI_API_KEY and optional OPENAI_BASE_URL. + * Model ids may still use an `openai/` prefix; it is stripped before + * the API call. */ import OpenAI from 'openai'; @@ -37,9 +37,10 @@ export function extractCompletionText(completion) { /** * @param {NodeJS.ProcessEnv} [env] + * @param {string} [modelId] * @returns {import('./types.js').LlmProvider} */ -export function createOpenAiProvider(env = process.env) { +export function createOpenAiProvider(env = process.env, modelId) { const apiKey = env.OPENAI_API_KEY; if (!apiKey) { const err = new Error('OPENAI_API_KEY is not configured'); @@ -48,7 +49,7 @@ export function createOpenAiProvider(env = process.env) { } const baseURL = optionalBaseUrl(env.OPENAI_BASE_URL, 'OPENAI_BASE_URL'); - const model = normalizeOpenAiModelId(env.OPENAI_MODEL) || DEFAULT_OPENAI_MODEL; + const model = normalizeOpenAiModelId(modelId) || DEFAULT_OPENAI_MODEL; const client = new OpenAI({ apiKey, diff --git a/lib/llm/provider.js b/lib/llm/provider.js index cf6d9dd..925aef7 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -1,42 +1,34 @@ /** - * Resolve an LLM provider from environment variables. + * Resolve an LLM provider from a session model ref (`provider/model-id`). * - * LLM_PROVIDER selects the adapter (default: anthropic). Additional - * providers can be added to the switch without changing eval-run. + * The prefix selects the adapter; the remainder is the API model id. + * API keys and base URLs still come from the environment. */ import { createAnthropicProvider } from './anthropic.js'; import { createGeminiProvider } from './gemini.js'; +import { parseModelRef } from './model-ref.js'; import { createOpenAiProvider } from './openai.js'; -/** - * Env var that must be set for the selected provider. - * @param {NodeJS.ProcessEnv} [env] - * @returns {string} - */ -export function requiredApiKeyName(env = process.env) { - const name = String(env.LLM_PROVIDER ?? 'anthropic').trim().toLowerCase() || 'anthropic'; - if (name === 'openai') return 'OPENAI_API_KEY'; - if (name === 'gemini') return 'GOOGLE_API_KEY'; - return 'ANTHROPIC_API_KEY'; -} +export { DEFAULT_MODEL_REF, parseModelRef, requiredApiKeyName } from './model-ref.js'; /** * @param {NodeJS.ProcessEnv} [env] + * @param {string} [modelRef] * @returns {import('./types.js').LlmProvider} */ -export function createLlmProvider(env = process.env) { - const name = String(env.LLM_PROVIDER ?? 'anthropic').trim().toLowerCase() || 'anthropic'; +export function createLlmProvider(env = process.env, modelRef) { + const parsed = parseModelRef(modelRef); - switch (name) { + switch (parsed.provider) { case 'anthropic': - return createAnthropicProvider(env); + return createAnthropicProvider(env, parsed.modelId); case 'openai': - return createOpenAiProvider(env); + return createOpenAiProvider(env, parsed.modelId); case 'gemini': - return createGeminiProvider(env); + return createGeminiProvider(env, parsed.modelId); default: { - const err = new Error(`Unsupported LLM_PROVIDER "${name}"`); + const err = new Error(`Unsupported model provider "${parsed.prefix}"`); err.code = 'LLM_UNSUPPORTED_PROVIDER'; throw err; } diff --git a/lib/session-config.js b/lib/session-config.js index dc9fca2..f126258 100644 --- a/lib/session-config.js +++ b/lib/session-config.js @@ -2,9 +2,22 @@ * Local session config for the Prompt Evaluation Simulator. * * session.config.json is machine-local (not checked in). It is not a secrets - * file — provider keys stay in .env. This module is browser-safe. + * file — provider keys stay in .env. `model` is a provider/model-id ref + * (e.g. openai/gpt-5.6-luna) and must be listed in `allowedModels`. + * This module is browser-safe. */ +import { DEFAULT_MODEL_REF, parseModelRef } from './llm/model-ref.js'; + +export { DEFAULT_MODEL_REF }; + +/** Catalog used when `allowedModels` is missing or empty after filtering. */ +export const DEFAULT_ALLOWED_MODELS = [ + DEFAULT_MODEL_REF, + 'openai/gpt-5.6-luna', + 'google/gemini-3.6-flash', +]; + /** UI limits when session.config.json is missing or omits `defaults`. Keep aligned with MIN/MAX_EVAL_* in eval-run and eval-compare. */ export const FALLBACK_DEFAULTS = { minRuns: 1, @@ -68,17 +81,73 @@ function normalizeCases(raw) { })); } +/** + * @param {unknown} raw + * @returns {string[]} + */ +export function normalizeAllowedModels(raw) { + if (!Array.isArray(raw)) return [...DEFAULT_ALLOWED_MODELS]; + const seen = new Set(); + /** @type {string[]} */ + const models = []; + for (const item of raw) { + if (typeof item !== 'string') continue; + const trimmed = item.trim(); + if (!trimmed || seen.has(trimmed)) continue; + try { + parseModelRef(trimmed); + } catch { + continue; + } + seen.add(trimmed); + models.push(trimmed); + } + return models.length > 0 ? models : [...DEFAULT_ALLOWED_MODELS]; +} + +/** + * @param {string} model + * @param {string[]} allowedModels + */ +export function assertAllowedModel(model, allowedModels) { + if (!Array.isArray(allowedModels) || !allowedModels.includes(model)) { + const err = new Error(`Model "${model}" is not in allowedModels`); + err.code = 'LLM_MODEL_NOT_ALLOWED'; + throw err; + } +} + +/** + * @param {unknown} rawModel + * @param {string[]} allowedModels + * @returns {string} + */ +function resolveConfiguredModel(rawModel, allowedModels) { + const model = typeof rawModel === 'string' && rawModel.trim() + ? rawModel.trim() + : ''; + if (model) return model; + if (allowedModels.includes(DEFAULT_MODEL_REF)) return DEFAULT_MODEL_REF; + return allowedModels[0]; +} + /** * Normalize a raw session.config.json object into a complete payload. * Missing or invalid fields use empty initial-session values and FALLBACK_DEFAULTS. + * `model` is only defaulted when omitted; a provided value is kept even if it is + * not in `allowedModels` so eval can reject the mismatch. * @param {unknown} raw * @returns {{ + * model: string, + * allowedModels: string[], * defaults: { minRuns: number, maxRuns: number, minCases: number, maxCases: number }, * initialSession: { promptA: string, promptB: string, cases: { input: string, expectedAnswer: string }[] }, * }} */ export function normalizeSessionConfig(raw) { const src = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {}; + const allowedModels = normalizeAllowedModels(src.allowedModels); + const model = resolveConfiguredModel(src.model, allowedModels); const defaultsSrc = src.defaults && typeof src.defaults === 'object' && !Array.isArray(src.defaults) ? src.defaults : {}; @@ -100,6 +169,8 @@ export function normalizeSessionConfig(raw) { ); return { + model, + allowedModels, defaults: { minRuns: runs.min, maxRuns: runs.max, diff --git a/server.js b/server.js index 03a5433..7051c5a 100644 --- a/server.js +++ b/server.js @@ -8,7 +8,7 @@ import { MAX_EVAL_RUNS, MIN_EVAL_RUNS } from './lib/eval-run.js'; import { runPromptComparison, MAX_EVAL_CASES } from './lib/eval-compare.js'; import { createLlmProvider, requiredApiKeyName } from './lib/llm/provider.js'; import { DEFAULT_METRIC_ID, isValidMetricId } from './lib/metrics/index.js'; -import { normalizeSessionConfig } from './lib/session-config.js'; +import { assertAllowedModel, normalizeSessionConfig } from './lib/session-config.js'; import { normalizeEvalSession } from './lib/eval-session.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -19,16 +19,32 @@ const PORT = Number.parseInt(process.env.PORT ?? '3000', 10) || 3000; /** @type {import('./lib/llm/types.js').LlmProvider | null | undefined} */ let cachedLlm; +/** @type {string | undefined} */ +let cachedModel; + +/** + * @returns {Promise<{ model: string, allowedModels: string[] }>} + */ +async function sessionLlmConfig() { + const config = normalizeSessionConfig(await readJsonFile(SESSION_CONFIG_FILE, {})); + return { model: config.model, allowedModels: config.allowedModels }; +} /** * Resolve the configured LLM provider. A missing provider API key is treated * as "not configured" so eval routes can return 503 without crashing startup. + * @param {string} model + * @param {string[]} allowedModels * @returns {import('./lib/llm/types.js').LlmProvider | null} */ -function getLlm() { - const keyName = requiredApiKeyName(process.env); +function getLlm(model, allowedModels) { + const keyName = requiredApiKeyName(model); + assertAllowedModel(model, allowedModels); if (!process.env[keyName]) return null; - if (!cachedLlm) cachedLlm = createLlmProvider(process.env); + if (!cachedLlm || cachedModel !== model) { + cachedLlm = createLlmProvider(process.env, model); + cachedModel = model; + } return cachedLlm; } @@ -36,13 +52,14 @@ function getLlm() { * Resolve a provider for an eval route without letting createLlmProvider() * throws escape Express 4 async handlers as unhandled rejections. * @param {import('express').Response} res - * @returns {import('./lib/llm/types.js').LlmProvider | null} + * @returns {Promise} */ -function resolveLlm(res) { +async function resolveLlm(res) { try { - const llm = getLlm(); + const { model, allowedModels } = await sessionLlmConfig(); + const llm = getLlm(model, allowedModels); if (!llm) { - res.status(503).json({ error: `${requiredApiKeyName()} is not configured` }); + res.status(503).json({ error: `${requiredApiKeyName(model)} is not configured` }); return null; } return llm; @@ -58,6 +75,7 @@ function resolveLlm(res) { /** Clear the cached provider. Used by tests when mocking createLlmProvider. */ export function resetLlmCache() { cachedLlm = undefined; + cachedModel = undefined; } // ── Middleware ──────────────────────────────────────────────── @@ -65,7 +83,7 @@ app.use(express.json()); app.use('/design-system', express.static(path.join(__dirname, 'design-system'))); app.use(express.static(path.join(__dirname, 'public'))); -// Local eval session defaults (prompts, cases, UI min/max). Not secrets — +// Local eval session defaults (model, prompts, cases, UI min/max). Not secrets — // those stay in .env. Missing file → empty initial session + built-in limits. app.get('/api/session-config', async (_req, res) => { const raw = await readJsonFile(SESSION_CONFIG_FILE, {}); @@ -106,7 +124,7 @@ app.put('/api/eval/session', async (req, res) => { // Evaluate Prompt A across cases; optionally compare with Prompt B under // identical conditions. Each run is an independent LLM complete() call. app.post('/api/eval/compare', async (req, res) => { - const llm = resolveLlm(res); + const llm = await resolveLlm(res); if (!llm) return; const { @@ -193,11 +211,18 @@ app.post('/api/eval/compare', async (req, res) => { }); if (process.env.NODE_ENV !== 'test') { - const server = app.listen(PORT, () => { + const server = app.listen(PORT, async () => { console.log(`Prompt Evaluation Simulator at http://localhost:${PORT}`); - const keyName = requiredApiKeyName(); - if (!process.env[keyName]) { - console.warn(`[WARN] ${keyName} is not set — evaluation will not work until it is configured.`); + try { + const { model, allowedModels } = await sessionLlmConfig(); + assertAllowedModel(model, allowedModels); + const keyName = requiredApiKeyName(model); + if (!process.env[keyName]) { + console.warn(`[WARN] ${keyName} is not set — evaluation will not work until it is configured.`); + } + } catch (err) { + const message = err instanceof Error && err.message ? err.message : 'LLM model is not configured'; + console.warn(`[WARN] ${message}`); } }); server.on('error', (err) => { diff --git a/session.config.example.json b/session.config.example.json index acfce2a..16dbdb1 100644 --- a/session.config.example.json +++ b/session.config.example.json @@ -1,4 +1,10 @@ { + "model": "anthropic/claude-haiku-4-5-20251001", + "allowedModels": [ + "anthropic/claude-haiku-4-5-20251001", + "openai/gpt-5.6-luna", + "google/gemini-3.6-flash" + ], "defaults": { "minRuns": 1, "maxRuns": 5, diff --git a/tests/llm-gemini.test.js b/tests/llm-gemini.test.js index 7d516c1..e736cc1 100644 --- a/tests/llm-gemini.test.js +++ b/tests/llm-gemini.test.js @@ -67,17 +67,16 @@ describe('createLlmProvider gemini', () => { constructorOpts.mockReset(); }); - it('selects gemini when LLM_PROVIDER=gemini', () => { + it('selects gemini from a google/ model ref', () => { const llm = createLlmProvider({ - LLM_PROVIDER: 'gemini', GOOGLE_API_KEY: 'sk-test', - }); + }, 'google/gemini-2.5-flash'); expect(llm.name).toBe('gemini'); expect(llm.model).toBe(DEFAULT_GEMINI_MODEL); }); it('throws when GOOGLE_API_KEY is missing', () => { - expect(() => createLlmProvider({ LLM_PROVIDER: 'gemini' })).toThrow(/GOOGLE_API_KEY/); + expect(() => createLlmProvider({}, 'google/gemini-2.5-flash')).toThrow(/GOOGLE_API_KEY/); try { createGeminiProvider({}); } catch (err) { @@ -85,7 +84,7 @@ describe('createLlmProvider gemini', () => { } }); - it('strips google/ from GOOGLE_MODEL and from complete() requests', async () => { + it('uses the model id from the session ref and strips google/ on complete()', async () => { const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); generateContent.mockResolvedValue({ text: 'Paris', @@ -93,11 +92,9 @@ describe('createLlmProvider gemini', () => { }); const llm = createLlmProvider({ - LLM_PROVIDER: 'gemini', GOOGLE_API_KEY: 'sk-test', GOOGLE_BASE_URL: 'https://generativelanguage.example.test', - GOOGLE_MODEL: 'google/gemini-2.5-flash', - }); + }, 'google/gemini-2.5-flash'); expect(llm.model).toBe('gemini-2.5-flash'); expect(constructorOpts).toHaveBeenCalledWith({ apiKey: 'sk-test', @@ -132,10 +129,9 @@ describe('createLlmProvider gemini', () => { generateContent.mockRejectedValueOnce(err); const llm = createLlmProvider({ - LLM_PROVIDER: 'gemini', GOOGLE_API_KEY: 'sk-test', GOOGLE_BASE_URL: 'https://generativelanguage.example.test', - }); + }, 'google/gemini-2.5-flash'); await expect(llm.complete({ model: 'gemini-2.5-flash', messages: [{ role: 'user', content: 'Hi' }], diff --git a/tests/llm-model-ref.test.js b/tests/llm-model-ref.test.js new file mode 100644 index 0000000..bc765fc --- /dev/null +++ b/tests/llm-model-ref.test.js @@ -0,0 +1,85 @@ +import { describe, it, expect } from 'vitest'; +import { + DEFAULT_MODEL_REF, + parseModelRef, + requiredApiKeyName, +} from '../lib/llm/model-ref.js'; + +describe('parseModelRef', () => { + it('defaults to anthropic when the ref is missing', () => { + expect(parseModelRef()).toMatchObject({ + provider: 'anthropic', + modelId: 'claude-sonnet-4-6', + prefix: 'anthropic', + raw: DEFAULT_MODEL_REF, + apiKeyName: 'ANTHROPIC_API_KEY', + }); + expect(parseModelRef('')).toMatchObject({ + provider: 'anthropic', + modelId: 'claude-sonnet-4-6', + }); + }); + + it('routes openai and google prefixes', () => { + expect(parseModelRef('openai/gpt-5.6-luna')).toMatchObject({ + provider: 'openai', + modelId: 'gpt-5.6-luna', + prefix: 'openai', + apiKeyName: 'OPENAI_API_KEY', + }); + expect(parseModelRef('google/gemini-3.6-flash')).toMatchObject({ + provider: 'gemini', + modelId: 'gemini-3.6-flash', + prefix: 'google', + apiKeyName: 'GOOGLE_API_KEY', + }); + expect(parseModelRef('gemini/gemini-2.5-flash')).toMatchObject({ + provider: 'gemini', + modelId: 'gemini-2.5-flash', + prefix: 'gemini', + }); + }); + + it('trims whitespace and lowercases the prefix', () => { + expect(parseModelRef(' OpenAI/gpt-5.6-luna ')).toMatchObject({ + provider: 'openai', + modelId: 'gpt-5.6-luna', + prefix: 'openai', + }); + }); + + it('keeps extra slashes in the model id', () => { + expect(parseModelRef('openai/ft:gpt-4o/custom')).toMatchObject({ + provider: 'openai', + modelId: 'ft:gpt-4o/custom', + }); + }); + + it('rejects a missing slash or empty model id', () => { + expect(() => parseModelRef('gpt-5.6-luna')).toThrow(/provider\/model-id/); + expect(() => parseModelRef('openai/')).toThrow(/provider\/model-id/); + try { + parseModelRef('bare-model'); + } catch (err) { + expect(err.code).toBe('LLM_INVALID_MODEL'); + } + }); + + it('rejects an unknown provider prefix', () => { + expect(() => parseModelRef('mistral/large')).toThrow(/Unsupported model provider "mistral"/); + try { + parseModelRef('mistral/large'); + } catch (err) { + expect(err.code).toBe('LLM_UNSUPPORTED_PROVIDER'); + } + }); +}); + +describe('requiredApiKeyName', () => { + it('maps a model ref to the provider API key env var', () => { + expect(requiredApiKeyName()).toBe('ANTHROPIC_API_KEY'); + expect(requiredApiKeyName(DEFAULT_MODEL_REF)).toBe('ANTHROPIC_API_KEY'); + expect(requiredApiKeyName('openai/gpt-5.6-luna')).toBe('OPENAI_API_KEY'); + expect(requiredApiKeyName('google/gemini-3.6-flash')).toBe('GOOGLE_API_KEY'); + }); +}); diff --git a/tests/llm-openai.test.js b/tests/llm-openai.test.js index 95b915c..0f215e4 100644 --- a/tests/llm-openai.test.js +++ b/tests/llm-openai.test.js @@ -59,17 +59,16 @@ describe('createLlmProvider openai', () => { constructorOpts.mockReset(); }); - it('selects openai when LLM_PROVIDER=openai', () => { + it('selects openai from an openai/ model ref', () => { const llm = createLlmProvider({ - LLM_PROVIDER: 'openai', OPENAI_API_KEY: 'sk-test', - }); + }, 'openai/gpt-4o'); expect(llm.name).toBe('openai'); expect(llm.model).toBe(DEFAULT_OPENAI_MODEL); }); it('throws when OPENAI_API_KEY is missing', () => { - expect(() => createLlmProvider({ LLM_PROVIDER: 'openai' })).toThrow(/OPENAI_API_KEY/); + expect(() => createLlmProvider({}, 'openai/gpt-4o')).toThrow(/OPENAI_API_KEY/); try { createOpenAiProvider({}); } catch (err) { @@ -77,7 +76,7 @@ describe('createLlmProvider openai', () => { } }); - it('strips openai/ from OPENAI_MODEL and from complete() requests', async () => { + it('uses the model id from the session ref and strips openai/ on complete()', async () => { const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); createMock.mockResolvedValue({ id: 'chatcmpl-1', @@ -85,11 +84,9 @@ describe('createLlmProvider openai', () => { }); const llm = createLlmProvider({ - LLM_PROVIDER: 'openai', OPENAI_API_KEY: 'sk-test', OPENAI_BASE_URL: 'https://api.example.test/v1', - OPENAI_MODEL: 'openai/gpt-4o', - }); + }, 'openai/gpt-4o'); expect(llm.model).toBe('gpt-4o'); expect(constructorOpts).toHaveBeenCalledWith({ apiKey: 'sk-test', @@ -124,10 +121,9 @@ describe('createLlmProvider openai', () => { createMock.mockRejectedValueOnce(err); const llm = createLlmProvider({ - LLM_PROVIDER: 'openai', OPENAI_API_KEY: 'sk-test', OPENAI_BASE_URL: 'https://api.example.test/v1', - }); + }, 'openai/gpt-4o'); await expect(llm.complete({ model: 'gpt-4o', messages: [{ role: 'user', content: 'Hi' }], diff --git a/tests/llm-provider.test.js b/tests/llm-provider.test.js index 5dd7359..ec3c9e9 100644 --- a/tests/llm-provider.test.js +++ b/tests/llm-provider.test.js @@ -20,6 +20,7 @@ const { DEFAULT_ANTHROPIC_MODEL, } = await import('../lib/llm/anthropic.js'); const { createLlmProvider, requiredApiKeyName } = await import('../lib/llm/provider.js'); +const { DEFAULT_MODEL_REF } = await import('../lib/llm/model-ref.js'); describe('normalizeAnthropicModelId', () => { it('strips an anthropic/ prefix', () => { @@ -54,21 +55,21 @@ describe('createLlmProvider', () => { expect(llm.model).toBe(DEFAULT_ANTHROPIC_MODEL); }); - it('throws on an unknown provider', () => { + it('throws on an unknown provider prefix', () => { expect(() => createLlmProvider({ - LLM_PROVIDER: 'mistral', ANTHROPIC_API_KEY: 'sk-test', - })).toThrow(/Unsupported LLM_PROVIDER "mistral"/); + }, 'mistral/large')).toThrow(/Unsupported model provider "mistral"/); }); - it('reports the API key env var for the selected provider', () => { - expect(requiredApiKeyName({})).toBe('ANTHROPIC_API_KEY'); - expect(requiredApiKeyName({ LLM_PROVIDER: 'openai' })).toBe('OPENAI_API_KEY'); - expect(requiredApiKeyName({ LLM_PROVIDER: 'gemini' })).toBe('GOOGLE_API_KEY'); + it('reports the API key env var for the selected model', () => { + expect(requiredApiKeyName()).toBe('ANTHROPIC_API_KEY'); + expect(requiredApiKeyName(DEFAULT_MODEL_REF)).toBe('ANTHROPIC_API_KEY'); + expect(requiredApiKeyName('openai/gpt-5.6-luna')).toBe('OPENAI_API_KEY'); + expect(requiredApiKeyName('google/gemini-3.6-flash')).toBe('GOOGLE_API_KEY'); }); it('throws when ANTHROPIC_API_KEY is missing', () => { - expect(() => createLlmProvider({ LLM_PROVIDER: 'anthropic' })).toThrow(/ANTHROPIC_API_KEY/); + expect(() => createLlmProvider({}, 'anthropic/claude-sonnet-4-6')).toThrow(/ANTHROPIC_API_KEY/); try { createAnthropicProvider({}); } catch (err) { @@ -76,7 +77,7 @@ describe('createLlmProvider', () => { } }); - it('strips anthropic/ from ANTHROPIC_MODEL and from complete() requests', async () => { + it('uses the model id from the session ref and strips anthropic/ on complete()', async () => { const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); createMock.mockResolvedValue({ id: 'msg-1', @@ -87,8 +88,7 @@ describe('createLlmProvider', () => { const llm = createLlmProvider({ ANTHROPIC_API_KEY: 'sk-test', ANTHROPIC_BASE_URL: 'https://api.example.test', - ANTHROPIC_MODEL: 'anthropic/claude-sonnet-4-6', - }); + }, 'anthropic/claude-sonnet-4-6'); expect(llm.model).toBe('claude-sonnet-4-6'); const result = await llm.complete({ diff --git a/tests/server.test.js b/tests/server.test.js index 7e0d838..b8b898c 100644 --- a/tests/server.test.js +++ b/tests/server.test.js @@ -136,6 +136,12 @@ describe('GET /api/session-config', () => { fs.readFile.mockRejectedValue(new Error('ENOENT')); const res = await request(app).get('/api/session-config'); expect(res.status).toBe(200); + expect(res.body.model).toBe('anthropic/claude-sonnet-4-6'); + expect(res.body.allowedModels).toEqual([ + 'anthropic/claude-sonnet-4-6', + 'openai/gpt-5.6-luna', + 'google/gemini-3.6-flash', + ]); expect(res.body.defaults).toEqual({ minRuns: 1, maxRuns: 5, @@ -153,6 +159,11 @@ describe('GET /api/session-config', () => { fs.readFile.mockImplementation(async (p) => { if (String(p).includes('session.config.json')) { return JSON.stringify({ + model: 'google/gemini-3.6-flash', + allowedModels: [ + 'anthropic/claude-sonnet-4-6', + 'google/gemini-3.6-flash', + ], defaults: { minRuns: 2, maxRuns: 4 }, initialSession: { promptA: 'Prompt A', @@ -165,6 +176,11 @@ describe('GET /api/session-config', () => { }); const res = await request(app).get('/api/session-config'); expect(res.status).toBe(200); + expect(res.body.model).toBe('google/gemini-3.6-flash'); + expect(res.body.allowedModels).toEqual([ + 'anthropic/claude-sonnet-4-6', + 'google/gemini-3.6-flash', + ]); expect(res.body.defaults.minRuns).toBe(2); expect(res.body.defaults.maxRuns).toBe(4); expect(res.body.initialSession.promptA).toBe('Prompt A'); @@ -178,6 +194,40 @@ describe('GET /api/session-config', () => { describe('POST /api/eval/compare', () => { beforeEach(() => { runPromptComparison.mockReset(); + fs.readFile.mockImplementation(async (p) => { + if (String(p).includes('session.config.json')) return '{}'; + throw new Error('ENOENT'); + }); + }); + + it('creates the LLM from the session.config.json model ref', async () => { + resetLlmCache(); + fs.readFile.mockImplementation(async (p) => { + if (String(p).includes('session.config.json')) { + return JSON.stringify({ model: 'openai/gpt-5.6-luna' }); + } + throw new Error('ENOENT'); + }); + const previousKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = 'test-openai-key'; + runPromptComparison.mockResolvedValue({ + conditions: { metricId: 'exact-match', runs: 1, caseCount: 1 }, + cases: [], + prompts: [], + comparison: { outcome: 'unscored', winnerId: null, means: {} }, + }); + + try { + const res = await request(app) + .post('/api/eval/compare') + .send({ promptA: 'Hi', input: 'x', runs: 1 }); + expect(res.status).toBe(200); + expect(createLlmProvider).toHaveBeenCalledWith(expect.anything(), 'openai/gpt-5.6-luna'); + } finally { + if (previousKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousKey; + resetLlmCache(); + } }); it('returns 400 when promptA is missing', async () => { @@ -268,7 +318,7 @@ describe('POST /api/eval/compare', () => { it('returns 503 when createLlmProvider throws a configuration error', async () => { resetLlmCache(); - const err = new Error('Unsupported LLM_PROVIDER "gemini"'); + const err = new Error('Unsupported model provider "mistral"'); err.code = 'LLM_UNSUPPORTED_PROVIDER'; createLlmProvider.mockImplementationOnce(() => { throw err; @@ -279,15 +329,19 @@ describe('POST /api/eval/compare', () => { .send({ promptA: 'Hi', input: 'x', runs: 1 }); expect(res.status).toBe(503); - expect(res.body.error).toMatch(/Unsupported LLM_PROVIDER/); + expect(res.body.error).toMatch(/Unsupported model provider/); expect(runPromptComparison).not.toHaveBeenCalled(); resetLlmCache(); }); - it('returns 503 when LLM_PROVIDER=openai and OPENAI_API_KEY is missing', async () => { - const previousProvider = process.env.LLM_PROVIDER; + it('returns 503 when model is openai/… and OPENAI_API_KEY is missing', async () => { + fs.readFile.mockImplementation(async (p) => { + if (String(p).includes('session.config.json')) { + return JSON.stringify({ model: 'openai/gpt-5.6-luna' }); + } + throw new Error('ENOENT'); + }); const previousKey = process.env.OPENAI_API_KEY; - process.env.LLM_PROVIDER = 'openai'; delete process.env.OPENAI_API_KEY; try { const res = await request(app) @@ -297,17 +351,19 @@ describe('POST /api/eval/compare', () => { expect(res.body.error).toMatch(/OPENAI_API_KEY/); expect(runPromptComparison).not.toHaveBeenCalled(); } finally { - if (previousProvider === undefined) delete process.env.LLM_PROVIDER; - else process.env.LLM_PROVIDER = previousProvider; if (previousKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousKey; } }); - it('returns 503 when LLM_PROVIDER=gemini and GOOGLE_API_KEY is missing', async () => { - const previousProvider = process.env.LLM_PROVIDER; + it('returns 503 when model is google/… and GOOGLE_API_KEY is missing', async () => { + fs.readFile.mockImplementation(async (p) => { + if (String(p).includes('session.config.json')) { + return JSON.stringify({ model: 'google/gemini-3.6-flash' }); + } + throw new Error('ENOENT'); + }); const previousKey = process.env.GOOGLE_API_KEY; - process.env.LLM_PROVIDER = 'gemini'; delete process.env.GOOGLE_API_KEY; try { const res = await request(app) @@ -317,10 +373,35 @@ describe('POST /api/eval/compare', () => { expect(res.body.error).toMatch(/GOOGLE_API_KEY/); expect(runPromptComparison).not.toHaveBeenCalled(); } finally { - if (previousProvider === undefined) delete process.env.LLM_PROVIDER; - else process.env.LLM_PROVIDER = previousProvider; if (previousKey === undefined) delete process.env.GOOGLE_API_KEY; else process.env.GOOGLE_API_KEY = previousKey; } }); + + it('returns 503 when model is not in allowedModels', async () => { + createLlmProvider.mockClear(); + fs.readFile.mockImplementation(async (p) => { + if (String(p).includes('session.config.json')) { + return JSON.stringify({ + model: 'openai/gpt-4o', + allowedModels: ['google/gemini-3.6-flash'], + }); + } + throw new Error('ENOENT'); + }); + const previousKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = 'test-openai-key'; + try { + const res = await request(app) + .post('/api/eval/compare') + .send({ promptA: 'Hi', input: 'x', runs: 1 }); + expect(res.status).toBe(503); + expect(res.body.error).toMatch(/not in allowedModels/); + expect(runPromptComparison).not.toHaveBeenCalled(); + expect(createLlmProvider).not.toHaveBeenCalled(); + } finally { + if (previousKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousKey; + } + }); }); diff --git a/tests/session-config.test.js b/tests/session-config.test.js index 57fff60..f90f30a 100644 --- a/tests/session-config.test.js +++ b/tests/session-config.test.js @@ -1,18 +1,53 @@ import { describe, it, expect } from 'vitest'; -import { FALLBACK_DEFAULTS, normalizeSessionConfig } from '../lib/session-config.js'; +import { + DEFAULT_ALLOWED_MODELS, + DEFAULT_MODEL_REF, + FALLBACK_DEFAULTS, + assertAllowedModel, + normalizeAllowedModels, + normalizeSessionConfig, +} from '../lib/session-config.js'; describe('normalizeSessionConfig', () => { it('returns fallback defaults and an empty session when raw is missing', () => { expect(normalizeSessionConfig(undefined)).toEqual({ + model: DEFAULT_MODEL_REF, + allowedModels: [...DEFAULT_ALLOWED_MODELS], defaults: { ...FALLBACK_DEFAULTS }, initialSession: { promptA: '', promptB: '', cases: [] }, }); expect(normalizeSessionConfig({})).toEqual({ + model: DEFAULT_MODEL_REF, + allowedModels: [...DEFAULT_ALLOWED_MODELS], defaults: { ...FALLBACK_DEFAULTS }, initialSession: { promptA: '', promptB: '', cases: [] }, }); }); + it('keeps a trimmed model ref and defaults a blank one', () => { + expect(normalizeSessionConfig({ + model: ' google/gemini-3.6-flash ', + }).model).toBe('google/gemini-3.6-flash'); + expect(normalizeSessionConfig({ model: ' ' }).model).toBe(DEFAULT_MODEL_REF); + expect(normalizeSessionConfig({ model: 12 }).model).toBe(DEFAULT_MODEL_REF); + }); + + it('defaults model to the first allowed entry when the default is not listed', () => { + expect(normalizeSessionConfig({ + allowedModels: ['openai/gpt-5.6-luna', 'google/gemini-3.6-flash'], + }).model).toBe('openai/gpt-5.6-luna'); + }); + + it('keeps a provided model even when it is not in allowedModels', () => { + const result = normalizeSessionConfig({ + model: 'openai/gpt-4o', + allowedModels: ['google/gemini-3.6-flash'], + }); + expect(result.model).toBe('openai/gpt-4o'); + expect(result.allowedModels).toEqual(['google/gemini-3.6-flash']); + expect(() => assertAllowedModel(result.model, result.allowedModels)).toThrow(/not in allowedModels/); + }); + it('applies optional default bounds', () => { const result = normalizeSessionConfig({ defaults: { minRuns: 2, maxRuns: 3, minCases: 1, maxCases: 2 }, @@ -95,3 +130,42 @@ describe('normalizeSessionConfig', () => { }); }); }); + +describe('normalizeAllowedModels', () => { + it('returns the default catalog when the value is missing or empty', () => { + expect(normalizeAllowedModels(undefined)).toEqual(DEFAULT_ALLOWED_MODELS); + expect(normalizeAllowedModels([])).toEqual(DEFAULT_ALLOWED_MODELS); + expect(normalizeAllowedModels(['', 12, 'not-a-model'])).toEqual(DEFAULT_ALLOWED_MODELS); + }); + + it('trims, drops invalid refs, and de-duplicates', () => { + expect(normalizeAllowedModels([ + ' openai/gpt-5.6-luna ', + 'openai/gpt-5.6-luna', + 'mistral/large', + 'google/gemini-3.6-flash', + ])).toEqual([ + 'openai/gpt-5.6-luna', + 'google/gemini-3.6-flash', + ]); + }); +}); + +describe('assertAllowedModel', () => { + it('accepts a model that is listed', () => { + expect(() => assertAllowedModel( + 'google/gemini-3.6-flash', + ['openai/gpt-5.6-luna', 'google/gemini-3.6-flash'], + )).not.toThrow(); + }); + + it('rejects a model that is not listed', () => { + try { + assertAllowedModel('openai/gpt-4o', ['google/gemini-3.6-flash']); + throw new Error('expected assertAllowedModel to throw'); + } catch (err) { + expect(err.code).toBe('LLM_MODEL_NOT_ALLOWED'); + expect(err.message).toMatch(/openai\/gpt-4o/); + } + }); +}); From db5709f17eb373e6a02593fc4fc4bbb4bfb99bb5 Mon Sep 17 00:00:00 2001 From: Brian Genisio Date: Thu, 3 Sep 2026 06:54:01 -0400 Subject: [PATCH 2/2] Match allowed models by canonical provider and id. Co-authored-by: Cursor --- lib/session-config.js | 25 ++++++++++++++++++++++++- tests/session-config.test.js | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/session-config.js b/lib/session-config.js index f126258..d823059 100644 --- a/lib/session-config.js +++ b/lib/session-config.js @@ -105,12 +105,35 @@ export function normalizeAllowedModels(raw) { return models.length > 0 ? models : [...DEFAULT_ALLOWED_MODELS]; } +/** + * @param {unknown} modelRef + * @returns {{ provider: string, modelId: string } | null} + */ +function tryParseModelRef(modelRef) { + if (typeof modelRef !== 'string' || !modelRef.trim()) return null; + try { + const parsed = parseModelRef(modelRef); + return { provider: parsed.provider, modelId: parsed.modelId }; + } catch { + return null; + } +} + /** * @param {string} model * @param {string[]} allowedModels */ export function assertAllowedModel(model, allowedModels) { - if (!Array.isArray(allowedModels) || !allowedModels.includes(model)) { + const requested = tryParseModelRef(model); + const allowed = Array.isArray(allowedModels) && requested + ? allowedModels.some((item) => { + const parsed = tryParseModelRef(item); + return parsed != null + && parsed.provider === requested.provider + && parsed.modelId === requested.modelId; + }) + : false; + if (!allowed) { const err = new Error(`Model "${model}" is not in allowedModels`); err.code = 'LLM_MODEL_NOT_ALLOWED'; throw err; diff --git a/tests/session-config.test.js b/tests/session-config.test.js index f90f30a..9f5ba05 100644 --- a/tests/session-config.test.js +++ b/tests/session-config.test.js @@ -168,4 +168,40 @@ describe('assertAllowedModel', () => { expect(err.message).toMatch(/openai\/gpt-4o/); } }); + + it('treats google/ and gemini/ as the same provider', () => { + expect(() => assertAllowedModel( + 'gemini/gemini-3.6-flash', + ['google/gemini-3.6-flash'], + )).not.toThrow(); + }); + + it('matches an allowed ref when the prefix casing differs', () => { + expect(() => assertAllowedModel( + 'Google/gemini-3.6-flash', + ['google/gemini-3.6-flash'], + )).not.toThrow(); + }); + + it('still rejects a different model id under the same provider', () => { + expect(() => assertAllowedModel( + 'google/gemini-2.5-flash', + ['google/gemini-3.6-flash'], + )).toThrow(/not in allowedModels/); + }); +}); + +describe('normalizeSessionConfig allowedModels', () => { + it('keeps raw configured refs rather than rewriting aliases', () => { + const result = normalizeSessionConfig({ + model: 'gemini/gemini-3.6-flash', + allowedModels: ['GEMINI/gemini-3.6-flash', 'openai/gpt-5.6-luna'], + }); + expect(result.model).toBe('gemini/gemini-3.6-flash'); + expect(result.allowedModels).toEqual([ + 'GEMINI/gemini-3.6-flash', + 'openai/gpt-5.6-luna', + ]); + expect(() => assertAllowedModel(result.model, result.allowedModels)).not.toThrow(); + }); });