Session config | Select the LLM from a model ref

.env.example

@@ -1,20 +1,14 @@
-# LLM provider: anthropic (default), openai, or gemini
-LLM_PROVIDER=anthropic
-
-# Anthropic (LLM_PROVIDER=anthropic)
+# Anthropic (session.config.json model: anthropic/…)
 ANTHROPIC_BASE_URL=https://api.anthropic.com
 ANTHROPIC_API_KEY=
-# ANTHROPIC_MODEL=claude-sonnet-4-6
 
-# OpenAI (LLM_PROVIDER=openai)
+# OpenAI (session.config.json model: openai/…)
 # OPENAI_BASE_URL=https://api.openai.com/v1
 # OPENAI_API_KEY=
-# OPENAI_MODEL=gpt-4o
 
-# Gemini (LLM_PROVIDER=gemini)
+# Gemini (session.config.json model: google/…)
 # GOOGLE_BASE_URL=https://generativelanguage.googleapis.com
 # GOOGLE_API_KEY=
-# GOOGLE_MODEL=gemini-2.5-flash
 
 # Optional: HTTP port (default 3000)
 # PORT=3000

README.md

@@ -22,14 +22,16 @@ cp .env.example .env
 cp session.config.example.json session.config.json
 ```
 
-Fill in `.env` for the provider you want:
+Fill in `.env` with the API key (and optional `*_BASE_URL`) for the provider you want. Choose the model in `session.config.json` as `provider/model-id`:
 
-- Anthropic (default): `ANTHROPIC_API_KEY`, optional `ANTHROPIC_BASE_URL` / `ANTHROPIC_MODEL`
-- OpenAI: set `LLM_PROVIDER=openai` and `OPENAI_API_KEY`, optional `OPENAI_BASE_URL` / `OPENAI_MODEL`
-- Gemini: set `LLM_PROVIDER=gemini` and `GOOGLE_API_KEY`, optional `GOOGLE_BASE_URL` / `GOOGLE_MODEL`
+- `anthropic/claude-sonnet-4-6` — needs `ANTHROPIC_API_KEY`, optional `ANTHROPIC_BASE_URL`
+- `openai/gpt-5.6-luna` — needs `OPENAI_API_KEY`, optional `OPENAI_BASE_URL`
+- `google/gemini-3.6-flash` — needs `GOOGLE_API_KEY`, optional `GOOGLE_BASE_URL` (`gemini/…` also routes to Gemini)
 
 `session.config.json` is separate from `.env`. It is local (not checked in) and holds **session defaults**, not secrets:
 
+- `model` (optional) — `provider/model-id` (default `anthropic/claude-sonnet-4-6`); must be listed in `allowedModels`
+- `allowedModels` (optional) — picker list of `provider/model-id` refs (defaults to Anthropic, OpenAI, and Gemini examples above)
 - `defaults` (optional) — `minRuns`, `maxRuns`, `minCases`, `maxCases` (each 1–5)
 - `initialSession` (optional) — `promptA`, `promptB`, and `cases` (`input` / `expectedAnswer`)
 

lib/llm/anthropic.js

@@ -1,9 +1,9 @@
 /**
  * Anthropic Messages API provider.
  *
- * Configured from ANTHROPIC_API_KEY and optional ANTHROPIC_BASE_URL /
- * ANTHROPIC_MODEL. Model ids may still use the Octavus-style
- * `anthropic/claude-…` prefix; it is stripped before the API call.
+ * Configured from ANTHROPIC_API_KEY and optional ANTHROPIC_BASE_URL.
+ * Model ids may still use an `anthropic/` prefix; it is stripped before
+ * the API call.
  */
 
 import Anthropic from '@anthropic-ai/sdk';
@@ -36,9 +36,10 @@ export function extractMessageText(message) {
 
 /**
  * @param {NodeJS.ProcessEnv} [env]
+ * @param {string} [modelId]
  * @returns {import('./types.js').LlmProvider}
  */
-export function createAnthropicProvider(env = process.env) {
+export function createAnthropicProvider(env = process.env, modelId) {
   const apiKey = env.ANTHROPIC_API_KEY;
   if (!apiKey) {
     const err = new Error('ANTHROPIC_API_KEY is not configured');
@@ -47,7 +48,7 @@ export function createAnthropicProvider(env = process.env) {
   }
 
   const baseURL = optionalBaseUrl(env.ANTHROPIC_BASE_URL, 'ANTHROPIC_BASE_URL');
-  const model = normalizeAnthropicModelId(env.ANTHROPIC_MODEL) || DEFAULT_ANTHROPIC_MODEL;
+  const model = normalizeAnthropicModelId(modelId) || DEFAULT_ANTHROPIC_MODEL;
 
   const client = new Anthropic({
     apiKey,

lib/llm/gemini.js

@@ -1,9 +1,9 @@
 /**
  * Google Gemini generateContent provider.
  *
- * Configured from GOOGLE_API_KEY and optional GOOGLE_BASE_URL /
- * GOOGLE_MODEL. Model ids may still use a `google/` or `gemini/`
- * prefix; it is stripped before the API call.
+ * Configured from GOOGLE_API_KEY and optional GOOGLE_BASE_URL.
+ * Model ids may still use a `google/` or `gemini/` prefix; it is
+ * stripped before the API call.
  */
 
 import { GoogleGenAI } from '@google/genai';
@@ -51,9 +51,10 @@ export function toGeminiContents(messages = []) {
 
 /**
  * @param {NodeJS.ProcessEnv} [env]
+ * @param {string} [modelId]
  * @returns {import('./types.js').LlmProvider}
  */
-export function createGeminiProvider(env = process.env) {
+export function createGeminiProvider(env = process.env, modelId) {
   const apiKey = env.GOOGLE_API_KEY;
   if (!apiKey) {
     const err = new Error('GOOGLE_API_KEY is not configured');
@@ -62,7 +63,7 @@ export function createGeminiProvider(env = process.env) {
   }
 
   const baseURL = optionalBaseUrl(env.GOOGLE_BASE_URL, 'GOOGLE_BASE_URL');
-  const model = normalizeGeminiModelId(env.GOOGLE_MODEL) || DEFAULT_GEMINI_MODEL;
+  const model = normalizeGeminiModelId(modelId) || DEFAULT_GEMINI_MODEL;
 
   const client = new GoogleGenAI({
     apiKey,

lib/llm/model-ref.js

@@ -0,0 +1,83 @@
+/**
+ * Parse a session model ref (`provider/model-id`) into a routed provider.
+ *
+ * Browser-safe — no SDK imports. Prefix aliases:
+ *   anthropic/… → anthropic
+ *   openai/…    → openai
+ *   google/…    → gemini
+ *   gemini/…    → gemini
+ */
+
+export const DEFAULT_MODEL_REF = 'anthropic/claude-sonnet-4-6';
+
+/** @type {Record<string, { name: string, apiKeyName: string }>} */
+const PROVIDERS = {
+  anthropic: { name: 'anthropic', apiKeyName: 'ANTHROPIC_API_KEY' },
+  openai: { name: 'openai', apiKeyName: 'OPENAI_API_KEY' },
+  google: { name: 'gemini', apiKeyName: 'GOOGLE_API_KEY' },
+  gemini: { name: 'gemini', apiKeyName: 'GOOGLE_API_KEY' },
+};
+
+/**
+ * @typedef {object} ParsedModelRef
+ * @property {string} provider
+ * @property {string} modelId
+ * @property {string} prefix
+ * @property {string} raw
+ * @property {string} apiKeyName
+ */
+
+/**
+ * @param {string} raw
+ * @param {string} [code]
+ * @returns {Error}
+ */
+function modelRefError(raw, code = 'LLM_INVALID_MODEL') {
+  const err = new Error(
+    code === 'LLM_UNSUPPORTED_PROVIDER'
+      ? `Unsupported model provider "${raw}"`
+      : `Model must use provider/model-id format (e.g. openai/gpt-5.6-luna), got "${raw}"`,
+  );
+  err.code = code;
+  return err;
+}
+
+/**
+ * @param {unknown} [modelRef]
+ * @returns {ParsedModelRef}
+ */
+export function parseModelRef(modelRef) {
+  const raw = String(modelRef ?? '').trim() || DEFAULT_MODEL_REF;
+  const slash = raw.indexOf('/');
+  if (slash <= 0 || slash === raw.length - 1) {
+    throw modelRefError(raw);
+  }
+
+  const prefix = raw.slice(0, slash).trim().toLowerCase();
+  const modelId = raw.slice(slash + 1).trim();
+  if (!prefix || !modelId) {
+    throw modelRefError(raw);
+  }
+
+  const provider = PROVIDERS[prefix];
+  if (!provider) {
+    throw modelRefError(prefix, 'LLM_UNSUPPORTED_PROVIDER');
+  }
+
+  return {
+    provider: provider.name,
+    modelId,
+    prefix,
+    raw,
+    apiKeyName: provider.apiKeyName,
+  };
+}
+
+/**
+ * Env var that must be set for the provider in a model ref.
+ * @param {unknown} [modelRef]
+ * @returns {string}
+ */
+export function requiredApiKeyName(modelRef = DEFAULT_MODEL_REF) {
+  return parseModelRef(modelRef).apiKeyName;
+}

lib/llm/openai.js

@@ -1,9 +1,9 @@
 /**
  * OpenAI Chat Completions API provider.
  *
- * Configured from OPENAI_API_KEY and optional OPENAI_BASE_URL /
- * OPENAI_MODEL. Model ids may still use an `openai/gpt-…` prefix;
- * it is stripped before the API call.
+ * Configured from OPENAI_API_KEY and optional OPENAI_BASE_URL.
+ * Model ids may still use an `openai/` prefix; it is stripped before
+ * the API call.
  */
 
 import OpenAI from 'openai';
@@ -37,9 +37,10 @@ export function extractCompletionText(completion) {
 
 /**
  * @param {NodeJS.ProcessEnv} [env]
+ * @param {string} [modelId]
  * @returns {import('./types.js').LlmProvider}
  */
-export function createOpenAiProvider(env = process.env) {
+export function createOpenAiProvider(env = process.env, modelId) {
   const apiKey = env.OPENAI_API_KEY;
   if (!apiKey) {
     const err = new Error('OPENAI_API_KEY is not configured');
@@ -48,7 +49,7 @@ export function createOpenAiProvider(env = process.env) {
   }
 
   const baseURL = optionalBaseUrl(env.OPENAI_BASE_URL, 'OPENAI_BASE_URL');
-  const model = normalizeOpenAiModelId(env.OPENAI_MODEL) || DEFAULT_OPENAI_MODEL;
+  const model = normalizeOpenAiModelId(modelId) || DEFAULT_OPENAI_MODEL;
 
   const client = new OpenAI({
     apiKey,

lib/llm/provider.js

@@ -1,42 +1,34 @@
 /**
- * Resolve an LLM provider from environment variables.
+ * Resolve an LLM provider from a session model ref (`provider/model-id`).
  *
- * LLM_PROVIDER selects the adapter (default: anthropic). Additional
- * providers can be added to the switch without changing eval-run.
+ * The prefix selects the adapter; the remainder is the API model id.
+ * API keys and base URLs still come from the environment.
  */
 
 import { createAnthropicProvider } from './anthropic.js';
 import { createGeminiProvider } from './gemini.js';
+import { parseModelRef } from './model-ref.js';
 import { createOpenAiProvider } from './openai.js';
 
-/**
- * Env var that must be set for the selected provider.
- * @param {NodeJS.ProcessEnv} [env]
- * @returns {string}
- */
-export function requiredApiKeyName(env = process.env) {
-  const name = String(env.LLM_PROVIDER ?? 'anthropic').trim().toLowerCase() || 'anthropic';
-  if (name === 'openai') return 'OPENAI_API_KEY';
-  if (name === 'gemini') return 'GOOGLE_API_KEY';
-  return 'ANTHROPIC_API_KEY';
-}
+export { DEFAULT_MODEL_REF, parseModelRef, requiredApiKeyName } from './model-ref.js';
 
 /**
  * @param {NodeJS.ProcessEnv} [env]
+ * @param {string} [modelRef]
  * @returns {import('./types.js').LlmProvider}
  */
-export function createLlmProvider(env = process.env) {
-  const name = String(env.LLM_PROVIDER ?? 'anthropic').trim().toLowerCase() || 'anthropic';
+export function createLlmProvider(env = process.env, modelRef) {
+  const parsed = parseModelRef(modelRef);
 
-  switch (name) {
+  switch (parsed.provider) {
     case 'anthropic':
-      return createAnthropicProvider(env);
+      return createAnthropicProvider(env, parsed.modelId);
     case 'openai':
-      return createOpenAiProvider(env);
+      return createOpenAiProvider(env, parsed.modelId);
     case 'gemini':
-      return createGeminiProvider(env);
+      return createGeminiProvider(env, parsed.modelId);
     default: {
-      const err = new Error(`Unsupported LLM_PROVIDER "${name}"`);
+      const err = new Error(`Unsupported model provider "${parsed.prefix}"`);
       err.code = 'LLM_UNSUPPORTED_PROVIDER';
       throw err;
     }