Session config | Let users pick an allowed model

README.md

@@ -32,6 +32,7 @@ Fill in `.env` with the API key (and optional `*_BASE_URL`) for the provider you
 
 - `model` (optional) — `provider/model-id` (default `anthropic/claude-sonnet-4-6`); must be listed in `allowedModels`
 - `allowedModels` (optional) — picker list of `provider/model-id` refs (defaults to Anthropic, OpenAI, and Gemini examples above)
+- `allowUserModelSelection` (optional) — when `true`, show a model picker and let the saved eval session override `model` with an entry from `allowedModels` (default `false`)
 - `defaults` (optional) — `minRuns`, `maxRuns`, `minCases`, `maxCases` (each 1–5)
 - `initialSession` (optional) — `promptA`, `promptB`, and `cases` (`input` / `expectedAnswer`)
 

lib/eval-session.js

@@ -15,6 +15,7 @@ import { FALLBACK_DEFAULTS } from './session-config.js';
 
 /**
  * @typedef {object} EvalSession
+ * @property {string} model
  * @property {string} promptA
  * @property {string} promptB
  * @property {boolean} compareMode
@@ -77,6 +78,7 @@ export function normalizeEvalSession(raw, limits = {}) {
   const metricId = isValidMetricId(src.metricId) ? src.metricId : DEFAULT_METRIC_ID;
 
   return {
+    model: typeof src.model === 'string' ? src.model.trim() : '',
     promptA: typeof src.promptA === 'string' ? src.promptA : '',
     promptB: typeof src.promptB === 'string' ? src.promptB : '',
     compareMode: src.compareMode === true,

lib/session-config.js

@@ -124,22 +124,30 @@ function tryParseModelRef(modelRef) {
  * @param {string[]} allowedModels
  */
 export function assertAllowedModel(model, allowedModels) {
-  const requested = tryParseModelRef(model);
-  const allowed = Array.isArray(allowedModels) && requested
-    ? allowedModels.some((item) => {
-      const parsed = tryParseModelRef(item);
-      return parsed != null
-        && parsed.provider === requested.provider
-        && parsed.modelId === requested.modelId;
-    })
-    : false;
-  if (!allowed) {
+  if (!findAllowedModel(model, allowedModels)) {
     const err = new Error(`Model "${model}" is not in allowedModels`);
     err.code = 'LLM_MODEL_NOT_ALLOWED';
     throw err;
   }
 }
 
+/**
+ * Return the raw allowlist entry equivalent to a requested model ref.
+ * @param {string} model
+ * @param {string[]} allowedModels
+ * @returns {string | undefined}
+ */
+export function findAllowedModel(model, allowedModels) {
+  const requested = tryParseModelRef(model);
+  if (!Array.isArray(allowedModels) || !requested) return undefined;
+  return allowedModels.find((item) => {
+    const parsed = tryParseModelRef(item);
+    return parsed != null
+      && parsed.provider === requested.provider
+      && parsed.modelId === requested.modelId;
+  });
+}
+
 /**
  * @param {unknown} rawModel
  * @param {string[]} allowedModels
@@ -163,6 +171,7 @@ function resolveConfiguredModel(rawModel, allowedModels) {
  * @returns {{
  *   model: string,
  *   allowedModels: string[],
+ *   allowUserModelSelection: boolean,
  *   defaults: { minRuns: number, maxRuns: number, minCases: number, maxCases: number },
  *   initialSession: { promptA: string, promptB: string, cases: { input: string, expectedAnswer: string }[] },
  * }}
@@ -194,6 +203,7 @@ export function normalizeSessionConfig(raw) {
   return {
     model,
     allowedModels,
+    allowUserModelSelection: src.allowUserModelSelection === true,
     defaults: {
       minRuns: runs.min,
       maxRuns: runs.max,

public/app.css

@@ -165,6 +165,11 @@ code {
   min-width: 12rem;
 }
 
+.eval-field--model {
+  flex: 1;
+  min-width: 16rem;
+}
+
 .eval-optional {
   font-weight: 400;
   color: var(--Colors-Text-Body-Light);

public/app.js

@@ -5,7 +5,12 @@
 
 import { isRenderableResult, normalizeEvalSession } from '../lib/eval-session.js';
 import { collectPromptScoresByCase } from '../lib/score-distribution.js';
-import { FALLBACK_DEFAULTS, normalizeSessionConfig } from '../lib/session-config.js';
+import { enqueueSessionsWrite } from '../lib/sessions-file.js';
+import {
+  FALLBACK_DEFAULTS,
+  findAllowedModel,
+  normalizeSessionConfig,
+} from '../lib/session-config.js';
 
 const promptAEl = document.getElementById('promptA');
 const promptBEl = document.getElementById('promptB');
@@ -21,6 +26,8 @@ const casesListEl = document.getElementById('casesList');
 const casesSummaryMeta = document.getElementById('casesSummaryMeta');
 const addCaseBtn = document.getElementById('addCaseBtn');
 const metricSelectEl = document.getElementById('metricSelect');
+const modelSelectWrap = document.getElementById('modelSelectWrap');
+const modelSelectEl = document.getElementById('modelSelect');
 const runCountEl = document.getElementById('runCount');
 const runCountLabel = document.getElementById('runCountLabel');
 const runBtn = document.getElementById('runBtn');
@@ -38,12 +45,16 @@ let MIN_RUNS = FALLBACK_DEFAULTS.minRuns;
 let MAX_RUNS = FALLBACK_DEFAULTS.maxRuns;
 let MIN_CASES = FALLBACK_DEFAULTS.minCases;
 let MAX_CASES = FALLBACK_DEFAULTS.maxCases;
+let ALLOW_USER_MODEL_SELECTION = false;
+let CONFIG_MODEL = '';
+let ALLOWED_MODELS = [];
 
 /** @type {ReturnType<typeof normalizeEvalSession>} */
 let session = normalizeEvalSession({});
 let persistEnabled = false;
 let saveTimer = null;
 const SAVE_DEBOUNCE_MS = 300;
+const persistWrite = { chain: Promise.resolve() };
 
 function sessionLimits() {
   return { minRuns: MIN_RUNS, maxRuns: MAX_RUNS, maxCases: MAX_CASES };
@@ -93,6 +104,7 @@ function setBusy(busy) {
   promptAEl.readOnly = busy;
   promptBEl.readOnly = busy;
   metricSelectEl.disabled = busy;
+  modelSelectEl.disabled = busy || !ALLOW_USER_MODEL_SELECTION;
   runCountEl.readOnly = busy;
   casesListEl.querySelectorAll('textarea, button').forEach((el) => {
     if (el.tagName === 'TEXTAREA') el.readOnly = busy;
@@ -433,7 +445,15 @@ async function runEvaluation() {
   showError('');
   pullSessionFromDom();
 
-  const { promptA, promptB, compareMode, cases, metricId, runs } = session;
+  const {
+    model,
+    promptA,
+    promptB,
+    compareMode,
+    cases,
+    metricId,
+    runs,
+  } = session;
   runCountEl.value = String(runs);
 
   if (!promptA.trim()) {
@@ -454,6 +474,7 @@ async function runEvaluation() {
   }
 
   const body = {
+    model,
     promptA,
     cases: cases.map((c, i) => ({
       id: c.id,
@@ -503,10 +524,33 @@ function applyDefaults(defaults) {
   }
 }
 
+function configureModelSelection(config) {
+  ALLOW_USER_MODEL_SELECTION = config.allowUserModelSelection;
+  CONFIG_MODEL = config.model;
+  ALLOWED_MODELS = config.allowedModels;
+
+  modelSelectEl.replaceChildren(...ALLOWED_MODELS.map((model) => {
+    const option = document.createElement('option');
+    option.value = model;
+    option.textContent = model;
+    return option;
+  }));
+  modelSelectWrap.hidden = !ALLOW_USER_MODEL_SELECTION;
+  modelSelectEl.disabled = !ALLOW_USER_MODEL_SELECTION;
+}
+
+function resolveSessionModel(savedModel) {
+  if (!ALLOW_USER_MODEL_SELECTION) return CONFIG_MODEL;
+  return findAllowedModel(savedModel, ALLOWED_MODELS)
+    ?? findAllowedModel(CONFIG_MODEL, ALLOWED_MODELS)
+    ?? CONFIG_MODEL;
+}
+
 function pullSessionFromDom() {
   syncCasesFromDom();
   session = normalizeEvalSession({
     ...session,
+    model: ALLOW_USER_MODEL_SELECTION ? modelSelectEl.value : CONFIG_MODEL,
     promptA: promptAEl.value,
     promptB: promptBEl.value,
     compareMode: session.compareMode,
@@ -518,6 +562,10 @@ function pullSessionFromDom() {
 }
 
 function applySessionToDom() {
+  session.model = resolveSessionModel(session.model);
+  if (ALLOW_USER_MODEL_SELECTION) {
+    modelSelectEl.value = session.model;
+  }
   promptAEl.value = session.promptA;
   promptBEl.value = session.promptB;
   if ([...metricSelectEl.options].some((opt) => opt.value === session.metricId)) {
@@ -530,6 +578,7 @@ function applySessionToDom() {
 
 function applyInitialSession(initial) {
   session = normalizeEvalSession({
+    model: resolveSessionModel(''),
     promptA: initial.promptA,
     promptB: initial.promptB,
     compareMode: initial.promptB.trim() !== '',
@@ -547,15 +596,18 @@ function applyInitialSession(initial) {
 async function persistSession() {
   if (!persistEnabled) return;
   pullSessionFromDom();
+  const snapshot = JSON.parse(JSON.stringify(session));
   try {
-    const res = await fetch('/api/eval/session', {
-      method: 'PUT',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(session),
-    });
-    if (!res.ok) {
-      console.error('[eval] Failed to persist session:', res.status);
-    }
+    await enqueueSessionsWrite(async () => {
+      const res = await fetch('/api/eval/session', {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(snapshot),
+      });
+      if (!res.ok) {
+        console.error('[eval] Failed to persist session:', res.status);
+      }
+    }, persistWrite);
   } catch (err) {
     console.error('[eval] Failed to persist session:', err);
   }
@@ -603,6 +655,7 @@ async function init() {
     loadEvalSession(),
   ]);
   applyDefaults(config.defaults);
+  configureModelSelection(config);
   if (saved) {
     session = normalizeEvalSession(saved, sessionLimits());
     applySessionToDom();
@@ -666,6 +719,11 @@ metricSelectEl.addEventListener('change', () => {
   persistSessionNow();
 });
 
+modelSelectEl.addEventListener('change', () => {
+  session.model = modelSelectEl.value;
+  persistSessionNow();
+});
+
 runBtn.addEventListener('click', () => {
   void runEvaluation();
 });

public/index.html

@@ -74,6 +74,11 @@ <h2 class="heading-xsmall" id="setupHeading">Prompt</h2>
         </div>
 
         <div class="eval-controls-row">
+          <label class="eval-field eval-field--model" id="modelSelectWrap" hidden>
+            <span class="body-xsmall eval-field__label">Model</span>
+            <select class="input" id="modelSelect"></select>
+          </label>
+
           <label class="eval-field eval-field--metric">
             <span class="body-xsmall eval-field__label">Metric</span>
             <select class="input" id="metricSelect">

server.js

@@ -8,7 +8,7 @@ import { MAX_EVAL_RUNS, MIN_EVAL_RUNS } from './lib/eval-run.js';
 import { runPromptComparison, MAX_EVAL_CASES } from './lib/eval-compare.js';
 import { createLlmProvider, requiredApiKeyName } from './lib/llm/provider.js';
 import { DEFAULT_METRIC_ID, isValidMetricId } from './lib/metrics/index.js';
-import { assertAllowedModel, normalizeSessionConfig } from './lib/session-config.js';
+import { assertAllowedModel, findAllowedModel, normalizeSessionConfig } from './lib/session-config.js';
 import { normalizeEvalSession } from './lib/eval-session.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -23,11 +23,15 @@ let cachedLlm;
 let cachedModel;
 
 /**
- * @returns {Promise<{ model: string, allowedModels: string[] }>}
+ * @returns {Promise<{ model: string, allowedModels: string[], allowUserModelSelection: boolean }>}
  */
 async function sessionLlmConfig() {
   const config = normalizeSessionConfig(await readJsonFile(SESSION_CONFIG_FILE, {}));
-  return { model: config.model, allowedModels: config.allowedModels };
+  return {
+    model: config.model,
+    allowedModels: config.allowedModels,
+    allowUserModelSelection: config.allowUserModelSelection,
+  };
 }
 
 /**
@@ -52,11 +56,19 @@ function getLlm(model, allowedModels) {
  * Resolve a provider for an eval route without letting createLlmProvider()
  * throws escape Express 4 async handlers as unhandled rejections.
  * @param {import('express').Response} res
+ * @param {unknown} requestedModel
  * @returns {Promise<import('./lib/llm/types.js').LlmProvider | null>}
  */
-async function resolveLlm(res) {
+async function resolveLlm(res, requestedModel) {
   try {
-    const { model, allowedModels } = await sessionLlmConfig();
+    const config = await sessionLlmConfig();
+    const requested = config.allowUserModelSelection
+      && typeof requestedModel === 'string'
+      && requestedModel.trim()
+      ? requestedModel.trim()
+      : config.model;
+    const { allowedModels } = config;
+    const model = findAllowedModel(requested, allowedModels) ?? requested;
     const llm = getLlm(model, allowedModels);
     if (!llm) {
       res.status(503).json({ error: `${requiredApiKeyName(model)} is not configured` });
@@ -124,7 +136,7 @@ app.put('/api/eval/session', async (req, res) => {
 // Evaluate Prompt A across cases; optionally compare with Prompt B under
 // identical conditions. Each run is an independent LLM complete() call.
 app.post('/api/eval/compare', async (req, res) => {
-  const llm = await resolveLlm(res);
+  const llm = await resolveLlm(res, req.body?.model);
   if (!llm) return;
 
   const {

session.config.example.json

@@ -5,6 +5,7 @@
     "openai/gpt-5.6-luna",
     "google/gemini-3.6-flash"
   ],
+  "allowUserModelSelection": true,
   "defaults": {
     "minRuns": 1,
     "maxRuns": 5,

tests/eval-session.test.js

@@ -6,6 +6,7 @@ import { FALLBACK_DEFAULTS } from '../lib/session-config.js';
 describe('normalizeEvalSession', () => {
   it('returns an empty session when raw is missing or invalid', () => {
     const empty = {
+      model: '',
       promptA: '',
       promptB: '',
       compareMode: false,
@@ -22,6 +23,7 @@ describe('normalizeEvalSession', () => {
   it('keeps valid fields and case ids', () => {
     const lastResult = { conditions: { runs: 1, caseCount: 1 }, prompts: [], cases: [], comparison: {} };
     const result = normalizeEvalSession({
+      model: '  google/gemini-3.6-flash  ',
       promptA: 'A',
       promptB: 'B',
       compareMode: true,
@@ -31,6 +33,7 @@ describe('normalizeEvalSession', () => {
       lastResult,
     });
     expect(result).toEqual({
+      model: 'google/gemini-3.6-flash',
       promptA: 'A',
       promptB: 'B',
       compareMode: true,