diff --git a/README.md b/README.md index fd41517..6cd047c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Fill in `.env` with the API key (and optional `*_BASE_URL`) for the provider you - `model` (optional) — `provider/model-id` (default `anthropic/claude-sonnet-4-6`); must be listed in `allowedModels` - `allowedModels` (optional) — picker list of `provider/model-id` refs (defaults to Anthropic, OpenAI, and Gemini examples above) +- `allowUserModelSelection` (optional) — when `true`, show a model picker and let the saved eval session override `model` with an entry from `allowedModels` (default `false`) - `defaults` (optional) — `minRuns`, `maxRuns`, `minCases`, `maxCases` (each 1–5) - `initialSession` (optional) — `promptA`, `promptB`, and `cases` (`input` / `expectedAnswer`) diff --git a/lib/eval-session.js b/lib/eval-session.js index 3ddf20b..aa4b7fe 100644 --- a/lib/eval-session.js +++ b/lib/eval-session.js @@ -15,6 +15,7 @@ import { FALLBACK_DEFAULTS } from './session-config.js'; /** * @typedef {object} EvalSession + * @property {string} model * @property {string} promptA * @property {string} promptB * @property {boolean} compareMode @@ -77,6 +78,7 @@ export function normalizeEvalSession(raw, limits = {}) { const metricId = isValidMetricId(src.metricId) ? src.metricId : DEFAULT_METRIC_ID; return { + model: typeof src.model === 'string' ? src.model.trim() : '', promptA: typeof src.promptA === 'string' ? src.promptA : '', promptB: typeof src.promptB === 'string' ? src.promptB : '', compareMode: src.compareMode === true, diff --git a/lib/session-config.js b/lib/session-config.js index d823059..714a77f 100644 --- a/lib/session-config.js +++ b/lib/session-config.js @@ -124,22 +124,30 @@ function tryParseModelRef(modelRef) { * @param {string[]} allowedModels */ export function assertAllowedModel(model, allowedModels) { - const requested = tryParseModelRef(model); - const allowed = Array.isArray(allowedModels) && requested - ? allowedModels.some((item) => { - const parsed = tryParseModelRef(item); - return parsed != null - && parsed.provider === requested.provider - && parsed.modelId === requested.modelId; - }) - : false; - if (!allowed) { + if (!findAllowedModel(model, allowedModels)) { const err = new Error(`Model "${model}" is not in allowedModels`); err.code = 'LLM_MODEL_NOT_ALLOWED'; throw err; } } +/** + * Return the raw allowlist entry equivalent to a requested model ref. + * @param {string} model + * @param {string[]} allowedModels + * @returns {string | undefined} + */ +export function findAllowedModel(model, allowedModels) { + const requested = tryParseModelRef(model); + if (!Array.isArray(allowedModels) || !requested) return undefined; + return allowedModels.find((item) => { + const parsed = tryParseModelRef(item); + return parsed != null + && parsed.provider === requested.provider + && parsed.modelId === requested.modelId; + }); +} + /** * @param {unknown} rawModel * @param {string[]} allowedModels @@ -163,6 +171,7 @@ function resolveConfiguredModel(rawModel, allowedModels) { * @returns {{ * model: string, * allowedModels: string[], + * allowUserModelSelection: boolean, * defaults: { minRuns: number, maxRuns: number, minCases: number, maxCases: number }, * initialSession: { promptA: string, promptB: string, cases: { input: string, expectedAnswer: string }[] }, * }} @@ -194,6 +203,7 @@ export function normalizeSessionConfig(raw) { return { model, allowedModels, + allowUserModelSelection: src.allowUserModelSelection === true, defaults: { minRuns: runs.min, maxRuns: runs.max, diff --git a/public/app.css b/public/app.css index dc335ae..014b0ef 100644 --- a/public/app.css +++ b/public/app.css @@ -165,6 +165,11 @@ code { min-width: 12rem; } +.eval-field--model { + flex: 1; + min-width: 16rem; +} + .eval-optional { font-weight: 400; color: var(--Colors-Text-Body-Light); diff --git a/public/app.js b/public/app.js index 6188368..191fd87 100644 --- a/public/app.js +++ b/public/app.js @@ -5,7 +5,12 @@ import { isRenderableResult, normalizeEvalSession } from '../lib/eval-session.js'; import { collectPromptScoresByCase } from '../lib/score-distribution.js'; -import { FALLBACK_DEFAULTS, normalizeSessionConfig } from '../lib/session-config.js'; +import { enqueueSessionsWrite } from '../lib/sessions-file.js'; +import { + FALLBACK_DEFAULTS, + findAllowedModel, + normalizeSessionConfig, +} from '../lib/session-config.js'; const promptAEl = document.getElementById('promptA'); const promptBEl = document.getElementById('promptB'); @@ -21,6 +26,8 @@ const casesListEl = document.getElementById('casesList'); const casesSummaryMeta = document.getElementById('casesSummaryMeta'); const addCaseBtn = document.getElementById('addCaseBtn'); const metricSelectEl = document.getElementById('metricSelect'); +const modelSelectWrap = document.getElementById('modelSelectWrap'); +const modelSelectEl = document.getElementById('modelSelect'); const runCountEl = document.getElementById('runCount'); const runCountLabel = document.getElementById('runCountLabel'); const runBtn = document.getElementById('runBtn'); @@ -38,12 +45,16 @@ let MIN_RUNS = FALLBACK_DEFAULTS.minRuns; let MAX_RUNS = FALLBACK_DEFAULTS.maxRuns; let MIN_CASES = FALLBACK_DEFAULTS.minCases; let MAX_CASES = FALLBACK_DEFAULTS.maxCases; +let ALLOW_USER_MODEL_SELECTION = false; +let CONFIG_MODEL = ''; +let ALLOWED_MODELS = []; /** @type {ReturnType} */ let session = normalizeEvalSession({}); let persistEnabled = false; let saveTimer = null; const SAVE_DEBOUNCE_MS = 300; +const persistWrite = { chain: Promise.resolve() }; function sessionLimits() { return { minRuns: MIN_RUNS, maxRuns: MAX_RUNS, maxCases: MAX_CASES }; @@ -93,6 +104,7 @@ function setBusy(busy) { promptAEl.readOnly = busy; promptBEl.readOnly = busy; metricSelectEl.disabled = busy; + modelSelectEl.disabled = busy || !ALLOW_USER_MODEL_SELECTION; runCountEl.readOnly = busy; casesListEl.querySelectorAll('textarea, button').forEach((el) => { if (el.tagName === 'TEXTAREA') el.readOnly = busy; @@ -433,7 +445,15 @@ async function runEvaluation() { showError(''); pullSessionFromDom(); - const { promptA, promptB, compareMode, cases, metricId, runs } = session; + const { + model, + promptA, + promptB, + compareMode, + cases, + metricId, + runs, + } = session; runCountEl.value = String(runs); if (!promptA.trim()) { @@ -454,6 +474,7 @@ async function runEvaluation() { } const body = { + model, promptA, cases: cases.map((c, i) => ({ id: c.id, @@ -503,10 +524,33 @@ function applyDefaults(defaults) { } } +function configureModelSelection(config) { + ALLOW_USER_MODEL_SELECTION = config.allowUserModelSelection; + CONFIG_MODEL = config.model; + ALLOWED_MODELS = config.allowedModels; + + modelSelectEl.replaceChildren(...ALLOWED_MODELS.map((model) => { + const option = document.createElement('option'); + option.value = model; + option.textContent = model; + return option; + })); + modelSelectWrap.hidden = !ALLOW_USER_MODEL_SELECTION; + modelSelectEl.disabled = !ALLOW_USER_MODEL_SELECTION; +} + +function resolveSessionModel(savedModel) { + if (!ALLOW_USER_MODEL_SELECTION) return CONFIG_MODEL; + return findAllowedModel(savedModel, ALLOWED_MODELS) + ?? findAllowedModel(CONFIG_MODEL, ALLOWED_MODELS) + ?? CONFIG_MODEL; +} + function pullSessionFromDom() { syncCasesFromDom(); session = normalizeEvalSession({ ...session, + model: ALLOW_USER_MODEL_SELECTION ? modelSelectEl.value : CONFIG_MODEL, promptA: promptAEl.value, promptB: promptBEl.value, compareMode: session.compareMode, @@ -518,6 +562,10 @@ function pullSessionFromDom() { } function applySessionToDom() { + session.model = resolveSessionModel(session.model); + if (ALLOW_USER_MODEL_SELECTION) { + modelSelectEl.value = session.model; + } promptAEl.value = session.promptA; promptBEl.value = session.promptB; if ([...metricSelectEl.options].some((opt) => opt.value === session.metricId)) { @@ -530,6 +578,7 @@ function applySessionToDom() { function applyInitialSession(initial) { session = normalizeEvalSession({ + model: resolveSessionModel(''), promptA: initial.promptA, promptB: initial.promptB, compareMode: initial.promptB.trim() !== '', @@ -547,15 +596,18 @@ function applyInitialSession(initial) { async function persistSession() { if (!persistEnabled) return; pullSessionFromDom(); + const snapshot = JSON.parse(JSON.stringify(session)); try { - const res = await fetch('/api/eval/session', { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(session), - }); - if (!res.ok) { - console.error('[eval] Failed to persist session:', res.status); - } + await enqueueSessionsWrite(async () => { + const res = await fetch('/api/eval/session', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(snapshot), + }); + if (!res.ok) { + console.error('[eval] Failed to persist session:', res.status); + } + }, persistWrite); } catch (err) { console.error('[eval] Failed to persist session:', err); } @@ -603,6 +655,7 @@ async function init() { loadEvalSession(), ]); applyDefaults(config.defaults); + configureModelSelection(config); if (saved) { session = normalizeEvalSession(saved, sessionLimits()); applySessionToDom(); @@ -666,6 +719,11 @@ metricSelectEl.addEventListener('change', () => { persistSessionNow(); }); +modelSelectEl.addEventListener('change', () => { + session.model = modelSelectEl.value; + persistSessionNow(); +}); + runBtn.addEventListener('click', () => { void runEvaluation(); }); diff --git a/public/index.html b/public/index.html index 86a65d4..bf8e1f5 100644 --- a/public/index.html +++ b/public/index.html @@ -74,6 +74,11 @@

Prompt

+ +