From 2ea6f28c86c7f5a15c6a7df1c854a81f558f2015 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:06:48 +0100 Subject: [PATCH 1/9] test: derive brittle test expectations from their source of truth --- .ai/runs/2026-11-11-robust-tests/HANDOFF.md | 3 + .ai/runs/2026-11-11-robust-tests/PLAN.md | 4 ++ test/editor-demand-wiring.test.ts | 44 +++++++-------- test/helpers/source-blocks.ts | 44 +++++++++++++++ test/install-skills.test.ts | 10 +++- test/lantern-macro-load.test.ts | 26 ++++----- test/lantern-permissions-triggers.test.ts | 62 +++++++++++++++++++-- test/lantern-sync-selftest.test.ts | 48 ++++++++++++---- test/project-window.test.ts | 31 ++--------- 9 files changed, 190 insertions(+), 82 deletions(-) create mode 100644 .ai/runs/2026-11-11-robust-tests/HANDOFF.md create mode 100644 .ai/runs/2026-11-11-robust-tests/PLAN.md create mode 100644 test/helpers/source-blocks.ts diff --git a/.ai/runs/2026-11-11-robust-tests/HANDOFF.md b/.ai/runs/2026-11-11-robust-tests/HANDOFF.md new file mode 100644 index 0000000..5d88817 --- /dev/null +++ b/.ai/runs/2026-11-11-robust-tests/HANDOFF.md @@ -0,0 +1,3 @@ +# Handoff + +All six tests rewritten, mutations verified. New helper test/helpers/source-blocks.ts. diff --git a/.ai/runs/2026-11-11-robust-tests/PLAN.md b/.ai/runs/2026-11-11-robust-tests/PLAN.md new file mode 100644 index 0000000..d5f81e5 --- /dev/null +++ b/.ai/runs/2026-11-11-robust-tests/PLAN.md @@ -0,0 +1,4 @@ +# Plan + +Rewrite six brittle tests to derive expectations from the registry, BUNDLE_SKILLS, model-floor.json +and real exported functions; prove each with a mutation. Plan unreviewed (unattended run). diff --git a/test/editor-demand-wiring.test.ts b/test/editor-demand-wiring.test.ts index db914cc..a89f75a 100644 --- a/test/editor-demand-wiring.test.ts +++ b/test/editor-demand-wiring.test.ts @@ -1,40 +1,38 @@ import fs from 'node:fs'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; +import { functionBody, ipcHandler, occurrences, within } from './helpers/source-blocks'; const MAIN = fs.readFileSync(path.join(__dirname, '..', 'src', 'main', 'main.ts'), 'utf8'); const WINDOWS = fs.readFileSync(path.join(__dirname, '..', 'src', 'main', 'windows.ts'), 'utf8'); -function bodyOf(source: string, header: string) { - const start = source.indexOf(header); - expect(start, `${header} exists`).toBeGreaterThan(-1); - const end = source.indexOf('\n}\n', start); - return source.slice(start, end); -} - -function handlerOf(channel: string) { - const start = MAIN.indexOf(`ipcMain.handle('${channel}'`); - expect(start, `${channel} handler exists`).toBeGreaterThan(-1); - return MAIN.slice(start, MAIN.indexOf('}));', start)); -} +const REQUEST = 'editorDemand.request('; +const KEEP_WARM = 'editorDemand.keepWarm('; +const ON_DEMAND_CHANNELS = ['ws:need-editor', 'ws:restart-editor']; describe('main.ts starts the editor server only on demand', () => { it('does not start or ask in init(), openWorkspace() or a project window', () => { - for (const body of [bodyOf(MAIN, 'async function init() {'), bodyOf(MAIN, 'function openWorkspace() {'), bodyOf(WINDOWS, 'function openProjectWindow(')]) { - expect(body).not.toMatch(/editorDemand\.|launchEditor\(|startEditor|warmEditor|serveweb\.start/); - } + const bodies = [functionBody(MAIN, 'init'), functionBody(MAIN, 'openWorkspace'), functionBody(WINDOWS, 'openProjectWindow')]; + for (const body of bodies) expect(body.text).not.toMatch(/editorDemand\.|launchEditor\(|startEditor|warmEditor|serveweb\.start/); + }); + + it('asks for the server only inside the need-editor and restart handlers, and each of them asks', () => { + const handlers = ON_DEMAND_CHANNELS.map((channel) => ipcHandler(MAIN, channel)); + for (const handler of handlers) expect(handler.text, 'handler asks').toContain(REQUEST); + const stray = occurrences(MAIN, REQUEST).filter((index) => !handlers.some((handler) => within(index, handler))); + expect(stray, 'request() outside the on-demand handlers').toEqual([]); }); - it('reaches the server start from the need-editor and restart channels alone', () => { - expect(handlerOf('ws:need-editor')).toContain('editorDemand.request()'); - expect(handlerOf('ws:restart-editor')).toContain('editorDemand.request()'); - expect([...MAIN.matchAll(/editorDemand\.request\(\)/g)]).toHaveLength(2); - expect([...MAIN.matchAll(/launchEditor\(/g)]).toHaveLength(1); - expect(MAIN).toContain('createEditorDemand(launchEditor)'); + it('hands launchEditor to the demand gate and calls it nowhere else', () => { + expect(MAIN).toMatch(/createEditorDemand\(\s*launchEditor\s*\)/); + const withoutGate = MAIN.replace(/function launchEditor\(/, '').replace(/createEditorDemand\(\s*launchEditor\s*\)/, ''); + expect(withoutGate).not.toContain('launchEditor'); }); it('only keeps the server warm from the scan, which waits for a first request', () => { - expect([...MAIN.matchAll(/editorDemand\.keepWarm\(\)/g)]).toHaveLength(1); - expect(bodyOf(MAIN, 'function runScan() {')).toContain('editorDemand.keepWarm()'); + const scan = functionBody(MAIN, 'runScan'); + const warm = occurrences(MAIN, KEEP_WARM); + expect(warm.length).toBeGreaterThan(0); + expect(warm.filter((index) => !within(index, scan))).toEqual([]); }); }); diff --git a/test/helpers/source-blocks.ts b/test/helpers/source-blocks.ts new file mode 100644 index 0000000..674326e --- /dev/null +++ b/test/helpers/source-blocks.ts @@ -0,0 +1,44 @@ +export interface SourceRange { + start: number; + end: number; + text: string; +} + +const CLOSERS: Record = { '(': ')', '{': '}', '[': ']' }; + +function balancedFrom(source: string, open: number): SourceRange { + const opener = source[open]; + const closer = CLOSERS[opener]; + let depth = 0; + for (let i = open; i < source.length; i++) { + if (source[i] === opener) depth++; + else if (source[i] === closer && --depth === 0) return { start: open, end: i + 1, text: source.slice(open, i + 1) }; + } + throw new Error(`unbalanced ${opener} at ${open}`); +} + +export function functionBody(source: string, name: string): SourceRange { + const at = source.search(new RegExp(`function ${name}\\(`)); + if (at < 0) throw new Error(`function ${name} not found`); + const params = balancedFrom(source, source.indexOf('(', at)); + return balancedFrom(source, source.indexOf('{', params.end)); +} + +export function callArguments(source: string, callee: string): SourceRange { + const at = source.indexOf(`${callee}(`); + if (at < 0) throw new Error(`${callee}( not found`); + return balancedFrom(source, at + callee.length); +} + +export function ipcHandler(source: string, channel: string): SourceRange { + const at = source.search(new RegExp(`ipcMain\\.handle\\(\\s*'${channel}'`)); + if (at < 0) throw new Error(`no ipcMain.handle for ${channel}`); + return balancedFrom(source, source.indexOf('(', at)); +} + +export function occurrences(source: string, needle: string | RegExp): number[] { + const pattern = typeof needle === 'string' ? new RegExp(needle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g') : new RegExp(needle.source, 'g'); + return [...source.matchAll(pattern)].map((m) => m.index ?? 0); +} + +export const within = (index: number, range: SourceRange) => index >= range.start && index < range.end; diff --git a/test/install-skills.test.ts b/test/install-skills.test.ts index d41d071..977330e 100644 --- a/test/install-skills.test.ts +++ b/test/install-skills.test.ts @@ -9,6 +9,7 @@ import { AGENT_NAMES, git, makeBundle, makeHome, makeLinker, makeRepo, snapshot const run = promisify(execFile); const SCRIPT = path.join(__dirname, '..', 'scripts', 'install-skills.mjs'); +const SKILLS_SOURCE = path.join(__dirname, '..', 'skills'); const SKILL_NAMES = BUNDLE_SKILLS.map((s) => s.name); let tmp: string; @@ -60,10 +61,13 @@ describe('install-skills', () => { expect(fs.existsSync(path.join(skills, 'lantern-sync', SKILL_MARKER))).toBe(true); }); - it('installs all sixteen bundled skills into a fresh project, lantern-local-loop with its nested files and executable loop.sh', async () => { + it('installs every bundled skill into a fresh project, lantern-local-loop with its nested files and executable loop.sh', async () => { const { stdout } = await install(project); - const all = ['lantern-setup', 'lantern-sync', 'lantern-delegate', 'lantern-delegate-complex', 'lantern-plan', 'lantern-plan-impact', 'lantern-plan-stages', 'lantern-plan-review', 'lantern-plan-handoff', 'lantern-commit', 'lantern-local-loop', 'lantern-no-comments', 'lantern-code-reviewer', 'lantern-jira', 'lantern-bitbucket', 'lantern-github']; - expect(SKILL_NAMES).toEqual(all); + const shipped = fs.readdirSync(SKILLS_SOURCE).filter((entry) => fs.existsSync(path.join(SKILLS_SOURCE, entry, 'SKILL.md'))).sort(); + const all = SKILL_NAMES; + expect(all.length).toBeGreaterThan(0); + expect(new Set(all).size).toBe(all.length); + expect([...all].sort()).toEqual(shipped); for (const name of all) { expect(stdout, name).toMatch(new RegExp(`installed +${name}\\b`)); expect(fs.existsSync(path.join(skills, name, 'SKILL.md')), name).toBe(true); diff --git a/test/lantern-macro-load.test.ts b/test/lantern-macro-load.test.ts index 39f4cc9..77e963f 100644 --- a/test/lantern-macro-load.test.ts +++ b/test/lantern-macro-load.test.ts @@ -6,6 +6,7 @@ import { loadMacro } from '../src/main/lantern'; import { MIN_STRONG_EDGE } from '../src/main/lantern-macro'; import { NO_MAP, readRegisteredMacro } from '../src/main/lantern-macro-load'; import { macroSystemsByPath } from '../src/main/lantern-macro-systems'; +import { ipcHandler } from './helpers/source-blocks'; import { macroFixtureBlocks, macroFixtureRecords, macroFixtureView } from './helpers/lantern-macro-fixture'; const FIXTURE_ROOT = path.join(__dirname, 'fixtures', 'lantern-macro'); @@ -17,14 +18,14 @@ afterEach(() => { }); describe('lantern:macro handler', () => { - it('returns deriveMacroView of the registered project: 14 blocks, 58 edges', async () => { + it('returns deriveMacroView of the registered project', async () => { const result = await readRegisteredMacro(registeredOnly(FIXTURE_ROOT), loadMacro, FIXTURE_ROOT); expect(result.ok).toBe(true); if (!result.ok) return; expect(result.view).toEqual(macroFixtureView()); - expect(result.view.blocks).toHaveLength(14); - expect(result.view.edges).toHaveLength(58); + expect(result.view.blocks.length).toBeGreaterThan(0); + expect(result.view.edges.length).toBeGreaterThan(0); expect(result.minStrongEdge).toBe(MIN_STRONG_EDGE); expect(result.texts).toEqual({}); }); @@ -156,23 +157,20 @@ describe('lantern:macro texts', () => { describe('lantern:macro wiring', () => { const mainSource = fs.readFileSync(path.join(__dirname, '..', 'src', 'main', 'main.ts'), 'utf8'); const preloadSource = fs.readFileSync(path.join(__dirname, '..', 'src', 'preload', 'workspace.ts'), 'utf8'); + const rendererSource = fs.readFileSync(path.join(__dirname, '..', 'src', 'renderer', 'features', 'lantern.ts'), 'utf8'); + const handler = () => ipcHandler(mainSource, 'lantern:macro').text; it('registers the channel behind the trusted-sender wrapper', () => { - expect(mainSource).toMatch(/ipcMain\.handle\('lantern:macro', fromWorkspace\(/); + expect(handler()).toMatch(/^\(\s*'lantern:macro',\s*fromWorkspace\(/); }); it('checks the project against the registered projects before reading', () => { - const handler = mainSource.slice(mainSource.indexOf("ipcMain.handle('lantern:macro'")); - expect(handler.slice(0, handler.indexOf('));'))).toContain('isRegisteredDir'); + expect(handler()).toMatch(/readRegisteredMacro\(\s*isRegisteredDir\b/); }); - it('exposes one wrapped invoke in the preload, not ipcRenderer itself', () => { - expect(preloadSource).toContain("lanternMacro: (project: string, lang?: string) => ipcRenderer.invoke('lantern:macro', project, lang)"); - }); - - it('passes the viewer language from the renderer to the main process', () => { - const rendererSource = fs.readFileSync(path.join(__dirname, '..', 'src', 'renderer', 'features', 'lantern.ts'), 'utf8'); - expect(mainSource).toMatch(/readRegisteredMacro\([^]*lantern\.loadMacro, project, lang\)/); - expect(rendererSource).toContain('window.ccmws.lanternMacro(project, app.lanternLang.get(project))'); + it('passes the viewer language from the preload and the renderer on to the reader', () => { + expect(preloadSource).toMatch(/lanternMacro:\s*\(project: string, lang\?: string\)\s*=>\s*ipcRenderer\.invoke\('lantern:macro', project, lang\)/); + expect(handler()).toMatch(/readRegisteredMacro\([^]*\bproject,\s*lang\s*\)/); + expect(rendererSource).toMatch(/lanternMacro\(project,\s*app\.lanternLang\.get\(project\)\)/); }); }); diff --git a/test/lantern-permissions-triggers.test.ts b/test/lantern-permissions-triggers.test.ts index 1982078..20076b1 100644 --- a/test/lantern-permissions-triggers.test.ts +++ b/test/lantern-permissions-triggers.test.ts @@ -2,16 +2,17 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { registerExtensionsIpc } from '../src/main/extensions-ipc'; import { extensionsPath, importExtensionsList, saveSkillLinker } from '../src/main/extensions'; import { forgetAttempts, installLanternBundle, installOnOpen, settled, settledFor, syncPermissionsOf } from '../src/main/bundle'; import { missingPermissions, requiredAllow } from '../skills/lantern-delegate/scripts/permissions.mjs'; +import { callArguments } from './helpers/source-blocks'; import { makeHome, makeLinker, makeRepo } from './support/bundle-fixtures'; vi.setConfig({ testTimeout: 30_000, hookTimeout: 30_000 }); const REAL_ROOT = path.join(__dirname, '..', 'skills'); const MAIN = fs.readFileSync(path.join(__dirname, '..', 'src', 'main', 'main.ts'), 'utf8'); -const EXTENSIONS_IPC = fs.readFileSync(path.join(__dirname, '..', 'src', 'main', 'extensions-ipc.ts'), 'utf8'); let tmp: string; let project: string; @@ -105,11 +106,60 @@ describe('changing the skill linker syncs the delegate rules', () => { expect(allow()).toContain(linkRule(linker)); }); - it('is wired into the Skills window handlers that save the linker', () => { - expect(EXTENSIONS_IPC).toMatch(/'extensions:skillLinker',\n\s*fromWorkspace\(async[^\n]*\n\s*const panel = saveSkillLinker\([^\n]*\n\s*await syncPermissionsFor\(req\?\.project\);\n\s*return panel;/); - expect(EXTENSIONS_IPC).toMatch(/const panel = importExtensionsList\([^\n]*\n\s*await syncPermissionsFor\(req\?\.project\);\n\s*return panel;/); - expect(MAIN).toMatch(/registerExtensionsIpc\(\{[^]*syncPermissionsFor,/); - expect(MAIN).toMatch(/const syncPermissionsFor = [^\n]*syncPermissionsOf\(project, \{ provider: settings\.provider\(project\) \}\)/); + describe('through the Skills window handlers', () => { + const SENDER = { id: 'workspace' }; + + function ipcHandlers(homeDir: string, pickList: () => Promise, onSync: (project: unknown) => void) { + const handlers = new Map Promise>(); + registerExtensionsIpc({ + handle: (channel, handler) => handlers.set(channel, handler as never), + fromWorkspace: (fn) => fn, + installedFor: () => [], + reinstallFor: () => null, + syncPermissionsFor: async (target) => onSync(target), + pickList, + home: homeDir, + }); + return (channel: string, ...args: unknown[]) => handlers.get(channel)!({ sender: SENDER }, ...args); + } + + it('syncs the rules of the open project after the linker is saved', async () => { + const homeDir = makeHome(path.join(tmp, 'home')); + const linker = makeLinker(path.join(tmp, 'linker-w')); + const savedWhenSynced: boolean[] = []; + const synced: unknown[] = []; + const invoke = ipcHandlers(homeDir, async () => null, (target) => { + synced.push(target); + savedWhenSynced.push(fs.existsSync(extensionsPath(homeDir))); + }); + await invoke('extensions:skillLinker', { project, value: linker }); + expect(synced).toEqual([project]); + expect(savedWhenSynced).toEqual([true]); + }); + + it('syncs the rules of the open project after a list is imported, and not when the picker is cancelled', async () => { + const homeDir = makeHome(path.join(tmp, 'home')); + const list = path.join(tmp, 'list.json'); + fs.writeFileSync(list, JSON.stringify({ skillLinker: makeLinker(path.join(tmp, 'linker-v')) })); + const synced: unknown[] = []; + const importedWhenSynced: boolean[] = []; + let picked: string | null = null; + const invoke = ipcHandlers(homeDir, async () => picked, (target) => { + synced.push(target); + importedWhenSynced.push(fs.existsSync(extensionsPath(homeDir))); + }); + await invoke('extensions:import', { project }); + expect(synced).toEqual([]); + picked = list; + await invoke('extensions:import', { project }); + expect(synced).toEqual([project]); + expect(importedWhenSynced).toEqual([true]); + }); + + it('is the sync main.ts hands to those handlers, bound to the provider of the project', () => { + expect(callArguments(MAIN, 'registerExtensionsIpc').text).toMatch(/[{,]\s*syncPermissionsFor\s*[,}]/); + expect(MAIN).toMatch(/const syncPermissionsFor\s*=[^;]*syncPermissionsOf\(\s*project,[^;]*settings\.provider\(project\)/); + }); }); }); diff --git a/test/lantern-sync-selftest.test.ts b/test/lantern-sync-selftest.test.ts index 5a3d795..f4087d0 100644 --- a/test/lantern-sync-selftest.test.ts +++ b/test/lantern-sync-selftest.test.ts @@ -1,4 +1,5 @@ import { execFile } from 'node:child_process'; +import fs from 'node:fs'; import path from 'node:path'; import { promisify } from 'node:util'; import { describe, expect, it } from 'vitest'; @@ -15,19 +16,44 @@ describe('lantern-sync self-test', () => { }); describe('Lantern mapping model policy', () => { - const modelCli = path.join(__dirname, '..', 'skills', 'lantern-setup', 'scripts', 'model.mjs'); + const setup = path.join(__dirname, '..', 'skills', 'lantern-setup'); + const modelCli = path.join(setup, 'scripts', 'model.mjs'); + const floor = JSON.parse(fs.readFileSync(path.join(setup, 'references', 'model-floor.json'), 'utf8')) as { + claude: { family: string; recommended?: string }[]; + allowedIds?: string[]; + codex?: { family: string }[]; + }; + const check = (id: string) => run(process.execPath, [modelCli, 'check', id]); + const versionId = (family: string, version: string) => `claude-${family}-${version.replace('.', '-')}`; + const bump = (version: string, by: number) => { + const [major, minor = 0] = version.split('.').map(Number); + return `${major}.${minor + by}`; + }; - it('allows exact Astra and the supported Sonnet, Opus and Fable ids, accepting older versions, rejecting aliases and Haiku', async () => { - for (const id of ['gpt-6-astra', 'claude-sonnet-5-5', 'claude-sonnet-5-6', 'claude-sonnet-6', 'claude-opus-5-5', 'claude-fable-5-1']) { - const { stdout } = await run(process.execPath, [modelCli, 'check', id]); - expect(stdout.trim()).toBe(`OK ${id}`); + it('allows every family of the floor and every exact id it lists', async () => { + expect(floor.claude.length).toBeGreaterThan(0); + for (const id of [...(floor.allowedIds ?? []), ...floor.claude.map(({ family }) => versionId(family, '9.9'))]) { + expect((await check(id)).stdout.trim(), id).toBe(`OK ${id}`); } - for (const id of ['claude-sonnet-5-4', 'claude-opus-5-0', 'claude-fable-5-0']) { - const { stdout } = await run(process.execPath, [modelCli, 'check', id]); - expect(stdout.trim()).toMatch(new RegExp(`^OK ${id} \\(below the recommended `)); - } - for (const id of ['gpt-6-sol', 'gpt-6.1-sol', 'gpt-6-terra', 'gpt-6-astra-preview', 'astra', 'terra', 'claude-haiku-5-5']) { - await expect(run(process.execPath, [modelCli, 'check', id])).rejects.toMatchObject({ code: 1 }); + }); + + it('accepts the recommended version and newer without advice, and an older one with advice', async () => { + for (const { family, recommended } of floor.claude.filter((entry) => entry.recommended)) { + for (const version of [recommended!, bump(recommended!, 1)]) { + const id = versionId(family, version); + expect((await check(id)).stdout.trim(), id).toBe(`OK ${id}`); + } + const older = versionId(family, '1.0'); + const { stdout } = await check(older); + expect(stdout.trim(), older).toMatch(new RegExp(`^OK ${older} \\(.*${family}`)); } }); + + it('rejects a Codex id the Claude route does not list, an id outside the floor and an alias', async () => { + const known = new Set([...floor.claude.map(({ family }) => family), ...(floor.allowedIds ?? [])]); + const outside = ['haiku', 'unlisted-family'].filter((family) => !known.has(family)); + const codexOnly = (floor.codex ?? []).map(({ family }) => `gpt-6-${family}`).filter((id) => !floor.allowedIds?.includes(id)); + const ids = [...codexOnly, ...outside.map((family) => versionId(family, '5.5')), ...floor.claude.map(({ family }) => family), `${floor.allowedIds?.[0] ?? 'exact-id'}-preview`]; + for (const id of ids) await expect(check(id), id).rejects.toMatchObject({ code: 1 }); + }); }); diff --git a/test/project-window.test.ts b/test/project-window.test.ts index e2d13f0..0aa0f1b 100644 --- a/test/project-window.test.ts +++ b/test/project-window.test.ts @@ -2,6 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { COMMANDS } from '../src/shared/commands'; import { SETTING_DEFAULTS } from '../src/shared/settings-registry'; import { mapOf } from './helpers/lantern-map'; @@ -342,6 +343,7 @@ describe('project window', () => { }); describe('always-on commands', () => { + const REGISTERED_ON_DEMAND = ['explain.popout']; const commands = () => import('../src/renderer/commands'); it('add folder runs the same bridge call from the button and the command', async () => { @@ -370,33 +372,12 @@ describe('always-on commands', () => { expect(window.ccmws.needEditor).toHaveBeenCalledTimes(1); }); - it('lists every always-on command after boot', async () => { + it('registers every command of the registry except the ones a feature registers on demand', async () => { await boot(''); const { availableCommands } = await commands(); - expect(availableCommands().sort()).toEqual( - [ - 'palette.open', - 'settings.open', - 'settings.openFile', - 'project.new', - 'project.addFolder', - 'project.filter', - 'editor.show', - 'editor.restart', - 'editor.devtools', - 'lantern.open', - 'lantern.problems', - 'lantern.previousFile', - 'lantern.nextFile', - 'lantern.refresh', - 'explain.open', - 'memory.open', - 'diagnostics.open', - 'skills.open', - 'skills.fixMicrophone', - 'integrations.open', - ].sort(), - ); + const alwaysOn = COMMANDS.map((c) => c.id as string).filter((id) => !REGISTERED_ON_DEMAND.includes(id)); + expect(alwaysOn.length).toBeGreaterThan(0); + expect(availableCommands()).toEqual(expect.arrayContaining(alwaysOn)); }); }); From 8c1335b44ba0111babece67bd9a5137d99eb2ce7 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:10:09 +0100 Subject: [PATCH 2/9] chore: update the Lantern map --- .lantern/lang/en/test/helpers.lantern.json | 3 ++- .lantern/map/test.lantern | 19 ++++++++++++------- .lantern/map/test/helpers.lantern | 3 +++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.lantern/lang/en/test/helpers.lantern.json b/.lantern/lang/en/test/helpers.lantern.json index fd4301e..16bdc69 100644 --- a/.lantern/lang/en/test/helpers.lantern.json +++ b/.lantern/lang/en/test/helpers.lantern.json @@ -36,7 +36,8 @@ "macro-lab-world.ts": "Generates the invented Orchard Market project (six areas, eighteen features, five platforms with an infrastructure one and two chat bots, a foundation band, a feature left to assign, a pull request across three areas with its Flow, outside systems with different counts and some talked to by the foundation) and derives its Macro payload with the real derivation.", "lantern-plan-world.ts": "builds a throwaway project from the shared plan fixtures, with or without a remote, and runs the plan and sync scripts against it for the plan tests", "lantern-vocab-project.ts": "writes a synthetic project with its map, features, platforms, systems list and areas to a temp folder, runs the lantern-setup scripts on it and snapshots its .lantern files", - "lantern-map.ts": "builders for map nodes, whole maps (with their plans and unreadable plan files) and PR records", + "source-blocks.ts": "finds a function body, an IPC handler's arguments or every occurrence of a call inside a source file by matching brackets, so wiring tests check where a call sits rather than counting matches", + "lantern-map.ts":"builders for map nodes, whole maps (with their plans and unreadable plan files) and PR records", "lantern-sections.ts": "section fixtures (RBAC, states) and a mount helper for the map view", "macro-sizes-world.ts": "Generates a small (4 areas, 8 features) and a large (7 areas, 24 features) invented project and derives their Macro payloads with the real derivation.", "memory-fixtures.ts": "reads the recorded ps, top, lsof and memory_pressure text and fakes the command runner", diff --git a/.lantern/map/test.lantern b/.lantern/map/test.lantern index b8082b1..85ff51b 100644 --- a/.lantern/map/test.lantern +++ b/.lantern/map/test.lantern @@ -347,10 +347,11 @@ test/dom.test.ts . test 54fb7da385e4 = src/renderer/lib/util.ts < src/shared/code-preview.ts -test/editor-demand-wiring.test.ts . test cfed95f7565a +test/editor-demand-wiring.test.ts . test 08482bc6b8e1 #feature serve-web = src/main/main.ts = src/main/windows.ts + = test/helpers/source-blocks.ts test/editor-tab.test.ts . test f3f282dd27ed #feature serve-web @@ -441,7 +442,7 @@ test/idle-tabs.test.ts . test 7dece3b876fc #feature serve-web = src/renderer/lib/idle-tabs.ts -test/install-skills.test.ts . test 646437f73dc3 +test/install-skills.test.ts . test e0e74002199c #feature skill-installer = scripts/install-skills.mjs = scripts/lantern-bundle.mjs @@ -877,7 +878,7 @@ test/lantern-macro-layout.test.ts . test 3fea77a5c0a1 = test/fixtures/lantern-macro-own-view.json = test/helpers/lantern-macro-fixture.ts#macroFixtureView -test/lantern-macro-load.test.ts . test 8da441c3f027 +test/lantern-macro-load.test.ts . test 8909cb92a510 #feature macro-view = src/main/lantern-macro-load.ts#readRegisteredMacro = src/main/lantern-macro-systems.ts#macroSystemsByPath @@ -887,6 +888,7 @@ test/lantern-macro-load.test.ts . test 8da441c3f027 = src/renderer/features/lantern.ts = test/helpers/lantern-macro-fixture.ts#macroFixtureBlocks = test/helpers/lantern-macro-fixture.ts#macroFixtureView + = test/helpers/source-blocks.ts test/lantern-macro-platform-uses-view.test.ts . test 192debd3300f #feature macro-view @@ -1024,13 +1026,14 @@ test/lantern-panel.test.ts . test 2e3ec75e2ee0 > src/renderer/lib/providers.ts = test/helpers/lantern-map.ts -test/lantern-permissions-triggers.test.ts . test 4ccf978663f8 +test/lantern-permissions-triggers.test.ts . test 98f18d350a91 #feature skill-installer = src/main/bundle.ts = src/main/extensions.ts#saveSkillLinker - < src/main/extensions-ipc.ts + = src/main/extensions-ipc.ts#registerExtensionsIpc < src/main/main.ts < skills/lantern-delegate/scripts/permissions.mjs#missingPermissions + = test/helpers/source-blocks.ts = test/support/bundle-fixtures.ts test/lantern-pixi-edge-style.test.ts . test b0bded696957 @@ -1608,8 +1611,9 @@ test/lantern-sync-record-mirror.test.ts . test 6ade1aeb2746 = test/helpers/git-repo.ts = test/helpers/lantern-plan-world.ts -test/lantern-sync-selftest.test.ts . test 8a7a15c03f9c +test/lantern-sync-selftest.test.ts . test 7debfe67ebf5 #feature map-sync + = skills/lantern-setup/references/model-floor.json = skills/lantern-setup/scripts/model.mjs = skills/lantern-sync/scripts/selftest.mjs @@ -1984,9 +1988,10 @@ test/project-release.test.ts . test fcaf35dd232c test/project-ident.test.ts . test c98778ad4b70 #feature projects-hub = src/main/project-ident/index.ts#projectIdent -test/project-window.test.ts . test b6e790e93fff +test/project-window.test.ts . test d030bbd7379a #feature projects-hub = src/renderer/commands/index.ts + < src/shared/commands/index.ts#COMMANDS < src/renderer/state.ts#app < src/renderer/workspace.css < src/renderer/workspace.html diff --git a/.lantern/map/test/helpers.lantern b/.lantern/map/test/helpers.lantern index 25e30bb..266bd89 100644 --- a/.lantern/map/test/helpers.lantern +++ b/.lantern/map/test/helpers.lantern @@ -217,5 +217,8 @@ test/helpers/updater-world.ts . test 577549414f03 #feature app-updates = src/main/updater/service.ts#createUpdater +test/helpers/source-blocks.ts . test 45cdb0e2dee8 +#feature serve-web + test/helpers/pr-rows.ts . test ef812afdf693 #feature pull-requests From 8edbd6c41d2df08638ade7770a5fa63afe4188c3 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:10:13 +0100 Subject: [PATCH 3/9] chore: remove the run folder --- .ai/runs/2026-11-11-robust-tests/HANDOFF.md | 3 --- .ai/runs/2026-11-11-robust-tests/PLAN.md | 4 ---- 2 files changed, 7 deletions(-) delete mode 100644 .ai/runs/2026-11-11-robust-tests/HANDOFF.md delete mode 100644 .ai/runs/2026-11-11-robust-tests/PLAN.md diff --git a/.ai/runs/2026-11-11-robust-tests/HANDOFF.md b/.ai/runs/2026-11-11-robust-tests/HANDOFF.md deleted file mode 100644 index 5d88817..0000000 --- a/.ai/runs/2026-11-11-robust-tests/HANDOFF.md +++ /dev/null @@ -1,3 +0,0 @@ -# Handoff - -All six tests rewritten, mutations verified. New helper test/helpers/source-blocks.ts. diff --git a/.ai/runs/2026-11-11-robust-tests/PLAN.md b/.ai/runs/2026-11-11-robust-tests/PLAN.md deleted file mode 100644 index d5f81e5..0000000 --- a/.ai/runs/2026-11-11-robust-tests/PLAN.md +++ /dev/null @@ -1,4 +0,0 @@ -# Plan - -Rewrite six brittle tests to derive expectations from the registry, BUNDLE_SKILLS, model-floor.json -and real exported functions; prove each with a mutation. Plan unreviewed (unattended run). From 5b66a6095e3d15028572168c5fefadd7b3482bff Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:10:56 +0100 Subject: [PATCH 4/9] chore: record pull request #33 in the Lantern map --- .lantern/prs/33.pr | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .lantern/prs/33.pr diff --git a/.lantern/prs/33.pr b/.lantern/prs/33.pr new file mode 100644 index 0000000..ef4aa72 --- /dev/null +++ b/.lantern/prs/33.pr @@ -0,0 +1,46 @@ +lantern-pr 1 +# 33 +#title Make six brittle tests derive their expectations from the source +#url https://github.com/MobileReality/lantern/pull/33 +#branch chore/robust-brittle-tests +#base main +#opened 2026-11-11 +#head 8edbd6c41d2df08638ade7770a5fa63afe4188c3 +#describe what Six brittle tests now derive expectations from the command registry, the skill bundle, model-floor.json and the real handlers. +#describe before Hard-coded 20-command and 16-skill lists, counted regex matches in main.ts. +#describe after Lists come from the source of truth; wiring is checked by where a call sits. +#describe verified typecheck, lint and the full test run are green; each contract was broken in source and its test failed. ++ test/helpers/source-blocks.ts +~ test/editor-demand-wiring.test.ts +~ test/install-skills.test.ts +~ test/lantern-macro-load.test.ts +~ test/lantern-permissions-triggers.test.ts +~ test/lantern-sync-selftest.test.ts +~ test/project-window.test.ts +#order test/helpers/source-blocks.ts +#order test/editor-demand-wiring.test.ts +#order test/lantern-permissions-triggers.test.ts +#order test/lantern-macro-load.test.ts +#order test/project-window.test.ts +#order test/install-skills.test.ts +#order test/lantern-sync-selftest.test.ts +#reason test/helpers/source-blocks.ts Shared bracket matcher so wiring tests locate handlers and function bodies without brittle regexes. +#reason test/editor-demand-wiring.test.ts Asserts where editor-server calls sit instead of counting regex matches, so renames and reorders do not break it. +#reason test/install-skills.test.ts Takes the skill list from the bundle and the skills folder instead of a hand-kept list of sixteen. +#reason test/lantern-macro-load.test.ts Drops the literal block and edge counts and checks the handler wiring by location, not exact text. +#reason test/lantern-permissions-triggers.test.ts Checks that saving the linker or importing a list syncs rules by calling the real handlers. +#reason test/lantern-sync-selftest.test.ts Drives the model policy check from model-floor.json instead of a pinned list of ids and a sentence. +#reason test/project-window.test.ts Expects the registry's commands to be registered instead of a hand-kept list of command names. +#story Tests stop breaking on unrelated edits +#step actor A developer renames or reorders code +#file test/helpers/source-blocks.ts +#step action runs the test suite +#step subject the six rewritten tests +#file test/editor-demand-wiring.test.ts +#file test/project-window.test.ts +#file test/install-skills.test.ts +#below interaction Sees a failure only when a real contract breaks +#file test/lantern-permissions-triggers.test.ts +#file test/lantern-macro-load.test.ts +#file test/lantern-sync-selftest.test.ts +#below consequence A new command or skill needed a list edit => The list comes from the registry or bundle From e911927bad393549ea24eb103097382c4575c438 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:15:26 +0100 Subject: [PATCH 5/9] test: tighten the command, loader, panel and model-advisory checks --- test/lantern-macro-load.test.ts | 2 +- test/lantern-permissions-triggers.test.ts | 6 ++++-- test/lantern-sync-selftest.test.ts | 4 ++-- test/project-window.test.ts | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/lantern-macro-load.test.ts b/test/lantern-macro-load.test.ts index 77e963f..a7d7829 100644 --- a/test/lantern-macro-load.test.ts +++ b/test/lantern-macro-load.test.ts @@ -170,7 +170,7 @@ describe('lantern:macro wiring', () => { it('passes the viewer language from the preload and the renderer on to the reader', () => { expect(preloadSource).toMatch(/lanternMacro:\s*\(project: string, lang\?: string\)\s*=>\s*ipcRenderer\.invoke\('lantern:macro', project, lang\)/); - expect(handler()).toMatch(/readRegisteredMacro\([^]*\bproject,\s*lang\s*\)/); + expect(handler()).toMatch(/readRegisteredMacro\([^]*lantern\.loadMacro,\s*project,\s*lang\s*\)/); expect(rendererSource).toMatch(/lanternMacro\(project,\s*app\.lanternLang\.get\(project\)\)/); }); }); diff --git a/test/lantern-permissions-triggers.test.ts b/test/lantern-permissions-triggers.test.ts index 20076b1..d592d79 100644 --- a/test/lantern-permissions-triggers.test.ts +++ b/test/lantern-permissions-triggers.test.ts @@ -132,7 +132,8 @@ describe('changing the skill linker syncs the delegate rules', () => { synced.push(target); savedWhenSynced.push(fs.existsSync(extensionsPath(homeDir))); }); - await invoke('extensions:skillLinker', { project, value: linker }); + const panel = await invoke('extensions:skillLinker', { project, value: linker }); + expect(panel).toBeTruthy(); expect(synced).toEqual([project]); expect(savedWhenSynced).toEqual([true]); }); @@ -151,7 +152,8 @@ describe('changing the skill linker syncs the delegate rules', () => { await invoke('extensions:import', { project }); expect(synced).toEqual([]); picked = list; - await invoke('extensions:import', { project }); + const panel = await invoke('extensions:import', { project }); + expect(panel).toBeTruthy(); expect(synced).toEqual([project]); expect(importedWhenSynced).toEqual([true]); }); diff --git a/test/lantern-sync-selftest.test.ts b/test/lantern-sync-selftest.test.ts index f4087d0..4d8072a 100644 --- a/test/lantern-sync-selftest.test.ts +++ b/test/lantern-sync-selftest.test.ts @@ -39,13 +39,13 @@ describe('Lantern mapping model policy', () => { it('accepts the recommended version and newer without advice, and an older one with advice', async () => { for (const { family, recommended } of floor.claude.filter((entry) => entry.recommended)) { - for (const version of [recommended!, bump(recommended!, 1)]) { + for (const version of [recommended!, bump(recommended!, 1), String(Number(recommended!.split('.')[0]) + 1)]) { const id = versionId(family, version); expect((await check(id)).stdout.trim(), id).toBe(`OK ${id}`); } const older = versionId(family, '1.0'); const { stdout } = await check(older); - expect(stdout.trim(), older).toMatch(new RegExp(`^OK ${older} \\(.*${family}`)); + expect(stdout.trim(), older).toMatch(new RegExp(`^OK ${older} \\(below the recommended ${family}`)); } }); diff --git a/test/project-window.test.ts b/test/project-window.test.ts index 0aa0f1b..02f8208 100644 --- a/test/project-window.test.ts +++ b/test/project-window.test.ts @@ -377,7 +377,7 @@ describe('always-on commands', () => { const { availableCommands } = await commands(); const alwaysOn = COMMANDS.map((c) => c.id as string).filter((id) => !REGISTERED_ON_DEMAND.includes(id)); expect(alwaysOn.length).toBeGreaterThan(0); - expect(availableCommands()).toEqual(expect.arrayContaining(alwaysOn)); + expect([...availableCommands()].sort()).toEqual([...alwaysOn].sort()); }); }); From 61b9e1974433982fdd222b62e5019efcf19ae2a4 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:19:55 +0100 Subject: [PATCH 6/9] test: probe a minor below the recommended model version --- test/lantern-sync-selftest.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/lantern-sync-selftest.test.ts b/test/lantern-sync-selftest.test.ts index 4d8072a..620cbbd 100644 --- a/test/lantern-sync-selftest.test.ts +++ b/test/lantern-sync-selftest.test.ts @@ -43,9 +43,13 @@ describe('Lantern mapping model policy', () => { const id = versionId(family, version); expect((await check(id)).stdout.trim(), id).toBe(`OK ${id}`); } - const older = versionId(family, '1.0'); - const { stdout } = await check(older); - expect(stdout.trim(), older).toMatch(new RegExp(`^OK ${older} \\(below the recommended ${family}`)); + const [, recommendedMinor = '0'] = recommended!.split('.'); + const olderVersions = ['1.0', ...(Number(recommendedMinor) > 0 ? [bump(recommended!, -1)] : [])]; + for (const version of olderVersions) { + const older = versionId(family, version); + const { stdout } = await check(older); + expect(stdout.trim(), older).toMatch(new RegExp(`^OK ${older} \\(below the recommended ${family}`)); + } } }); @@ -53,7 +57,9 @@ describe('Lantern mapping model policy', () => { const known = new Set([...floor.claude.map(({ family }) => family), ...(floor.allowedIds ?? [])]); const outside = ['haiku', 'unlisted-family'].filter((family) => !known.has(family)); const codexOnly = (floor.codex ?? []).map(({ family }) => `gpt-6-${family}`).filter((id) => !floor.allowedIds?.includes(id)); - const ids = [...codexOnly, ...outside.map((family) => versionId(family, '5.5')), ...floor.claude.map(({ family }) => family), `${floor.allowedIds?.[0] ?? 'exact-id'}-preview`]; + const codexVersioned = (floor.codex ?? []).map(({ family }) => `gpt-6.1-${family}`).filter((id) => !floor.allowedIds?.includes(id)); + const codexBare = (floor.codex ?? []).map(({ family }) => family); + const ids = [...codexOnly, ...codexVersioned, ...codexBare, ...outside.map((family) => versionId(family, '5.5')), ...floor.claude.map(({ family }) => family), `${floor.allowedIds?.[0] ?? 'exact-id'}-preview`]; for (const id of ids) await expect(check(id), id).rejects.toMatchObject({ code: 1 }); }); }); From 22c4fa26190ce84c9da064b6396d25c6ca494032 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:27:22 +0100 Subject: [PATCH 7/9] test: check the linker and import handlers sit behind the trusted-sender wrapper --- test/lantern-permissions-triggers.test.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/lantern-permissions-triggers.test.ts b/test/lantern-permissions-triggers.test.ts index d592d79..7d94b6c 100644 --- a/test/lantern-permissions-triggers.test.ts +++ b/test/lantern-permissions-triggers.test.ts @@ -108,21 +108,34 @@ describe('changing the skill linker syncs the delegate rules', () => { describe('through the Skills window handlers', () => { const SENDER = { id: 'workspace' }; + const trustedOnly = new WeakSet(); function ipcHandlers(homeDir: string, pickList: () => Promise, onSync: (project: unknown) => void) { const handlers = new Map Promise>(); registerExtensionsIpc({ handle: (channel, handler) => handlers.set(channel, handler as never), - fromWorkspace: (fn) => fn, + fromWorkspace: (fn) => { + const wrapped = (...args: Parameters) => fn(...args); + trustedOnly.add(wrapped); + return wrapped; + }, installedFor: () => [], reinstallFor: () => null, syncPermissionsFor: async (target) => onSync(target), pickList, home: homeDir, }); - return (channel: string, ...args: unknown[]) => handlers.get(channel)!({ sender: SENDER }, ...args); + return Object.assign((channel: string, ...args: unknown[]) => handlers.get(channel)!({ sender: SENDER }, ...args), { + isTrustedOnly: (channel: string) => trustedOnly.has(handlers.get(channel) as never), + }); } + it('registers the linker and import handlers behind the trusted-sender wrapper', () => { + const invoke = ipcHandlers(makeHome(path.join(tmp, 'home')), async () => null, () => {}); + expect(invoke.isTrustedOnly('extensions:skillLinker')).toBe(true); + expect(invoke.isTrustedOnly('extensions:import')).toBe(true); + }); + it('syncs the rules of the open project after the linker is saved', async () => { const homeDir = makeHome(path.join(tmp, 'home')); const linker = makeLinker(path.join(tmp, 'linker-w')); From 85663c052422fad59201e8dd61813c34835b5c79 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:35:29 +0100 Subject: [PATCH 8/9] chore: update the Lantern map --- .lantern/map/test.lantern | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.lantern/map/test.lantern b/.lantern/map/test.lantern index 85ff51b..50b1dd3 100644 --- a/.lantern/map/test.lantern +++ b/.lantern/map/test.lantern @@ -878,7 +878,7 @@ test/lantern-macro-layout.test.ts . test 3fea77a5c0a1 = test/fixtures/lantern-macro-own-view.json = test/helpers/lantern-macro-fixture.ts#macroFixtureView -test/lantern-macro-load.test.ts . test 8909cb92a510 +test/lantern-macro-load.test.ts . test 0deea99dce39 #feature macro-view = src/main/lantern-macro-load.ts#readRegisteredMacro = src/main/lantern-macro-systems.ts#macroSystemsByPath @@ -1026,7 +1026,7 @@ test/lantern-panel.test.ts . test 2e3ec75e2ee0 > src/renderer/lib/providers.ts = test/helpers/lantern-map.ts -test/lantern-permissions-triggers.test.ts . test 98f18d350a91 +test/lantern-permissions-triggers.test.ts . test 74de70063f74 #feature skill-installer = src/main/bundle.ts = src/main/extensions.ts#saveSkillLinker @@ -1611,7 +1611,7 @@ test/lantern-sync-record-mirror.test.ts . test 6ade1aeb2746 = test/helpers/git-repo.ts = test/helpers/lantern-plan-world.ts -test/lantern-sync-selftest.test.ts . test 7debfe67ebf5 +test/lantern-sync-selftest.test.ts . test 6f4ec74e1322 #feature map-sync = skills/lantern-setup/references/model-floor.json = skills/lantern-setup/scripts/model.mjs @@ -1988,7 +1988,7 @@ test/project-release.test.ts . test fcaf35dd232c test/project-ident.test.ts . test c98778ad4b70 #feature projects-hub = src/main/project-ident/index.ts#projectIdent -test/project-window.test.ts . test d030bbd7379a +test/project-window.test.ts . test 85f60c1b2ebb #feature projects-hub = src/renderer/commands/index.ts < src/shared/commands/index.ts#COMMANDS From 5b4b81e27b86347f8b25927dc1480266cc99e311 Mon Sep 17 00:00:00 2001 From: Marcin Sadowski Date: Wed, 11 Nov 2026 19:35:34 +0100 Subject: [PATCH 9/9] chore: record pull request #33 in the Lantern map --- .lantern/prs/33.pr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.lantern/prs/33.pr b/.lantern/prs/33.pr index ef4aa72..cf3c12c 100644 --- a/.lantern/prs/33.pr +++ b/.lantern/prs/33.pr @@ -5,7 +5,7 @@ lantern-pr 1 #branch chore/robust-brittle-tests #base main #opened 2026-11-11 -#head 8edbd6c41d2df08638ade7770a5fa63afe4188c3 +#head 22c4fa26190ce84c9da064b6396d25c6ca494032 #describe what Six brittle tests now derive expectations from the command registry, the skill bundle, model-floor.json and the real handlers. #describe before Hard-coded 20-command and 16-skill lists, counted regex matches in main.ts. #describe after Lists come from the source of truth; wiring is checked by where a call sits.