From 95dd7f36dcbb3162d3b1f2ba85b2d841fe2179d8 Mon Sep 17 00:00:00 2001 From: Theo Ephraim Date: Mon, 7 Sep 2026 19:04:17 -0700 Subject: [PATCH] feat(diagnostics): token and rule labels for expected-X messages The emitted engine's $missing diagnostics name what was missing by its grammar name: a token NUM reads as "expected 'NUM'", a rule Value as "expected Value". For a language whose grammar names are internal identifiers (DEC_VALUE_TEXT) that text leaks straight to editor users. token(pattern, { label }) and rule(fn, { label }) substitute a display string in exactly those messages and nowhere else: leaf tokenTypes, ruleNameOf, the CST, and every derived artifact keep the grammar name, so a grammar that adds labels parses byte-identically. A labelled token renders bare (expected a number); an unlabelled one keeps the quoted name, so existing output is unchanged. Gate: test/diagnostic-labels.ts (tiny grammar plus TypeScript's Expr rule relabelled, tree identity across labels). --- README.md | 2 + src/api.ts | 10 ++++ src/emit-parser.ts | 14 ++++- src/types.ts | 3 + test/check.ts | 1 + test/diagnostic-labels.ts | 117 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 test/diagnostic-labels.ts diff --git a/README.md b/README.md index beb35ea..a6d2412 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,8 @@ export default defineGrammar({ }); ``` +**Diagnostic labels.** The total parser's `expected …` messages name the missing token or rule by its grammar name (`expected 'Number'`, `expected Expr`). When a grammar name is an internal identifier rather than a word an end user should read, give it a display label: `token(pattern, { label: 'a number' })` / `rule(fn, { label: 'an expression' })` render `expected a number` / `expected an expression`. Labels change message text only: leaf `tokenType`s, `ruleNameOf`, the CST, and every derived artifact keep the grammar name, so adding labels parses byte-identically (`test/diagnostic-labels.ts`). + Token patterns are **combinators, not regular expressions** — `seq` / `oneOf` / `range` / `noneOf` / `plus` / `star` / `altPattern` / `optPattern` / … assemble a structured pattern IR (regex is a *derived* backend, not the source of truth). A bare `RegExp` is not a valid token pattern: `token(/…/)` is a `TS2345` type error. Coming from regex: | RegExp | Combinator | diff --git a/src/api.ts b/src/api.ts index 99d693f..b822e30 100644 --- a/src/api.ts +++ b/src/api.ts @@ -14,6 +14,10 @@ export { interface TokenOptions { skip?: boolean; + // Display name for `expected …` diagnostics (the emitted engine's $missing rows): a token + // `NUM` reads as `expected 'NUM'` by default; `label: 'a number'` renders `expected a number`. + // Messages ONLY: leaf tokenTypes, scopes, and every derived artifact keep the grammar name. + label?: string; scope?: string; escape?: TokenPattern; // Highlight-only interpolation regions for ordinary string tokens (e.g. env-spec `${…}` / `$(…)`). @@ -77,6 +81,10 @@ export function token(pattern: TokenPattern, opts?: TokenOptions): TokenRef { interface RuleOptions { type?: boolean; + // Display name for `expected …` diagnostics (a missing required rule): `expected Value` by + // default, `expected a value` with `label: 'a value'`. Messages only; `ruleNameOf`, the CST, + // and every derived artifact keep the rule name. + label?: string; } type Element = string | TokenRef | RuleRef | Marker | Combinator; @@ -555,6 +563,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin blockOnly: tok.opts.blockOnly, flags, scope: tok.opts.scope, + label: tok.opts.label, escapePattern: tok.opts.escape, interpolation: tok.opts.interpolation ? (Array.isArray(tok.opts.interpolation) ? tok.opts.interpolation : [tok.opts.interpolation]).map((i) => ({ ...i })) @@ -599,6 +608,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin name, body: convertAlternatives(alts, names), flags: r.opts.type ? ['type'] : [], + label: r.opts.label, }; }); diff --git a/src/emit-parser.ts b/src/emit-parser.ts index 74feb92..e82b64c 100644 --- a/src/emit-parser.ts +++ b/src/emit-parser.ts @@ -1164,6 +1164,8 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string // Every token is BORN with tok.k (type kind) + tok.t (literal kind) and the stamp // flags — one monomorphic shape, one allocation, no post-pass. e.emit(`const TYPE_KIND = new Map(${J([...st.typeKind])});`); + // Token diagnostic labels (TokenDecl.label): name → display string, for "expected …" only. + e.emit(`const TOKEN_LABELS = new Map(${J(grammar.tokens.filter(t => t.label !== undefined).map(t => [t.name, t.label!]))});`); e.emit(`const LIT_KW = new Map(${J([...st.kwLitKind])});`); e.emit(`const LIT_PU = new Map(${J([...st.puLitKind])});`); e.emit(`const K_PUNCT = ${st.KIND_PUNCT};`); @@ -1288,6 +1290,10 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string // node's rule name so trees stay byte-identical to the base grammar. Identical to // RULE_NAMES when no rule is forked (the common case). e.emit(`const RULE_DISPLAY = ${J([...grammar.rules.map(r => r.canon ?? r.name), '$template', '$error', '$missing'])};`); + // Diagnostic LABELS: what a `$missing` row's "expected …" names for a missing required RULE. + // `RuleDecl.label` substitutes a display string there and nowhere else (RULE_DISPLAY stays the + // node's reported rule name), so a grammar that adds labels parses byte-identically. + e.emit(`const RULE_LABELS = ${J([...grammar.rules.map(r => r.label ?? r.canon ?? r.name), '$template', '$error', '$missing'])};`); e.emit(`const RID_TEMPLATE = ${grammar.rules.length};`); e.emit(`const RID_ERROR = ${grammar.rules.length + 1};`); e.emit(`const RID_MISSING = ${grammar.rules.length + 2};`); @@ -2530,6 +2536,10 @@ function tokTextAt(i: number) { // The k → type-name inverse, for reconstructing a token object (tokenAt). const K_NAMES: string[] = []; for (const [n, k] of TYPE_KIND) K_NAMES[k] = n; +// The k → diagnostic-label inverse: a labelled token renders bare (expected a number), an +// unlabelled one keeps the quoted grammar name (expected 'NUM'). +const K_LABELS: string[] = []; +for (const [n, k] of TYPE_KIND) K_LABELS[k] = TOKEN_LABELS.get(n) ?? "'" + n + "'"; // A per-token object view over the columns (gates / debugging — the parser never builds these). export function tokenAt(i: number) { return { @@ -2930,9 +2940,9 @@ function missLit(v: number) { function missEntry(v: number, kb: number): Diag { let message; if (v >= 1 << 21) message = 'expected ' + VSETS[v >>> 21]; - else if (v >= RULE_MISS_BASE) message = 'expected ' + RULE_DISPLAY[v - RULE_MISS_BASE]; + else if (v >= RULE_MISS_BASE) message = 'expected ' + RULE_LABELS[v - RULE_MISS_BASE]; else if (v > 0) message = "expected '" + LIT_NAMES[v] + "'"; - else message = "expected '" + (K_NAMES[-v] ?? '?') + "'"; + else message = 'expected ' + (K_LABELS[-v] ?? "'?'"); return { offset: kb, end: kb, message }; } function collectErrRows(id: number, charBase: number, tokBase: number) { diff --git a/src/types.ts b/src/types.ts index bf2817b..0d2bed1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,6 +21,7 @@ export interface TokenDecl { pattern: TokenPattern; flags: string[]; scope?: string; // @scope(...) override + label?: string; // display name for `expected …` diagnostics only (see api.ts TokenOptions.label) escapePattern?: TokenPattern; // @escape pattern — escape sequence pattern (highlight only) interpolation?: StringInterpolation[]; // highlight-only interpolation regions inside a string token (e.g. `${…}` / `$(…)`) // Highlight-only: this comment-scoped token matches only the INTRODUCER (e.g. a bare `#`) @@ -553,6 +554,8 @@ export interface RuleDecl { // parser keeps the distinct `name` for its memo/adoption rule identity, but reports // `canon` as the node's rule name so trees stay byte-identical to the base grammar. canon?: string; + // Display name for `expected …` diagnostics only (see api.ts RuleOptions.label). + label?: string; } export interface CstGrammar { diff --git a/test/check.ts b/test/check.ts index 6dca4b6..6aa91f9 100644 --- a/test/check.ts +++ b/test/check.ts @@ -19,6 +19,7 @@ const GATES: Gate[] = [ { group: 'core', name: 'agnostic', args: ['test/agnostic.ts'] }, { group: 'core', name: 'left-recursion', args: ['test/left-recursion.ts'] }, { group: 'core', name: 'newline-mode', args: ['test/newline-mode.ts'] }, + { group: 'core', name: 'diagnostic-labels', args: ['test/diagnostic-labels.ts'] }, { group: 'core', name: 'interpolation-metadata', args: ['test/interpolation-metadata.ts'] }, { group: 'core', name: 'refactor-guard', args: ['test/refactor-guard.ts'] }, { group: 'core', name: 'cst-text-invariant', args: ['test/cst-text-invariant.ts'] }, diff --git a/test/diagnostic-labels.ts b/test/diagnostic-labels.ts new file mode 100644 index 0000000..38c6b6b --- /dev/null +++ b/test/diagnostic-labels.ts @@ -0,0 +1,117 @@ +// Gate: human-readable LABELS for `$missing` diagnostics. +// +// The emitted engine's "expected X" messages name what a required position was missing. By +// default X is the raw grammar name (a token `NUM` reads as `expected 'NUM'`, a rule `Value` +// as `expected Value`), which is fine for a language whose grammar names are already words but +// leaks internal identifiers (`DEC_VALUE_TEXT`) to end users of any real editor. `token(p, +// { label })` and `rule(fn, { label })` substitute a display string in exactly those messages +// and NOWHERE else: leaf `tokenType`s, `ruleNameOf`, the CST, and every other artifact keep +// the grammar name, so a grammar that adds labels parses byte-identically. +// +// Run with: node test/diagnostic-labels.ts +import { writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { emitParser, jsTarget } from '../src/emit.ts'; +import { createParser } from '../src/gen-parser.ts'; +import { objectify } from './emitted-obj.ts'; +import { token, rule, defineGrammar, many, opt, plus, oneOf, range } from '../src/api.ts'; +import { generateTmLanguage } from '../src/gen-tm.ts'; +import { generateTreeSitter } from '../src/gen-treesitter.ts'; +import { generateLanguageConfig } from '../src/gen-vscode-config.ts'; + +let ok = 0, fail = 0; +const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log(' ✗', label); } }; + +type Diag = { offset: number; end: number; message: string }; + +function build(labelled: boolean) { + const WS = token(plus(oneOf(' ', '\t')), { skip: true }); + const IDENT = token(plus(oneOf(range('a', 'z'))), { identifier: true }); + const NUM = token(plus(oneOf(range('0', '9'))), labelled ? { label: 'a number' } : {}); + const SEMI = token(';', {}); + const Value = rule(() => [[NUM], [IDENT, '(', opt(NUM), ')']], labelled ? { label: 'a value' } : {}); + // `opt('=', Value)`: once the optional group has consumed `=` it is committed, so a missing + // Value synthesizes a $missing row (the tsc-style rule the engine derives; see TOTAL-PARSING.md). + const Stmt = rule(() => [[IDENT, opt('=', Value), SEMI]]); + const Program = rule(() => [[many(Stmt)]]); + return defineGrammar({ name: 'labels', tokens: { WS, IDENT, NUM, SEMI }, rules: { Value, Stmt, Program }, entry: Program }); +} + +const plain = build(false); +const labelled = build(true); + +// ── 1. defineGrammar carries the labels onto the declarations (and only when given) ── +check('token label lands on TokenDecl', labelled.tokens.find(t => t.name === 'NUM')?.label === 'a number'); +check('rule label lands on RuleDecl', labelled.rules.find(r => r.name === 'Value')?.label === 'a value'); +check('an unlabelled token has no label', plain.tokens.find(t => t.name === 'NUM')?.label === undefined); +check('an unlabelled rule has no label', plain.rules.find(r => r.name === 'Value')?.label === undefined); + +// ── 2. The emitted engine renders labels in `expected …` messages ── +const dir = tmpdir(); +async function load(g: ReturnType, tag: string) { + const file = join(dir, `monogram-labels-${tag}-${process.pid}.ts`); + writeFileSync(file, emitParser(g, jsTarget)); + const em = await import(file + '?v=' + Date.now()); + return em.createParser() as { parse(s: string): { root: number; errors: Diag[] }; visit(c: unknown, fns: object): void; tree: any }; +} +const pp = await load(plain, 'plain'); +const pl = await load(labelled, 'labelled'); +const msgs = (p: typeof pp, src: string) => p.parse(src).errors.map(e => e.message); + +// A required TOKEN missing: `a = 1` (no `;`) and `a = f(` (the `)` is a literal, unaffected). +check("plain: missing named token → expected 'SEMI'", msgs(pp, 'a = 1').includes("expected 'SEMI'")); +check("labelled: unlabelled token keeps the quoted grammar name", msgs(pl, 'a = 1').includes("expected 'SEMI'")); + +// A required RULE missing is exercised on the TypeScript grammar below (2b): whether a tiny +// grammar synthesizes the rule or absorbs the statement is the recovery engine's call, not +// this gate's subject. + +// A labelled TOKEN missing: `a = f(1` is a literal `)`; use `a = f(` + `;`? The optional NUM never +// synthesizes, so exercise the token label through a grammar position where NUM is required: +{ + const WS = token(plus(oneOf(' ', '\t')), { skip: true }); + const NUM = token(plus(oneOf(range('0', '9'))), { label: 'a number' }); + const Pair = rule(() => [[NUM, ',', NUM]]); + const Top = rule(() => [[many(Pair)]]); + const g = defineGrammar({ name: 'labels2', tokens: { WS, NUM }, rules: { Pair, Top }, entry: Top }); + const p = await load(g, 'pair'); + check('labelled: missing token → expected a number (unquoted label)', msgs(p, '1,').includes('expected a number')); + check("labelled: the quoted raw token name is gone", !msgs(p, '1,').includes("expected 'NUM'")); +} + +const tree = (p: typeof pp, src: string) => { const c = p.parse(src); return JSON.stringify(objectify(p.tree, (fns: any) => p.visit(c, fns))); }; + +// ── 2b. A real grammar: label TypeScript's Expr rule and read `const a = ;` ── +{ + const ts = (await import('../typescript.ts')).default; + const labelledTs = { ...ts, rules: ts.rules.map((r: any) => r.name === 'Expr' ? { ...r, label: 'an expression' } : r) }; + const p0 = await load(ts as any, 'ts-plain'); + const p1 = await load(labelledTs as any, 'ts-labelled'); + check('typescript: default message is expected Expr', msgs(p0, 'const a = ;').includes('expected Expr')); + check('typescript: labelled message is expected an expression', msgs(p1, 'const a = ;').includes('expected an expression')); + check('typescript: labelled tree is byte-identical', tree(p0, 'const a = ;\nfoo(1, [2, 3]);') === tree(p1, 'const a = ;\nfoo(1, [2, 3]);')); +} + +// ── 3. Labels change messages ONLY: trees, leaf token types, and literal messages are identical ── +for (const src of ['a = 1;', 'a = f(2);', 'a = ;', 'a = f(', 'a = 1']) { + check(`byte-identical tree for ${JSON.stringify(src)}`, tree(pp, src) === tree(pl, src)); +} +check("literal messages unchanged: expected ')'", msgs(pl, 'a = f(').includes("expected ')'")); +check('related info unchanged', JSON.stringify(pl.parse('a = f(').errors).includes("to match this '('")); +check('valid input has no errors under labels', pl.parse('a = 1; b = f(2);').errors.length === 0); + +// ── 3b. Every derived artifact is unaffected: labels are not scopes, captures, or names ── +check('TextMate grammar identical with and without labels', JSON.stringify(generateTmLanguage(plain)) === JSON.stringify(generateTmLanguage(labelled))); +check('tree-sitter output identical with and without labels', JSON.stringify(generateTreeSitter(plain, 'labels')) === JSON.stringify(generateTreeSitter(labelled, 'labels'))); +check('language-configuration identical with and without labels', JSON.stringify(generateLanguageConfig(plain)) === JSON.stringify(generateLanguageConfig(labelled))); +check('leaf tokenTypes keep the grammar name (no label leaks into the tree)', tree(pl, 'a = 1;').includes('"tokenType":"NUM"') && !tree(pl, 'a = 1;').includes('a number')); + +// ── 4. The interpreter is unaffected (it has no expected-X diagnostics to label) ── +const interp = createParser(labelled); +let threw = ''; +try { interp.parse('a = ;'); } catch (e) { threw = (e as Error).message; } +check('interpreter still rejects with its own message', threw.startsWith('Parse error at offset')); + +console.log(`\n${ok}/${ok + fail} diagnostic-label checks pass${fail ? '' : ' ✓'}`); +if (fail) process.exit(1);