diff --git a/README.md b/README.md index beb35ea..b41ab15 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,8 @@ A new language is **one grammar file** on the unchanged engine: 2. **Prove it as a parser** against the language's own official test suite, measured **bidirectionally** (accept what the reference accepts, reject what it rejects). 3. **Drop in the official TextMate grammar** as the baseline, so highlighter coverage is measured against what you're replacing, not asserted. +One rule the engine enforces at `defineGrammar` time: a rule other than the entry may not be able to match the **empty string**. A rule never succeeds with an empty match (an empty alternative cannot win longest-match), so a nullable rule's empty case is unreachable and a reference to it fails silently wherever it would have matched nothing. Write the repetition inline at the use site (`[many(NL), Stmt]`, not `[Filler, Stmt]` with `Filler = rule(() => [[many(NL)]])`), or make the rule non-empty and wrap references in `opt(...)`. A grammar that reaches emptiness through alternatives on purpose (YAML: `key:` with no node) sets `allowNullableRules: true` to keep the declarations; it changes no parse. + The lexer, CST types, and all three highlighters fall out of step 1; a *dialect* (`.tsx`/`.jsx`, or a markup dialect on [`html.ts`](html.ts)) reuses a base grammar's rules by name in a few lines. The conformance/highlighter harnesses are currently TypeScript-specific (they call `tsc` and read VS Code's grammar) — point them at your own reference compiler. ## Known differences from the official highlighter diff --git a/src/api.ts b/src/api.ts index 99d693f..223a791 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,4 +1,5 @@ import type { LedPrec, CstGrammar, TokenDecl, PrecLevel, PrecOperator, RuleDecl, RuleExpr, MarkupConfig, IndentConfig, NewlineConfig, StringInterpolation, TokenPattern } from './types.ts'; +import { computeNullableRules, findEntryRule } from './grammar-analysis.ts'; import { altPattern, anyChar, followedBy, never, noneOf, notFollowedBy, notPrecededBy, oneOf, optPattern, plus, precededBy, range, repeat, @@ -528,6 +529,12 @@ interface GrammarConfig { aliasScopes?: { scope: string; file: string }[]; // extra grammars re-exposing this one under another scopeName (e.g. text.html.derivative) canonicalRepoNames?: Record; // official repo KEY NAME → structural key(s) for the SAME construct; gen-tm RENAMES the structural key (or synthesises a union wrapper) to emit the official name natively (the 限制器; see CstGrammar.canonicalRepoNames) manifest?: import('./types.ts').ContributesManifest; // VS Code `contributes` packaging (emits a pasteable snippet) + // A rule never succeeds with an EMPTY match (the longest-match loop keeps an alternative only + // when it advanced), so a nullable non-entry rule's empty case is unreachable and a reference + // to it FAILS wherever it would have matched nothing. `defineGrammar` rejects such rules; a + // grammar that deliberately reaches emptiness through ALTERNATIVES instead (YAML: `key:` with + // no node, `{a: }`) sets this to keep the nullable declarations. It changes no parse. + allowNullableRules?: boolean; } export function defineGrammar(config: GrammarConfig): CstGrammar & { name: string; scopeName?: string } { @@ -627,5 +634,25 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin }); return { token: tokenName, within, scope: entry.scope }; }); - return { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, contextualScopes, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest }; + const grammar = { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, contextualScopes, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest }; + + // Nullable NON-ENTRY rules are a definition error (see GrammarConfig.allowNullableRules): the + // engine never returns an empty match for a rule, so such a rule's empty case is unreachable + // and references to it fail silently where the author expected an empty match. + if (!config.allowNullableRules) { + const entryName = config.entry ? names.get(config.entry) : findEntryRule(grammar); // `entry` may be omitted: the last rule is the entry + const nullable = [...computeNullableRules(grammar).nullableRules].filter((n) => n !== entryName); + if (nullable.length > 0) { + const list = nullable.map((n) => `'${n}'`).join(', '); + throw new Error( + `Rule${nullable.length > 1 ? 's' : ''} ${list} can match the empty string, but a rule never succeeds with an ` + + `empty match (an empty alternative cannot win longest-match), so the empty case is unreachable: a ` + + `reference to it fails wherever it would have matched nothing. Inline the combinator at the use site ` + + `(e.g. \`many(X)\` in the sequence instead of a \`rule(() => [[many(X)]])\`), or make the rule non-empty ` + + `and wrap references in \`opt(...)\` where emptiness is intended. If emptiness is deliberately handled by ` + + `alternatives, set \`allowNullableRules: true\` on the grammar.`, + ); + } + } + return grammar; } diff --git a/src/grammar-analysis.ts b/src/grammar-analysis.ts index b1c9933..304d592 100644 --- a/src/grammar-analysis.ts +++ b/src/grammar-analysis.ts @@ -45,6 +45,42 @@ export function findEntryRule(grammar: CstGrammar): string { * Derive the full STRUCTURAL analysis, returned as plain data + live closures. Both engines * call this once and destructure; their downstream code keeps its own local names. */ +/** + * The rules that can derive the EMPTY string (fixpoint over the rule bodies), plus the expression + * predicate behind it. Single-sourced here because two consumers need the same answer: the + * analysis below (left-corner edges, alt dispatch) and `defineGrammar`, which REJECTS a nullable + * non-entry rule at definition time. The engine never lets a rule succeed with an empty match + * (`parseNonRec` keeps an alternative only when it advanced, `pos > bestPos`), so a nullable + * rule's empty case is unreachable: a reference to it FAILS wherever it would have matched + * nothing, which silently drops whole parses (`[Filler, Stmt]` with a `Filler = rule(() => + * [[many(NL)]])` rejects every `Stmt`). The entry rule is exempt (an empty document is handled + * by the driver). See `allowNullableRules` on the grammar config for the deliberate case. + */ +export function computeNullableRules(grammar: CstGrammar): { nullableRules: Set; exprNullable: (e: RuleExpr) => boolean } { + const tokenNames = new Set(grammar.tokens.map(t => t.name)); + const nullableRules = new Set(); + function exprNullable(e: RuleExpr): boolean { + switch (e.type) { + case 'literal': return false; + case 'ref': return tokenNames.has(e.name) ? false : nullableRules.has(e.name); + case 'seq': return e.items.every(exprNullable); + case 'alt': return e.items.some(exprNullable); + case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true; + case 'group': return exprNullable(e.body); + case 'not': return true; // zero-width assertion: consumes nothing + case 'sep': return true; // sep matches zero elements + default: return true; // op/prefix/postfix markers don't consume + } + } + for (let changed = true; changed; ) { + changed = false; + for (const rule of grammar.rules) { + if (!nullableRules.has(rule.name) && exprNullable(rule.body)) { nullableRules.add(rule.name); changed = true; } + } + } + return { nullableRules, exprNullable }; +} + export function analyzeGrammar(grammar: CstGrammar) { const tokenNames = new Set(grammar.tokens.map(t => t.name)); @@ -164,26 +200,7 @@ export function analyzeGrammar(grammar: CstGrammar) { // // Nullability feeds the left-corner edges (a nullable leftmost element passes through to the // next), so compute it first. op/prefix/postfix consume an operator token → left-edge BARRIERS. - const nullableRules = new Set(); - function exprNullable(e: RuleExpr): boolean { - switch (e.type) { - case 'literal': return false; - case 'ref': return tokenNames.has(e.name) ? false : nullableRules.has(e.name); - case 'seq': return e.items.every(exprNullable); - case 'alt': return e.items.some(exprNullable); - case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true; - case 'group': return exprNullable(e.body); - case 'not': return true; // zero-width assertion: consumes nothing - case 'sep': return true; // sep matches zero elements - default: return true; // op/prefix/postfix markers don't consume - } - } - for (let changed = true; changed; ) { - changed = false; - for (const rule of grammar.rules) { - if (!nullableRules.has(rule.name) && exprNullable(rule.body)) { nullableRules.add(rule.name); changed = true; } - } - } + const { nullableRules, exprNullable } = computeNullableRules(grammar); // The set of rules reachable at the LEFT CORNER of an expression: every rule ref that could be // the leftmost symbol, looking through nullable prefixes and stopping at the first non-nullable diff --git a/test/check.ts b/test/check.ts index 6dca4b6..f0be247 100644 --- a/test/check.ts +++ b/test/check.ts @@ -18,6 +18,7 @@ interface Gate { group: string; name: string; args: string[] } const GATES: Gate[] = [ { group: 'core', name: 'agnostic', args: ['test/agnostic.ts'] }, { group: 'core', name: 'left-recursion', args: ['test/left-recursion.ts'] }, + { group: 'core', name: 'nullable-rules', args: ['test/nullable-rules.ts'] }, { group: 'core', name: 'newline-mode', args: ['test/newline-mode.ts'] }, { group: 'core', name: 'interpolation-metadata', args: ['test/interpolation-metadata.ts'] }, { group: 'core', name: 'refactor-guard', args: ['test/refactor-guard.ts'] }, diff --git a/test/nullable-rules.ts b/test/nullable-rules.ts new file mode 100644 index 0000000..57f9344 --- /dev/null +++ b/test/nullable-rules.ts @@ -0,0 +1,100 @@ +// Gate: nullable NON-ENTRY rules are rejected at definition time. +// +// The engine never returns an EMPTY match for a rule: the longest-match loop keeps an +// alternative only when it advanced (`pos > bestPos`), so a rule that can derive the empty +// string has an unreachable empty case, and a reference to it FAILS wherever it would have +// matched nothing. Written as a "filler" rule that is exactly the silent failure mode: +// `Stmt = [Filler, Ident]` with `Filler = rule(() => [[many(NL)]])` rejects a plain `a`. +// `defineGrammar` now names the rule and the fix; `allowNullableRules: true` keeps the +// declaration for a grammar that reaches emptiness through alternatives (YAML). +// +// Run with: node test/nullable-rules.ts +import { writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createParser } from '../src/gen-parser.ts'; +import { computeNullableRules } from '../src/grammar-analysis.ts'; +import { emitParser, jsTarget } from '../src/emit.ts'; +import { token, rule, defineGrammar, many, opt, plus, oneOf, range } from '../src/api.ts'; + +let ok = 0, fail = 0; +const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log(' ✗', label); } }; + +const WS = token(plus(oneOf(' ', '\t')), { skip: true }); +const IDENT = token(plus(oneOf(range('a', 'z'))), { identifier: true }); +const NL = token(';', {}); // stands in for a line terminator (a real `\n` is skipped as whitespace here) + +function filler(allow: boolean) { + const Filler = rule(() => [[many(NL)]]); // NULLABLE: zero or more terminators + const Stmt = rule(() => [[Filler, IDENT, opt(',', IDENT)]]); + const Program = rule(() => [[many(Stmt)]]); // the entry may be nullable + return defineGrammar({ name: 'filler', tokens: { WS, IDENT, NL }, rules: { Filler, Stmt, Program }, entry: Program, ...(allow ? { allowNullableRules: true } : {}) }); +} +function inlined() { + const Stmt = rule(() => [[many(NL), IDENT, opt(',', IDENT)]]); // the same language, filler inlined + const Program = rule(() => [[many(Stmt)]]); + return defineGrammar({ name: 'inlined', tokens: { WS, IDENT, NL }, rules: { Stmt, Program }, entry: Program }); +} + +// ── 1. defineGrammar rejects the nullable non-entry rule, by name, with the fix ── +let msg = ''; +try { filler(false); } catch (e) { msg = (e as Error).message; } +check('a nullable non-entry rule is rejected', msg !== ''); +check('the message names the rule', msg.includes("'Filler'")); +check('the message explains the unreachable empty match', /empty match/.test(msg) && /unreachable/.test(msg)); +check('the message points at the inline fix', msg.includes('many(X)')); +check('the message points at the opt-out', msg.includes('allowNullableRules')); + +// ── 2. A nullable ENTRY rule is fine (an empty document is handled by the driver) ── +let entryOk = true; +try { inlined(); } catch { entryOk = false; } +check('a nullable entry rule (Program = many(Stmt)) is accepted', entryOk); +const gi = inlined(); +check('the inlined grammar reports no nullable non-entry rule', [...computeNullableRules(gi).nullableRules].filter(n => n !== 'Program').length === 0); +check('the inlined form parses what the filler form silently rejected', (() => { try { return createParser(gi).parse(';;a,b;c').rule === 'Program'; } catch { return false; } })()); + +// `entry` may be omitted (the LAST rule is then the entry, as findEntryRule resolves it): the +// exemption must follow that rule, and only that rule. +{ + let implicitOk = true; + try { + const Stmt = rule(() => [[IDENT]]); const Program = rule(() => [[many(Stmt)]]); + defineGrammar({ name: 'implicit', tokens: { WS, IDENT, NL }, rules: { Stmt, Program } } as any); + } catch { implicitOk = false; } + check('with `entry` omitted, a nullable LAST rule is the implicit entry and is accepted', implicitOk); + let implicitBad = false; + try { + const Filler = rule(() => [[many(NL)]]); const Program = rule(() => [[Filler, IDENT]]); + defineGrammar({ name: 'implicit2', tokens: { WS, IDENT, NL }, rules: { Filler, Program } } as any); + } catch { implicitBad = true; } + check('with `entry` omitted, a nullable non-last rule is still rejected', implicitBad); +} + +// ── 3. The opt-out keeps the declaration and changes no parse (the empty case stays unreachable) ── +const gf = filler(true); +check('allowNullableRules keeps the grammar buildable', gf.rules.some(r => r.name === 'Filler')); +check('computeNullableRules still reports it', computeNullableRules(gf).nullableRules.has('Filler')); +const interp = createParser(gf); +const rejects = (s: string) => { try { interp.parse(s); return false; } catch { return true; } }; +check('documented behaviour: the filler grammar rejects a plain statement (empty Filler never matches)', rejects('a')); +check('documented behaviour: the same grammar ACCEPTS `;a` (Filler matched non-empty), which is the inconsistency the check exists to surface', !rejects(';a')); +{ + const file = join(tmpdir(), `monogram-nullable-${process.pid}.ts`); + writeFileSync(file, emitParser(gf, jsTarget)); + const p = (await import(file + '?v=' + Date.now())).createParser(); + check('the emitted engine agrees (total parse reports the statement as unexpected)', p.parse('a').errors.length > 0); +} + +// ── 4. The shipped grammars: YAML opts out explicitly; the rest have no nullable non-entry rule ── +for (const [name, entry] of [['typescript', 'Program'], ['javascript', 'Program'], ['html', 'Document'], ['yaml', 'Stream']] as const) { + let g: any; let loaded = true; + try { g = (await import(`../${name}.ts`)).default; } catch { loaded = false; } + check(`${name}.ts still defines`, loaded); + if (!loaded) continue; + const nonEntry = [...computeNullableRules(g).nullableRules].filter(n => n !== entry); + if (name === 'yaml') check('yaml declares its nullable rules deliberately (Node, FlowNode, …)', nonEntry.includes('Node') && nonEntry.includes('FlowNode')); + else check(`${name} has no nullable non-entry rule`, nonEntry.length === 0); +} + +console.log(`\n${ok}/${ok + fail} nullable-rule checks pass${fail ? '' : ' ✓'}`); +if (fail) process.exit(1); diff --git a/test/tm-completeness.ts b/test/tm-completeness.ts index 8f6e56f..c8d1439 100644 --- a/test/tm-completeness.ts +++ b/test/tm-completeness.ts @@ -123,6 +123,9 @@ function checkRuleExprClosure(): void { g = defineGrammar({ name: 'closure', tokens: { A, B }, rules: { Leaf, Refs, Quant, Alt, Sep, Group, Nots, Markers, Pratt, Entry }, entry: Entry, + // Sep and Nots can derive the empty string by construction (this grammar exists to reach every + // combinator, not to parse); the nullable-rule definition check is not the subject of this lemma. + allowNullableRules: true, }); } catch (e) { threw = true; g = null as any; } check('Lemma A1: toRuleExpr is total (no throw lowering every combinator)', !threw, threw ? 'defineGrammar threw' : ''); diff --git a/yaml.ts b/yaml.ts index 84fa6c8..2a59031 100644 --- a/yaml.ts +++ b/yaml.ts @@ -672,5 +672,10 @@ export default defineGrammar({ FlowNode, FlowExplicit, FlowMapEntry, FlowMapping, FlowSeqEntry, FlowSeqKey, FlowSequence, Scalar, BlockKeyScalar, DocFold, InlineDocNode, ExplicitDocBody, AfterDocEnd, NextDoc, Stream, }, entry: Stream, + // Node / FlowNode / FlowMapEntry / FlowSeqEntry / AfterDocEnd can derive the empty string. The + // engine never returns an empty match for a rule, so their empty cases are unreachable and YAML + // reaches emptiness through ALTERNATIVES instead (`key:` with no node, `{a: }`, an empty doc). + // Declaring that is what keeps the definition honest without rewriting five rules. + allowNullableRules: true, indent, });