feat(grammar): reject nullable non-entry rules at definition time

README.md

@@ -369,6 +369,8 @@ A new language is **one grammar file** on the unchanged engine:
 2. **Prove it as a parser** against the language's own official test suite, measured **bidirectionally** (accept what the reference accepts, reject what it rejects).
 3. **Drop in the official TextMate grammar** as the baseline, so highlighter coverage is measured against what you're replacing, not asserted.
 
+One rule the engine enforces at `defineGrammar` time: a rule other than the entry may not be able to match the **empty string**. A rule never succeeds with an empty match (an empty alternative cannot win longest-match), so a nullable rule's empty case is unreachable and a reference to it fails silently wherever it would have matched nothing. Write the repetition inline at the use site (`[many(NL), Stmt]`, not `[Filler, Stmt]` with `Filler = rule(() => [[many(NL)]])`), or make the rule non-empty and wrap references in `opt(...)`. A grammar that reaches emptiness through alternatives on purpose (YAML: `key:` with no node) sets `allowNullableRules: true` to keep the declarations; it changes no parse.
+
 The lexer, CST types, and all three highlighters fall out of step 1; a *dialect* (`.tsx`/`.jsx`, or a markup dialect on [`html.ts`](html.ts)) reuses a base grammar's rules by name in a few lines. The conformance/highlighter harnesses are currently TypeScript-specific (they call `tsc` and read VS Code's grammar) — point them at your own reference compiler.
 
 ## Known differences from the official highlighter

src/api.ts

@@ -1,4 +1,5 @@
 import type { LedPrec, CstGrammar, TokenDecl, PrecLevel, PrecOperator, RuleDecl, RuleExpr, MarkupConfig, IndentConfig, NewlineConfig, StringInterpolation, TokenPattern } from './types.ts';
+import { computeNullableRules, findEntryRule } from './grammar-analysis.ts';
 import {
   altPattern, anyChar, followedBy, never, noneOf, notFollowedBy,
   notPrecededBy, oneOf, optPattern, plus, precededBy, range, repeat,
@@ -528,6 +529,12 @@ interface GrammarConfig {
   aliasScopes?: { scope: string; file: string }[];  // extra grammars re-exposing this one under another scopeName (e.g. text.html.derivative)
   canonicalRepoNames?: Record<string, string | string[]>;  // official repo KEY NAME → structural key(s) for the SAME construct; gen-tm RENAMES the structural key (or synthesises a union wrapper) to emit the official name natively (the 限制器; see CstGrammar.canonicalRepoNames)
   manifest?: import('./types.ts').ContributesManifest;  // VS Code `contributes` packaging (emits a pasteable snippet)
+  // A rule never succeeds with an EMPTY match (the longest-match loop keeps an alternative only
+  // when it advanced), so a nullable non-entry rule's empty case is unreachable and a reference
+  // to it FAILS wherever it would have matched nothing. `defineGrammar` rejects such rules; a
+  // grammar that deliberately reaches emptiness through ALTERNATIVES instead (YAML: `key:` with
+  // no node, `{a: }`) sets this to keep the nullable declarations. It changes no parse.
+  allowNullableRules?: boolean;
 }
 
 export function defineGrammar(config: GrammarConfig): CstGrammar & { name: string; scopeName?: string } {
@@ -627,5 +634,25 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
     });
     return { token: tokenName, within, scope: entry.scope };
   });
-  return { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, contextualScopes, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest };
+  const grammar = { name: config.name, scopeName: config.scopeName, tokens, precs, ledPrecs: config.ledPrec, rules, scopeOverrides, contextualScopes, markup: config.markup, indent: config.indent, newline: config.newline, expressionRule: config.expression ? names.get(config.expression) : undefined, aliasScopes: config.aliasScopes, canonicalRepoNames: config.canonicalRepoNames, manifest: config.manifest };
+
+  // Nullable NON-ENTRY rules are a definition error (see GrammarConfig.allowNullableRules): the
+  // engine never returns an empty match for a rule, so such a rule's empty case is unreachable
+  // and references to it fail silently where the author expected an empty match.
+  if (!config.allowNullableRules) {
+    const entryName = config.entry ? names.get(config.entry) : findEntryRule(grammar);   // `entry` may be omitted: the last rule is the entry
+    const nullable = [...computeNullableRules(grammar).nullableRules].filter((n) => n !== entryName);
+    if (nullable.length > 0) {
+      const list = nullable.map((n) => `'${n}'`).join(', ');
+      throw new Error(
+        `Rule${nullable.length > 1 ? 's' : ''} ${list} can match the empty string, but a rule never succeeds with an `
+        + `empty match (an empty alternative cannot win longest-match), so the empty case is unreachable: a `
+        + `reference to it fails wherever it would have matched nothing. Inline the combinator at the use site `
+        + `(e.g. \`many(X)\` in the sequence instead of a \`rule(() => [[many(X)]])\`), or make the rule non-empty `
+        + `and wrap references in \`opt(...)\` where emptiness is intended. If emptiness is deliberately handled by `
+        + `alternatives, set \`allowNullableRules: true\` on the grammar.`,
+      );
+    }
+  }
+  return grammar;
 }

src/grammar-analysis.ts

@@ -45,6 +45,42 @@ export function findEntryRule(grammar: CstGrammar): string {
  * Derive the full STRUCTURAL analysis, returned as plain data + live closures. Both engines
  * call this once and destructure; their downstream code keeps its own local names.
  */
+/**
+ * The rules that can derive the EMPTY string (fixpoint over the rule bodies), plus the expression
+ * predicate behind it. Single-sourced here because two consumers need the same answer: the
+ * analysis below (left-corner edges, alt dispatch) and `defineGrammar`, which REJECTS a nullable
+ * non-entry rule at definition time. The engine never lets a rule succeed with an empty match
+ * (`parseNonRec` keeps an alternative only when it advanced, `pos > bestPos`), so a nullable
+ * rule's empty case is unreachable: a reference to it FAILS wherever it would have matched
+ * nothing, which silently drops whole parses (`[Filler, Stmt]` with a `Filler = rule(() =>
+ * [[many(NL)]])` rejects every `Stmt`). The entry rule is exempt (an empty document is handled
+ * by the driver). See `allowNullableRules` on the grammar config for the deliberate case.
+ */
+export function computeNullableRules(grammar: CstGrammar): { nullableRules: Set<string>; exprNullable: (e: RuleExpr) => boolean } {
+  const tokenNames = new Set(grammar.tokens.map(t => t.name));
+  const nullableRules = new Set<string>();
+  function exprNullable(e: RuleExpr): boolean {
+    switch (e.type) {
+      case 'literal': return false;
+      case 'ref': return tokenNames.has(e.name) ? false : nullableRules.has(e.name);
+      case 'seq': return e.items.every(exprNullable);
+      case 'alt': return e.items.some(exprNullable);
+      case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true;
+      case 'group': return exprNullable(e.body);
+      case 'not': return true;                                   // zero-width assertion: consumes nothing
+      case 'sep': return true;                                   // sep matches zero elements
+      default: return true;                                      // op/prefix/postfix markers don't consume
+    }
+  }
+  for (let changed = true; changed; ) {
+    changed = false;
+    for (const rule of grammar.rules) {
+      if (!nullableRules.has(rule.name) && exprNullable(rule.body)) { nullableRules.add(rule.name); changed = true; }
+    }
+  }
+  return { nullableRules, exprNullable };
+}
+
 export function analyzeGrammar(grammar: CstGrammar) {
   const tokenNames = new Set(grammar.tokens.map(t => t.name));
 
@@ -164,26 +200,7 @@ export function analyzeGrammar(grammar: CstGrammar) {
   //
   // Nullability feeds the left-corner edges (a nullable leftmost element passes through to the
   // next), so compute it first. op/prefix/postfix consume an operator token → left-edge BARRIERS.
-  const nullableRules = new Set<string>();
-  function exprNullable(e: RuleExpr): boolean {
-    switch (e.type) {
-      case 'literal': return false;
-      case 'ref': return tokenNames.has(e.name) ? false : nullableRules.has(e.name);
-      case 'seq': return e.items.every(exprNullable);
-      case 'alt': return e.items.some(exprNullable);
-      case 'quantifier': return e.kind === '+' ? exprNullable(e.body) : true;
-      case 'group': return exprNullable(e.body);
-      case 'not': return true;                                   // zero-width assertion: consumes nothing
-      case 'sep': return true;                                   // sep matches zero elements
-      default: return true;                                      // op/prefix/postfix markers don't consume
-    }
-  }
-  for (let changed = true; changed; ) {
-    changed = false;
-    for (const rule of grammar.rules) {
-      if (!nullableRules.has(rule.name) && exprNullable(rule.body)) { nullableRules.add(rule.name); changed = true; }
-    }
-  }
+  const { nullableRules, exprNullable } = computeNullableRules(grammar);
 
   // The set of rules reachable at the LEFT CORNER of an expression: every rule ref that could be
   // the leftmost symbol, looking through nullable prefixes and stopping at the first non-nullable

test/check.ts

@@ -18,6 +18,7 @@ interface Gate { group: string; name: string; args: string[] }
 const GATES: Gate[] = [
   { group: 'core', name: 'agnostic', args: ['test/agnostic.ts'] },
   { group: 'core', name: 'left-recursion', args: ['test/left-recursion.ts'] },
+  { group: 'core', name: 'nullable-rules', args: ['test/nullable-rules.ts'] },
   { group: 'core', name: 'newline-mode', args: ['test/newline-mode.ts'] },
   { group: 'core', name: 'interpolation-metadata', args: ['test/interpolation-metadata.ts'] },
   { group: 'core', name: 'refactor-guard', args: ['test/refactor-guard.ts'] },

test/nullable-rules.ts

@@ -0,0 +1,100 @@
+// Gate: nullable NON-ENTRY rules are rejected at definition time.
+//
+// The engine never returns an EMPTY match for a rule: the longest-match loop keeps an
+// alternative only when it advanced (`pos > bestPos`), so a rule that can derive the empty
+// string has an unreachable empty case, and a reference to it FAILS wherever it would have
+// matched nothing. Written as a "filler" rule that is exactly the silent failure mode:
+// `Stmt = [Filler, Ident]` with `Filler = rule(() => [[many(NL)]])` rejects a plain `a`.
+// `defineGrammar` now names the rule and the fix; `allowNullableRules: true` keeps the
+// declaration for a grammar that reaches emptiness through alternatives (YAML).
+//
+// Run with: node test/nullable-rules.ts
+import { writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createParser } from '../src/gen-parser.ts';
+import { computeNullableRules } from '../src/grammar-analysis.ts';
+import { emitParser, jsTarget } from '../src/emit.ts';
+import { token, rule, defineGrammar, many, opt, plus, oneOf, range } from '../src/api.ts';
+
+let ok = 0, fail = 0;
+const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log('  ✗', label); } };
+
+const WS = token(plus(oneOf(' ', '\t')), { skip: true });
+const IDENT = token(plus(oneOf(range('a', 'z'))), { identifier: true });
+const NL = token(';', {});   // stands in for a line terminator (a real `\n` is skipped as whitespace here)
+
+function filler(allow: boolean) {
+  const Filler = rule(() => [[many(NL)]]);                       // NULLABLE: zero or more terminators
+  const Stmt = rule(() => [[Filler, IDENT, opt(',', IDENT)]]);
+  const Program = rule(() => [[many(Stmt)]]);                     // the entry may be nullable
+  return defineGrammar({ name: 'filler', tokens: { WS, IDENT, NL }, rules: { Filler, Stmt, Program }, entry: Program, ...(allow ? { allowNullableRules: true } : {}) });
+}
+function inlined() {
+  const Stmt = rule(() => [[many(NL), IDENT, opt(',', IDENT)]]);  // the same language, filler inlined
+  const Program = rule(() => [[many(Stmt)]]);
+  return defineGrammar({ name: 'inlined', tokens: { WS, IDENT, NL }, rules: { Stmt, Program }, entry: Program });
+}
+
+// ── 1. defineGrammar rejects the nullable non-entry rule, by name, with the fix ──
+let msg = '';
+try { filler(false); } catch (e) { msg = (e as Error).message; }
+check('a nullable non-entry rule is rejected', msg !== '');
+check('the message names the rule', msg.includes("'Filler'"));
+check('the message explains the unreachable empty match', /empty match/.test(msg) && /unreachable/.test(msg));
+check('the message points at the inline fix', msg.includes('many(X)'));
+check('the message points at the opt-out', msg.includes('allowNullableRules'));
+
+// ── 2. A nullable ENTRY rule is fine (an empty document is handled by the driver) ──
+let entryOk = true;
+try { inlined(); } catch { entryOk = false; }
+check('a nullable entry rule (Program = many(Stmt)) is accepted', entryOk);
+const gi = inlined();
+check('the inlined grammar reports no nullable non-entry rule', [...computeNullableRules(gi).nullableRules].filter(n => n !== 'Program').length === 0);
+check('the inlined form parses what the filler form silently rejected', (() => { try { return createParser(gi).parse(';;a,b;c').rule === 'Program'; } catch { return false; } })());
+
+// `entry` may be omitted (the LAST rule is then the entry, as findEntryRule resolves it): the
+// exemption must follow that rule, and only that rule.
+{
+  let implicitOk = true;
+  try {
+    const Stmt = rule(() => [[IDENT]]); const Program = rule(() => [[many(Stmt)]]);
+    defineGrammar({ name: 'implicit', tokens: { WS, IDENT, NL }, rules: { Stmt, Program } } as any);
+  } catch { implicitOk = false; }
+  check('with `entry` omitted, a nullable LAST rule is the implicit entry and is accepted', implicitOk);
+  let implicitBad = false;
+  try {
+    const Filler = rule(() => [[many(NL)]]); const Program = rule(() => [[Filler, IDENT]]);
+    defineGrammar({ name: 'implicit2', tokens: { WS, IDENT, NL }, rules: { Filler, Program } } as any);
+  } catch { implicitBad = true; }
+  check('with `entry` omitted, a nullable non-last rule is still rejected', implicitBad);
+}
+
+// ── 3. The opt-out keeps the declaration and changes no parse (the empty case stays unreachable) ──
+const gf = filler(true);
+check('allowNullableRules keeps the grammar buildable', gf.rules.some(r => r.name === 'Filler'));
+check('computeNullableRules still reports it', computeNullableRules(gf).nullableRules.has('Filler'));
+const interp = createParser(gf);
+const rejects = (s: string) => { try { interp.parse(s); return false; } catch { return true; } };
+check('documented behaviour: the filler grammar rejects a plain statement (empty Filler never matches)', rejects('a'));
+check('documented behaviour: the same grammar ACCEPTS `;a` (Filler matched non-empty), which is the inconsistency the check exists to surface', !rejects(';a'));
+{
+  const file = join(tmpdir(), `monogram-nullable-${process.pid}.ts`);
+  writeFileSync(file, emitParser(gf, jsTarget));
+  const p = (await import(file + '?v=' + Date.now())).createParser();
+  check('the emitted engine agrees (total parse reports the statement as unexpected)', p.parse('a').errors.length > 0);
+}
+
+// ── 4. The shipped grammars: YAML opts out explicitly; the rest have no nullable non-entry rule ──
+for (const [name, entry] of [['typescript', 'Program'], ['javascript', 'Program'], ['html', 'Document'], ['yaml', 'Stream']] as const) {
+  let g: any; let loaded = true;
+  try { g = (await import(`../${name}.ts`)).default; } catch { loaded = false; }
+  check(`${name}.ts still defines`, loaded);
+  if (!loaded) continue;
+  const nonEntry = [...computeNullableRules(g).nullableRules].filter(n => n !== entry);
+  if (name === 'yaml') check('yaml declares its nullable rules deliberately (Node, FlowNode, …)', nonEntry.includes('Node') && nonEntry.includes('FlowNode'));
+  else check(`${name} has no nullable non-entry rule`, nonEntry.length === 0);
+}
+
+console.log(`\n${ok}/${ok + fail} nullable-rule checks pass${fail ? '' : ' ✓'}`);
+if (fail) process.exit(1);

test/tm-completeness.ts

@@ -123,6 +123,9 @@ function checkRuleExprClosure(): void {
     g = defineGrammar({
       name: 'closure', tokens: { A, B },
       rules: { Leaf, Refs, Quant, Alt, Sep, Group, Nots, Markers, Pratt, Entry }, entry: Entry,
+      // Sep and Nots can derive the empty string by construction (this grammar exists to reach every
+      // combinator, not to parse); the nullable-rule definition check is not the subject of this lemma.
+      allowNullableRules: true,
     });
   } catch (e) { threw = true; g = null as any; }
   check('Lemma A1: toRuleExpr is total (no throw lowering every combinator)', !threw, threw ? 'defineGrammar threw' : '');

yaml.ts

@@ -672,5 +672,10 @@ export default defineGrammar({
     FlowNode, FlowExplicit, FlowMapEntry, FlowMapping, FlowSeqEntry, FlowSeqKey, FlowSequence, Scalar, BlockKeyScalar, DocFold, InlineDocNode, ExplicitDocBody, AfterDocEnd, NextDoc, Stream,
   },
   entry: Stream,
+  // Node / FlowNode / FlowMapEntry / FlowSeqEntry / AfterDocEnd can derive the empty string. The
+  // engine never returns an empty match for a rule, so their empty cases are unreachable and YAML
+  // reaches emptiness through ALTERNATIVES instead (`key:` with no node, `{a: }`, an empty doc).
+  // Declaring that is what keeps the definition honest without rewriting five rules.
+  allowNullableRules: true,
   indent,
 });