feat(emit): standalone output for createLexer-fallback grammars

README.md

@@ -353,6 +353,8 @@ const cst    = parse(tokens);   // same tokens → CST — no re-lexing
 
 `emitLexer(grammar, target)` is the second public emitter: a **standalone tokenizer module** (the same lexer `emitParser` embeds, alone). Go and Rust expose the same `tokenize`/`parse` pair (Rust's `tokenize` returns a token struct that carries the source slice, as it keeps no globals). The CLI shape `test/portable-targets.ts` runs (stdin → CST JSON) is a *harness* wrapper — `target.emitRunner()`, appended by the gate to make the library executable — not part of the parser.
 
+A grammar whose lexing is one of the data-driven state machines (`indent`, `newline`, `markup`) has no specialized lexer to embed, so its `jsTarget` output reaches the shared `createLexer` runtime at load time. By default that is an `import` of this repo's `src/gen-lexer.ts` by absolute path, which is right for the in-repo gates and wrong for shipping: pass `emitParser(grammar, jsTarget, { lexerRuntime: 'inline' })` to copy the runtime into the module (standalone, no import, type-checks alone) or `{ lexerRuntime: { import: 'your/specifier' } }` to write a specifier you own (`test/emit-standalone.ts`).
+
 `jsTarget` (the optimized JS path) has a different emitted shape by design: a zero-materialization arena parser — `parse(source)` returns an arena node handle (traverse with `visit`/`tree`, read tokens via `tokenAt`) and adds incremental `parseEdited`. Its lexer is fused into the arena pipeline, so it has no standalone tokenizer (`emitLexer(grammar, jsTarget)` is `null`).
 
 The proof is the full languages: the real [`javascript.ts`](javascript.ts) and [`typescript.ts`](typescript.ts) grammars — including the `[Await]/[Yield]` fork, left recursion, the regex/division and template state machines, arrow functions, and the TS type grammar — emit to **TypeScript, Go, and Rust**, and every emitted parser agrees with the reference interpreter on accept/reject outcomes (plus a rule-skeleton guard on tiny inputs). [`test/portable-targets.ts`](test/portable-targets.ts) compiles and runs all three for sixteen grammars (the two real languages plus focused fixtures) on every CI run. The Rust output reaches [oxc](https://github.com/oxc-project/oxc) throughput and the Go output beats [tsgo](https://github.com/microsoft/typescript-go) on the same corpus (an arena keeps both near zero-allocation). Byte-based Go/Rust use UTF-8 offsets — identical to the JS interpreter's for ASCII; non-ASCII offset units differ inherently.

src/emit-parser.ts

@@ -28,7 +28,7 @@ import type { CstGrammar, RuleExpr, RuleDecl } from './types.ts';
 import { isKeywordLiteral, collectLiterals } from './grammar-utils.ts';
 import { analyzeGrammar, findEntryRule, type Sec } from './grammar-analysis.ts';
 import { emitSoaLexer } from './emit-lexer.ts';
-import type { Target } from './emit.ts';
+import type { Target, EmitOptions } from './emit.ts';
 import { withAwaitYield } from './await-yield-fork.ts';
 
 // ── Static analysis ──
@@ -1115,7 +1115,7 @@ export function emitJsLexer(grammar: CstGrammar): string | null {
   });
 }
 
-export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string {
+export function emitJsParser(grammar: CstGrammar, lexSrc: string | null, opts?: EmitOptions): string {
   // [Await]/[Yield] context: name-fork the body-reachable rule closure into $A/$Y/$AY
   // families (see await-yield-fork.ts). No-op for a grammar with no ctx markers. Done
   // HERE (not at grammar export) so the forks exist ONLY in the parser's rule identity
@@ -1154,7 +1154,9 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string
   const st = a.symtab;
   e.soa = lexSrc !== null;
   if (!lexSrc) {
-    e.emit(`import { createLexer } from ${J(resolveLexerImport())};`);
+    const rt = opts?.lexerRuntime ?? 'import';
+    if (rt === 'inline') e.emit(inlineLexerRuntime());
+    else e.emit(`import { createLexer } from ${J(typeof rt === 'object' ? rt.import : resolveLexerImport())};`);
     e.emit(``);
     e.emit(`const LEX_GRAMMAR = ${J(lexGrammar)};`);
   }
@@ -1331,9 +1333,37 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string
 // (e.g. /tmp). resolveLexerImport returns that absolute specifier.
 import { fileURLToPath } from 'node:url';
 import { dirname, resolve as pathResolve } from 'node:path';
+import { readFileSync } from 'node:fs';
 const __dir = dirname(fileURLToPath(import.meta.url));
 function resolveLexerImport(): string { return pathResolve(__dir, 'gen-lexer.ts'); }
 
+// `lexerRuntime: 'inline'` — the createLexer runtime copied INTO the emitted module so the output is
+// standalone (no import at load time; see EmitOptions). The four sources are concatenated verbatim
+// inside one function scope: their `import` lines go (they only import each other and the types,
+// all of which are now in scope) and their `export` keywords go (nothing leaves the scope but
+// createLexer). Read from THIS repo at emit time — the runtime files stay the single source, so an
+// engine fix reaches inlined consumers on their next emit, exactly like the import path.
+const INLINE_LEXER_SOURCES = ['types.ts', 'grammar-utils.ts', 'token-pattern.ts', 'gen-lexer.ts'];
+function inlineLexerRuntime(): string {
+  const parts = INLINE_LEXER_SOURCES.map((file) => {
+    const src = readFileSync(pathResolve(__dir, file), 'utf8')
+      .split('\n')
+      .filter((line) => !/^import\s.*from\s+'\.\/[^']+\.ts';\s*$/.test(line))   // sibling imports only (single-line by convention here)
+      .map((line) => line.replace(/^export\s+(?=(?:function|const|let|class|interface|type|abstract\s+class)\b)/, ''))
+      .join('\n');
+    if (/^(import|export)\b/m.test(src)) throw new Error(`inlineLexerRuntime: unexpected module syntax left in src/${file} (only sibling imports and declaration exports are inlineable)`);
+    return `// ── src/${file} (verbatim; declarations un-exported) ──\n${src}`;
+  });
+  return [
+    `// ── Inlined lexer runtime (lexerRuntime: 'inline'): ${INLINE_LEXER_SOURCES.map((f) => 'src/' + f).join(', ')} ──`,
+    `// Copied verbatim at emit time so this module needs no import to lex. Nothing but createLexer escapes the scope.`,
+    `const createLexer = (() => {`,
+    ...parts,
+    `return createLexer;`,
+    `})();`,
+  ].join('\n');
+}
+
 // ── Runtime: the generic engine state + control loops, emitted verbatim ──
 // These are copied from gen-parser.ts so their semantics are byte-identical. The
 // ONLY change: where the interpreter called matchExpr(alt)/matchSeq(items) per arm,

src/emit.ts

@@ -12,6 +12,22 @@
 // `tsTarget`/`goTarget`/`rustTarget` (emit-portable.ts + target-*.ts).
 import type { CstGrammar } from './types.ts';
 
+// Per-emit options. Only `jsTarget` reads them today; the portable targets embed their own lexer
+// and have nothing to resolve.
+export interface EmitOptions {
+  // How a createLexer-FALLBACK grammar (indent / newline / markup: the data-driven lexer state
+  // machines are interpreter-only, so `embedLexer` is null) reaches the lexer runtime at load time:
+  //   'import' (default) — `import { createLexer } from "<absolute path to this repo's src/gen-lexer.ts>"`,
+  //                        resolved at emit time; the emitted file runs from anywhere on THIS machine.
+  //   'inline'           — the runtime (src/gen-lexer.ts + the two helpers it uses + the type
+  //                        declarations) is copied verbatim into the module. The output is then
+  //                        STANDALONE: no import, no path, no dependency on monogram at load time.
+  //   { import: spec }   — a caller-supplied specifier (a package entry, a relative path the caller
+  //                        controls). The caller owns making `spec` resolve to gen-lexer's exports.
+  // Self-contained grammars (token-stream languages) embed a specialized lexer and ignore this.
+  lexerRuntime?: 'import' | 'inline' | { import: string };
+}
+
 export interface Target {
   name: string;
   ext: string;                                                  // emitted file extension (no dot)
@@ -22,7 +38,7 @@ export interface Target {
   // null where the lexer is not separable from the parser: jsTarget fuses lexing into its arena
   // pipeline (no token list), so there is no standalone tokenizer to emit.
   emitLexer(grammar: CstGrammar): string | null;
-  emitParser(grammar: CstGrammar, lexerSrc: string | null): string;   // the parser LIBRARY (exports `tokenize` + `parse`; no I/O)
+  emitParser(grammar: CstGrammar, lexerSrc: string | null, opts?: EmitOptions): string;   // the parser LIBRARY (exports `tokenize` + `parse`; no I/O)
   // A standalone CLI harness (stdin → CST JSON) APPENDED to the library to make it executable —
   // needed to run the compiled go/rust (and ts) parsers for verification. Not part of the parser.
   emitRunner?(): string;
@@ -34,8 +50,8 @@ export function emitLexer(grammar: CstGrammar, target: Target): string | null {
   return target.emitLexer(grammar);
 }
 
-export function emitParser(grammar: CstGrammar, target: Target): string {
-  return target.emitParser(grammar, target.embedLexer(grammar));
+export function emitParser(grammar: CstGrammar, target: Target, opts?: EmitOptions): string {
+  return target.emitParser(grammar, target.embedLexer(grammar), opts);
 }
 
 export { jsTarget } from './emit-parser.ts';

test/check.ts

@@ -29,6 +29,7 @@ const GATES: Gate[] = [
   { group: 'emit-parity', name: 'emit-reject-messages', args: ['test/emit-reject-messages.ts'] },
   { group: 'emit-parity', name: 'emit-lexer-verify', args: ['test/emit-lexer-verify.ts'] },
   { group: 'emit-parity', name: 'emit-tsc-gate', args: ['test/emit-tsc-gate.ts'] },
+  { group: 'emit-parity', name: 'emit-standalone', args: ['test/emit-standalone.ts'] },
   { group: 'emit-parity', name: 'portable-targets', args: ['test/portable-targets.ts'] },
   { group: 'emit-parity', name: 'unicode-parity', args: ['test/unicode-parity.ts'] },
   { group: 'emit-parity', name: 'ast-builder', args: ['test/ast-builder.ts'] },

test/emit-standalone.ts

@@ -0,0 +1,101 @@
+// Gate: a createLexer-FALLBACK grammar (indent / newline / markup) can be emitted as a STANDALONE
+// module — `emitParser(grammar, jsTarget, { lexerRuntime: 'inline' })`.
+//
+// By default such a grammar's emitted parser imports the data-driven lexer runtime from THIS
+// repo by absolute path (`import { createLexer } from "…/src/gen-lexer.ts"`), which is right for
+// the in-repo gates and wrong for anyone shipping the emitted file: it only loads on the machine
+// that emitted it. 'inline' copies the runtime into the module; `{ import: spec }` writes a
+// caller-owned specifier instead. This gate proves, for YAML and HTML:
+//   1. the inline module has NO import statement at all, and loads from a directory outside the repo;
+//   2. it parses byte-identically to the interpreter (the same trees the import path produces);
+//   3. it type-checks under `tsc --strict` WITHOUT --allowImportingTsExtensions (no .ts import remains);
+//   4. `{ import }` writes exactly the given specifier, and the default is unchanged.
+//
+// Run with: node test/emit-standalone.ts
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { execFileSync } from 'node:child_process';
+import { emitParser, jsTarget } from '../src/emit.ts';
+import { createParser } from '../src/gen-parser.ts';
+import { objectify } from './emitted-obj.ts';
+import type { CstGrammar } from '../src/types.ts';
+
+let ok = 0, fail = 0;
+const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log('  ✗', label); } };
+
+const SAMPLES: Record<string, string[]> = {
+  yaml: ['a: 1\nb:\n  - x\n  - y\nc: {k: v, n: [1, 2]}\n', '# comment\nkey: "quoted"\nblock: |\n  line one\n  line two\n', 'a:\n'],
+  html: ['<div class="a">hi<br>there</div>', '<!-- c --><p>a < b</p><script>if (a < b) {}</script>', '<ul><li>one</li><li>two</li></ul>', '<p>unclosed <b>bold'],
+};
+
+// A directory OUTSIDE the repo: a relative or absolute import into src/ cannot resolve from here by accident.
+const dir = mkdtempSync(join(tmpdir(), 'monogram-standalone-'));
+const TSC_FLAGS = ['--strict', '--noEmit', '--target', 'ES2022', '--module', 'ES2022', '--moduleResolution', 'Bundler', '--skipLibCheck'];
+
+for (const [name, samples] of Object.entries(SAMPLES)) {
+  let grammar: CstGrammar;
+  try { grammar = (await import(`../${name}.ts`)).default; } catch { console.log(`  ${name}: (grammar not present — skipped)`); continue; }
+  check(`${name}: is a createLexer-fallback grammar (embedLexer is null)`, jsTarget.embedLexer(grammar) === null);
+
+  // ── default: unchanged (absolute import into this repo) ──
+  const dflt = emitParser(grammar, jsTarget);
+  check(`${name}: default still imports createLexer by absolute path`, /^import \{ createLexer \} from "\/.*\/src\/gen-lexer\.ts";$/m.test(dflt));
+
+  // ── { import: spec }: the caller's specifier, verbatim ──
+  const spec = emitParser(grammar, jsTarget, { lexerRuntime: { import: 'monogram/lexer' } });
+  check(`${name}: { import } writes the given specifier`, spec.includes('import { createLexer } from "monogram/lexer";'));
+  check(`${name}: { import } output is otherwise the default output`, spec.replace('"monogram/lexer"', '__X__') === dflt.replace(/"\/.*\/src\/gen-lexer\.ts"/, '__X__'));
+
+  // ── 'inline': standalone ──
+  const inline = emitParser(grammar, jsTarget, { lexerRuntime: 'inline' });
+  check(`${name}: inline output has no import statement`, !/^import\s/m.test(inline));
+  check(`${name}: inline output has no export but the parser API (no runtime export leaked)`, !/^export (function|const) (createLexer|collectLiterals|tokenPatternSource)\b/m.test(inline));
+  const file = join(dir, `${name}.ts`);
+  writeFileSync(file, inline);
+  let mod: any = null;
+  try { mod = await import(file + '?v=' + Date.now()); } catch (e) { console.log(`  ${name}: inline module failed to load: ${(e as Error).message.split('\n')[0]}`); }
+  check(`${name}: inline module loads from outside the repo`, !!mod);
+  if (!mod) continue;
+
+  // The claim is inline ≡ the default (import-path) emitted engine, tree AND errors, on valid and
+  // broken input alike; and where that engine parses cleanly, ≡ the interpreter too (emit-parser-verify
+  // owns emitted-vs-interpreter parity in general; recovery output is the emitted engine's own).
+  const dfltFile = join(dir, `${name}-default-import.ts`);
+  writeFileSync(dfltFile, dflt.replace(/^import \{ createLexer \} from "(\/.*\/src\/gen-lexer\.ts)";$/m, (_m, p) => `import { createLexer } from ${JSON.stringify(p)};`));
+  const pd = (await import(dfltFile + '?v=' + Date.now())).createParser();
+  const interp = createParser(grammar);
+  const p = mod.createParser();
+  const treeOf = (parser: any, src: string) => { const cst = parser.parse(src); const obj = objectify(parser.tree, (fns: any) => parser.visit(cst, fns)); return JSON.stringify({ ...obj, errors: cst.errors }); };
+  for (const src of samples) {
+    const viaInline = treeOf(p, src), viaImport = treeOf(pd, src);
+    check(`${name}: inline ≡ default emitted engine for ${JSON.stringify(src.slice(0, 24))}`, viaInline === viaImport);
+    if (JSON.parse(viaImport).errors.length === 0) {
+      check(`${name}: inline ≡ interpreter on the clean parse of ${JSON.stringify(src.slice(0, 24))}`, viaInline === JSON.stringify(interp.parseTotal(src)));
+    }
+  }
+
+  try {
+    execFileSync('npx', ['tsc', ...TSC_FLAGS, file], { stdio: 'pipe' });
+    check(`${name}: inline module type-checks (tsc --strict, no --allowImportingTsExtensions)`, true);
+  } catch (e: any) {
+    const log = (e.stdout?.toString() ?? '') + (e.stderr?.toString() ?? '');
+    console.log(log.split('\n').slice(0, 8).join('\n'));
+    check(`${name}: inline module type-checks (tsc --strict, no --allowImportingTsExtensions)`, false);
+  }
+}
+
+// ── A self-contained grammar (specialized lexer embedded) ignores the option: byte-identical output ──
+{
+  let ts: CstGrammar | null = null;
+  try { ts = (await import('../typescript.ts')).default; } catch { console.log('  typescript: (grammar not present — skipped)'); }
+  if (ts) {
+    check('typescript: embeds its own lexer (not a fallback grammar)', jsTarget.embedLexer(ts) !== null);
+    const base = emitParser(ts, jsTarget);
+    check("typescript: lexerRuntime 'inline' is a no-op (byte-identical output)", emitParser(ts, jsTarget, { lexerRuntime: 'inline' }) === base);
+    check("typescript: lexerRuntime { import } is a no-op (byte-identical output)", emitParser(ts, jsTarget, { lexerRuntime: { import: 'monogram/lexer' } }) === base);
+  }
+}
+
+console.log(`\n${ok}/${ok + fail} standalone-emit checks pass${fail ? '' : ' ✓'}`);
+if (fail) process.exit(1);