From 5e8bd4415cca669fcd8147964586f740e4ba2f22 Mon Sep 17 00:00:00 2001 From: Theo Ephraim Date: Mon, 7 Sep 2026 19:10:56 -0700 Subject: [PATCH] feat(emit): standalone output for createLexer-fallback grammars A grammar whose lexing is one of the data-driven state machines (indent, newline, markup) has no specialized lexer to embed, so its jsTarget output reaches the shared createLexer runtime at load time. Until now that was always an import of this repo's src/gen-lexer.ts by ABSOLUTE path: right for the in-repo gates, wrong for anyone shipping the emitted file, which only loaded on the machine that emitted it. emitParser(grammar, target, opts) gains opts.lexerRuntime: 'import' (default) the absolute-path import, unchanged 'inline' gen-lexer.ts plus the two helpers it uses and the type declarations copied verbatim into the module, scoped so only createLexer escapes; the output is standalone (no import) and type-checks alone { import: spec } a caller-owned specifier written verbatim Gate: test/emit-standalone.ts proves for YAML and HTML that the inline module loads from a directory outside the repo, parses tree-and-errors identically to the default emitted engine (and to the interpreter on clean parses), and passes tsc --strict without --allowImportingTsExtensions. --- README.md | 2 + src/emit-parser.ts | 36 ++++++++++++-- src/emit.ts | 22 +++++++-- test/check.ts | 1 + test/emit-standalone.ts | 101 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 test/emit-standalone.ts diff --git a/README.md b/README.md index beb35ea..15fa49c 100644 --- a/README.md +++ b/README.md @@ -353,6 +353,8 @@ const cst = parse(tokens); // same tokens → CST — no re-lexing `emitLexer(grammar, target)` is the second public emitter: a **standalone tokenizer module** (the same lexer `emitParser` embeds, alone). Go and Rust expose the same `tokenize`/`parse` pair (Rust's `tokenize` returns a token struct that carries the source slice, as it keeps no globals). The CLI shape `test/portable-targets.ts` runs (stdin → CST JSON) is a *harness* wrapper — `target.emitRunner()`, appended by the gate to make the library executable — not part of the parser. +A grammar whose lexing is one of the data-driven state machines (`indent`, `newline`, `markup`) has no specialized lexer to embed, so its `jsTarget` output reaches the shared `createLexer` runtime at load time. By default that is an `import` of this repo's `src/gen-lexer.ts` by absolute path, which is right for the in-repo gates and wrong for shipping: pass `emitParser(grammar, jsTarget, { lexerRuntime: 'inline' })` to copy the runtime into the module (standalone, no import, type-checks alone) or `{ lexerRuntime: { import: 'your/specifier' } }` to write a specifier you own (`test/emit-standalone.ts`). + `jsTarget` (the optimized JS path) has a different emitted shape by design: a zero-materialization arena parser — `parse(source)` returns an arena node handle (traverse with `visit`/`tree`, read tokens via `tokenAt`) and adds incremental `parseEdited`. Its lexer is fused into the arena pipeline, so it has no standalone tokenizer (`emitLexer(grammar, jsTarget)` is `null`). The proof is the full languages: the real [`javascript.ts`](javascript.ts) and [`typescript.ts`](typescript.ts) grammars — including the `[Await]/[Yield]` fork, left recursion, the regex/division and template state machines, arrow functions, and the TS type grammar — emit to **TypeScript, Go, and Rust**, and every emitted parser agrees with the reference interpreter on accept/reject outcomes (plus a rule-skeleton guard on tiny inputs). [`test/portable-targets.ts`](test/portable-targets.ts) compiles and runs all three for sixteen grammars (the two real languages plus focused fixtures) on every CI run. The Rust output reaches [oxc](https://github.com/oxc-project/oxc) throughput and the Go output beats [tsgo](https://github.com/microsoft/typescript-go) on the same corpus (an arena keeps both near zero-allocation). Byte-based Go/Rust use UTF-8 offsets — identical to the JS interpreter's for ASCII; non-ASCII offset units differ inherently. diff --git a/src/emit-parser.ts b/src/emit-parser.ts index 74feb92..0fee49b 100644 --- a/src/emit-parser.ts +++ b/src/emit-parser.ts @@ -28,7 +28,7 @@ import type { CstGrammar, RuleExpr, RuleDecl } from './types.ts'; import { isKeywordLiteral, collectLiterals } from './grammar-utils.ts'; import { analyzeGrammar, findEntryRule, type Sec } from './grammar-analysis.ts'; import { emitSoaLexer } from './emit-lexer.ts'; -import type { Target } from './emit.ts'; +import type { Target, EmitOptions } from './emit.ts'; import { withAwaitYield } from './await-yield-fork.ts'; // ── Static analysis ── @@ -1115,7 +1115,7 @@ export function emitJsLexer(grammar: CstGrammar): string | null { }); } -export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string { +export function emitJsParser(grammar: CstGrammar, lexSrc: string | null, opts?: EmitOptions): string { // [Await]/[Yield] context: name-fork the body-reachable rule closure into $A/$Y/$AY // families (see await-yield-fork.ts). No-op for a grammar with no ctx markers. Done // HERE (not at grammar export) so the forks exist ONLY in the parser's rule identity @@ -1154,7 +1154,9 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string const st = a.symtab; e.soa = lexSrc !== null; if (!lexSrc) { - e.emit(`import { createLexer } from ${J(resolveLexerImport())};`); + const rt = opts?.lexerRuntime ?? 'import'; + if (rt === 'inline') e.emit(inlineLexerRuntime()); + else e.emit(`import { createLexer } from ${J(typeof rt === 'object' ? rt.import : resolveLexerImport())};`); e.emit(``); e.emit(`const LEX_GRAMMAR = ${J(lexGrammar)};`); } @@ -1331,9 +1333,37 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string // (e.g. /tmp). resolveLexerImport returns that absolute specifier. import { fileURLToPath } from 'node:url'; import { dirname, resolve as pathResolve } from 'node:path'; +import { readFileSync } from 'node:fs'; const __dir = dirname(fileURLToPath(import.meta.url)); function resolveLexerImport(): string { return pathResolve(__dir, 'gen-lexer.ts'); } +// `lexerRuntime: 'inline'` — the createLexer runtime copied INTO the emitted module so the output is +// standalone (no import at load time; see EmitOptions). The four sources are concatenated verbatim +// inside one function scope: their `import` lines go (they only import each other and the types, +// all of which are now in scope) and their `export` keywords go (nothing leaves the scope but +// createLexer). Read from THIS repo at emit time — the runtime files stay the single source, so an +// engine fix reaches inlined consumers on their next emit, exactly like the import path. +const INLINE_LEXER_SOURCES = ['types.ts', 'grammar-utils.ts', 'token-pattern.ts', 'gen-lexer.ts']; +function inlineLexerRuntime(): string { + const parts = INLINE_LEXER_SOURCES.map((file) => { + const src = readFileSync(pathResolve(__dir, file), 'utf8') + .split('\n') + .filter((line) => !/^import\s.*from\s+'\.\/[^']+\.ts';\s*$/.test(line)) // sibling imports only (single-line by convention here) + .map((line) => line.replace(/^export\s+(?=(?:function|const|let|class|interface|type|abstract\s+class)\b)/, '')) + .join('\n'); + if (/^(import|export)\b/m.test(src)) throw new Error(`inlineLexerRuntime: unexpected module syntax left in src/${file} (only sibling imports and declaration exports are inlineable)`); + return `// ── src/${file} (verbatim; declarations un-exported) ──\n${src}`; + }); + return [ + `// ── Inlined lexer runtime (lexerRuntime: 'inline'): ${INLINE_LEXER_SOURCES.map((f) => 'src/' + f).join(', ')} ──`, + `// Copied verbatim at emit time so this module needs no import to lex. Nothing but createLexer escapes the scope.`, + `const createLexer = (() => {`, + ...parts, + `return createLexer;`, + `})();`, + ].join('\n'); +} + // ── Runtime: the generic engine state + control loops, emitted verbatim ── // These are copied from gen-parser.ts so their semantics are byte-identical. The // ONLY change: where the interpreter called matchExpr(alt)/matchSeq(items) per arm, diff --git a/src/emit.ts b/src/emit.ts index fc85a2f..1329b7f 100644 --- a/src/emit.ts +++ b/src/emit.ts @@ -12,6 +12,22 @@ // `tsTarget`/`goTarget`/`rustTarget` (emit-portable.ts + target-*.ts). import type { CstGrammar } from './types.ts'; +// Per-emit options. Only `jsTarget` reads them today; the portable targets embed their own lexer +// and have nothing to resolve. +export interface EmitOptions { + // How a createLexer-FALLBACK grammar (indent / newline / markup: the data-driven lexer state + // machines are interpreter-only, so `embedLexer` is null) reaches the lexer runtime at load time: + // 'import' (default) — `import { createLexer } from ""`, + // resolved at emit time; the emitted file runs from anywhere on THIS machine. + // 'inline' — the runtime (src/gen-lexer.ts + the two helpers it uses + the type + // declarations) is copied verbatim into the module. The output is then + // STANDALONE: no import, no path, no dependency on monogram at load time. + // { import: spec } — a caller-supplied specifier (a package entry, a relative path the caller + // controls). The caller owns making `spec` resolve to gen-lexer's exports. + // Self-contained grammars (token-stream languages) embed a specialized lexer and ignore this. + lexerRuntime?: 'import' | 'inline' | { import: string }; +} + export interface Target { name: string; ext: string; // emitted file extension (no dot) @@ -22,7 +38,7 @@ export interface Target { // null where the lexer is not separable from the parser: jsTarget fuses lexing into its arena // pipeline (no token list), so there is no standalone tokenizer to emit. emitLexer(grammar: CstGrammar): string | null; - emitParser(grammar: CstGrammar, lexerSrc: string | null): string; // the parser LIBRARY (exports `tokenize` + `parse`; no I/O) + emitParser(grammar: CstGrammar, lexerSrc: string | null, opts?: EmitOptions): string; // the parser LIBRARY (exports `tokenize` + `parse`; no I/O) // A standalone CLI harness (stdin → CST JSON) APPENDED to the library to make it executable — // needed to run the compiled go/rust (and ts) parsers for verification. Not part of the parser. emitRunner?(): string; @@ -34,8 +50,8 @@ export function emitLexer(grammar: CstGrammar, target: Target): string | null { return target.emitLexer(grammar); } -export function emitParser(grammar: CstGrammar, target: Target): string { - return target.emitParser(grammar, target.embedLexer(grammar)); +export function emitParser(grammar: CstGrammar, target: Target, opts?: EmitOptions): string { + return target.emitParser(grammar, target.embedLexer(grammar), opts); } export { jsTarget } from './emit-parser.ts'; diff --git a/test/check.ts b/test/check.ts index 6dca4b6..7d926c2 100644 --- a/test/check.ts +++ b/test/check.ts @@ -29,6 +29,7 @@ const GATES: Gate[] = [ { group: 'emit-parity', name: 'emit-reject-messages', args: ['test/emit-reject-messages.ts'] }, { group: 'emit-parity', name: 'emit-lexer-verify', args: ['test/emit-lexer-verify.ts'] }, { group: 'emit-parity', name: 'emit-tsc-gate', args: ['test/emit-tsc-gate.ts'] }, + { group: 'emit-parity', name: 'emit-standalone', args: ['test/emit-standalone.ts'] }, { group: 'emit-parity', name: 'portable-targets', args: ['test/portable-targets.ts'] }, { group: 'emit-parity', name: 'unicode-parity', args: ['test/unicode-parity.ts'] }, { group: 'emit-parity', name: 'ast-builder', args: ['test/ast-builder.ts'] }, diff --git a/test/emit-standalone.ts b/test/emit-standalone.ts new file mode 100644 index 0000000..23e89b0 --- /dev/null +++ b/test/emit-standalone.ts @@ -0,0 +1,101 @@ +// Gate: a createLexer-FALLBACK grammar (indent / newline / markup) can be emitted as a STANDALONE +// module — `emitParser(grammar, jsTarget, { lexerRuntime: 'inline' })`. +// +// By default such a grammar's emitted parser imports the data-driven lexer runtime from THIS +// repo by absolute path (`import { createLexer } from "…/src/gen-lexer.ts"`), which is right for +// the in-repo gates and wrong for anyone shipping the emitted file: it only loads on the machine +// that emitted it. 'inline' copies the runtime into the module; `{ import: spec }` writes a +// caller-owned specifier instead. This gate proves, for YAML and HTML: +// 1. the inline module has NO import statement at all, and loads from a directory outside the repo; +// 2. it parses byte-identically to the interpreter (the same trees the import path produces); +// 3. it type-checks under `tsc --strict` WITHOUT --allowImportingTsExtensions (no .ts import remains); +// 4. `{ import }` writes exactly the given specifier, and the default is unchanged. +// +// Run with: node test/emit-standalone.ts +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { emitParser, jsTarget } from '../src/emit.ts'; +import { createParser } from '../src/gen-parser.ts'; +import { objectify } from './emitted-obj.ts'; +import type { CstGrammar } from '../src/types.ts'; + +let ok = 0, fail = 0; +const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log(' ✗', label); } }; + +const SAMPLES: Record = { + yaml: ['a: 1\nb:\n - x\n - y\nc: {k: v, n: [1, 2]}\n', '# comment\nkey: "quoted"\nblock: |\n line one\n line two\n', 'a:\n'], + html: ['
hi
there
', '

a < b

', '
  • one
  • two
', '

unclosed bold'], +}; + +// A directory OUTSIDE the repo: a relative or absolute import into src/ cannot resolve from here by accident. +const dir = mkdtempSync(join(tmpdir(), 'monogram-standalone-')); +const TSC_FLAGS = ['--strict', '--noEmit', '--target', 'ES2022', '--module', 'ES2022', '--moduleResolution', 'Bundler', '--skipLibCheck']; + +for (const [name, samples] of Object.entries(SAMPLES)) { + let grammar: CstGrammar; + try { grammar = (await import(`../${name}.ts`)).default; } catch { console.log(` ${name}: (grammar not present — skipped)`); continue; } + check(`${name}: is a createLexer-fallback grammar (embedLexer is null)`, jsTarget.embedLexer(grammar) === null); + + // ── default: unchanged (absolute import into this repo) ── + const dflt = emitParser(grammar, jsTarget); + check(`${name}: default still imports createLexer by absolute path`, /^import \{ createLexer \} from "\/.*\/src\/gen-lexer\.ts";$/m.test(dflt)); + + // ── { import: spec }: the caller's specifier, verbatim ── + const spec = emitParser(grammar, jsTarget, { lexerRuntime: { import: 'monogram/lexer' } }); + check(`${name}: { import } writes the given specifier`, spec.includes('import { createLexer } from "monogram/lexer";')); + check(`${name}: { import } output is otherwise the default output`, spec.replace('"monogram/lexer"', '__X__') === dflt.replace(/"\/.*\/src\/gen-lexer\.ts"/, '__X__')); + + // ── 'inline': standalone ── + const inline = emitParser(grammar, jsTarget, { lexerRuntime: 'inline' }); + check(`${name}: inline output has no import statement`, !/^import\s/m.test(inline)); + check(`${name}: inline output has no export but the parser API (no runtime export leaked)`, !/^export (function|const) (createLexer|collectLiterals|tokenPatternSource)\b/m.test(inline)); + const file = join(dir, `${name}.ts`); + writeFileSync(file, inline); + let mod: any = null; + try { mod = await import(file + '?v=' + Date.now()); } catch (e) { console.log(` ${name}: inline module failed to load: ${(e as Error).message.split('\n')[0]}`); } + check(`${name}: inline module loads from outside the repo`, !!mod); + if (!mod) continue; + + // The claim is inline ≡ the default (import-path) emitted engine, tree AND errors, on valid and + // broken input alike; and where that engine parses cleanly, ≡ the interpreter too (emit-parser-verify + // owns emitted-vs-interpreter parity in general; recovery output is the emitted engine's own). + const dfltFile = join(dir, `${name}-default-import.ts`); + writeFileSync(dfltFile, dflt.replace(/^import \{ createLexer \} from "(\/.*\/src\/gen-lexer\.ts)";$/m, (_m, p) => `import { createLexer } from ${JSON.stringify(p)};`)); + const pd = (await import(dfltFile + '?v=' + Date.now())).createParser(); + const interp = createParser(grammar); + const p = mod.createParser(); + const treeOf = (parser: any, src: string) => { const cst = parser.parse(src); const obj = objectify(parser.tree, (fns: any) => parser.visit(cst, fns)); return JSON.stringify({ ...obj, errors: cst.errors }); }; + for (const src of samples) { + const viaInline = treeOf(p, src), viaImport = treeOf(pd, src); + check(`${name}: inline ≡ default emitted engine for ${JSON.stringify(src.slice(0, 24))}`, viaInline === viaImport); + if (JSON.parse(viaImport).errors.length === 0) { + check(`${name}: inline ≡ interpreter on the clean parse of ${JSON.stringify(src.slice(0, 24))}`, viaInline === JSON.stringify(interp.parseTotal(src))); + } + } + + try { + execFileSync('npx', ['tsc', ...TSC_FLAGS, file], { stdio: 'pipe' }); + check(`${name}: inline module type-checks (tsc --strict, no --allowImportingTsExtensions)`, true); + } catch (e: any) { + const log = (e.stdout?.toString() ?? '') + (e.stderr?.toString() ?? ''); + console.log(log.split('\n').slice(0, 8).join('\n')); + check(`${name}: inline module type-checks (tsc --strict, no --allowImportingTsExtensions)`, false); + } +} + +// ── A self-contained grammar (specialized lexer embedded) ignores the option: byte-identical output ── +{ + let ts: CstGrammar | null = null; + try { ts = (await import('../typescript.ts')).default; } catch { console.log(' typescript: (grammar not present — skipped)'); } + if (ts) { + check('typescript: embeds its own lexer (not a fallback grammar)', jsTarget.embedLexer(ts) !== null); + const base = emitParser(ts, jsTarget); + check("typescript: lexerRuntime 'inline' is a no-op (byte-identical output)", emitParser(ts, jsTarget, { lexerRuntime: 'inline' }) === base); + check("typescript: lexerRuntime { import } is a no-op (byte-identical output)", emitParser(ts, jsTarget, { lexerRuntime: { import: 'monogram/lexer' } }) === base); + } +} + +console.log(`\n${ok}/${ok + fail} standalone-emit checks pass${fail ? '' : ' ✓'}`); +if (fail) process.exit(1);