diff --git a/README.md b/README.md index beb35ea..15fa49c 100644 --- a/README.md +++ b/README.md @@ -353,6 +353,8 @@ const cst = parse(tokens); // same tokens → CST — no re-lexing `emitLexer(grammar, target)` is the second public emitter: a **standalone tokenizer module** (the same lexer `emitParser` embeds, alone). Go and Rust expose the same `tokenize`/`parse` pair (Rust's `tokenize` returns a token struct that carries the source slice, as it keeps no globals). The CLI shape `test/portable-targets.ts` runs (stdin → CST JSON) is a *harness* wrapper — `target.emitRunner()`, appended by the gate to make the library executable — not part of the parser. +A grammar whose lexing is one of the data-driven state machines (`indent`, `newline`, `markup`) has no specialized lexer to embed, so its `jsTarget` output reaches the shared `createLexer` runtime at load time. By default that is an `import` of this repo's `src/gen-lexer.ts` by absolute path, which is right for the in-repo gates and wrong for shipping: pass `emitParser(grammar, jsTarget, { lexerRuntime: 'inline' })` to copy the runtime into the module (standalone, no import, type-checks alone) or `{ lexerRuntime: { import: 'your/specifier' } }` to write a specifier you own (`test/emit-standalone.ts`). + `jsTarget` (the optimized JS path) has a different emitted shape by design: a zero-materialization arena parser — `parse(source)` returns an arena node handle (traverse with `visit`/`tree`, read tokens via `tokenAt`) and adds incremental `parseEdited`. Its lexer is fused into the arena pipeline, so it has no standalone tokenizer (`emitLexer(grammar, jsTarget)` is `null`). The proof is the full languages: the real [`javascript.ts`](javascript.ts) and [`typescript.ts`](typescript.ts) grammars — including the `[Await]/[Yield]` fork, left recursion, the regex/division and template state machines, arrow functions, and the TS type grammar — emit to **TypeScript, Go, and Rust**, and every emitted parser agrees with the reference interpreter on accept/reject outcomes (plus a rule-skeleton guard on tiny inputs). [`test/portable-targets.ts`](test/portable-targets.ts) compiles and runs all three for sixteen grammars (the two real languages plus focused fixtures) on every CI run. The Rust output reaches [oxc](https://github.com/oxc-project/oxc) throughput and the Go output beats [tsgo](https://github.com/microsoft/typescript-go) on the same corpus (an arena keeps both near zero-allocation). Byte-based Go/Rust use UTF-8 offsets — identical to the JS interpreter's for ASCII; non-ASCII offset units differ inherently. diff --git a/src/emit-parser.ts b/src/emit-parser.ts index 74feb92..0fee49b 100644 --- a/src/emit-parser.ts +++ b/src/emit-parser.ts @@ -28,7 +28,7 @@ import type { CstGrammar, RuleExpr, RuleDecl } from './types.ts'; import { isKeywordLiteral, collectLiterals } from './grammar-utils.ts'; import { analyzeGrammar, findEntryRule, type Sec } from './grammar-analysis.ts'; import { emitSoaLexer } from './emit-lexer.ts'; -import type { Target } from './emit.ts'; +import type { Target, EmitOptions } from './emit.ts'; import { withAwaitYield } from './await-yield-fork.ts'; // ── Static analysis ── @@ -1115,7 +1115,7 @@ export function emitJsLexer(grammar: CstGrammar): string | null { }); } -export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string { +export function emitJsParser(grammar: CstGrammar, lexSrc: string | null, opts?: EmitOptions): string { // [Await]/[Yield] context: name-fork the body-reachable rule closure into $A/$Y/$AY // families (see await-yield-fork.ts). No-op for a grammar with no ctx markers. Done // HERE (not at grammar export) so the forks exist ONLY in the parser's rule identity @@ -1154,7 +1154,9 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string const st = a.symtab; e.soa = lexSrc !== null; if (!lexSrc) { - e.emit(`import { createLexer } from ${J(resolveLexerImport())};`); + const rt = opts?.lexerRuntime ?? 'import'; + if (rt === 'inline') e.emit(inlineLexerRuntime()); + else e.emit(`import { createLexer } from ${J(typeof rt === 'object' ? rt.import : resolveLexerImport())};`); e.emit(``); e.emit(`const LEX_GRAMMAR = ${J(lexGrammar)};`); } @@ -1331,9 +1333,37 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string // (e.g. /tmp). resolveLexerImport returns that absolute specifier. import { fileURLToPath } from 'node:url'; import { dirname, resolve as pathResolve } from 'node:path'; +import { readFileSync } from 'node:fs'; const __dir = dirname(fileURLToPath(import.meta.url)); function resolveLexerImport(): string { return pathResolve(__dir, 'gen-lexer.ts'); } +// `lexerRuntime: 'inline'` — the createLexer runtime copied INTO the emitted module so the output is +// standalone (no import at load time; see EmitOptions). The four sources are concatenated verbatim +// inside one function scope: their `import` lines go (they only import each other and the types, +// all of which are now in scope) and their `export` keywords go (nothing leaves the scope but +// createLexer). Read from THIS repo at emit time — the runtime files stay the single source, so an +// engine fix reaches inlined consumers on their next emit, exactly like the import path. +const INLINE_LEXER_SOURCES = ['types.ts', 'grammar-utils.ts', 'token-pattern.ts', 'gen-lexer.ts']; +function inlineLexerRuntime(): string { + const parts = INLINE_LEXER_SOURCES.map((file) => { + const src = readFileSync(pathResolve(__dir, file), 'utf8') + .split('\n') + .filter((line) => !/^import\s.*from\s+'\.\/[^']+\.ts';\s*$/.test(line)) // sibling imports only (single-line by convention here) + .map((line) => line.replace(/^export\s+(?=(?:function|const|let|class|interface|type|abstract\s+class)\b)/, '')) + .join('\n'); + if (/^(import|export)\b/m.test(src)) throw new Error(`inlineLexerRuntime: unexpected module syntax left in src/${file} (only sibling imports and declaration exports are inlineable)`); + return `// ── src/${file} (verbatim; declarations un-exported) ──\n${src}`; + }); + return [ + `// ── Inlined lexer runtime (lexerRuntime: 'inline'): ${INLINE_LEXER_SOURCES.map((f) => 'src/' + f).join(', ')} ──`, + `// Copied verbatim at emit time so this module needs no import to lex. Nothing but createLexer escapes the scope.`, + `const createLexer = (() => {`, + ...parts, + `return createLexer;`, + `})();`, + ].join('\n'); +} + // ── Runtime: the generic engine state + control loops, emitted verbatim ── // These are copied from gen-parser.ts so their semantics are byte-identical. The // ONLY change: where the interpreter called matchExpr(alt)/matchSeq(items) per arm, diff --git a/src/emit.ts b/src/emit.ts index fc85a2f..1329b7f 100644 --- a/src/emit.ts +++ b/src/emit.ts @@ -12,6 +12,22 @@ // `tsTarget`/`goTarget`/`rustTarget` (emit-portable.ts + target-*.ts). import type { CstGrammar } from './types.ts'; +// Per-emit options. Only `jsTarget` reads them today; the portable targets embed their own lexer +// and have nothing to resolve. +export interface EmitOptions { + // How a createLexer-FALLBACK grammar (indent / newline / markup: the data-driven lexer state + // machines are interpreter-only, so `embedLexer` is null) reaches the lexer runtime at load time: + // 'import' (default) — `import { createLexer } from ""`, + // resolved at emit time; the emitted file runs from anywhere on THIS machine. + // 'inline' — the runtime (src/gen-lexer.ts + the two helpers it uses + the type + // declarations) is copied verbatim into the module. The output is then + // STANDALONE: no import, no path, no dependency on monogram at load time. + // { import: spec } — a caller-supplied specifier (a package entry, a relative path the caller + // controls). The caller owns making `spec` resolve to gen-lexer's exports. + // Self-contained grammars (token-stream languages) embed a specialized lexer and ignore this. + lexerRuntime?: 'import' | 'inline' | { import: string }; +} + export interface Target { name: string; ext: string; // emitted file extension (no dot) @@ -22,7 +38,7 @@ export interface Target { // null where the lexer is not separable from the parser: jsTarget fuses lexing into its arena // pipeline (no token list), so there is no standalone tokenizer to emit. emitLexer(grammar: CstGrammar): string | null; - emitParser(grammar: CstGrammar, lexerSrc: string | null): string; // the parser LIBRARY (exports `tokenize` + `parse`; no I/O) + emitParser(grammar: CstGrammar, lexerSrc: string | null, opts?: EmitOptions): string; // the parser LIBRARY (exports `tokenize` + `parse`; no I/O) // A standalone CLI harness (stdin → CST JSON) APPENDED to the library to make it executable — // needed to run the compiled go/rust (and ts) parsers for verification. Not part of the parser. emitRunner?(): string; @@ -34,8 +50,8 @@ export function emitLexer(grammar: CstGrammar, target: Target): string | null { return target.emitLexer(grammar); } -export function emitParser(grammar: CstGrammar, target: Target): string { - return target.emitParser(grammar, target.embedLexer(grammar)); +export function emitParser(grammar: CstGrammar, target: Target, opts?: EmitOptions): string { + return target.emitParser(grammar, target.embedLexer(grammar), opts); } export { jsTarget } from './emit-parser.ts'; diff --git a/test/check.ts b/test/check.ts index 6dca4b6..7d926c2 100644 --- a/test/check.ts +++ b/test/check.ts @@ -29,6 +29,7 @@ const GATES: Gate[] = [ { group: 'emit-parity', name: 'emit-reject-messages', args: ['test/emit-reject-messages.ts'] }, { group: 'emit-parity', name: 'emit-lexer-verify', args: ['test/emit-lexer-verify.ts'] }, { group: 'emit-parity', name: 'emit-tsc-gate', args: ['test/emit-tsc-gate.ts'] }, + { group: 'emit-parity', name: 'emit-standalone', args: ['test/emit-standalone.ts'] }, { group: 'emit-parity', name: 'portable-targets', args: ['test/portable-targets.ts'] }, { group: 'emit-parity', name: 'unicode-parity', args: ['test/unicode-parity.ts'] }, { group: 'emit-parity', name: 'ast-builder', args: ['test/ast-builder.ts'] }, diff --git a/test/emit-standalone.ts b/test/emit-standalone.ts new file mode 100644 index 0000000..23e89b0 --- /dev/null +++ b/test/emit-standalone.ts @@ -0,0 +1,101 @@ +// Gate: a createLexer-FALLBACK grammar (indent / newline / markup) can be emitted as a STANDALONE +// module — `emitParser(grammar, jsTarget, { lexerRuntime: 'inline' })`. +// +// By default such a grammar's emitted parser imports the data-driven lexer runtime from THIS +// repo by absolute path (`import { createLexer } from "…/src/gen-lexer.ts"`), which is right for +// the in-repo gates and wrong for anyone shipping the emitted file: it only loads on the machine +// that emitted it. 'inline' copies the runtime into the module; `{ import: spec }` writes a +// caller-owned specifier instead. This gate proves, for YAML and HTML: +// 1. the inline module has NO import statement at all, and loads from a directory outside the repo; +// 2. it parses byte-identically to the interpreter (the same trees the import path produces); +// 3. it type-checks under `tsc --strict` WITHOUT --allowImportingTsExtensions (no .ts import remains); +// 4. `{ import }` writes exactly the given specifier, and the default is unchanged. +// +// Run with: node test/emit-standalone.ts +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { emitParser, jsTarget } from '../src/emit.ts'; +import { createParser } from '../src/gen-parser.ts'; +import { objectify } from './emitted-obj.ts'; +import type { CstGrammar } from '../src/types.ts'; + +let ok = 0, fail = 0; +const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log(' ✗', label); } }; + +const SAMPLES: Record = { + yaml: ['a: 1\nb:\n - x\n - y\nc: {k: v, n: [1, 2]}\n', '# comment\nkey: "quoted"\nblock: |\n line one\n line two\n', 'a:\n'], + html: ['
hi
there
', '

a < b

', '', '

unclosed bold'], +}; + +// A directory OUTSIDE the repo: a relative or absolute import into src/ cannot resolve from here by accident. +const dir = mkdtempSync(join(tmpdir(), 'monogram-standalone-')); +const TSC_FLAGS = ['--strict', '--noEmit', '--target', 'ES2022', '--module', 'ES2022', '--moduleResolution', 'Bundler', '--skipLibCheck']; + +for (const [name, samples] of Object.entries(SAMPLES)) { + let grammar: CstGrammar; + try { grammar = (await import(`../${name}.ts`)).default; } catch { console.log(` ${name}: (grammar not present — skipped)`); continue; } + check(`${name}: is a createLexer-fallback grammar (embedLexer is null)`, jsTarget.embedLexer(grammar) === null); + + // ── default: unchanged (absolute import into this repo) ── + const dflt = emitParser(grammar, jsTarget); + check(`${name}: default still imports createLexer by absolute path`, /^import \{ createLexer \} from "\/.*\/src\/gen-lexer\.ts";$/m.test(dflt)); + + // ── { import: spec }: the caller's specifier, verbatim ── + const spec = emitParser(grammar, jsTarget, { lexerRuntime: { import: 'monogram/lexer' } }); + check(`${name}: { import } writes the given specifier`, spec.includes('import { createLexer } from "monogram/lexer";')); + check(`${name}: { import } output is otherwise the default output`, spec.replace('"monogram/lexer"', '__X__') === dflt.replace(/"\/.*\/src\/gen-lexer\.ts"/, '__X__')); + + // ── 'inline': standalone ── + const inline = emitParser(grammar, jsTarget, { lexerRuntime: 'inline' }); + check(`${name}: inline output has no import statement`, !/^import\s/m.test(inline)); + check(`${name}: inline output has no export but the parser API (no runtime export leaked)`, !/^export (function|const) (createLexer|collectLiterals|tokenPatternSource)\b/m.test(inline)); + const file = join(dir, `${name}.ts`); + writeFileSync(file, inline); + let mod: any = null; + try { mod = await import(file + '?v=' + Date.now()); } catch (e) { console.log(` ${name}: inline module failed to load: ${(e as Error).message.split('\n')[0]}`); } + check(`${name}: inline module loads from outside the repo`, !!mod); + if (!mod) continue; + + // The claim is inline ≡ the default (import-path) emitted engine, tree AND errors, on valid and + // broken input alike; and where that engine parses cleanly, ≡ the interpreter too (emit-parser-verify + // owns emitted-vs-interpreter parity in general; recovery output is the emitted engine's own). + const dfltFile = join(dir, `${name}-default-import.ts`); + writeFileSync(dfltFile, dflt.replace(/^import \{ createLexer \} from "(\/.*\/src\/gen-lexer\.ts)";$/m, (_m, p) => `import { createLexer } from ${JSON.stringify(p)};`)); + const pd = (await import(dfltFile + '?v=' + Date.now())).createParser(); + const interp = createParser(grammar); + const p = mod.createParser(); + const treeOf = (parser: any, src: string) => { const cst = parser.parse(src); const obj = objectify(parser.tree, (fns: any) => parser.visit(cst, fns)); return JSON.stringify({ ...obj, errors: cst.errors }); }; + for (const src of samples) { + const viaInline = treeOf(p, src), viaImport = treeOf(pd, src); + check(`${name}: inline ≡ default emitted engine for ${JSON.stringify(src.slice(0, 24))}`, viaInline === viaImport); + if (JSON.parse(viaImport).errors.length === 0) { + check(`${name}: inline ≡ interpreter on the clean parse of ${JSON.stringify(src.slice(0, 24))}`, viaInline === JSON.stringify(interp.parseTotal(src))); + } + } + + try { + execFileSync('npx', ['tsc', ...TSC_FLAGS, file], { stdio: 'pipe' }); + check(`${name}: inline module type-checks (tsc --strict, no --allowImportingTsExtensions)`, true); + } catch (e: any) { + const log = (e.stdout?.toString() ?? '') + (e.stderr?.toString() ?? ''); + console.log(log.split('\n').slice(0, 8).join('\n')); + check(`${name}: inline module type-checks (tsc --strict, no --allowImportingTsExtensions)`, false); + } +} + +// ── A self-contained grammar (specialized lexer embedded) ignores the option: byte-identical output ── +{ + let ts: CstGrammar | null = null; + try { ts = (await import('../typescript.ts')).default; } catch { console.log(' typescript: (grammar not present — skipped)'); } + if (ts) { + check('typescript: embeds its own lexer (not a fallback grammar)', jsTarget.embedLexer(ts) !== null); + const base = emitParser(ts, jsTarget); + check("typescript: lexerRuntime 'inline' is a no-op (byte-identical output)", emitParser(ts, jsTarget, { lexerRuntime: 'inline' }) === base); + check("typescript: lexerRuntime { import } is a no-op (byte-identical output)", emitParser(ts, jsTarget, { lexerRuntime: { import: 'monogram/lexer' } }) === base); + } +} + +console.log(`\n${ok}/${ok + fail} standalone-emit checks pass${fail ? '' : ' ✓'}`); +if (fail) process.exit(1);