feat(newline): 'terminator' mode, a NEWLINE at every line break

newline mode was built for dotenv-style grammars, but its NEWLINE is a
statement SEPARATOR: one token between two content-bearing lines, placed
at the start of the later line, with blank and comment-only lines
collapsed and nothing before the first or after the last line. A grammar
whose AST keeps blank lines and comments as lines (env-spec: File =
many(Line), Line = Stmt NEWLINE? | NEWLINE) cannot be written on that,
which is why env-spec sits on indent mode today with INDENT/DEDENT
filler in every rule and an indent-stack tree-sitter scanner it never
uses.

NewlineConfig.mode: 'terminator' emits one NEWLINE at EVERY block-context
line break, placed at the break: blank lines, comment-only lines,
leading and trailing breaks included; breaks inside flow delimiters stay
suspended; a final line with no break has none. Positions are exact per
line, so a blank-line node maps to its own line. This is also the shape
the derived tree-sitter scanner already has (stateless, one NEWLINE per
break where the grammar permits one), so the two agree more closely than
in separator mode. 'separator' stays the default and is byte-identical.

The emitted engine inherits the mode through the createLexer fallback.
The portable TS/Go/Rust newline lexers on the portable-newline-mode
branch will need the same three emission points once that lands.

Gate: test/newline-mode.ts section 7.

Author: theoephraim

src/gen-lexer.ts

@@ -306,6 +306,10 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
   const kVoidNameTok = kOf(markup?.voidNameToken ?? null);
   const tTagOpen = markup ? (puLitOf.get(markup.tagOpen) ?? 0) : 0;
   const kNewlineModeTok = kOf(newline?.token ?? null);
+  // newline-only 'terminator' mode: a NEWLINE at EVERY block-context line break, placed at the break
+  // (see NewlineConfig.mode). The line-start boundary emission below is then skipped; the three
+  // places a break is consumed (a content line's break, a blank line, a tab-blank line) emit instead.
+  const nlTerminators = !indent && newline?.mode === 'terminator';
   const kIndentTok = kOf(indent?.indentToken ?? null), kDedentTok = kOf(indent?.dedentToken ?? null), kIndentNewlineTok = kOf(indent?.newlineToken ?? null);
   const kBlockScalarTok = kOf(indent?.blockScalar?.token ?? null);
   const kRawBlockTok = kOf(indent?.rawBlock?.token ?? null);
@@ -682,6 +686,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
         const ch = source[p];
         if (p >= source.length) { pos = p; lineStart = false; continue; }   // EOF — final DEDENTs emitted after the loop
         if (ch === '\n' || ch === '\r') {                                   // blank line — ignored for structure
+          if (nlTerminators) push(mkNamed(newline!.token, '', p, kNewlineModeTok));   // …but a line break all the same
           pos = p + 1; if (ch === '\r' && source[pos] === '\n') pos++;
           continue;                                                         // still at a line start
         }
@@ -696,6 +701,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
           let b = p; while (b < source.length && (source[b] === ' ' || source[b] === '\t')) b++;
           const bc = source[b];
           if (b >= source.length || bc === '\n' || bc === '\r') {
+            if (nlTerminators && bc !== undefined) push(mkNamed(newline!.token, '', b, kNewlineModeTok));
             pos = b; if (bc === '\r' && source[pos + 1] === '\n') pos += 2; else if (bc !== undefined) pos++;
             continue;
           }
@@ -728,7 +734,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
         // ── newline-only mode: no indent stack — emit ONE NEWLINE at this real line boundary (a
         // leading boundary before any content is suppressed via emittedContent) and move on. ──
         if (!indent) {
-          if (emittedContent) push(mkNamed(newline!.token, '', pos, kNewlineModeTok));
+          if (emittedContent && !nlTerminators) push(mkNamed(newline!.token, '', pos, kNewlineModeTok));
           lineStart = false;
           atLineLead = true;
           continue;
@@ -811,6 +817,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) {
           pos++; continue;
         }
         if (c === '\n' || c === '\r') {
+          if (nlTerminators && flowDepth === 0) push(mkNamed(newline!.token, '', pos, kNewlineModeTok));
           pos++; if (c === '\r' && source[pos] === '\n') pos++;
           if (flowDepth === 0) lineStart = true;
           else if (indent) {

src/types.ts

@@ -438,7 +438,19 @@ export interface NewlineConfig {
   token: string;        // token TYPE emitted at each significant line boundary (engine-emitted, like the indent tokens)
   flowOpen?: string[];  // punctuation that SUSPENDS newline significance while open (e.g. ['(', '[', '{'])
   flowClose?: string[]; // matching closers (e.g. [')', ']', '}'])
-  comment?: string;     // line-comment introducer; a comment-only line emits no NEWLINE (e.g. '#')
+  comment?: string;     // line-comment introducer; a comment-only line emits no NEWLINE in 'separator' mode (e.g. '#')
+  // WHAT a NEWLINE token means:
+  //   'separator' (default) — one NEWLINE between two content-bearing lines, placed at the start of
+  //     the later line's content. Blank and comment-only lines collapse into it; nothing is emitted
+  //     before the first content or after the last. The right shape for a statement SEPARATOR.
+  //   'terminator' — one NEWLINE at EVERY line break outside flow delimiters, placed AT the break
+  //     (zero-width, before the `\n` / `\r\n`): blank lines, comment-only lines, leading and
+  //     trailing breaks included; only a final line with no break has none. The right shape for a
+  //     grammar whose AST keeps blank lines and comments as lines (dotenv / env-spec: `File =
+  //     many(Line)`, `Line = [Stmt, opt(NEWLINE)] | [NEWLINE]`), and the shape the derived
+  //     tree-sitter scanner already has (it is stateless: one NEWLINE per break where the grammar
+  //     permits one). Positions are exact per line, so a blank-line node maps to its own line.
+  mode?: 'separator' | 'terminator';
 }
 
 export interface PrecOperator {

test/newline-mode.ts

@@ -167,5 +167,52 @@ if (hasCli()) {
   console.log('\ntree-sitter CLI not found — structural validation only (not a failure).');
 }
 
+
+// ── 7. `mode: 'terminator'`: one NEWLINE at EVERY block-context line break, placed at the break ──
+//      (blank / comment-only / leading / trailing breaks included; none inside flow; none at EOF
+//      without a break). The shape a grammar that keeps blank lines and comments as AST lines needs.
+{
+  const LineT = rule(() => [[Stmt, opt(Newline)], [Newline]]);                 // Line = Stmt NEWLINE? | NEWLINE (a blank line)
+  const ProgramT = rule(() => [[many(LineT)]]);
+  const gT = defineGrammar({
+    name: 'envspec-t', scopeName: 'source.envspec-t',
+    tokens: { Comment, Ident, Newline },
+    rules: { Value, Stmt, Line: LineT, Program: ProgramT }, entry: ProgramT,
+    newline: { ...newline, mode: 'terminator' },
+  });
+  const lexT = createLexer(gT).tokenize;
+  const nls = (s: string) => lexT(s).filter(t => t.type === 'Newline').map(t => t.offset);
+  check("terminator: one NEWLINE between two statements, AT the break", JSON.stringify(nls('A=1\nB=2')) === '[3]');
+  check("terminator: every blank line is its own NEWLINE", JSON.stringify(nls('A=1\n\n\nB=2')) === '[3,4,5]');
+  check("terminator: leading breaks are emitted", JSON.stringify(nls('\n\nA=1')) === '[0,1]');
+  check("terminator: a trailing break is emitted", JSON.stringify(nls('A=1\n')) === '[3]');
+  check("terminator: no break, no NEWLINE", JSON.stringify(nls('A=1')) === '[]');
+  check("terminator: a comment-only line still breaks", JSON.stringify(nls('A=1\n# note\nB=2')) === '[3,10]');
+  check("terminator: a whitespace-only line (tabs) is a break", JSON.stringify(nls('A=1\n \t\nB=2')) === '[3,6]');
+  check("terminator: CRLF counts once, at the CR", JSON.stringify(nls('A=1\r\n\r\nB=2')) === '[3,5]');
+  check("terminator: breaks INSIDE flow ( … ) stay suspended", JSON.stringify(nls('A=fn(1,\n2)\nB=3')) === '[10]');
+  check("terminator: still no INDENT/DEDENT", !lexT('A=1\n\nB=2').some(t => t.type === 'Indent' || t.type === 'Dedent'));
+  check("separator (default) is unchanged by the option's existence", countNL('A=1\n\n\nB=2') === 1 && tokenize('\n\nA=1')[0]?.type !== 'Newline');
+
+  const parseT = createParser(gT).parse;
+  const acceptsT = (s: string) => { try { return parseT(s).rule !== undefined; } catch { return false; } };
+  check('terminator: parses statements with blank lines between and a trailing break', acceptsT('A=1\n\nB=2\n'));
+  check('terminator: parses leading blank lines', acceptsT('\n\nA=1'));
+  check('terminator: parses a lone statement without a break', acceptsT('A=1'));
+  check('terminator: parses a flow value spanning lines', acceptsT('A=fn(1,\n2)\nB=3'));
+  check('terminator: still rejects a malformed statement', !acceptsT('A B'));
+  // A blank line is its own Line node, at its own offset (what an editor maps to a line number).
+  const cst = parseT('A=1\n\nB=2') as any;
+  const nlLeaves = JSON.stringify(cst).match(/"tokenType":"Newline","offset":(\d+)/g) ?? [];
+  check('terminator: the blank line and the statement break are distinct leaves at 3 and 4', nlLeaves.length === 2 && nlLeaves[0].endsWith(':3') && nlLeaves[1].endsWith(':4'));
+
+  // The generators do not care which mode is set (the tree-sitter scanner is already per-break).
+  check('terminator: TextMate generates', Object.keys(generateTmLanguage(gT).repository).length > 0);
+  check('terminator: Monarch generates', !!generateMonarch(gT).tokenizer.root);
+  const tsT = generateTreeSitter(gT, 'envspec_t');
+  check('terminator: tree-sitter externals still include newline', tsT.externalTokens.includes('newline'));
+  check('terminator: tree-sitter scanner unchanged (stateless per-break scan_newline)', tsT.scannerC.includes('scan_newline'));
+}
+
 console.log(fail === 0 ? `\n${ok}/${ok} newline-mode checks pass` : `\n${fail} of ${ok + fail} FAILED`);
 process.exit(fail === 0 ? 0 : 1);