Mirror #3226: parse TS variance annotations

CHANGELOG.md

@@ -53,6 +53,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- TypeScript variance annotations like `interface Stream<out A, in E>` or `<in out T>` now parse. Before, each one was a parse error, and in a large file enough of them could make the whole file fail to parse and show up with almost no symbols. Re-index TypeScript projects after upgrading.
 - In TypeScript, an awaited call with type arguments like `await fetchApi<User>("/me")` or `await api.get<T>(url)` now links to the function it calls. Before, the call was dropped, so typed fetch wrappers looked like they had no callers. (#2586)
 - In JavaScript and TypeScript, a call or `new` in a parameter's default value, like `function renderWithNote(note = buildNote())`, a destructured `{ size = defaultSize() }` or `constructor(private svc = new Service())`, now counts as the function's own call. Before, it was dropped, so `buildNote` was missing that caller and impact on it came back incomplete. Re-index JavaScript and TypeScript projects after upgrading. Thanks @inth3shadows for the report and the pull request. (#2588)
 - In React, a component rendered at module scope, in a top-level constant or inside a test's `it(…)` callback, like `createRoot(el).render(<App />)` or `render(<Card />)`, now counts as rendered there, so an app's entry reaches its root component and a component's tests show among its callers. (#2587)

README.md

@@ -2,11 +2,16 @@
 > **This is a personal fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph).**
 > It exists to carry fixes for bugs that put wrong edges in the graph, found while using CodeGraph
 > as the ground truth for other tools. Wrong edges are worse than missing ones: a
-> fabricated call makes every tool built on the graph confidently wrong. The fix it carries now:
+> fabricated call makes every tool built on the graph confidently wrong. The fixes it carries now:
 >
 > - **Impact is fast again.** `codegraph impact`, `codegraph_impact` and the viewer's blast
 >   radius had become 2–3x slower on busy symbols; the answer itself was unchanged
 >   (upstream [#3249](https://github.com/colbymchenry/codegraph/pull/3249), fixes #3248).
+> - **TypeScript variance annotations parse.** `interface Stream<out A, in E>` and other
+>   `in` / `out` type parameters no longer turn the declaration, and in large files the whole
+>   file, into a parse error that indexes as almost nothing (upstream
+>   [#3226](https://github.com/colbymchenry/codegraph/pull/3226); the grammar's own fix PRs,
+>   tree-sitter/tree-sitter-typescript#361 and #364, are unmerged). Re-index TypeScript projects after updating.
 >
 > Earlier fork fixes are now in upstream: `codegraph sync` on an index from an older engine
 > (#1842, which upstream made a warning rather than a refusal), imports onto an exported

__tests__/fixtures/ts-syntax/variance.ts

@@ -1,4 +1,4 @@
-// Known gap: variance annotations on type parameters (TS 4.7), tree-sitter-typescript#370.
+// Variance annotations (TS 4.7): fixed by the vendored grammar patch (#3226; upstream tree-sitter-typescript#370). Must keep parsing.
 export interface Producer<out T> { get(): T; }
 export interface Consumer<in T> { set(v: T): void; }
 export interface State<in out T> { get(): T; set(v: T): void; }

__tests__/typescript-variance-parse.test.ts

@@ -0,0 +1,147 @@
+/**
+ * TS 4.7 variance annotations (`interface Stream<out A, in E>`, `<in out T>`)
+ * were parse errors in tree-sitter-typescript v0.23.2 (upstream #370), and in
+ * a big file such errors can pile up into a whole-file ERROR. The vendored
+ * grammar (wasm and kernel) is patched; see docs/grammars/tree-sitter-typescript.md.
+ *
+ * The patch makes `in`/`out` keywords where a type parameter starts, so the
+ * cases below also pin that `out` still works as a name where real code uses
+ * it: the upstream fix PRs as written break `type F<out> = out[]`, and
+ * accepting `out` as a type name everywhere broke `[out: T]` tuple labels
+ * (Effect's JsonSchema.ts). Known limit, by choice: a type or JSX tag named
+ * `out`/`in` right after an expression `<` (`<out>x`, `<out>…</out>`,
+ * `<out.Provider>`) no longer parses.
+ */
+import { describe, it, expect, afterAll, beforeAll } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import type { Node } from 'web-tree-sitter';
+import { CodeGraph } from '../src';
+import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars';
+
+beforeAll(async () => {
+  await initGrammars();
+  await loadGrammarsForLanguages(['typescript', 'tsx']);
+});
+
+function parse(language: 'typescript' | 'tsx', src: string) {
+  return getParser(language)!.parse(src)!.rootNode;
+}
+
+/** The modifier tokens of every type_parameter, and its name. */
+function typeParams(root: Node): string[] {
+  return root.descendantsOfType('type_parameter').map((p) => {
+    const mods: string[] = [];
+    for (let i = 0; i < p.childCount; i++) {
+      const c = p.child(i)!;
+      if (!c.isNamed && ['in', 'out', 'const'].includes(c.type)) mods.push(c.type);
+    }
+    return [...mods, p.childForFieldName('name')!.text].join(' ');
+  });
+}
+
+describe('TypeScript variance annotations', () => {
+  it.each(['typescript', 'tsx'] as const)('%s: reads in / out / in out / const before the name', (language) => {
+    const cases: Array<[string, string[]]> = [
+      ['interface E<out A, in B, in out C> {}', ['out A', 'in B', 'in out C']],
+      ['class C<const in T> {}', ['const in T']],
+      ['class C<in const T> {}', ['in const T']],
+      ['export interface Channel<\n  out OutElem,\n  in InElem = unknown\n> {}', ['out OutElem', 'in InElem']],
+    ];
+    for (const [src, want] of cases) {
+      const tree = parse(language, src);
+      expect(tree.hasError, src).toBe(false);
+      expect(typeParams(tree), src).toEqual(want);
+    }
+  });
+
+  it.each(['typescript', 'tsx'] as const)('%s: still takes `out` as a parameter name', (language) => {
+    const cases: Array<[string, string[]]> = [
+      ['type F<out> = out[];', ['out']],
+      ['type H<out = number> = out;', ['out']],
+      ['type I<out, T> = [out, T];', ['out', 'T']],
+      ['type J<in out> = out;', ['in out']],
+      ['type K<out out> = out;', ['out out']],
+    ];
+    for (const [src, want] of cases) {
+      const tree = parse(language, src);
+      expect(tree.hasError, src).toBe(false);
+      expect(typeParams(tree), src).toEqual(want);
+    }
+  });
+
+  it('tsx: still takes `out` as a generic arrow parameter', () => {
+    const tree = parse('tsx', 'const c = <out,>(x: out) => x;');
+    expect(tree.hasError).toBe(false);
+    expect(typeParams(tree)).toEqual(['out']);
+  });
+
+  it('keeps `out` usable as a name outside type parameters', () => {
+    const tuple = parse('typescript', 'type P = readonly [out: number, value: string]');
+    expect(tuple.hasError).toBe(false);
+    expect(tuple.descendantsOfType('required_parameter').map((p) => p.childForFieldName('name')!.text)).toEqual(['out', 'value']);
+
+    const call = parse('typescript', 'f<out>(x)');
+    expect(call.hasError).toBe(false);
+    const callExpr = call.descendantsOfType('call_expression');
+    expect(callExpr).toHaveLength(1);
+    expect(callExpr[0]!.childForFieldName('type_arguments')!.text).toBe('<out>');
+
+    const mapped = parse('typescript', 'type M = { [out in K]: 1 }');
+    expect(mapped.hasError).toBe(false);
+    expect(mapped.descendantsOfType('mapped_type_clause').map((m) => m.childForFieldName('name')!.text)).toEqual(['out']);
+
+    const index = parse('typescript', 'type N = { [out: string]: 1 }');
+    expect(index.hasError).toBe(false);
+    expect(index.descendantsOfType('index_signature')).toHaveLength(1);
+
+    const predicate = parse('typescript', 'function isOut(out: unknown): out is string { return true }');
+    expect(predicate.hasError).toBe(false);
+    expect(predicate.descendantsOfType('type_predicate').map((p) => p.childForFieldName('name')!.text)).toEqual(['out']);
+
+    for (const src of [
+      'type Q = (out: number) => void',
+      'let out = 1; out++; const o = { out }; function g(out: number) { return out }',
+      'interface I { out: number; in: string }',
+      'enum E { out, in }',
+      'for (const k in obj) {}\nconst b = "a" in obj;',
+    ]) {
+      expect(parse('typescript', src).hasError, src).toBe(false);
+    }
+  });
+});
+
+describe('TypeScript variance annotations: extraction', () => {
+  const source = `export interface Stream<out A, in out E = never> {
+  map<B>(f: (a: A) => B): Stream<B, E>
+}
+
+export function runStream<A>(s: Stream<A>): A[] {
+  return []
+}
+`;
+  let root = '';
+  let cg: CodeGraph;
+
+  beforeAll(async () => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-ts-variance-'));
+    fs.mkdirSync(path.join(root, 'src'), { recursive: true });
+    fs.writeFileSync(path.join(root, 'src/stream.ts'), source);
+    cg = await CodeGraph.init(root, { index: true });
+  });
+
+  afterAll(() => {
+    cg?.close();
+    if (root) fs.rmSync(root, { recursive: true, force: true });
+  });
+
+  it('parses the file clean and extracts its declarations', () => {
+    // A small file recovers locally even on the old grammar, so the parse
+    // check is what fails without the patch; the extraction check guards
+    // that the new `in`/`out` tokens don't disturb the extractor.
+    expect(parse('typescript', source).hasError).toBe(false);
+    const names = cg.getNodesInFile('src/stream.ts').map((n) => `${n.kind}:${n.name}`);
+    expect(names).toEqual(expect.arrayContaining(['interface:Stream', 'method:map', 'function:runStream']));
+  });
+});

codegraph-kernel/Cargo.toml

@@ -20,7 +20,8 @@ regex = "1"
 # path loads (tree-sitter-wasms npm package / src/extraction/wasm/). The
 # kernel-grammar-parity test asserts node-kind-table equality at test time;
 # bump these together with the wasm side or that gate fails.
-tree-sitter-typescript = "0.23"
+# typescript + tsx: vendored C (grammars/typescript, compiled in build.rs) —
+# v0.23.2 carries docs/grammars/tree-sitter-typescript.patch, regenerated.
 tree-sitter-javascript = "0.25"
 tree-sitter-java = "0.23"
 tree-sitter-python = "0.23"
@@ -58,7 +59,7 @@ tree-sitter-r = "=1.2.0"
 tree-sitter-luau = "=1.2.0"
 
 # tree-sitter-language: the version-agnostic LanguageFn shim for the vendored
-# grammar C (kotlin, lua, scala, dart, swift — see build.rs).
+# grammar C (kotlin, lua, scala, dart, swift, typescript, tsx — see build.rs).
 tree-sitter-language = "0.1"
 
 # Stack-bounds queries for the walker stack guard (src/stack.rs, #1581):

codegraph-kernel/build.rs

@@ -112,4 +112,32 @@ fn main() {
     swift.flag_if_supported("-utf-8"); // msvc
     swift.compile("tree-sitter-swift");
     println!("cargo:rerun-if-changed=grammars/swift");
+
+    // TypeScript + TSX grammars — vendored C (sixth vendored-grammar-C
+    // language): tree-sitter/tree-sitter-typescript v0.23.2 (f975a62) with
+    // docs/grammars/tree-sitter-typescript.patch applied to the grammar source,
+    // then REGENERATED with tree-sitter-cli 0.25.10 (ABI 15) — the patch
+    // touches grammar rules, so the checked-in parser.c can't be reused. The
+    // vendored wasm is built from these same generated sources. Layout mirrors
+    // the upstream repo so scanner.c's `../../common/scanner.h` resolves
+    // unedited. The patch: TS 4.7 variance annotations (`<in T>`, `<out T>`).
+    //   common/scanner.h           da66ef2bd14a3f7ea743e25ba068c6c9aae2c3509db200ff80c4a0e6116e564c (unpatched)
+    //   typescript/src/parser.c    eafc3e8cd89fbbfdcec9182b22446f19c31756f3c8c1212cfd7bffdb4b761424
+    //   tsx/src/parser.c           232fe5b8178180307a40b393713d82074920977b147638bae898874085bcee86
+    // Flags crib the crate's bindings/rust/build.rs (c11). Each parser.c is
+    // ~9 MB.
+    for dialect in ["typescript", "tsx"] {
+        let src = format!("grammars/typescript/{dialect}/src");
+        let mut ts = cc::Build::new();
+        ts.std("c11");
+        ts.include(&src);
+        ts.file(format!("{src}/parser.c"));
+        ts.file(format!("{src}/scanner.c"));
+        ts.flag_if_supported("-Wno-unused-parameter");
+        ts.flag_if_supported("-Wno-unused-but-set-variable");
+        ts.flag_if_supported("-Wno-trigraphs");
+        ts.flag_if_supported("-utf-8"); // msvc
+        ts.compile(&format!("tree-sitter-{dialect}"));
+    }
+    println!("cargo:rerun-if-changed=grammars/typescript");
 }

codegraph-kernel/grammars/typescript/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.