diff --git a/.claude/skills/add-lang/SKILL.md b/.claude/skills/add-lang/SKILL.md index 37cbdce556..d71d15430a 100644 --- a/.claude/skills/add-lang/SKILL.md +++ b/.claude/skills/add-lang/SKILL.md @@ -60,11 +60,14 @@ still be unusable:** ```bash node scripts/add-lang/check-grammar.mjs path/to/valid-sample. ``` -It prints the grammar's ABI version and parses a valid sample many times in a -multi-grammar runtime. If it **FAILs** (ERROR trees on valid code — an old ABI +It prints the grammar's ABI version and runs it the way indexing does: in a +child process with indexing's V8 flags, beside the vendored grammars, parsing a +valid sample many times and once at the 1 MiB file-size limit. If it **FAILs** +(ERROR trees, or later parses giving a different tree, on valid code — an old ABI corrupting the shared WASM heap, which silently drops nested calls/imports on every file after the first; e.g. the tree-sitter-wasms **Lua** grammar is ABI 13 -and fails), do NOT use that wasm. **Vendor a newer (ABI 14/15) build instead:** +and fails — or the process running it crashing or hanging, even after every +parse finished), do NOT use that wasm. **Vendor a newer (ABI 14/15) build instead:** ```bash npm pack @tree-sitter-grammars/tree-sitter- # often ships a prebuilt *.wasm # or build one: npx tree-sitter build --wasm (needs Docker/emscripten) @@ -131,7 +134,7 @@ npm run build # tsc + copy-assets (copies any vendored *.wasm into di ``` Index a small sample repo and check extraction: ```bash -( cd && codegraph init -i ) +( cd && codegraph init ) node scripts/add-lang/verify-extraction.mjs ``` `verify-extraction.mjs` fails (exit 1) if the language isn't detected or only diff --git a/.cursor/rules/codegraph.mdc b/.cursor/rules/codegraph.mdc index 17d144a600..a51308252c 100644 --- a/.cursor/rules/codegraph.mdc +++ b/.cursor/rules/codegraph.mdc @@ -20,5 +20,5 @@ Reach for `codegraph_explore` before grep/find or Read for any **structural** qu ### If `.codegraph/` doesn't exist -The MCP server returns "not initialized." Ask the user: *"I notice this project doesn't have CodeGraph initialized. Want me to run `codegraph init -i` to build the index?"* +The MCP tools say the project isn't indexed. Indexing is the user's decision: tell them they can run `codegraph init` to build the index, but don't run it yourself. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..a63fc54f57 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Third-party tree-sitter grammars that the native kernel compiles from source +# (codegraph-kernel/build.rs). Keep all of them out of GitHub's language +# statistics, and collapse only the generated parser.c files in pull-request +# diffs. The hand-written scanner.c files are where C gets patched or added, +# so they stay expanded for review. +# The /** is needed: an attributes pattern that names a directory does not +# reach the files inside it. +codegraph-kernel/grammars/** linguist-vendored +codegraph-kernel/grammars/**/parser.c linguist-generated diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index b66dde951a..88a00efb06 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -24,9 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Build with Astro - uses: withastro/action@v3 + uses: withastro/action@v6 with: path: site node-version: 22 @@ -40,4 +40,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 diff --git a/AGENTS.md b/AGENTS.md index d46e3cc675..88d0dfb26d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,15 +47,14 @@ leaves behind, and asserts the seam: nothing outside `lib/adapter.js` may reach package is **prepared, not published** — `ui/package.json` carries `"private": true` deliberately, and `scripts/pack-npm.sh` only packs a tarball when `CODEGRAPH_PACK_UI=1`. -Tests run as **two vitest projects** (`vitest.workspace.mts`): `engine` (node) and `ui` (jsdom, the +Tests run as **two Vitest projects** (`test.projects` in `vitest.config.mts`): `engine` (node) and `ui` (jsdom, the Svelte plugin, `resolve.conditions: ['browser']`) for the single `__tests__/ui-package.test.ts`. `npm test` still runs both. The split is not cosmetic — `browser` is a package-resolution condition, and applied globally it hands the engine's suites the browser builds of `web-tree-sitter` and friends. The root config (`vitest.config.mts`, `.mts` because the plugin is -ESM-only and the repo is CJS) is the shared base; note that a workspace project **concatenates** -the base's `include` with its own, which is why the `ui` project does not `extends` it. +ESM-only and the repo is CJS) is the shared base; note that a project with `extends: true` **concatenates** the base's `include` with its own, which is why the `ui` project sets `extends: false`. -Node engines: `>=20.0.0 <25.0.0`. There is a hard exit on Node 25.x and below 20 (see `src/bin/node-version-check.ts`). +Node engines: `>=20.0.0 <25.0.0`. There is a hard exit on Node 25.x and below 20 (see `src/bin/node-version-check.ts`). That range is what users run. Working in this repo needs Node 22.12 or newer: `devEngines` makes `npm ci` and `npm run` stop on anything older, for Vite 7 and Vitest 4. ## Architecture @@ -76,17 +75,18 @@ The public API surface is `src/index.ts` — the `CodeGraph` class wires all the ### Module layout - `src/index.ts` — `CodeGraph` class: `init`/`open`/`close`, `indexAll`, `sync`, `searchNodes`, `getCallers`/`getCallees`, `getImpactRadius`, `buildContext`, `watch`/`unwatch`. + **It must stay light to load:** a read command, a query worker and the prompt hook all load it, and never index. What only indexing needs (extraction, resolution, the watcher) goes through `engine()` (`src/lazy-engine.ts`, listed in `src/engine.ts`), never an import of its own; `__tests__/lazy-engine.test.ts` fails otherwise. - `src/db/` — `DatabaseConnection`, `QueryBuilder` (prepared statements), `schema.sql`, `sqlite-adapter.ts`. Backed by Node's built-in **`node:sqlite`** (`DatabaseSync`) — real SQLite with WAL + FTS5, exposed through a thin better-sqlite3-shaped adapter. The bundled runtime always ships Node ≥22.5, so `node:sqlite` is always available: **no native build step and no wasm fallback**. (Running from source needs Node ≥22.5.) `codegraph status` reports the live backend (`node-sqlite`, the sole backend). - `src/extraction/` — `ExtractionOrchestrator`, tree-sitter wrappers, per-language extractors under `languages/` (one file per language), plus standalone extractors for non-tree-sitter formats (`svelte-extractor.ts`, `vue-extractor.ts`, `liquid-extractor.ts`, `dfm-extractor.ts` for Delphi). `parse-worker.ts` runs heavy parsing off the main thread. -- `src/resolution/` — `ReferenceResolver` orchestrates `import-resolver.ts` (with `path-aliases.ts` for tsconfig path aliases + cargo workspace member globs), `name-matcher.ts`, and `frameworks/` (Express, Laravel, Rails, FastAPI, Django, Flask, Spring, Gin, Axum, ASP.NET, Vapor, React Router, Next.js — `nextjs.ts`: pages and `route.ts` handlers from files, `router.push` / `redirect` / `NextResponse.redirect` as `navigates` edges, with `next-router-synthesizer.ts` for `` — Expo Router, SvelteKit, Vue/Nuxt, Cargo workspaces). Frameworks emit `route` nodes and `references` edges. `callback-synthesizer.ts` holds the whole-graph synthesis passes (`SYNTH_PASSES`, merged in registry order — first-seen wins a duplicate pair) with the language gates; `tier-synthesizer.ts` is the cross-tier pass (a client's literal `fetch`/`axios` path onto its own route, a queue job onto its consumer, a bus / socket event onto its handler — `channel`, `tier`, `registeredAt` on every edge; registered before the in-process emitter pass so its more specific edge wins); `synth-utils.ts` has the helpers they share (`enclosingFn`, `enclosingValue`, `makeLineAt`). Express's `postExtract` composes `app.use('/prefix', router)` mounts onto a mounted file's route names, idempotently (the original path stays in `qualifiedName`). +- `src/resolution/` — `ReferenceResolver` orchestrates `import-resolver.ts` (with `path-aliases.ts` for tsconfig path aliases + cargo workspace member globs), `name-matcher.ts`, and `frameworks/` (Express, Laravel, CodeIgniter 3, Rails, FastAPI, Django, Flask, Spring, Gin, Axum, ASP.NET, Vapor, React Router, Next.js — `nextjs.ts`: pages and `route.ts` handlers from files, `router.push` / `redirect` / `NextResponse.redirect` as `navigates` edges, with `next-router-synthesizer.ts` for `` and for the routes a proxy's `config.matcher` accepts, read by `next-proxy.ts` — Expo Router, SvelteKit, Vue/Nuxt, Cargo workspaces). Frameworks emit `route` nodes and `references` edges. `callback-synthesizer.ts` holds the whole-graph synthesis passes (`SYNTH_PASSES`, merged in registry order — first-seen wins a duplicate pair) with the language gates; `tier-synthesizer.ts` is the cross-tier pass (a client's literal `fetch`/`axios` path onto its own route, a queue job onto its consumer, a bus / socket event onto its handler — `channel`, `tier`, `registeredAt` on every edge; registered before the in-process emitter pass so its more specific edge wins); `synth-utils.ts` has the helpers they share (`enclosingFn`, `enclosingValue`, `makeLineAt`). Express's `postExtract` composes `app.use('/prefix', router)` mounts onto a mounted file's route names, idempotently (the original path stays in `qualifiedName`). - `src/graph/` — `GraphTraverser` (BFS/DFS, impact radius, path finding) and `GraphQueryManager` (high-level queries), plus the shared query-time derivations more than one surface renders: `named-symbol-flow.ts` (the one path finder, behind `codegraph_explore`'s Flow section and the viewer's Flow strip), `dynamic-boundary-report.ts` (where the graph stops), `type-hierarchy.ts` (ancestors/subtypes and the implementation count explore prints and the viewer draws), - `dead-code.ts` (unreferenced symbols, and every reason a candidate is NOT claimed), `synthesized-hop.ts` (how a synthesized hop reads, in explore's Flow, the `codegraph_node` trail and ContextBuilder's call paths — a new synthesizer's wording goes here). A derivation that two callers render must live here, not in `ToolHandler` — two derivations eventually disagree. + `dead-code.ts` (unreferenced symbols, and every reason a candidate is NOT claimed), `synthesized-hop.ts` (how a synthesized hop reads, in explore's Flow, the `codegraph_node` trail and ContextBuilder's call paths — a new synthesizer's wording goes here), `file-cycles.ts` (circular imports, in `codegraph cycles` and the viewer's map). A derivation that two callers render must live here, not in `ToolHandler` — two derivations eventually disagree. - `src/context/` — `ContextBuilder` + formatter for markdown/JSON output. - `src/search/` — full-text query parser and helpers for FTS5. - `src/sync/` — `FileWatcher` (native FSEvents/inotify/RDCW) with debounce + filter, and git-hook helpers. - `src/mcp/` — MCP server (`MCPServer`, `tools.ts`, `transport.ts`). `server-instructions.ts` is what the server returns in the MCP `initialize` response — keep it in sync with the user-facing tool guidance. - `src/installer/` — see below. -- `src/bin/codegraph.ts` — CLI (commander). Subcommands: `install`, `init`, `uninit`, `index`, `sync`, `status`, `query`, `files`, `context`, `affected`, `serve --mcp`. +- `src/bin/codegraph.ts` — CLI (commander). Subcommands: `install`, `uninstall`, `init`, `uninit`, `index`, `sync`, `status`, `unlock`, `query`, `explore`, `context`, `node`, `files`, `callers`, `callees`, `impact`, `affected`, `cycles`, `daemon`, `telemetry`, `upgrade`, `version`. Hidden from `--help`: `serve --mcp`, `prompt-hook`, and `ui`/`web` unless `CODEGRAPH_UI=1`. - `src/ui/` — terminal UI (shimmer progress, worker). - `src/ui-server/` -- read-only JSON API for the `codegraph ui` browser viewer (`api/`: `node`, `flow`, `map`, `screens`, `steps`, `deadcode`, `trails`, `program`, ...) plus static server; Svelte viewer lives in `ui/` (see `docs/design/codegraph-ui-design-spec.md`). `screens`/`steps`/`program` share one fold (`via`/`when` via `graph/branch-guards.ts`); `api/effects.ts` curates calls that leave the index; `api/route-roots.ts` names where a route's code starts. Derivations rendered by more than one surface belong in `src/graph/`, not `ToolHandler`. **Not released yet:** `codegraph ui` / `web` are refused (and hidden from `--help`) unless `CODEGRAPH_UI=1` is set — `src/bin/viewer-gate.ts`. Viewer changelog entries wait in `docs/viewer-launch-changelog.md`, not under `[Unreleased]`. At launch: delete the gate, move those entries back, drop the "not released yet" notes from `site/`. @@ -103,11 +103,13 @@ Defined in `src/types.ts`. Both extractors and resolvers must use these exact st `src/installer/` is the entry point for `codegraph install` (and the bare `codegraph`/`npx @colbymchenry/codegraph` invocation). Architecture: - `targets/registry.ts` lists every supported agent. -- `targets/types.ts` defines the `AgentTarget` interface — adding a 5th agent (Continue, Zed, Windsurf…) is **one new file in `targets/` + one entry in `registry.ts`**. Each target owns its config-file location and MCP-server JSON/TOML/JSONC writing. (Targets no longer write an instructions file — see below.) -- Current targets: `claude.ts`, `cursor.ts`, `codex.ts`, `opencode.ts`. -- `targets/toml.ts` is a hand-rolled TOML serializer scoped to `[mcp_servers.codegraph]` (used by Codex). Sibling tables and `[[array_of_tables]]` are preserved verbatim. No new dependency. +- `targets/types.ts` defines the `AgentTarget` interface — adding another agent (Continue, Zed…) is **one new file in `targets/` + one entry in `registry.ts`**. Each target owns its config-file location and MCP-server JSON/TOML/JSONC writing. (Claude, Codex, Gemini, opencode, Droid, Qwen, DSH, Grok, Kimi and Pi also write a short instructions-file block — see below.) +- Current targets: `claude.ts`, `cursor.ts`, `codex.ts`, `opencode.ts`, `hermes.ts`, `gemini.ts`, `antigravity.ts`, `kiro.ts`, `copilot-vscode.ts`, `copilot-cli.ts`, `copilot-jetbrains.ts`, `bob.ts`, `droid.ts`, `windsurf.ts`, `qwen.ts`, `dsh.ts`, `grok.ts`, `kimi.ts`, `qoder.ts`, `pi.ts`. +- `targets/toml.ts` is a hand-rolled TOML serializer scoped to `[mcp_servers.codegraph]` (used by Codex and Grok, through `writeTomlTable` / `deleteTomlTable`). Sibling tables and `[[array_of_tables]]` are preserved verbatim. No new dependency. +- `targets/json-config.ts` edits `mcpServers.codegraph` in place through `jsonc-parser`, for Bob, Droid, Windsurf, Qwen, Kimi, Qoder and Pi. An entry already in the file is the user's: agents write switches into it and users pin a `command` or `cwd`, so a re-install or `--refresh` only fills in missing keys. DSH's and Grok's writers keep to the same rule. Use it for a new agent with that file shape; a file with nothing left in it is deleted on uninstall, through `removeFileUnlessLinked`. +- `*_HOME`-style overrides (`QWEN_HOME`, `KIMI_CODE_HOME`, `GROK_HOME`, `DSH_HOME`, `QODER_CONFIG_DIR`, `PI_CODING_AGENT_DIR`) go through `configDirFromEnv` in `targets/shared.ts`; each must also be cleared in `__tests__/setup-home-sandbox.ts` and in the suite's `setHome`. - opencode reads `opencode.jsonc` by default; the installer prefers existing `.jsonc`, falls back to `.json`, and creates `.jsonc` for greenfield installs. Edits are surgical via `jsonc-parser` so user comments and formatting survive install/re-install/uninstall round-trips. The MCP entry is OpenCode 2's native `mcp.servers.codegraph` with `disabled: false` and `codemode: false` (so `codegraph_explore` stays on the native tool list); a pre-#1698 `mcp.codegraph` + `enabled` entry is migrated on re-install and removed by uninstall. -- `instructions-template.ts` no longer holds an instructions body — it exports only the ``/`` markers. The installer **stopped writing** a `## CodeGraph` block into each agent's instructions file (`CLAUDE.md` / `~/.codex/AGENTS.md` / `~/.config/opencode/AGENTS.md` / `~/.gemini/GEMINI.md` / `.cursor/rules/codegraph.mdc` / Kiro steering doc) because it duplicated the MCP `initialize` instructions verbatim (issue #529). Each target's `install` (self-heal on upgrade) and `uninstall` use the markers to **strip** a block a previous install left behind. `server-instructions.ts` is the single source of truth for agent-facing guidance. +- `instructions-template.ts` holds `CODEGRAPH_INSTRUCTIONS_BLOCK`, a deliberately short marker-fenced (``/``) `## CodeGraph` block. The Claude, Codex, Gemini, opencode, Droid, Qwen, DSH, Grok, Kimi and Pi targets write it into their instructions file (`CLAUDE.md` / `~/.codex/AGENTS.md` / `~/.gemini/GEMINI.md` / `~/.config/opencode/AGENTS.md` / `~/.factory/AGENTS.md` / `~/.qwen/QWEN.md` / `~/.dsh/AGENTS.md` / `~/.grok/rules/codegraph.md` / `~/.kimi-code/AGENTS.md` / `~/.pi/agent/AGENTS.md`) via `upsertInstructionsEntry`, and `uninstall` removes it. History: #529 removed the old full playbook because it duplicated the MCP `initialize` instructions; #704 brought back this small block because the instructions file reaches two audiences `initialize` cannot — Task-tool subagents and non-MCP harnesses (the `codegraph explore` CLI). Cursor and Kiro write no block; their `install` only strips one a previous install left behind. Bob, Windsurf and Qoder are MCP config only. Keep the block to command names and the two surfaces; tool guidance itself still belongs in `server-instructions.ts`. - All installer changes need matching coverage in `__tests__/installer-targets.test.ts` — there are ~47 parameterized contract tests covering install idempotency, sibling preservation, uninstall reverses install, byte-equal re-runs returning `unchanged`, and partial-state recovery for Codex. ### Cursor MCP working-directory quirk @@ -116,7 +118,7 @@ Cursor launches MCP subprocesses with the wrong cwd and doesn't pass `rootUri` i ### MCP server instructions -`src/mcp/server-instructions.ts` is sent back to the agent in the MCP `initialize` response. This is the *first* thing every agent sees about how to use the tools, and as of issue #529 it is the **single source of truth** for agent-facing tool guidance — the installer no longer writes a duplicate `## CodeGraph` instructions block into `CLAUDE.md` / `AGENTS.md` / `.cursor/rules/codegraph.mdc`. Edit tool guidance here and nowhere else. +`src/mcp/server-instructions.ts` is sent back to the agent in the MCP `initialize` response. This is the *first* thing every agent sees about how to use the tools, and as of issue #529 it is the **single source of truth** for agent-facing tool guidance. The installer's instructions-file block (`instructions-template.ts`, see above) only names the tool and the CLI for subagents and non-MCP agents; it does not repeat this guidance. Edit tool guidance here and nowhere else. ## Retrieval performance & dynamic-dispatch coverage (do not regress) @@ -174,7 +176,7 @@ Full methodology (feedback metrics, CLI contamination guard, Sonnet/`--effort hi Tests live in `__tests__/` and mirror the module they cover. Notable ones beyond the obvious: -- `installer-targets.test.ts` — parameterized contract suite across all 4 agent targets (see installer notes above). +- `installer-targets.test.ts` — parameterized contract suite across every registered target (`ALL_TARGETS`); see the installer notes above. - `evaluation/` — `runner.ts` + `test-cases.ts` exercise codegraph against synthetic projects and score the results; run via `npm run eval` (builds first). Not part of `npm test`. - `sqlite-backend.test.ts` / `node-sqlite-backend.test.ts` — pin that `node:sqlite` is the sole backend: `getBackend()` reports `node-sqlite` and the DB comes up in WAL. - `pr19-improvements.test.ts`, `frameworks-integration.test.ts` — regression coverage for specific past PRs/incidents; don't rename these, the names anchor to git history. @@ -279,7 +281,7 @@ publish actions on shared state. Write the files, hand the user the commands. ## House rules - The `0.7.x` line is in active multi-agent rollout. Any change to `src/installer/` (especially `targets/`) needs corresponding test coverage and a CHANGELOG entry — installer regressions break every new install silently. -- When changing what the MCP tools do or how agents should use them, edit `src/mcp/server-instructions.ts` — it is the **single source of truth** for agent-facing tool guidance (issue #529). The installer no longer writes a duplicate instructions block into `CLAUDE.md` / `AGENTS.md` / `GEMINI.md` / `.cursor/rules/codegraph.mdc` / Kiro steering, so there's nothing to keep in sync anymore. (The repo's own checked-in `.cursor/rules/codegraph.mdc` is dogfooding config — update it too if you use Cursor on this repo, but it ships nowhere.) +- When changing what the MCP tools do or how agents should use them, edit `src/mcp/server-instructions.ts` — it is the **single source of truth** for agent-facing tool guidance (issue #529). The installer's short instructions-file block (`instructions-template.ts`) only names `codegraph_explore` and the `codegraph explore` CLI; update it only when a tool or CLI command is renamed or removed. (The repo's own checked-in `.cursor/rules/codegraph.mdc` is dogfooding config — update it too if you use Cursor on this repo, but it ships nowhere.) - **Before adding or extending a router, a web framework, or a language's `WHEN` rules, read `docs/design/framework-coverage.md`.** It is the standing answer to "what is supported and what is left" across the three axes (route nodes → Entry points, `navigates` edges → Screens, branch-guard rules → the `WHEN` labels), with what each remaining item needs, the traps that have already cost debugging time, and the queries to re-verify it. Update it in the same change that moves a row. - CodeGraph provides **code context**, not product requirements. For new features, ask the user about UX, edge cases, and acceptance criteria — the graph won't tell you. - **When the user references issues, PR comments, or external reports, anchor them to a date and version before drawing conclusions.** Check the comment's `createdAt` against: diff --git a/BUNDLING.md b/BUNDLING.md index dc21ab5312..90d53ac9d1 100644 --- a/BUNDLING.md +++ b/BUNDLING.md @@ -33,6 +33,10 @@ scripts/build-bundle.sh linux-x64 # -> release/codegraph-linux-x64.ta scripts/build-bundle.sh win32-x64 # -> release/codegraph-win32-x64.zip ``` +The Node runtime is the official nodejs.org release named by the script's +default `NODE_VERSION` (or its second argument). The download is refused +unless it matches the SHA-256 listed in that release's `SHASUMS256.txt`. + Because dropping better-sqlite3 left **zero native addons**, building a bundle is pure file-packaging — **any** target builds on **any** OS (the whole matrix builds on one Linux runner). Cross-compilation isn't a concern; only *run-testing* a diff --git a/CHANGELOG.md b/CHANGELOG.md index 6968a89507..bc8c2f441a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,49 +38,199 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). extraction paths. Previously it got no callable node, so nothing could call it and the calls in its body were attributed to the file instead. +### Security + +- CodeGraph's refusal to use a sensitive folder such as `~/.ssh`, `~/.config`, `/etc` or `C:\Windows` as a project can no longer be sidestepped: a link or junction that leads to one, a differently capitalized name on Windows and macOS, a `\\?\` spelling on Windows, and a path to a missing folder inside one are now refused like the folder itself. Before, any of these let an AI agent's query read the files inside. Thanks @mixxer for finding it. +- The MCP server no longer uses a sensitive folder such as `~/.ssh`, `~/.config` or `/etc` as a session's default project, whether the folder came from your AI agent's workspace, the folder the server was started in, or `--path`. Before, only a folder an agent asked for by path was refused, so a session started in one that held an index answered from the files inside and kept it synced; now that session has no default project, and the agent can still query any other project by path. + +### New Features + +- `codegraph files --format package` and `codegraph_files` with `format: "package"` now list a project's files by the package that owns them, the nearest `package.json` or `composer.json` above each file that names one, so a monorepo splits into its workspace members with no setup and files nothing claims get a section of their own (#1871). Thanks @LuisAlbertoMK. +- In C#, a MediatR `Send` or `Publish` now reaches a handler written against the project's own interface, like `ICommandHandler` or `IQueryHandler`, or one from a package, such as the Mediator library or Ardalis.SharedKernel, with nothing to configure. A command built by a mapper, like `mapper.Map(request)`, is read as what it is, and two solutions kept side by side in one repository no longer link to each other's handlers. Thanks @PressXtoChris. +- `codegraph files --json` now marks each file as a test file or not, `--tests` and `--no-tests` filter the list in any format, and `codegraph node --file` says "test file" in its header, so your own tooling can ask CodeGraph instead of keeping a copy of its naming rules. Thanks @maxmilian. (#1877) +- `codegraph install` now sets up IBM Bob, for Bob IDE and Bob Shell: it adds the CodeGraph MCP server to `~/.bob/settings/mcp.json`, or to `.bob/mcp.json` in a project. Thanks @QuentinDstl. +- ASP.NET Razor views now link to the JavaScript their inline `\n`); + } + cg = CodeGraph.initSync(testDir); + await cg.indexAll(); + }); + + afterEach(() => { + if (cg) cg.destroy(); + if (fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true }); + }); + + it('lists views in their own slot, past the cap on code callers', async () => { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query: 'fmt' }); + const text = res.content[0]!.text; + expect(text).toMatch(/\+1 more; 2 views: `Views\/(?:Edit|Index)\.cshtml`, `Views\/(?:Edit|Index)\.cshtml`/); + }); + + it('reads the same when only views depend on a symbol', async () => { + fs.writeFileSync(path.join(testDir, 'src', 'only.js'), 'export function onlyViews() { return 1; }\n'); + fs.writeFileSync(path.join(testDir, 'Views', 'Solo.cshtml'), "\n"); + await cg.sync(); + const res = await new ToolHandler(cg).execute('codegraph_explore', { query: 'onlyViews' }); + expect(res.content[0]!.text).toMatch(/1 caller in 1 view: `Views\/Solo\.cshtml`/); + }); +}); diff --git a/__tests__/explore-cross-call-dedup.test.ts b/__tests__/explore-cross-call-dedup.test.ts index ce208dc95f..e18cc4371b 100644 --- a/__tests__/explore-cross-call-dedup.test.ts +++ b/__tests__/explore-cross-call-dedup.test.ts @@ -42,6 +42,37 @@ const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'payroll-go'); const QUERY = 'how does payroll cycle create and calculate payslips?'; const POINTER = 'Already sent earlier in this conversation'; +/** + * Give every file of the copied fixture LF line endings. A Windows checkout + * with `core.autocrlf` writes the fixture with CRLF, and explore's per-file + * character budget counts each line's `\r`, so there a file renders fewer + * lines and the second call meets different held spans than on Linux or macOS + * (#2756). The suite tests one scenario on every checkout. + */ +function normalizeLineEndings(dir: string): void { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) { + normalizeLineEndings(file); + continue; + } + // latin1 maps every byte to one char, so the rewrite changes nothing but the `\r`s. + const text = fs.readFileSync(file, 'latin1'); + if (text.includes('\r\n')) fs.writeFileSync(file, text.replace(/\r\n/g, '\n'), 'latin1'); + } +} + +/** Consecutive line numbers grouped into runs: [3, 4, 5, 9] → [[3, 4, 5], [9]]. */ +function consecutiveRuns(lines: Iterable): number[][] { + const runs: number[][] = []; + for (const n of [...lines].sort((a, b) => a - b)) { + const run = runs[runs.length - 1]; + if (run && n === run[run.length - 1]! + 1) run.push(n); + else runs.push([n]); + } + return runs; +} + describe('dedup configuration', () => { it('defaults off and requires an explicit truthy opt-in', () => { const previous = process.env.CODEGRAPH_EXPLORE_DEDUP; @@ -213,6 +244,7 @@ describe('a second call against a real index', () => { testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cg18-')); fs.cpSync(FIXTURE_SRC, testDir, { recursive: true }); fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true }); + normalizeLineEndings(testDir); cg = CodeGraph.initSync(testDir); await cg.indexAll(); handler = new ToolHandler(cg); @@ -266,10 +298,16 @@ describe('a second call against a real index', () => { expect(second).toContain(POINTER); for (const [file, lines] of before) { const repeated = [...(after.get(file) ?? [])].filter((n) => lines.has(n)); - if (repeated.length === 0) continue; - // The only sanctioned repeat is the anti-abandonment restore, which fires - // ONLY when the call found nothing new to say — and this one did. - throw new Error(`call 2 re-sent ${file} lines ${repeated.slice(0, 5).join(',')}`); + // Two repeats are sanctioned. The anti-abandonment restore fires ONLY + // when the call found nothing new to say, and this one did. And rule 3 + // of the range algebra: a held run shorter than MIN_COVERED_LINES stays + // inside the span around it rather than becoming a pointer (cycle.go's + // 7-line `RunResult` struct, held by call 1, comes back inside call 2's + // render of the file's head, #2756). A longer run is a real duplicate. + for (const run of consecutiveRuns(repeated)) { + if (run.length < EXPLORE_DEDUP.MIN_COVERED_LINES) continue; + throw new Error(`call 2 re-sent ${file} lines ${run[0]}-${run[run.length - 1]} (${run.length} lines)`); + } } }, 120_000); diff --git a/__tests__/explore-exact-target.test.ts b/__tests__/explore-exact-target.test.ts index d54d36798f..b1d11958f3 100644 --- a/__tests__/explore-exact-target.test.ts +++ b/__tests__/explore-exact-target.test.ts @@ -107,7 +107,7 @@ class SQLCompiler: def get_select(self, with_col_aliases=False): # GET_SELECT_BODY -${filler('sel', 34)} +${filler('sel', 60)} return [] def get_qualify_sql(self): @@ -235,10 +235,14 @@ class IExact: return result.content?.[0]?.text ?? ''; }; - it('fixture sanity: the flow runs as_sql → get_qualify_sql → get_select and compiler.py goes focused', async () => { + it('fixture sanity: the flow runs as_sql → pre_sql_setup → setup_query → get_select and compiler.py goes focused', async () => { const text = await explore('SQLCompiler.as_sql pre_sql_setup get_select'); expect(text).toContain('**Flow (call path among the symbols you queried)'); - expect(text).toMatch(/1\. as_sql \(compiler\.py:\d+\)[\s\S]*2\. get_qualify_sql[\s\S]*3\. get_select/); + // Through every symbol the query names (#2970), as on django itself: the + // shorter route through the unnamed `get_qualify_sql` skipped `pre_sql_setup`. + // The named and spine bodies outgrow compiler.py's share, as django's do + // (18,390 chars against 8,895), which is what makes it a per-symbol view. + expect(text).toMatch(/1\. as_sql \(compiler\.py:\d+\)[\s\S]*2\. pre_sql_setup[\s\S]*3\. setup_query[\s\S]*4\. get_select/); expect(sectionFor(text, COMPILER), 'the family file renders as a per-symbol view').toContain('· focused'); }); diff --git a/__tests__/explore-long-query.test.ts b/__tests__/explore-long-query.test.ts new file mode 100644 index 0000000000..5f635564fe --- /dev/null +++ b/__tests__/explore-long-query.test.ts @@ -0,0 +1,390 @@ +/** + * Long `codegraph_explore` queries on a large index (#2620). + * + * The prompt hook hands explore the whole prompt, so a pasted log, document + * or source file reaches it as a query of hundreds of words, and every word + * cost several reads that each scanned the index: on a 560k-node index such + * a query took one to four minutes and the hook gave up. These tests pin the + * two halves of the fix: + * + * - the per-word reads give exactly the rows they gave before, now from + * memory (the name-substring lookups, the fuzzy fallback's name list, the + * LIKE fallback's "nothing to find" check), read once per database state; + * - a query searches at most MAX_QUERY_SYMBOLS names and MAX_QUERY_TERMS + * terms, code-shaped words first, so its cost no longer grows with its + * length. + */ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src'; +import { QueryBuilder } from '../src/db/queries'; +import { MAX_QUERY_SYMBOLS, MAX_QUERY_TERMS } from '../src/context'; +import { scorePathRelevance, extractSearchTerms, normalizeNameToken, isTestFile } from '../src/search/query-utils'; +import type { SqliteDatabase } from '../src/db/sqlite-adapter'; + +function queriesOf(cg: CodeGraph): QueryBuilder { + return (cg as unknown as { queries: QueryBuilder }).queries; +} +function rawDb(cg: CodeGraph): SqliteDatabase { + return (cg as unknown as { db: { getDb(): SqliteDatabase } }).db.getDb(); +} + +/** Count the statements run whose SQL matches `pattern`, through the adapter. */ +function countStatements(db: SqliteDatabase, pattern: RegExp): { count: number } { + const counter = { count: 0 }; + const prepare = db.prepare.bind(db); + vi.spyOn(db, 'prepare').mockImplementation((sql: string) => { + const stmt = prepare(sql); + if (!pattern.test(sql)) return stmt; + const wrap = any>(fn: T): T => ((...args: any[]) => { counter.count++; return fn(...args); }) as T; + return { ...stmt, all: wrap(stmt.all.bind(stmt)), get: wrap(stmt.get.bind(stmt)), iterate: wrap(stmt.iterate.bind(stmt)), run: stmt.run.bind(stmt) }; + }); + return counter; +} + +const DEF = ['class', 'interface', 'struct', 'union', 'trait', 'protocol', 'enum', 'type_alias']; +const CALL = ['function', 'method', 'component']; +const SEED = ['function', 'method', 'component', 'variable', 'constant']; + +/** The lookup as it was before the index: a LIKE scan, ties left to the plan. */ +function likeScan(db: SqliteDatabase, substring: string, kinds: string[], excludePrefix: boolean, limit: number): string[] { + let sql = 'SELECT nodes.*, 1.0 as score FROM nodes WHERE name LIKE ?'; + const params: (string | number)[] = [`%${substring}%`]; + if (excludePrefix) { sql += ' AND name NOT LIKE ?'; params.push(`${substring}%`); } + sql += ` AND kind IN (${kinds.map(() => '?').join(',')}) ORDER BY length(name) ASC LIMIT ?`; + return (db.prepare(sql).all(...params, ...kinds, limit) as Array<{ id: string }>).map((row) => row.id); +} + +/** Many names of equal length across kinds and files, so ties decide the order. */ +function writeTieProject(dir: string): void { + const files: Record = {}; + for (let f = 0; f < 6; f++) { + const lines: string[] = []; + for (let i = 0; i < 12; i++) { + const n = (f * 7 + i * 5) % 12; + lines.push(`export function load${n}Item${f}(): void {}`); + lines.push(`export class Item${n}Loader${f} { loadItem${n}(): void {} }`); + lines.push(`export interface Item${n}Shape${f} { loadItem${n}Shape(): void; }`); + lines.push(`export const ITEM_${n}_LOADER_${f} = ${i};`); + } + lines.push('export function load_item_snake(): void {}'); + lines.push('export function loadXitemXsnake(): void {}'); + lines.push('export class LOADITEMLOUD {}'); + lines.push('export function größeLaden(): void {}'); + lines.push('export function GrößeLaden(): void {}'); + lines.push('export function ÉtatLoader(): void {}'); + lines.push('export function étatLoader(): void {}'); + lines.push('export function 𝑥ItemLoader(): void {}'); + files[`src/part${(f * 5) % 6}/items${f}.ts`] = lines.join('\n') + '\n'; + } + for (const [rel, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), text); + } +} + +const PATTERNS = [ + 'Item', 'item', 'ITEM', 'Loader', 'Load', 'load', 'Item1', 'Item1loader', 'Loaditem', 'Shape', 'Oader', 'O', + 'Load_item', 'Item_', '_', 'I_em', 'Load_i', 'Item_1', 'Größe', 'größe', 'GRÖSSE', 'État', 'état', 'Etat', + '𝑥', '𝑥item', 'Xitem', 'Loud', 'Missingword', 'Item.', 'Item(', +]; + +describe('name-substring lookups give the rows a LIKE scan gives (#2620)', () => { + let dir: string; + const open: CodeGraph[] = []; + afterEach(() => { + for (const cg of open.splice(0)) cg.close(); + vi.restoreAllMocks(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + async function setup(): Promise { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-substr-')); + writeTieProject(dir); + const cg = await CodeGraph.init(dir, { index: true }); + open.push(cg); + return cg; + } + + function expectSameRows(cg: CodeGraph, plan: 'rowid' | 'index'): void { + const q = queriesOf(cg); + const db = rawDb(cg); + let compared = 0; + for (const substring of PATTERNS) { + for (const kinds of [DEF, CALL, SEED, ['method'], ['constant', 'function']]) { + for (const excludePrefix of [false, true]) { + for (const limit of [200, 7, 1]) { + const expected = likeScan(db, substring, kinds, excludePrefix, limit); + const actual = q.findNodesByNameSubstring(substring, { kinds: kinds as never, excludePrefix, limit }).map((r) => r.node.id); + expect(actual, `${substring} ${kinds.join(',')} excludePrefix=${excludePrefix} limit=${limit}`).toEqual(expected); + compared++; + } + } + } + } + expect(compared).toBeGreaterThan(900); + // The answers came from the index, under the plan SQLite chose. + const memo = (q as unknown as { readMemos: Map } | null }> }) + .readMemos.get('nameSubstringIndex'); + expect(memo?.value).toBeTruthy(); + const plans = [...memo!.value!.plans.values()]; + expect(plans).toContain(plan); + expect(plans).not.toContain(null); + } + + it('matches when SQLite scans the table (a small index)', async () => { + const cg = await setup(); + const plan = rawDb(cg).prepare( + `EXPLAIN QUERY PLAN SELECT * FROM nodes WHERE name LIKE '%x%' AND kind IN (${DEF.map(() => '?').join(',')}) ORDER BY length(name) LIMIT 200`, + ).all(...DEF) as Array<{ detail: string }>; + expect(plan.map((r) => r.detail)).toContain('SCAN nodes'); + expectSameRows(cg, 'rowid'); + }); + + it('matches when SQLite reads through the kind index (a large index)', async () => { + const cg = await setup(); + const db = rawDb(cg); + // The statistics of a large index (vscode's), so the planner reads by kind. + db.exec('ANALYZE'); + db.exec("UPDATE sqlite_stat1 SET stat = '562636' || substr(stat, instr(stat, ' ')) WHERE tbl = 'nodes' AND instr(stat, ' ') > 0"); + db.exec("UPDATE sqlite_stat1 SET stat = '562636 1001 3 1 1' WHERE idx = 'idx_nodes_kind'"); + db.exec('ANALYZE sqlite_schema'); + const plan = db.prepare( + `EXPLAIN QUERY PLAN SELECT * FROM nodes WHERE name LIKE '%x%' AND kind IN (${DEF.map(() => '?').join(',')}) ORDER BY length(name) LIMIT 200`, + ).all(...DEF) as Array<{ detail: string }>; + expect(plan.map((r) => r.detail).some((d) => d.startsWith('SEARCH nodes USING INDEX idx_nodes_kind'))).toBe(true); + expectSameRows(cg, 'index'); + }); + + it('asks SQLite inside a transaction, and for a lookup without a kind filter', async () => { + const cg = await setup(); + const q = queriesOf(cg); + const db = rawDb(cg); + const scans = countStatements(db, /^\s*SELECT nodes\.\*, 1\.0 as score\s+FROM nodes\s+WHERE name LIKE \?/); + db.exec('BEGIN'); + try { + expect(q.findNodesByNameSubstring('Item', { kinds: DEF as never, limit: 50 }).map((r) => r.node.id)) + .toEqual(likeScan(db, 'Item', DEF, false, 50)); + } finally { + db.exec('ROLLBACK'); + } + expect(scans.count).toBe(2); // the lookup itself, and the comparison + q.findNodesByNameSubstring('Item', { limit: 50 }); + expect(scans.count).toBe(3); + q.findNodesByNameSubstring('Item', { kinds: DEF as never, limit: 50 }); + expect(scans.count).toBe(3); + }); +}); + +describe('per-word reads happen once per database state (#2620)', () => { + let dir: string; + const open: CodeGraph[] = []; + afterEach(() => { + for (const cg of open.splice(0)) cg.close(); + vi.restoreAllMocks(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + async function setup(): Promise { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-perword-')); + fs.mkdirSync(path.join(dir, 'src', 'zorbtown'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'src', 'zorbtown', 'orders.ts'), [ + 'export class OrderStateMachine {', + ' submitOrder(): boolean { return this.transition("submitted"); }', + ' transition(state: string): boolean { return state.length > 0; }', + '}', + 'export function linkSessions(ids: string[]): string[] { return ids; }', + ].join('\n') + '\n'); + const cg = await CodeGraph.init(dir, { index: true }); + open.push(cg); + return cg; + } + + it('reads the name list once for every word the fuzzy fallback tries', async () => { + const cg = await setup(); + const reads = vi.spyOn(queriesOf(cg), 'getAllNodeNames'); + for (const word of ['submitordr', 'transitoin', 'linksesions', 'qwertyuiop', 'zxcvbnmasd']) cg.searchNodes(word); + expect(reads).toHaveBeenCalledTimes(1); + // A typo still finds its symbol. + expect(cg.searchNodes('transitoin').map((r) => r.node.name)).toContain('transition'); + }); + + it('reads the names again after a sync or another connection\'s write', async () => { + const cg = await setup(); + expect(cg.searchNodes('ordermachne').map((r) => r.node.name)).not.toContain('OrderMachine'); + fs.writeFileSync(path.join(dir, 'src', 'zorbtown', 'more.ts'), 'export class OrderMachine {}\n'); + await cg.sync(); + expect(cg.searchNodes('ordermachne').map((r) => r.node.name)).toContain('OrderMachine'); + + const reader = await CodeGraph.open(dir); + open.push(reader); + expect(reader.searchNodes('invoicemachne').map((r) => r.node.name)).not.toContain('InvoiceMachine'); + fs.writeFileSync(path.join(dir, 'src', 'zorbtown', 'invoice.ts'), 'export class InvoiceMachine {}\n'); + await cg.sync(); + expect(reader.searchNodes('invoicemachne').map((r) => r.node.name)).toContain('InvoiceMachine'); + }); + + it('reads the names fresh inside a transaction, and does not keep them', async () => { + const cg = await setup(); + const db = rawDb(cg); + expect(cg.searchNodes('linksesions').map((r) => r.node.name)).toContain('linkSessions'); + db.exec('BEGIN'); + try { + db.exec("DELETE FROM nodes WHERE name = 'linkSessions'"); + expect(cg.searchNodes('linksesions').map((r) => r.node.name)).not.toContain('linkSessions'); + } finally { + db.exec('ROLLBACK'); + } + expect(cg.searchNodes('linksesions').map((r) => r.node.name)).toContain('linkSessions'); + }); + + it('skips the LIKE scan for a word no name or qualified name holds, and still finds one inside a qualified name', async () => { + const cg = await setup(); + const scans = countStatements(rawDb(cg), /qualified_name LIKE \?/); + // No token starts with these, so FTS misses and the LIKE fallback runs. + expect(cg.searchNodes('qwzxv')).toEqual([]); + expect(cg.searchNodes('plmokn')).toEqual([]); + expect(scans.count).toBe(0); + // "rbtow" is inside the folder name in a file's qualified name (its path), + // and inside no symbol name. + const inPath = cg.searchNodes('rbtow'); + expect(inPath.map((r) => r.node.filePath)).toContain('src/zorbtown/orders.ts'); + // "tateMach" is inside a name. + expect(cg.searchNodes('tatemach').map((r) => r.node.name)).toContain('OrderStateMachine'); + expect(scans.count).toBe(2); + }); +}); + +describe('a long query searches a bounded number of words (#2620)', () => { + let dir: string; + const open: CodeGraph[] = []; + afterEach(() => { + for (const cg of open.splice(0)) cg.close(); + vi.restoreAllMocks(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + async function setup(): Promise { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-longquery-')); + fs.writeFileSync(path.join(dir, 'orders.ts'), [ + 'export class OrderStateMachine {', + ' submitOrder(): boolean { return this.transition("submitted"); }', + ' transition(state: string): boolean { return state.length > 0; }', + '}', + 'export function reconcileLedger(): void {}', + 'export function zebrafinchquota(): void {}', + ].join('\n') + '\n'); + const cg = await CodeGraph.init(dir, { index: true }); + open.push(cg); + return cg; + } + + /** Distinct lowercase filler words that name nothing in the project. */ + function filler(n: number): string { + const words: string[] = []; + for (let i = 0; i < n; i++) words.push(`word${String.fromCharCode(97 + (i % 26))}${String.fromCharCode(97 + Math.floor(i / 26) % 26)}x`); + return words.join(' '); + } + + it('runs a bounded number of searches however long the query is', async () => { + const cg = await setup(); + const q = queriesOf(cg); + const search = vi.spyOn(q, 'searchNodes'); + const substring = vi.spyOn(q, 'findNodesByNameSubstring'); + await cg.findRelevantContext(`how does submitOrder work? ${filler(400)}`); + // A name is searched as a definition prefix (with its stems) and by four + // substring lookups; a term once. + expect(search.mock.calls.length).toBeLessThanOrEqual(MAX_QUERY_SYMBOLS * 4 + MAX_QUERY_TERMS); + expect(substring.mock.calls.length).toBeLessThanOrEqual(MAX_QUERY_SYMBOLS * 4); + }); + + it('searches a bag of symbol names whole', async () => { + const cg = await setup(); + const search = vi.spyOn(queriesOf(cg), 'searchNodes'); + const bag = 'AuthService AuthController TokenStore SessionManager UserRepository PasswordHasher JwtStrategy ' + + 'RefreshTokenService LoginHandler LogoutHandler AuthGuard RolesGuard PermissionService AuditLogger ' + + 'RateLimiter CsrfMiddleware reconcileLedger'; + expect(extractSearchTerms(bag).length).toBeGreaterThan(64); + expect(extractSearchTerms(bag).length).toBeLessThanOrEqual(MAX_QUERY_TERMS); + await cg.findRelevantContext(bag); + const searched = new Set(search.mock.calls.map((c) => String(c[0]).toLowerCase())); + for (const term of extractSearchTerms(bag)) expect(searched).toContain(term); + }); + + it('still searches every word of a query within the bound', async () => { + const cg = await setup(); + const search = vi.spyOn(queriesOf(cg), 'searchNodes'); + await cg.findRelevantContext(`${filler(20)} zebrafinchquota`); + const searched = new Set(search.mock.calls.map((c) => String(c[0]).toLowerCase())); + expect(searched).toContain('zebrafinchquota'); + expect(searched).toContain('wordaax'); + }); + + it('keeps the identifiers of a long paste, wherever they are in it', async () => { + const cg = await setup(); + const result = await cg.findRelevantContext(`${filler(300)} where does reconcileLedger run?`); + expect([...result.nodes.values()].map((n) => n.name)).toContain('reconcileLedger'); + }); + + it('drops the plain words past the bound', async () => { + const cg = await setup(); + const search = vi.spyOn(queriesOf(cg), 'searchNodes'); + await cg.findRelevantContext(`${filler(300)} zebrafinchquota`); + const searched = new Set(search.mock.calls.map((c) => String(c[0]).toLowerCase())); + expect(searched).not.toContain('zebrafinchquota'); + }); +}); + +describe('path relevance scores are unchanged by the per-query cache (#2620)', () => { + /** scorePathRelevance as it was, re-splitting the query for every path. */ + function reference(filePath: string, query: string, projectNameTokens?: Set, isDeprioritized?: boolean): number { + const pathLower = filePath.toLowerCase(); + const fileName = path.basename(filePath).toLowerCase(); + const dirName = path.dirname(filePath).toLowerCase(); + let score = 0; + const allWords = query.split(/\s+/).filter((w) => w.length > 0); + if (allWords.length === 0) return 0; + const words = projectNameTokens && projectNameTokens.size > 0 + ? allWords.filter((w) => !projectNameTokens.has(normalizeNameToken(w))) + : allWords; + const scored = words.length > 0 ? words : allWords; + for (const word of scored) { + const subtokens = extractSearchTerms(word, { stems: false }); + if (subtokens.length === 0) continue; + if (subtokens.some((t) => fileName.includes(t))) score += 10; + if (subtokens.some((t) => dirName.includes(t))) score += 5; + else if (subtokens.some((t) => pathLower.includes(t))) score += 3; + } + const queryLower = query.toLowerCase(); + const isTestQuery = queryLower.includes('test') || queryLower.includes('spec'); + if ((!isTestQuery && isTestFile(filePath)) || isDeprioritized === true) score -= 15; + return score; + } + + it('scores every path as before, repeated words and project names included', () => { + const queries = [ + 'how does the order service submit an order', + 'OrderService.submit order ORDER order_service submitOrder', + 'SuperBizAgent agent routes for the agent', + 'test the order service', ' ', '', 'spec', 'the of and', + Array.from({ length: 200 }, (_, i) => ['order', 'service', 'ledger', 'Submit', 'superbizagent'][i % 5]).join(' '), + ]; + const paths = [ + 'src/orders/order-service.ts', 'src/orders/__tests__/order-service.test.ts', 'SuperBizAgent/agent/routes.ts', + 'lib/ledger.rb', 'packages/submit/index.js', 'examples/order/main.go', 'README.md', 'a', + ]; + const tokens = [undefined, new Set(), new Set(['superbizagent'])]; + for (const query of queries) { + for (const filePath of paths) { + for (const projectNameTokens of tokens) { + for (const dep of [undefined, true, false]) { + expect(scorePathRelevance(filePath, query, projectNameTokens, dep), `${query} | ${filePath}`) + .toBe(reference(filePath, query, projectNameTokens, dep)); + } + } + } + } + }); +}); diff --git a/__tests__/explore-output-budget.test.ts b/__tests__/explore-output-budget.test.ts index efe302fb75..7843bebd2a 100644 --- a/__tests__/explore-output-budget.test.ts +++ b/__tests__/explore-output-budget.test.ts @@ -197,7 +197,10 @@ describe('codegraph_explore output respects the adaptive budget', () => { const result = await handler.execute('codegraph_explore', { query: 'Session method helper' }); const text = result.content?.[0]?.text ?? ''; expect(text).not.toContain('### Additional relevant files'); - expect(text).not.toContain('Complete source code is included above'); + // The completeness note, which reads "Complete source for 1 file is + // included above…" on this fixture ("Verbatim source for N files…" after + // a trim). + expect(text).not.toContain('is included above'); expect(text).not.toContain('advisory only, NOT a quota'); }); @@ -213,6 +216,9 @@ describe('codegraph_explore output respects the adaptive budget', () => { const text = result.content?.[0]?.text ?? ''; expect(text).toContain('advisory only, NOT a quota'); expect(text).toContain('extra calls are never rejected or rate-limited'); + // The completeness note is on here, so the small tier's check for its + // absence above still names text the note has. + expect(text).toContain('is included above'); expect(text).not.toContain('remaining calls'); expect(text).not.toContain('Synthesize once'); } finally { diff --git a/__tests__/explore-queried-oversize-symbol.test.ts b/__tests__/explore-queried-oversize-symbol.test.ts new file mode 100644 index 0000000000..d9eb00fe5f --- /dev/null +++ b/__tests__/explore-queried-oversize-symbol.test.ts @@ -0,0 +1,223 @@ +/** + * A symbol the query asks for renders even when it is too long for its file's + * share of the answer, and the block says where it was cut (#2486). + * + * `shrinkCluster` keeps the first-ranked member of an oversize cluster however + * big it is and lets the ceiling window it (CG-30). Among members of one + * importance, the shorter span ranked first. So in a file that is one long + * function plus the line that calls it, + * + * def build_report(...): # 200 statements, 15 KB + * ... + * if __name__ == '__main__': + * path = build_report() # a search hit too: its signature names it + * + * the one-line `path` took that guarantee, the function then did not fit beside + * it, and `codegraph_explore build_report` returned the file's last six lines, + * under a header calling the block the verbatim source. Asking for the file by + * path did the same. A body the window does cut now carries the line range and + * the explore query that returns the rest, since asking for the symbol by name + * again returns the same window. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +const STATEMENTS = 200; +const DEF = "def build_report(output_path: str = 'reports/report.pdf'):"; + +/** The issue's own reproducer, line for line. */ +function reportSource(): string { + return [ + '"""Generate the phase-1 validation report as PDF."""', + 'from reportlab.platypus import SimpleDocTemplate', + '', + '', + DEF, + ' """Build the phase-1 validation report as a PDF."""', + ' doc = SimpleDocTemplate(output_path)', + ' story = []', + ...Array.from({ length: STATEMENTS }, (_, i) => + ` story.append(Paragraph(f'section ${i} of the report', styles['BodyText']))`), + ' doc.build(story)', + ' return output_path', + '', + '', + "if __name__ == '__main__':", + ' path = build_report()', + ' print(path)', + '', + ].join('\n'); +} + +/** + * The long function, then a 27-line function the search matches only through + * its docstring. Not a fragment, so nothing but rank decides which of the two + * the cluster keeps. + */ +function publishSource(): string { + return [ + '"""Report generation."""', + '', + '', + DEF, + ' """Build the phase-1 validation report as a PDF."""', + ' story = []', + ...Array.from({ length: 260 }, (_, i) => + ` story.append(Paragraph(f'section ${i} of the report body', styles['BodyText']))`), + ' return output_path', + '', + '', + 'def publish():', + ' """Run build_report and copy the result where readers find it."""', + ' target = build_report()', + ...Array.from({ length: 24 }, (_, i) => ` print('publishing step ${i} of the release checklist', target)`), + ' return target', + '', + ].join('\n'); +} + +/** A constant the query can name, right above a long function that has nothing to do with it. */ +function limitsSource(): string { + return [ + '"""Limits for the report."""', + '', + 'MAX_SECTIONS = 5', + '', + 'def render_everything(doc):', + ...Array.from({ length: STATEMENTS }, (_, i) => + ` doc.write(f'unrelated statement number ${i} of the long neighbour function')`), + ' return doc', + '', + ].join('\n'); +} + +type Project = { dir: string; cg: CodeGraph }; + +async function project(files: Record): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-2486-')); + for (const [rel, text] of Object.entries(files)) fs.writeFileSync(path.join(dir, rel), text); + const cg = CodeGraph.initSync(dir); + await cg.indexAll(); + return { dir, cg }; +} + +function destroy(p: Project | undefined): void { + if (!p) return; + p.cg.destroy(); + fs.rmSync(p.dir, { recursive: true, force: true }); +} + +/** A handler per call: a session that already holds a file's lines elides them. */ +async function explore(p: Project, query: string): Promise { + const result = await new ToolHandler(p.cg).execute('codegraph_explore', { query }); + return result.content?.[0]?.text ?? ''; +} + +/** The `\t` form explore renders a source line in. */ +const rendered = (n: number, line: string): string => `${n}\t${line}`; + +describe('#2486 — the queried symbol renders when it alone overruns its file\'s budget', () => { + let report: Project | undefined; + let byName = ''; + // The default tool surface: a shell that exports an allowlist must not change what this asserts. + const allowlist = process.env.CODEGRAPH_MCP_TOOLS; + + beforeAll(async () => { + delete process.env.CODEGRAPH_MCP_TOOLS; + report = await project({ 'report_gen.py': reportSource() }); + byName = await explore(report, 'build_report'); + }, 120_000); + afterAll(() => { + destroy(report); + if (allowlist === undefined) delete process.env.CODEGRAPH_MCP_TOOLS; + else process.env.CODEGRAPH_MCP_TOOLS = allowlist; + }); + + it('fixture shape: one function far bigger than the whole response budget, beside a one-line search hit', () => { + const nodes = report!.cg.getNodesInFile('report_gen.py'); + const fn = nodes.find((n) => n.name === 'build_report' && n.kind === 'function')!; + expect(fn.startLine).toBe(5); + expect(fn.endLine).toBe(STATEMENTS + 10); + expect(reportSource().length).toBeGreaterThan(13_000); + // The search matches the variable through its signature, `= build_report()`. + const hit = nodes.find((n) => n.name === 'path')!; + expect(hit.startLine).toBe(hit.endLine); + expect(hit.startLine - fn.endLine).toBeLessThan(7); + }); + + it('renders the definition and the head of its body, not the file tail', () => { + expect(byName).toContain(rendered(5, DEF)); + expect(byName).toContain("section 0 of the report'"); + expect(byName.length).toBeLessThanOrEqual(19_500); + }); + + it('says inside the block where the body was cut, and that query returns the rest', async () => { + const note = byName.match(/… lines (\d+)-(\d+) of `build_report` elided — codegraph_explore `(report_gen\.py:\d+-\d+)` returns them/); + expect(note, 'no cut note in:\n' + byName.slice(-1500)).not.toBeNull(); + const [, from, to, followUp] = note!; + expect(Number(to)).toBe(STATEMENTS + 10); + // The note sits right after the last line shown. + expect(byName).toContain(`${Number(from) - 1}\t`); + expect(byName).not.toContain(`\n${from}\t`); + const rest = await explore(report!, followUp!); + expect(rest).toContain(rendered(Number(from), ` story.append(Paragraph(f'section ${Number(from) - 9} of the report', styles['BodyText']))`)); + expect(rest).toContain(rendered(STATEMENTS + 10, ' return output_path')); + }); + + it('names no follow-up tool the default surface does not list', () => { + expect(byName).toContain('Some file sections were trimmed for size'); + expect(byName).not.toContain('codegraph_node'); + }); + + it('a file asked for by path renders from its first definition, not its tail', async () => { + const byPath = await explore(report!, 'report_gen.py'); + expect(byPath).toContain(rendered(5, DEF)); + expect(byPath).toContain("section 0 of the report'"); + expect(byPath).toContain('of `build_report` elided — codegraph_explore `report_gen.py:'); + expect(byPath.length).toBeLessThanOrEqual(19_500); + }); + + it('a query naming the one-liner too still renders the function', async () => { + const both = await explore(report!, 'build_report path'); + expect(both).toContain(rendered(5, DEF)); + expect(both).toContain("section 0 of the report'"); + }); + + it('still names codegraph_node when the tool is served', async () => { + process.env.CODEGRAPH_MCP_TOOLS = 'explore,node'; + try { + expect(await explore(report!, 'build_report')).toContain('(or `codegraph_node`)'); + } finally { + delete process.env.CODEGRAPH_MCP_TOOLS; + } + }); +}); + +describe('#2486 — a name the query spelled outranks a search hit of the same importance', () => { + let publish: Project | undefined; + afterAll(() => destroy(publish)); + + it('renders the named function, not only the docstring match beside it', async () => { + publish = await project({ 'report_gen.py': publishSource() }); + const text = await explore(publish, 'build_report'); + expect(text).toContain(rendered(4, DEF)); + expect(text).toContain("section 0 of the report body'"); + expect(text).toContain('of `build_report` elided — codegraph_explore `report_gen.py:'); + }, 120_000); +}); + +describe('#2486 — what the query did not ask for is left as it was', () => { + let limits: Project | undefined; + afterAll(() => destroy(limits)); + + it('a one-line constant the query names renders without the long function beside it', async () => { + limits = await project({ 'limits.py': limitsSource() }); + const text = await explore(limits, 'MAX_SECTIONS'); + expect(text).toContain(rendered(3, 'MAX_SECTIONS = 5')); + expect(text).not.toContain('unrelated statement number 100'); + }, 120_000); +}); diff --git a/__tests__/explore-synth-link-endpoints.test.ts b/__tests__/explore-synth-link-endpoints.test.ts new file mode 100644 index 0000000000..96f7e91ba6 --- /dev/null +++ b/__tests__/explore-synth-link-endpoints.test.ts @@ -0,0 +1,313 @@ +/** + * Regression: codegraph_explore's "Dynamic-dispatch links among your symbols" + * section lists each synthesized LINK, not each pair of names. + * + * It used to dedupe its lines by `source name > target name`. Every + * `interface → impl` link of one interface method reads `area → area`, so an + * interface method with several implementations printed ONE line, and which + * implementation survived was decided by edge order, not by the query: naming + * `Square.area` showed Triangle's link and hid Square's (gin's + * `setter.TrySet`, prometheus's `Queryable.Querier`). Links are now keyed by + * their endpoints, and a hop already on the rendered Flow chain is still left + * out. When more links exist than the 6-line cap holds, links into test files + * through an end the query didn't name go last, and the rest go by how + * pointedly the query named their ends, so the cap never drops an + * implementation the query names for one it doesn't. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +// Assertions read RAW codegraph_explore output; managed offload would replace it. +let prevOffloadDisable: string | undefined; +beforeAll(() => { + prevOffloadDisable = process.env.CODEGRAPH_OFFLOAD_DISABLE; + process.env.CODEGRAPH_OFFLOAD_DISABLE = '1'; +}); +afterAll(() => { + if (prevOffloadDisable === undefined) delete process.env.CODEGRAPH_OFFLOAD_DISABLE; + else process.env.CODEGRAPH_OFFLOAD_DISABLE = prevOffloadDisable; +}); + +const SECTION = '**Dynamic-dispatch links among your symbols**'; + +/** The `- a → b [...]` lines of the dynamic-dispatch links section. */ +function linkLines(text: string): string[] { + const at = text.indexOf(SECTION); + if (at < 0) return []; + const out: string[] = []; + for (const line of text.slice(at + SECTION.length).split(/\r?\n/)) { + if (line.startsWith('- ')) out.push(line); + else if (out.length > 0) break; + } + return out; +} + +/** The rendered Flow section, up to the next section header. */ +function flowSection(text: string): string { + const at = text.indexOf('**Flow (call path'); + if (at < 0) return ''; + const end = text.indexOf(SECTION, at); + return text.slice(at, end < 0 ? undefined : end); +} + +describe('codegraph_explore — dynamic-dispatch links are listed per link', () => { + let dir = ''; + let cg: CodeGraph | undefined; + + afterEach(() => { + cg?.destroy(); + cg = undefined; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + dir = ''; + }); + + /** + * `Shape.area` with one implementation per name. `Circle.area` calls + * `scale`, so `report → Shape.area → Circle.area → scale` is the longest + * chain among the query's symbols and the Flow section runs through Circle, + * never through the implementation the query names. `testDoubles` go in + * `fakes/.test.ts`. + */ + async function indexShapes(implementations: string[], testDoubles: string[] = []): Promise { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explore-synth-links-')); + const write = (rel: string, content: string) => { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), content); + }; + write('shape.ts', 'export interface Shape {\n area(): number;\n}\n'); + write('scale.ts', 'export function scale(n: number): number {\n return n * 2;\n}\n'); + write( + 'report.ts', + "import type { Shape } from './shape';\n\nexport function report(shape: Shape): number {\n return shape.area();\n}\n" + ); + write( + 'circle.ts', + "import type { Shape } from './shape';\nimport { scale } from './scale';\n\nexport class Circle implements Shape {\n area(): number {\n return scale(3);\n }\n}\n" + ); + for (const name of implementations) { + write( + `${name.toLowerCase()}.ts`, + `import type { Shape } from './shape';\n\nexport class ${name} implements Shape {\n area(): number {\n return 1;\n }\n}\n` + ); + } + for (const name of testDoubles) { + write( + `fakes/${name.toLowerCase()}.test.ts`, + `import type { Shape } from '../shape';\n\nexport class ${name} implements Shape {\n area(): number {\n return 0;\n }\n}\n` + ); + } + cg = CodeGraph.initSync(dir, { config: { include: ['**/*.ts'], exclude: [] } }); + await cg.indexAll(); + return new ToolHandler(cg); + } + + /** `Shape.area`'s interface → impl links, in the order the graph returns them. */ + function implementationsInEdgeOrder(): string[] { + const iface = cg!.getNodesByName('area').find((n) => n.qualifiedName === 'Shape::area'); + expect(iface).toBeDefined(); + return cg! + .getOutgoingEdges(iface!.id) + .filter((e) => e.provenance === 'heuristic' && e.metadata?.synthesizedBy === 'interface-impl') + .map((e) => cg!.getNode(e.target)?.qualifiedName.split('::')[0] ?? ''); + } + + it('lists the named implementation and every other off-chain one, the named one first', async () => { + const handler = await indexShapes(['Square', 'Triangle']); + expect(implementationsInEdgeOrder().sort()).toEqual(['Circle', 'Square', 'Triangle']); + + const res = await handler.execute('codegraph_explore', { query: 'report Shape.area Square.area scale' }); + const text = res.content[0].text as string; + + // Precondition: the Flow runs through Circle, so Square's link is not on it. + const flow = flowSection(text); + expect(flow).toMatch(/\d\. area \(circle\.ts:\d+\)/); + expect(flow).not.toContain('square.ts'); + + const links = linkLines(text); + const square = links.findIndex((l) => l.includes('@square.ts:')); + const triangle = links.findIndex((l) => l.includes('@triangle.ts:')); + // Distinct implementations are distinct links — the name pair `area → area` + // no longer folds them into one line. + expect(square).toBeGreaterThanOrEqual(0); + expect(triangle).toBeGreaterThanOrEqual(0); + // The implementation the query named leads. + expect(square).toBeLessThan(triangle); + // `area → area` names neither side; each end is named by its owner. + expect(links[square]).toMatch(/^- Shape\.area → Square\.area\s+\[dynamic: interface → impl @square\.ts:4\]$/); + expect(links[triangle]).toMatch(/^- Shape\.area → Triangle\.area\s+\[dynamic: interface → impl @triangle\.ts:4\]$/); + // The chain's own hop is rendered in the Flow, not repeated here. + expect(links.some((l) => l.includes('circle.ts'))).toBe(false); + expect(links).toHaveLength(2); + }, 120_000); + + it('keeps the 6-line cap and spends its first line on the named implementation', async () => { + const others = ['Square', 'Triangle', 'Pentagon', 'Hexagon', 'Heptagon', 'Octagon', 'Nonagon', 'Decagon']; + const handler = await indexShapes(others); + // Name the implementation edge order would list LAST, so only the + // named-first rule can bring it under the cap. + const order = implementationsInEdgeOrder().filter((owner) => owner !== 'Circle'); + expect([...order].sort()).toEqual([...others].sort()); + const last = order[order.length - 1]!; + + const res = await handler.execute('codegraph_explore', { query: `report Shape.area ${last}.area scale` }); + const text = res.content[0].text as string; + expect(flowSection(text)).toMatch(/\d\. area \(circle\.ts:\d+\)/); + + const links = linkLines(text); + expect(links).toHaveLength(6); + expect(links[0]).toMatch(new RegExp(`^- Shape\\.area → ${last}\\.area\\s+\\[dynamic: interface → impl @${last.toLowerCase()}\\.ts:4\\]$`)); + // The other five are five distinct implementations, none of them the chain's. + const rest = links.slice(1).map((l) => /→ (\w+)\.area/.exec(l)?.[1]); + expect(new Set(rest).size).toBe(5); + for (const owner of rest) expect(order).toContain(owner); + expect(rest).not.toContain(last); + expect(links.some((l) => l.includes('circle.ts'))).toBe(false); + }, 120_000); + + it('lists links into test files after the rest when the cap forces a choice', async () => { + // prometheus's `Queryable.Querier` has about as many test doubles as real + // implementations. Explore's answer leaves test files out, and here they only + // get the lines the real implementations leave. + const production = ['Square', 'Triangle', 'Pentagon', 'Hexagon', 'Heptagon', 'Octagon']; + const doubles = ['FakeA', 'FakeB', 'FakeC', 'FakeD', 'FakeE', 'FakeF']; + const handler = await indexShapes(production, doubles); + // Precondition: in edge order, a test double would take one of the six lines. + const order = implementationsInEdgeOrder().filter((owner) => owner !== 'Circle'); + expect([...order].sort()).toEqual([...production, ...doubles].sort()); + expect(order.slice(0, 6).some((owner) => doubles.includes(owner))).toBe(true); + + const res = await handler.execute('codegraph_explore', { query: 'report Shape.area scale' }); + const text = res.content[0].text as string; + expect(flowSection(text)).toMatch(/\d\. area \(circle\.ts:\d+\)/); + const links = linkLines(text); + expect(links).toHaveLength(6); + expect(links.map((l) => /→ (\w+)\.area/.exec(l)?.[1]).sort()).toEqual([...production].sort()); + }, 120_000); + + it("puts the links of an interface the query picks out before links among an ambiguous name's candidates", async () => { + // prometheus's `Discovery.refresh` matches a `Discovery` type in each discovery + // package. In `Discovery.refresh fetchApps Decoder.Decode`, their links to each + // other filled the cap and hid the one link into `fetchApps`, which the query + // names exactly. + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explore-synth-links-ambiguous-')); + const write = (rel: string, content: string) => { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), content); + }; + write('iface.ts', 'export interface Discovery {\n refresh(): number;\n}\n'); + for (let i = 1; i <= 5; i++) { + write( + `pkg${i}/discovery.ts`, + `import type { Discovery as Base } from '../iface';\n\nexport class Discovery implements Base {\n refresh(): number {\n return ${i};\n }\n}\n` + ); + } + write('refresher.ts', 'export interface Refresher {\n refresh(): number;\n}\n'); + for (const name of ['Hcloud', 'Robot']) { + write( + `${name.toLowerCase()}.ts`, + `import type { Refresher } from './refresher';\n\nexport class ${name} implements Refresher {\n refresh(): number {\n return 0;\n }\n}\n` + ); + } + cg = CodeGraph.initSync(dir, { config: { include: ['**/*.ts'], exclude: [] } }); + await cg.indexAll(); + // Precondition: the interface `Discovery` links to all five classes of its name. + const discoveryLinks = cg + .getNodesByName('refresh') + .filter((n) => n.qualifiedName === 'Discovery::refresh') + .flatMap((n) => cg!.getOutgoingEdges(n.id).filter((e) => e.metadata?.synthesizedBy === 'interface-impl')); + expect(discoveryLinks).toHaveLength(5); + + const res = await new ToolHandler(cg).execute('codegraph_explore', { + query: 'Discovery.refresh Refresher.refresh Hcloud.refresh', + }); + const links = linkLines(res.content[0].text as string); + expect(links).toHaveLength(6); + expect(links[0]).toBe('- Refresher.refresh → Hcloud.refresh [dynamic: interface → impl @hcloud.ts:4]'); + expect(links[1]).toBe('- Refresher.refresh → Robot.refresh [dynamic: interface → impl @robot.ts:4]'); + for (const l of links.slice(2)) { + expect(l).toMatch(/^- Discovery\.refresh → Discovery\.refresh {3}\[dynamic: interface → impl @pkg\d\/discovery\.ts:4\]$/); + } + }, 120_000); + + it('lets the named symbols take turns, so one long fan-out does not fill the list', async () => { + // prometheus's `refresh.Discovery.refresh` hands its func field to 28 + // discovery packages, and in a query that also names hetzner's + // `refresher.refresh` those links filled the list before robotDiscovery's. + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explore-synth-links-turns-')); + const write = (rel: string, content: string) => fs.writeFileSync(path.join(dir, rel), content); + const family = (iface: string, classes: string[]) => { + write(`${iface.toLowerCase()}.ts`, `export interface ${iface} {\n run(): number;\n}\n`); + for (const name of classes) { + write( + `${name.toLowerCase()}.ts`, + `import type { ${iface} } from './${iface.toLowerCase()}';\n\nexport class ${name} implements ${iface} {\n run(): number {\n return 1;\n }\n}\n` + ); + } + }; + family('Wide', ['WideA', 'WideB', 'WideC', 'WideD', 'WideE', 'WideF', 'WideG', 'WideH']); + family('Narrow', ['NarrowA', 'NarrowB']); + cg = CodeGraph.initSync(dir, { config: { include: ['**/*.ts'], exclude: [] } }); + await cg.indexAll(); + + const res = await new ToolHandler(cg).execute('codegraph_explore', { query: 'Wide.run Narrow.run' }); + const links = linkLines(res.content[0].text as string); + expect(links).toHaveLength(6); + // Both of Narrow's links make the list, taking turns with Wide's. + expect(links.map((l) => /^- (\w+)\.run →/.exec(l)?.[1])).toEqual(['Wide', 'Narrow', 'Wide', 'Narrow', 'Wide', 'Wide']); + expect(links.filter((l) => l.startsWith('- Narrow.run → ')).map((l) => /→ (\w+)\.run/.exec(l)?.[1]).sort()).toEqual(['NarrowA', 'NarrowB']); + }, 120_000); + + it('prints two links that read the same once, and none that reads like a Flow hop', async () => { + // gin declares `Binding` once per build tag (binding.go, binding_nomsgpack.go), + // so formBinding.Bind is linked from two `Binding.Bind` nodes and both links + // print as the same line. + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explore-synth-links-go-')); + const write = (rel: string, content: string) => fs.writeFileSync(path.join(dir, rel), content); + write('go.mod', 'module example.com/binding\n\ngo 1.22\n'); + const iface = 'package binding\n\ntype Binding interface {\n\tBind(req string) error\n}\n'; + write('binding.go', `//go:build !nomsgpack\n\n${iface}`); + write('binding_nomsgpack.go', `//go:build nomsgpack\n\n${iface}`); + write( + 'form.go', + 'package binding\n\ntype formBinding struct{}\n\nfunc (b formBinding) Bind(req string) error {\n\treturn mapForm(req)\n}\n\nfunc mapForm(req string) error {\n\treturn nil\n}\n' + ); + write( + 'query.go', + 'package binding\n\ntype queryBinding struct{}\n\nfunc (b queryBinding) Bind(req string) error {\n\treturn mapForm(req)\n}\n' + ); + cg = CodeGraph.initSync(dir, { config: { include: ['**/*.go'], exclude: [] } }); + await cg.indexAll(); + + // Precondition: each declaration links to both implementations. + const declarations = cg.getNodesByName('Bind').filter((n) => n.qualifiedName === 'Binding::Bind'); + expect(declarations).toHaveLength(2); + for (const d of declarations) { + const targets = cg + .getOutgoingEdges(d.id) + .filter((e) => e.metadata?.synthesizedBy === 'interface-impl') + .map((e) => cg!.getNode(e.target)?.qualifiedName); + expect(targets.sort()).toEqual(['formBinding::Bind', 'queryBinding::Bind']); + } + + const handler = new ToolHandler(cg); + // Two symbols make no Flow: each implementation's link, once. + const pair = await handler.execute('codegraph_explore', { query: 'Binding.Bind formBinding.Bind' }); + expect(linkLines(pair.content[0].text as string)).toEqual([ + '- Binding.Bind → formBinding.Bind [dynamic: interface → impl @form.go:5]', + '- Binding.Bind → queryBinding.Bind [dynamic: interface → impl @query.go:5]', + ]); + + // The Flow runs through one declaration's link to formBinding.Bind. The other + // declaration's link reads exactly like that hop, so it is not listed under it. + const res = await handler.execute('codegraph_explore', { query: 'Binding.Bind formBinding.Bind mapForm' }); + const text = res.content[0].text as string; + expect(flowSection(text)).toMatch(/\d\. Bind \(binding(_nomsgpack)?\.go:\d+\)\n\s+↓ dynamic: interface → impl @form\.go:5\n\d\. Bind \(form\.go:5\)/); + expect(linkLines(text)).toEqual([ + '- Binding.Bind → queryBinding.Bind [dynamic: interface → impl @query.go:5]', + ]); + }, 120_000); +}); diff --git a/__tests__/explore-unindexed-names.test.ts b/__tests__/explore-unindexed-names.test.ts new file mode 100644 index 0000000000..ed9c8429b8 --- /dev/null +++ b/__tests__/explore-unindexed-names.test.ts @@ -0,0 +1,196 @@ +/** + * Names the query spells that the index has never seen are said plainly, + * instead of answering with code that only matched the query's other words + * (#2462). + * + * A .NET repo kept real tool projects in a root `build/` folder, which the + * default skip list leaves unindexed. `codegraph_explore "ReleaseNotesBuilder + * Program Main ChangelogParser PersistentData Entry AddFeatures"` came back as + * an unrelated `Program.Main` and a C++ `Parser`, with nothing to say that four + * of the names matched nothing. The agent read the wrong code. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler, isIdentifierShapedToken } from '../src/mcp/tools'; + +const QUERY = 'ReleaseNotesBuilder Program Main ChangelogParser PersistentData Entry AddFeatures'; +const MISSING = ['ReleaseNotesBuilder', 'ChangelogParser', 'PersistentData', 'AddFeatures']; + +/** The report's layout: the product under src/, its release tooling under a root build/. */ +const FILES: Record = { + '.gitignore': 'bin/\nobj/\n', + 'src/Agent/Program.cs': `namespace Agent +{ + public static class Program + { + public static int Main(string[] args) + { + var host = new AgentHost(); + return host.Run(args); + } + } + + public class AgentHost + { + public int Run(string[] args) { return args.Length; } + } +} +`, + 'src/native/parser.cpp': `#include +namespace native { +class Parser { +public: + explicit Parser(std::string text) : text_(std::move(text)) {} + int parse() { return static_cast(text_.size()); } +private: + std::string text_; +}; +} // namespace native +`, + // `ChangelogWriter` and `auditTrail` are seen only through a docstring and a + // folder name, never as a symbol. + 'src/auditTrail/trail.ts': `/** Hands the audit entries to the ChangelogWriter. */ +export function recordTrail(): number { return 1; } +`, + // Indexed as `handle`, with no `app.` on it, as express's own is. + 'src/server/app.js': `const app = {}; +app.handle = function handle(req, res) { return res; }; +module.exports = app; +`, + 'build/ReleaseNotesBuilder/Program.cs': `namespace ReleaseNotesBuilder +{ + public static class Program + { + public static int Main(string[] args) + { + var parser = new ChangelogParser(args[0]); + var data = PersistentData.Load(args[1]); + foreach (var entry in parser.Entries()) entry.AddFeatures(data); + return 0; + } + } +} +`, + 'build/ReleaseNotesBuilder/ChangelogParser.cs': `using System.Collections.Generic; +namespace ReleaseNotesBuilder +{ + public class ChangelogParser + { + private readonly string _path; + public ChangelogParser(string path) { _path = path; } + public IEnumerable Entries() { yield return new Entry(_path); } + } +} +`, + 'build/ReleaseNotesBuilder/Entry.cs': `using System.Collections.Generic; +namespace ReleaseNotesBuilder +{ + public class Entry + { + private readonly List _features = new List(); + public Entry(string title) { } + public void AddFeatures(PersistentData data) { _features.AddRange(data.Features); } + } +} +`, + 'build/ReleaseNotesBuilder/PersistentData.cs': `using System.Collections.Generic; +namespace ReleaseNotesBuilder +{ + public class PersistentData + { + public List Features { get; } = new List(); + public static PersistentData Load(string path) { return new PersistentData(); } + } +} +`, +}; + +describe('#2462 — explore says which named symbols are not in the index', () => { + let dir: string; + let cg: CodeGraph; + const explore = async (query: string) => { + const result = await new ToolHandler(cg).execute('codegraph_explore', { query }); + return { text: result.content?.[0]?.text ?? '', isError: result.isError === true }; + }; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-2462-')); + for (const [rel, text] of Object.entries(FILES)) { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), text); + } + cg = CodeGraph.initSync(dir); + await cg.indexAll(); + }, 120_000); + + afterAll(() => { + cg?.destroy(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('fixture shape: the root build/ folder is skipped by default, so its names are not indexed', () => { + expect(cg.getFiles().map((f) => f.path).filter((p) => p.startsWith('build/'))).toEqual([]); + expect(cg.getNodesByName('Main').map((n) => n.filePath)).toEqual(['src/Agent/Program.cs']); + }); + + it('names the missing symbols above the unrelated matches, success-shaped', async () => { + const { text, isError } = await explore(QUERY); + expect(isError).toBe(false); + expect(text).toContain(`**Not in the index:** ${MISSING.map((n) => `\`${n}\``).join(', ')}.`); + expect(text).toContain("so the code below matches the query's other words, not them"); + // Above the source, where it is read first. + expect(text.indexOf('**Not in the index:**')).toBeLessThan(text.indexOf('**`src/Agent/Program.cs`**')); + // Where to look, never "Read"; including a skipped folder is the user's call. + expect(text).toContain('`codegraph status` shows what is.'); + expect(text).toContain('the user can include that folder by adding a re-include such as `!/build/` to `.gitignore`.'); + expect(text).not.toMatch(/\bRead (?:the|these|those|it)\b/); + }); + + it('never names a single word, which may be English, nor a name the index has', async () => { + const { text } = await explore(QUERY); + const note = text.split('\n').find((l) => l.startsWith('> **Not in the index:**'))!; + for (const word of ['Program', 'Main', 'Entry']) expect(note).not.toContain(`\`${word}\``); + expect((await explore('Program Main AgentHost Run')).text).not.toContain('Not in the index'); + expect((await explore('How does the agent host run its main program?')).text).not.toContain('Not in the index'); + }); + + it('a name seen only in a docstring or a folder name is in the index', () => { + expect(cg.getUnseenNames(['ChangelogWriter', 'auditTrail', 'agenthost', 'Agent.Program', ...MISSING])) + .toEqual(MISSING); + }); + + it('a qualified name is in the index when its member is, whatever the receiver', () => { + // `app.handle` is indexed as `handle`; only a member never seen makes it missing. + expect(cg.getUnseenNames(['app.handle', 'Zorblax.frobnicate', 'pkg::recordTrail'])).toEqual(['Zorblax.frobnicate']); + }); + + it('says so too when nothing matched at all', async () => { + const { text, isError } = await explore('ReleaseNotesBuilder PersistentData'); + expect(isError).toBe(false); + expect(text).toContain('No relevant code found'); + expect(text).toContain('**Not in the index:** `ReleaseNotesBuilder`, `PersistentData`.'); + expect(text).not.toContain('the code below'); + }); + + it('goes away once the folder is re-included and synced', async () => { + fs.writeFileSync(path.join(dir, '.gitignore'), 'bin/\nobj/\n!/build/\n'); + await cg.sync(); + const { text } = await explore(QUERY); + expect(text).not.toContain('Not in the index'); + expect(text).toContain('build/ReleaseNotesBuilder/ChangelogParser.cs'); + }, 120_000); +}); + +describe('isIdentifierShapedToken', () => { + it('takes names written like code and leaves single words alone', () => { + for (const t of ['ChangelogParser', 'addFeatures', 'build_report', 'MAX_SIZE', 'Session.request', 'pkg::run', '$store.items']) { + expect(isIdentifierShapedToken(t), t).toBe(true); + } + for (const t of ['Program', 'Main', 'How', 'parser', 'SQL', 'i18n', 'v1.2']) { + expect(isIdentifierShapedToken(t), t).toBe(false); + } + }); +}); diff --git a/__tests__/expo-router.test.ts b/__tests__/expo-router.test.ts index 3463e111d2..5ac1c939c6 100644 --- a/__tests__/expo-router.test.ts +++ b/__tests__/expo-router.test.ts @@ -741,8 +741,10 @@ export default function ReviewScreen() { // One transition into /capture/review, and it comes from /capture. expect(links.map((l) => screens.screens.find((s) => s.id === l.from)?.path)).toEqual(['/capture']); - // Written right there: no chain, and no Swift guard smuggled in. - expect(links[0]!.via).toEqual([]); + // Written in this screen's own file, reached through the memo wrapper it + // renders, and no Swift guard smuggled in. + expect(links[0]!.via.map((v) => v.name)).toEqual(['MemoizedCaptureComponent', 'CaptureComponent']); + expect(links[0]!.via.every((v) => v.language !== 'swift')).toBe(true); expect(links[0]!.when).toBe('!isRetakeBatchActive'); expect(links[0]!.sites[0]!.file).toBe('src/app/capture/index.tsx'); // …and /capture/review is not left feeding only itself. diff --git a/__tests__/expression-receiver-calls.test.ts b/__tests__/expression-receiver-calls.test.ts index e22de1f745..3e8aeb083f 100644 --- a/__tests__/expression-receiver-calls.test.ts +++ b/__tests__/expression-receiver-calls.test.ts @@ -24,6 +24,13 @@ const files: Record = { 'export class GraphAdapter { async map(req?: string) { return req ?? \'\'; } }\n' + 'export class Runner { run() { return 1; } }\n' + 'export class Base { hello() { return 1; } }\n', + // A project `push` an element of a field must not reach (`this.listeners[type].push`). + 'src/stack.ts': 'export interface UndoStack { push: (entry: string) => void; }\n', + 'src/bus.ts': + 'export class Bus {\n' + + ' private listeners: Record = {};\n' + + ' on(type: string, handler: string) { this.listeners[type].push(handler); }\n' + + '}\n', 'src/use.ts': "import { GraphAdapter, Runner, Base } from './adapter';\n" + "import { helper } from './helper.js';\n" + @@ -84,6 +91,10 @@ describe('expression receivers', () => { expect(cg.getCallers(node('map', 'adapter.ts').id).map(({ node: n }) => n.name)).toEqual(['typed']); }); + it('TS: a method on an element of a field does not bind by its bare name', () => { + expect(calleesOf('on', 'bus.ts')).toEqual([]); + }); + it('Rust: `v.iter().map()` does not bind to a TypeScript class method', () => { expect(calleesOf('lens', 'lib.rs')).toEqual([]); expect(calleesOf('opt', 'lib.rs')).toEqual([]); @@ -116,6 +127,9 @@ describe('expression receivers', () => { ' f().list.map(g);\n' + ' (() => 1).call(null);\n' + ' this.a.b.run();\n' + + ' this.listeners[type].push(h);\n' + + ' this[k].stop();\n' + + " this['svc'].start();\n" + ' new Runner().go();\n' + ' window.Api.start();\n' + '}\n'; @@ -124,7 +138,7 @@ describe('expression receivers', () => { .map((r) => r.referenceName); expect(refs).toEqual([ 'list().map', 'list', 'x.run', 'y.run', 'x.stop', 'getTarget().install', 'getTarget', - 'f', 'run', 'go', 'start', + 'f', 'run', 'this.listeners[].push', 'this.svc.start', 'new Runner().go', 'start', ]); }); }); diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index 1ceb2271e1..cc6323eb91 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -764,6 +764,86 @@ function main() { const calls = result.unresolvedReferences.filter((r) => r.referenceKind === 'calls'); expect(calls.some((c) => c.referenceName === 'processData')).toBe(true); }); + + // tree-sitter-typescript parses `await f(x)` as a call whose callee is + // `await f`, so the ref was named `await f` and matched nothing (#2586). + it.each([['user.ts'], ['user.tsx']])('an awaited call with type arguments names its callee: %s (#2586)', (file) => { + const code = [ + 'export async function awaited() { return await fetchApi("/me"); }', + 'export async function assigned() { const u = await fetchApi("/me"); return u; }', + 'export async function viaNamespace() { return await api.fetchApi("/me"); }', + 'export async function viaThis() { return await this.load("/me"); }', + '', + ].join('\n'); + const result = extractFromSource(file, code); + const calls = result.unresolvedReferences.filter((r) => r.referenceKind === 'calls'); + const byFn = (name: string) => { + const fn = result.nodes.find((n) => n.name === name)!; + return calls.filter((c) => c.fromNodeId === fn.id).map((c) => c.referenceName); + }; + expect(byFn('awaited')).toEqual(['fetchApi']); + expect(byFn('assigned')).toEqual(['fetchApi']); + expect(byFn('viaNamespace')).toEqual(['api.fetchApi']); + expect(byFn('viaThis')).toEqual(['load']); + expect(calls.some((c) => c.referenceName.startsWith('await'))).toBe(false); + // The reference sits where the callee starts, as for `await f(x)` — not on + // the `await`, which the call node starts at. + const lines = code.split('\n'); + const columnOf = (name: string) => { + const fn = result.nodes.find((n) => n.name === name)!; + return calls.find((c) => c.fromNodeId === fn.id)!.column; + }; + expect(columnOf('awaited')).toBe(lines[0]!.indexOf('fetchApi')); + expect(columnOf('viaNamespace')).toBe(lines[2]!.indexOf('api.fetchApi')); + }); + + it('an awaited generic call to a name a package exports links to nothing in the project (#2586)', async () => { + const dir = createTempDir(); + try { + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name: 'app', dependencies: { 'idb-keyval': '6.2.1' } })); + fs.mkdirSync(path.join(dir, 'src')); + // excalidraw's `await get(key, store)` from idb-keyval, beside + // a project class whose static `get` the name alone would match. + fs.writeFileSync(path.join(dir, 'src', 'storage.ts'), + 'import { get } from "idb-keyval";\nexport async function load() { return await get("k"); }\n'); + fs.writeFileSync(path.join(dir, 'src', 'cache.ts'), 'export class Cache {\n static get = (s: string) => s;\n}\n'); + const cg = await CodeGraph.init(dir, { index: true }); + try { + const load = cg.getNodesByKind('function').find((n) => n.name === 'load')!; + expect(cg.getOutgoingEdges(load.id).filter((e) => e.kind === 'calls')).toEqual([]); + } finally { + cg.close(); + } + } finally { + cleanupTempDir(dir); + } + }, 60_000); + + it.each([['user.ts'], ['user.tsx'], ['user.ets']])('an awaited generic call peels every await and skips comments: %s (#2586)', (file) => { + const code = [ + 'export async function chainedParen() { return (await fetchApi("/me")).json(); }', + 'export async function chainedThen() { return await api.get("/me").then(r => r); }', + 'export async function commented() { return await /* typed */ fetchApi("/me"); }', + 'export async function lineCommented() { return await // typed\n fetchApi("/me"); }', + 'export async function doubled() { return await await fetchApi("/me"); }', + '', + ].join('\n'); + const result = extractFromSource(file, code); + const calls = result.unresolvedReferences.filter((r) => r.referenceKind === 'calls'); + const byFn = (name: string) => { + const fn = result.nodes.find((n) => n.name === name)!; + return calls.filter((c) => c.fromNodeId === fn.id).map((c) => c.referenceName).sort(); + }; + expect(byFn('commented')).toEqual(['fetchApi']); + expect(byFn('lineCommented')).toEqual(['fetchApi']); + expect(byFn('doubled')).toEqual(['fetchApi']); + expect(calls.some((c) => /^await|^\/[*/]/.test(c.referenceName))).toBe(false); + // ArkTS keeps its own chain encoding; TS/TSX name the inner callee. + if (file !== 'user.ets') { + expect(byFn('chainedParen')).toEqual(['fetchApi', 'fetchApi().json']); + expect(byFn('chainedThen')).toEqual(['api.get', 'api.get().then']); + } + }); }); describe('Arrow Function Export Extraction', () => { @@ -1590,7 +1670,7 @@ impl From for Own { ).toBe(true); }); - it('keeps the owner shape for `self.()` and `self..()`, and collapses every other receiver (#1585, #1861)', () => { + it('keeps the owner shape for `self.()` and `self..()`, a longer receiver chain whole, and collapses a parenthesized receiver (#1585, #1861, #2836)', () => { const code = ` pub struct Outer { pub inner: Inner, pub deep: Deep } impl Outer { @@ -1609,22 +1689,23 @@ impl Outer { const calls = result.unresolvedReferences .filter((r) => r.referenceKind === 'calls') .map((r) => r.referenceName); - // Two shapes keep an owner the resolver can act on: the single-hop field - // receiver, whose type it reads off the owner struct (#1585), and the bare - // `self` receiver, whose type is the calling method's own owner (#1861). - // `self.make().run()` contributes `self.make` — the inner call — and its - // OUTER call collapses, because a method's return type is not read here. + // The single-hop field receiver keeps its owner, whose type the resolver + // reads off the owner struct (#1585), and so does the bare `self` + // receiver, whose type is the calling method's own owner (#1861). A + // longer chain of fields and method calls is kept whole, a call written + // `name()` (#2836): `self.make().run()` contributes `self.make`, the inner + // call, and `self.make().run`. expect(calls.filter((c) => c.startsWith('self.')).sort()).toEqual([ + 'self.deep.inner.run', 'self.inner.run', 'self.make', + 'self.make().run', 'self.run', ]); // A local receiver keeps its name as before… expect(calls).toContain('local.run'); - // …and the deeper chain, the call receiver and the parenthesized receiver - // still collapse to the method name. `self.run()` no longer does, so this - // is three rather than four. - expect(calls.filter((c) => c === 'run')).toHaveLength(3); + // …and the parenthesized receiver still collapses to the method name. + expect(calls.filter((c) => c === 'run')).toHaveLength(1); const outerRun = result.nodes.find((n) => n.qualifiedName === 'Outer::run'); expect(outerRun).toBeDefined(); const fieldRef = result.unresolvedReferences.find((r) => r.referenceName === 'self.inner.run'); @@ -5245,6 +5326,87 @@ import 'package:flutter/material.dart'; expect(vars).toContain('heading'); }); + /* `{% capture name %}…{% endcapture %}` declares a variable as `assign` + does; only assigns were read (#2498). */ + it.each([ + ['{% capture captured %}Hello{% endcapture %}', 'captured', 1, 0], + ['{%- capture trimmed -%}Hi{%- endcapture -%}', 'trimmed', 1, 0], + ['{% capture tight-%}Hi{% endcapture %}', 'tight', 1, 0], + ["

{% capture 'single' %}Hi{% endcapture %}

", 'single', 1, 3], + ['{% capture "double" %}Hi{% endcapture %}', 'double', 1, 0], + ['{% liquid\n capture bare\n echo "Hi"\n endcapture\n%}', 'bare', 2, 2], + ['{% liquid\r\n capture crlf # a note\r\n endcapture\r\n%}', 'crlf', 2, 2], + ["{%- liquid\n capture 'quoted_line'\n endcapture\n-%}", 'quoted_line', 2, 2], + // A name with `-` in it is one name; the `-` of `-%}` trims whitespace. + ['{% capture title-text %}Hi{% endcapture %}', 'title-text', 1, 0], + ['{% capture title-text-%}Hi{% endcapture %}', 'title-text', 1, 0], + ['{%- capture title-text -%}Hi{%- endcapture -%}', 'title-text', 1, 0], + ["{% capture 'hero-alt' %}Hi{% endcapture %}", 'hero-alt', 1, 0], + ['{% liquid\n capture hero-alt\n endcapture\n%}', 'hero-alt', 2, 2], + ] as const)('reads a capture declaration from %j', (code, name, line, column) => { + const result = extractFromSource('sections/card.liquid', code); + const variables = result.nodes.filter((n) => n.kind === 'variable'); + expect(variables.map((n) => [n.name, n.qualifiedName, n.startLine, n.startColumn])) + .toEqual([[name, `sections/card.liquid::${name}`, line, column]]); + const file = result.nodes.find((n) => n.kind === 'file')!; + expect(result.edges).toContainEqual({ source: file.id, target: variables[0]!.id, kind: 'contains' }); + }); + + it('reads no capture from comments, raw regions, prose or a tag without a name', () => { + const code = [ + '{% comment %}{% capture ghost %}x{% endcapture %}{% endcomment %}', + '{%- raw -%}{% capture ghost %}x{% endcapture %}{%- endraw -%}', + '{% # capture ghost %}', + '{% liquid', ' # capture ghost', ' comment', ' capture ghost', ' endcomment', '%}', + '{% capture %}x{% endcapture %}', + '

We capture ghost moments.

', + '{% assign text = "{% capture ghost %}" %}', + ].join('\n'); + const result = extractFromSource('sections/card.liquid', code); + expect(result.nodes.filter((n) => n.kind === 'variable').map((n) => n.name)).toEqual(['text']); + }); + + it('reads an assign name with `-` in it whole', () => { + const code = [ + "{% assign title-text = 'Sale' %}", + '{%- assign page-title=page.title -%}', + '{% liquid', + ' assign hero-image = section.settings.image', + '%}', + ].join('\n'); + const variables = extractFromSource('sections/card.liquid', code).nodes.filter((n) => n.kind === 'variable'); + expect(variables.map((n) => [n.name, n.startLine])).toEqual([['title-text', 1], ['page-title', 2], ['hero-image', 4]]); + }); + + it('never cuts a name short', () => { + // Liquid allows `.` in a name too, which neither rule reads; such a name + // gets no node rather than a wrong one. + const code = [ + '{% capture ghost.name %}x{% endcapture %}', + '{% capture ghost-name.x %}x{% endcapture %}', + '{% assign ghost.name = 1 %}', + '{% assign ghost-name.x = 1 %}', + ].join('\n'); + expect(extractFromSource('sections/card.liquid', code).nodes.filter((n) => n.kind === 'variable')).toEqual([]); + }); + + it('keeps an assign and captures of the same name on one line apart', () => { + const code = '{% assign title = 1 %}{% capture title %}one{% endcapture %}{% capture title %}two{% endcapture %}'; + const variables = extractFromSource('sections/card.liquid', code).nodes.filter((n) => n.kind === 'variable'); + expect(variables.map((n) => [n.name, n.startColumn])) + .toEqual([['title', 0], ['title', code.indexOf('{% capture')], ['title', code.lastIndexOf('{% capture')]]); + expect(new Set(variables.map((n) => n.id)).size).toBe(3); + // The assign keeps the id it had before captures were read. + expect(variables[0]!.id).toBe(generateNodeId('sections/card.liquid', 'variable', 'title', 1)); + }); + + it('still reads the tags inside a capture', () => { + const code = "{% capture card_html %}{% render 'card' %}{% assign inner = 1 %}{% endcapture %}"; + const result = extractFromSource('sections/product.liquid', code); + expect(result.nodes.filter((n) => n.kind === 'variable').map((n) => n.name).sort()).toEqual(['card_html', 'inner']); + expect(result.unresolvedReferences.map((r) => r.referenceName)).toEqual(['snippets/card.liquid']); + }); + it('should extract section inside a {% liquid %} block', () => { const code = ['{% liquid', " section 'header'", '%}'].join('\n'); const result = extractFromSource('layout/theme.liquid', code); @@ -5337,6 +5499,40 @@ import 'package:flutter/material.dart'; .toEqual([['snippets/live.liquid', 18], ['snippets/after.liquid', 20]]); }); + /* A layout renders a section group, the JSON file `sections/.json`, + with the plural tag; only the singular one was read (#2494). */ + it.each([ + "{% sections 'header-group' %}", + '{%- sections "header-group" -%}', + "{% liquid\n sections 'header-group'\n%}", + ])('reads a section group from %j', (code) => { + const result = extractFromSource('layout/theme.liquid', code); + expect(result.nodes.filter((n) => n.kind === 'import').map((n) => n.name)).toEqual(['header-group']); + expect(result.unresolvedReferences.map((r) => r.referenceName)).toEqual(['sections/header-group.json']); + }); + + it('keeps a section and a section group of the same name apart', () => { + const result = extractFromSource('layout/theme.liquid', "{% section 'header' %}{% sections 'header' %}"); + expect(result.unresolvedReferences.map((r) => [r.referenceName, r.line, r.column])) + .toEqual([['sections/header.liquid', 1, 0], ['sections/header.json', 1, 22]]); + expect(result.nodes.filter((n) => n.kind === 'component').map((n) => n.qualifiedName)) + .toEqual(['layout/theme.liquid::section:header', 'layout/theme.liquid::sections:header']); + expect(new Set(result.nodes.map((n) => n.id)).size).toBe(result.nodes.length); + }); + + it.each(['comment', 'raw'])('indexes no schema written inside %s', (tag) => { + // An example schema in a comment is not the section's, and a lone + // opening tag in one must not swallow the real schema after it (#2494). + const code = [ + `{% ${tag} %}`, '{% schema %}', '{"name": "Ghost"}', '{% endschema %}', `{% end${tag} %}`, + `{%- ${tag} -%}{% schema %}{%- end${tag} -%}`, + '{% schema %}', '{"name": "Live"}', '{% endschema %}', + ].join('\n'); + const result = extractFromSource('sections/header.liquid', code); + expect(result.nodes.filter((n) => n.kind === 'constant').map((n) => [n.name, n.startLine])) + .toEqual([['Live', 7]]); + }); + it('should extract multiple imports', () => { const code = ` {% section 'header' %} @@ -6931,13 +7127,15 @@ describe('Cross-language type/import gate (RN name collisions)', () => { // TS name-match — so a TS ref to `TestRunner` crossed web→jvm. The ref here // is intentionally NOT imported: a clean relative import would mask the bug // by resolving via the import map before the framework strategy can win. + // So the TS type is a global, declared in a script `.d.ts`: the one kind of + // type a module names without importing it. fs.writeFileSync( path.join(tempDir, 'package.json'), JSON.stringify({ dependencies: { 'react-native': '*' } }) ); fs.writeFileSync( - path.join(tempDir, 'useTests.ts'), - `export type TestRunner = { run: () => void };\n` + path.join(tempDir, 'useTests.d.ts'), + `type TestRunner = { run: () => void };\n` ); fs.writeFileSync( path.join(tempDir, 'basic.tsx'), @@ -7109,6 +7307,113 @@ describe('Razor / Blazor markup extraction', () => { if (fs.existsSync(tempDir)) fs.rmSync(tempDir, { recursive: true, force: true }); }); + /** The files of everything that calls or references the named function. */ + const dependents = async (name: string): Promise => { + cg = CodeGraph.initSync(tempDir); + await cg.indexAll(); + const target = cg.getNodesByKind('function').find((n) => n.name === name); + expect(target, name).toBeDefined(); + return [...cg.getImpactRadius(target!.id, 1).nodes.values()].map((n) => n.filePath ?? '').filter((p) => !p.endsWith('.js') && !p.endsWith('.ts')); + }; + + it('links what a view\'s inline script calls to the JavaScript that defines it', async () => { + fs.writeFileSync(path.join(tempDir, 'site.js'), 'export function showToast(msg) { return msg; }\n'); + fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'Views/Order.cshtml'), + "@model OrderViewModel\n
Order
\n\n" + ); + expect(await dependents('showToast')).toContain('Views/Order.cshtml'); + const ref = cg.getOutgoingEdges(cg.getNodesByName('Order')[0]!.id).find((e) => e.kind === 'calls'); + expect(ref?.line, 'a reference keeps its line in the view').toBe(5); + }); + + it('reads a global helper called from a script that holds Razor', async () => { + fs.writeFileSync(path.join(tempDir, 'common.js'), 'function displayNotice(text) { return text; }\nvar AjaxCart = { add: function add(url) { return url; } };\n'); + fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'Views/Cart.cshtml'), [ + '@model CartModel', + '', + ].join('\n')); + expect(await dependents('displayNotice')).toContain('Views/Cart.cshtml'); + const refs = (cg as any).db.db.prepare(`SELECT reference_name n FROM unresolved_refs WHERE file_path = 'Views/Cart.cshtml' AND language = 'javascript'`).all().map((r: { n: string }) => r.n); + for (const name of refs) expect(name, 'Razor reads as no JavaScript').not.toMatch(/Html|Json|Model|Url\.Action/); + }); + + it('leaves template, data and sourced script blocks alone, and calls no block\'s own function another file\'s', async () => { + fs.writeFileSync(path.join(tempDir, 'lib.js'), 'function fromTemplate() {}\nfunction ownName() {}\nfunction fromSrc() {}\n'); + fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'Views/Page.cshtml'), [ + '', + '', + '', + '@* *@', + '', + '', + ].join('\n')); + cg = CodeGraph.initSync(tempDir); + await cg.indexAll(); + for (const name of ['fromTemplate', 'ownName', 'fromSrc']) { + const target = cg.getNodesByKind('function').find((n) => n.name === name)!; + expect(cg.getCallers(target.id).map((c) => c.node.name), name).toEqual([]); + } + expect(cg.getNodesInFile('Views/Page.cshtml').filter((n) => n.kind === 'function'), 'a view stays one component node').toEqual([]); + }); + + it('leaves a global two bundles define for the page to decide', async () => { + fs.mkdirSync(path.join(tempDir, 'wwwroot/js'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'wwwroot/js/admin.common.js'), 'function setLocation(url) { return url; }\nfunction onlyAdmin() {}\n'); + fs.writeFileSync(path.join(tempDir, 'wwwroot/js/public.common.js'), 'function setLocation(url) { return url; }\n'); + fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'Views/Page.cshtml'), "\n"); + cg = CodeGraph.initSync(tempDir); + await cg.indexAll(); + const callees = cg.getOutgoingEdges(cg.getNodesByName('Page')[0]!.id).filter((e) => e.kind === 'calls').map((e) => cg.getNode(e.target)?.name); + expect(callees).toEqual(['onlyAdmin']); + }); + + it('reads a block as TypeScript when it says so', async () => { + fs.writeFileSync(path.join(tempDir, 'helper.ts'), 'export function typedHelper(n: number) { return n; }\n'); + fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'Views/Typed.cshtml'), + "\n" + ); + expect(await dependents('typedHelper')).toContain('Views/Typed.cshtml'); + }); + + it('keeps a view\'s script links through a sync that re-reads only the view', async () => { + fs.writeFileSync(path.join(tempDir, 'site.js'), 'function firstHelper() {}\nfunction secondHelper() {}\n'); + fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, 'Views/Page.cshtml'), '\n'); + expect(await dependents('firstHelper')).toContain('Views/Page.cshtml'); + fs.writeFileSync(path.join(tempDir, 'Views/Page.cshtml'), '\n'); + await cg.sync(); + const second = cg.getNodesByKind('function').find((n) => n.name === 'secondHelper')!; + expect(cg.getCallers(second.id).map((c) => c.node.filePath)).toEqual(['Views/Page.cshtml']); + }); + it('links @model and Blazor component tags to their C# types; ignores HTML elements', async () => { fs.mkdirSync(path.join(tempDir, 'Views'), { recursive: true }); fs.writeFileSync( @@ -7421,6 +7726,24 @@ describe('Liquid Shopify JSON template section resolution', () => { expect(cg.getNodesByKind('variable').some((n) => n.name === 'heading')).toBe(true); }); + it('indexes capture declarations as variables that search finds, in or out of a Shopify theme', async () => { + // A captured variable could not be found, while an assigned one could (#2498). + fs.mkdirSync(path.join(tempDir, '_includes'), { recursive: true }); + fs.writeFileSync(path.join(tempDir, '_includes/card.liquid'), [ + "{% assign existing = 'ok' %}", + '{% capture captured %}Hello{% endcapture %}', + '{% liquid', ' capture bare_capture', " echo 'x'", ' endcapture', '%}', + ].join('\n')); + cg = CodeGraph.initSync(tempDir); + await cg.indexAll(); + + for (const [name, line] of [['existing', 1], ['captured', 2], ['bare_capture', 4]] as const) { + const hit = cg.searchNodes(name, { kinds: ['variable'] }).find((r) => r.node.name === name); + expect(hit?.node, name).toMatchObject({ filePath: '_includes/card.liquid', language: 'liquid', startLine: line }); + expect(cg.getIncomingEdges(hit!.node.id)).toContainEqual(expect.objectContaining({ kind: 'contains' })); + } + }); + it('links a Shopify JSON template section `type` to its sections/.liquid', async () => { // Shopify OS 2.0 templates are JSON, referencing sections by `type` — not // a `{% section %}` Liquid tag — so a section used only from a JSON template @@ -7669,6 +7992,97 @@ describe('Liquid Shopify JSON template section resolution', () => { expect(cg.getFileDependencies('sections/header.liquid')).toEqual(['snippets/price.liquid']); }, 60_000); + // The notice Shopify's theme editor writes at the top of every JSON template + // and section group it saves. + const editorNotice = [ + '/*', + ' * ------------------------------------------------------------', + ' * IMPORTANT: The contents of this file are auto-generated.', + ' * ------------------------------------------------------------', + ' */', + ]; + const referencesFrom = (file: string): Array<[string | undefined, number | undefined, number | undefined]> => { + const fileNode = cg.getNodesInFile(file).find((n) => n.kind === 'file')!; + return cg.getOutgoingEdges(fileNode.id) + .filter((e) => e.kind === 'references') + .map((e): [string | undefined, number | undefined, number | undefined] => [cg.getNode(e.target)?.filePath, e.line, e.column]) + .sort((a, b) => a[1]! - b[1]!); + }; + + it.each(['\n', '\r\n'])('links the sections an editor-saved template and section group list, where each `type` is (%j)', async (newline) => { + // JSON.parse rejects the editor's comment, so such a file linked none of + // its sections, and a file without one linked them at line 1 (#2494). + writeFiles({ + 'layout/theme.liquid': `{% sections 'header-group' %}{{ content_for_layout }}\n`, + 'sections/header-group.json': [ + ...editorNotice, '{', ' "type": "header",', ' "name": "Header",', ' "sections": {', + ' "bar": { "type": "announcement-bar" },', ' "header": {', ' "type": "header"', ' }', + ' },', ' "order": ["bar", "header"]', '}', + ].join(newline), + 'sections/announcement-bar.liquid': `
\n`, + 'sections/header.liquid': `
\n`, + 'sections/main-product.liquid': `
{{ product.title }}
\n`, + 'templates/product.json': [ + ...editorNotice, '{', ' "sections": {', ' "main": {', ' "type": "main-product",', + ' "settings": { "note": "https://example.com/*not-a-comment*/ // nor this" }', ' },', + ' "again": { "type": "main-product" }', ' },', ' "order": ["main", "again"]', '}', + ].join(newline), + 'templates/index.json': `{\n "sections": {\n "main": { "type": "header" }\n },\n "order": ["main"]\n}\n`, + }); + cg = CodeGraph.initSync(tempDir); + await cg.indexAll(); + cg.resolveReferences(); + + expect(referencesFrom('templates/product.json')).toEqual([['sections/main-product.liquid', 9, 14]]); + expect(referencesFrom('sections/header-group.json')).toEqual([ + ['sections/announcement-bar.liquid', 10, 21], + ['sections/header.liquid', 12, 14], + ]); + expect(referencesFrom('templates/index.json')).toEqual([['sections/header.liquid', 3, 22]]); + expect(referencesFrom('layout/theme.liquid')).toEqual([['sections/header-group.json', 1, 0]]); + }); + + it('reads a JSON template with trailing commas, which Shopify accepts in theme JSON', () => { + const text = '// a note\n{"sections": {"main": {"type": "main-product",}, "hero": {"type": "hero"},}, "order": ["main", "hero",],}'; + expect(extractFromSource('templates/product.json', text, 'liquid').unresolvedReferences.map((r) => r.referenceName)) + .toEqual(['sections/main-product.liquid', 'sections/hero.liquid']); + }); + + it.each([ + ['a missing brace', '{"sections": {"main": {"type": "main-product"}}'], + ['an unclosed comment', '/* saved by the editor {"sections": {"main": {"type": "main-product"}}}'], + ['a missing comma', '{"sections": {"main": {"type": "main-product"} "hero": {"type": "hero"}}}'], + ])('links nothing from a JSON template with %s', (_case, text) => { + expect(extractFromSource('templates/product.json', text, 'liquid').unresolvedReferences).toEqual([]); + }); + + it("links a layout's section groups inside its own theme, and on to the sections they list", async () => { + // `{% sections 'header-group' %}` renders the theme's + // sections/header-group.json, whose sections render snippets (#2494). + writeFiles({ + ...severalThemes, + 'layout/theme.liquid': `{% sections 'header-group' %}{{ content_for_layout }}\n`, + 'sections/header-group.json': [...editorNotice, jsonTemplate('header')].join('\n'), + // This theme has no header group of its own, and the root theme's is not its. + 'examples/tailwind/layout/theme.liquid': `{% liquid\n sections 'header-group'\n%}{{ content_for_layout }}\n`, + }); + cg = CodeGraph.initSync(tempDir); + await cg.indexAll(); + cg.resolveReferences(); + + const linksOf = (file: string): string[] => cg.getFileDependencies(file).sort(); + expect(linksOf('layout/theme.liquid')).toEqual(['sections/header-group.json']); + expect(linksOf('sections/header-group.json')).toEqual(['sections/header.liquid']); + expect(linksOf('sections/header.liquid')).toEqual(['snippets/price.liquid']); + expect(linksOf('examples/tailwind/layout/theme.liquid')).toEqual([]); + + // Once the theme has a header group of its own, sync links the layout to it. + writeFiles({ 'examples/tailwind/sections/header-group.json': [...editorNotice, jsonTemplate('header')].join('\n') }); + await cg.sync(); + expect(linksOf('examples/tailwind/layout/theme.liquid')).toEqual(['examples/tailwind/sections/header-group.json']); + expect(linksOf('examples/tailwind/sections/header-group.json')).toEqual(['examples/tailwind/sections/header.liquid']); + }, 60_000); + it('finds the theme a file is in from the theme folder it sits in', () => { const markers = new Set(['layout/theme.liquid', 'examples/tailwind/config/settings_schema.json']); const exists = (relativePath: string): boolean => markers.has(relativePath); @@ -9797,6 +10211,66 @@ const value = 42; ); expect(containEdges.length).toBeGreaterThan(0); }); + + it('should not let a self-closing +`; + const result = extractFromSource('Page.astro', code); + + expect(result.errors).toEqual([]); + const ready = result.nodes.find((n) => n.name === 'ready'); + expect(ready).toBeDefined(); + expect(ready?.startLine).toBe(8); + // The markup between the two scripts is still scanned. + const call = result.unresolvedReferences.find( + (ref) => ref.referenceKind === 'calls' && ref.referenceName === 'formatDate' && ref.line === 6 + ); + expect(call).toBeDefined(); + }); + + it('should not let a self-closing +`; + const result = extractFromSource('Styled.astro', code); + + const call = result.unresolvedReferences.find( + (ref) => ref.referenceKind === 'calls' && ref.referenceName === 'formatDate' && ref.line === 2 + ); + expect(call).toBeDefined(); + }); + + it("still reads a call in a self-closing +`; + const result = extractFromSource('Schema.astro', code); + + expect(result.errors).toEqual([]); + expect(result.nodes.find((n) => n.name === 'ready')?.startLine).toBe(7); + const call = result.unresolvedReferences.find( + (ref) => ref.referenceKind === 'calls' && ref.referenceName === 'serializeJsonLd' && ref.line === 5 + ); + expect(call).toBeDefined(); + }); }); describe('Instantiates + Decorates edge extraction', () => { @@ -13458,6 +13932,300 @@ describe('C/C++ kernel-port preParse blanks (R7a)', () => { } }); + it('Go: a method carries the exportedness of its name, like a function', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-go-method-exported-')); + try { + fs.writeFileSync( + path.join(dir, 'log.go'), + 'package log\n\ntype writer struct{}\n\nfunc (w *writer) Close() error { return nil }\n\nfunc (w *writer) flush() {}\n\nfunc Open() *writer { return &writer{} }\n\nfunc helper() {}\n' + ); + const cg = await CodeGraph.init(dir, { index: true }); + try { + const flag = (name: string) => + cg.getNodesByKind('method').concat(cg.getNodesByKind('function')).find((n) => n.name === name)!.isExported; + expect(flag('Close')).toBe(true); + expect(flag('flush')).toBe(false); + expect(flag('Open')).toBe(true); + expect(flag('helper')).toBe(false); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('Go: a call through a package stays on its function, not an exported method of that name', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-go-package-member-')); + try { + fs.writeFileSync(path.join(dir, 'go.mod'), 'module example.com/app\n\ngo 1.22\n'); + fs.mkdirSync(path.join(dir, 'applog')); + // The method comes first, so a lookup by name meets it before the function. + fs.writeFileSync( + path.join(dir, 'applog', 'log.go'), + 'package applog\n\ntype Logger struct{}\n\nfunc (l *Logger) Infof(format string) {}\n\nfunc Infof(format string) {}\n' + ); + fs.mkdirSync(path.join(dir, 'svc')); + fs.writeFileSync( + path.join(dir, 'svc', 'use.go'), + 'package svc\n\nimport "example.com/app/applog"\n\nfunc Use(l *applog.Logger) {\n\tapplog.Infof("x")\n\tl.Infof("y")\n}\n' + ); + const cg = await CodeGraph.init(dir, { index: true }); + try { + const use = cg.getNodesByKind('function').find((n) => n.name === 'Use')!; + const calls = cg.getOutgoingEdgesFrom([use.id], ['calls']).map((e) => { + const target = cg.getNode(e.target)!; + return `${e.line} ${target.kind} ${target.qualifiedName}`; + }).sort(); + expect(calls).toEqual(['6 function Infof', '7 method Logger::Infof']); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('blankCUnbalancedConditionalBranches opens a damaged #if group, one branch kept only when they do not balance, offsets kept', async () => { + const { blankCUnbalancedConditionalBranches } = await import('../src/extraction/languages/c-cpp'); + const src = [ + 'void f(int id)', + '{', + ' if (id == 1) {', + ' a(id);', + ' }', + '#if defined(USE_V) || \\', + ' defined(USE_W)', + ' else if (id == 2) { // "{" in a comment', + ' b(id);', + ' }', + '#else', + ' else if (id == 3) {', + ' c(id);', + ' }', + '#endif', + '#ifdef USE_X', + ' d(id);', + '#else', + ' e(id);', + '#endif', + '}', + '#if 0', + 'void dead(void) {', + '#else', + 'void live(void) {', + '#endif', + '}', + ].join('\n'); + const out = blankCUnbalancedConditionalBranches(src); + expect(out.length).toBe(src.length); + expect(out.split('\n').length).toBe(src.split('\n').length); + const lines = out.split('\n'); + // The `else`-led group's branches each balance, so both stay, read in a + // row (`else if (id == 2) {…} else if (id == 3) {…}` is still C); only + // its directive lines, continuation included, become spaces. + expect(lines[5]).toBe(' '.repeat(src.split('\n')[5]!.length)); + expect(lines[6]).toBe(' '.repeat(src.split('\n')[6]!.length)); + expect(lines[7]).toBe(' else if (id == 2) { // "{" in a comment'); + expect(lines[8]).toBe(' b(id);'); + expect(lines[10]).toBe(' '); + expect(lines[11]).toBe(' else if (id == 3) {'); + expect(lines[12]).toBe(' c(id);'); + expect(lines[14]).toBe(' '); + // A balanced group is untouched, directives included. + expect(lines[15]).toBe('#ifdef USE_X'); + expect(lines[16]).toBe(' d(id);'); + expect(lines[18]).toBe(' e(id);'); + expect(lines[19]).toBe('#endif'); + // `#if 0` keeps the live branch instead. + expect(out).not.toContain('dead'); + expect(lines[24]).toBe('void live(void) {'); + // `#if 1` is live as written: its first branch stays, not its `#else`. + const on = blankCUnbalancedConditionalBranches( + ['void g(int id) {', ' if (id) { a(); }', '#if 1', ' else if (id == 2) { live(); }', '#else', ' else if (id == 3) { dead(); }', '#endif', '}'].join('\n') + ); + expect(on).toContain('else if (id == 2) { live(); }'); + expect(on).not.toContain('dead'); + // Arms that close the block above and open their own (`} else if (…) {`) + // balance too: under a flag, both builds' arms stay (betaflight's + // `#if ENABLE_RESCUE_PLAN` in failsafe.c), so neither one's calls are lost. + const arms = blankCUnbalancedConditionalBranches( + ['void f(int a) {', ' if (a == 1) {', ' one();', '#if ENABLE_X', ' } else if (a == 2) {', ' two();', '#else', ' } else if (a == 3) {', ' three();', '#endif', ' }', '}'].join('\n') + ).split('\n'); + expect(arms[3]).toBe(' '.repeat('#if ENABLE_X'.length)); + expect(arms[5]).toBe(' two();'); + expect(arms[8]).toBe(' three();'); + // A brace inside a block comment is not code. Lua's loadlib.c opens a + // banner with `** {====` inside a multi-line comment and closes it with + // `/* }==== */`: the group is whole, and every platform's functions stay. + const lua = [ + '#if defined(LUA_DL_DLOPEN)', + 'static void *ll_load (const char *path) { return dlopen(path); }', + '#elif defined(LUA_DL_DLL)', + '/*', + '** {======================================================', + '** This is an implementation of loadlib for Windows', + '** =======================================================', + '*/', + 'static void *ll_load (const char *path) { return LoadLibraryA(path); }', + '/* }====================================================== */', + '#else', + 'static void *ll_load (const char *path) { return NULL; }', + '#endif', + ].join('\n'); + expect(blankCUnbalancedConditionalBranches(lua)).toBe(lua); + // A `\` ending a line inside a block comment still continues the + // `#define` (jemalloc's tsd.h): the macro body is not code of the group. + const guarded = [ + '#ifndef GUARD_H', + '#define GUARD_H', + '#define O(n)\t\t\t\t\\', + 'static int n##_get(void) {\t\t\\', + '\t/*\t\t\t\t\t\\', + '\t * a comment inside the body\t\\', + '\t */\t\t\t\t\t\\', + '\treturn 0;\t\t\t\t\\', + '}', + 'O(a)', + '#endif', + ].join('\n'); + expect(blankCUnbalancedConditionalBranches(guarded)).toBe(guarded); + // No conditional group at all: identity. + const plain = 'int g(void) {\n#define X 1\n return X;\n}\n'; + expect(blankCUnbalancedConditionalBranches(plain)).toBe(plain); + }); + + it('blankCUnbalancedConditionalBranches: two function heads under #ifdef/#else keep the #else head; expression fragments keep their only branch', async () => { + const { blankCUnbalancedConditionalBranches } = await import('../src/extraction/languages/c-cpp'); + // jq's main.c: `umain` for WIN32, `main` otherwise — `main` must survive. + const heads = [ + '#ifdef WIN32', + 'int umain(int argc, char* argv[]) {', + '#else /*}*/', + 'int main(int argc, char* argv[]) {', + '#endif', + ' return run(argc, argv);', + '}', + ].join('\n'); + const h = blankCUnbalancedConditionalBranches(heads).split('\n'); + expect(h[1]).toBe(' '.repeat('int umain(int argc, char* argv[]) {'.length)); + expect(h[3]).toBe('int main(int argc, char* argv[]) {'); + expect(h[0]).toBe(' '); + // `#ifndef` is the inverse: its first branch is the default build. + const inv = heads.replace('#ifdef WIN32', '#ifndef POSIX'); + expect(blankCUnbalancedConditionalBranches(inv).split('\n')[1]).toBe('int umain(int argc, char* argv[]) {'); + // A directive right under the kept branch's own is a line of that branch: + // only the `#if`/`#else` line itself (and its `\` continuations) goes. + const withIncludes = [ + '#if defined(FEATURE) && \\', + ' defined(OTHER)', + '#include "feature.h"', + 'int f(int x) {', + '#else', + '#include "plain.h"', + 'int g(int x) {', + '#endif', + ' return x;', + '}', + ]; + const inc = blankCUnbalancedConditionalBranches(withIncludes.join('\n')).split('\n'); + expect(inc[1]).toBe(' '.repeat(withIncludes[1]!.length)); + expect(inc[2]).toBe(' '.repeat(withIncludes[2]!.length)); + expect(inc[5]).toBe('#include "plain.h"'); + expect(inc[6]).toBe('int g(int x) {'); + // A branch that is a fragment of an `if (…)` condition (STM32 HAL), or + // that opens with `} else if` (betaflight cli.c): directives blanked, the + // only branch kept, so the statement parses as written for that build. + const frag = [ + 'int check(op_t *h1, op_t *h4)', + '{', + ' if ((h1 == NULL)', + '#if defined(STM32G474xx)', + ' || (h4 == NULL)', + '#endif', + ' )', + ' {', + ' return 1;', + ' }', + ' if (', + '#if !defined(USE_FLASH)', + ' isEmpty(h1) ||', + '#endif', + ' strncasecmp(h1, "rom", 3) == 0) {', + ' reboot(1);', + '#if defined(USE_FLASH)', + ' } else if (isEmpty(h1)) {', + ' reboot(2);', + '#endif', + ' } else {', + ' reboot(0);', + ' }', + ' return 0;', + '}', + ].join('\n'); + const f = blankCUnbalancedConditionalBranches(frag).split('\n'); + expect(f[3]).toBe(' '.repeat('#if defined(STM32G474xx)'.length)); + expect(f[4]).toBe(' || (h4 == NULL)'); + expect(f[5]).toBe(' '); + expect(f[12]).toBe(' isEmpty(h1) ||'); + expect(f[17]).toBe(' } else if (isEmpty(h1)) {'); + expect(f[16]).toBe(' '.repeat('#if defined(USE_FLASH)'.length)); + }); + + it('a #if branch beginning with `else` no longer files the rest of the file under the enclosing function', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-c-ifelse-')); + try { + fs.writeFileSync( + path.join(dir, 'current.c'), + [ + 'void currentMeterRead(int id, meter_t *meter)', + '{', + ' if (id == 1) {', + ' adcRead(meter);', + ' }', + '#ifdef USE_VIRTUAL', + ' else if (id == 2) {', + ' virtualRead(meter);', + ' }', + '#endif', + ' else {', + ' resetMeter(meter);', + ' }', + '}', + '', + 'static bool markBusy(void)', + '{', + ' ATOMIC_BLOCK(NVIC_PRIO_MAX) {', + ' busy = true;', + ' }', + ' return true;', + '}', + '', + 'int after(void)', + '{', + ' return 1;', + '}', + '', + ].join('\n') + ); + const cg = await CodeGraph.init(dir, { index: true }); + try { + const fns = cg.getNodesByKind('function').filter((n) => n.filePath === 'current.c'); + const byName = Object.fromEntries(fns.map((n) => [n.name, n])); + expect(Object.keys(byName).sort()).toEqual(['after', 'currentMeterRead', 'markBusy']); + expect(byName.currentMeterRead!.endLine).toBe(14); + expect(byName.markBusy!.qualifiedName).toBe('markBusy'); + expect(byName.markBusy!.endLine).toBe(22); + expect(byName.after!.qualifiedName).toBe('after'); + expect(byName.after!.startLine).toBe(24); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it('blankCStatementMacroCalls blanks indented iterator macros, keeps the block', async () => { const { blankCStatementMacroCalls } = await import('../src/extraction/languages/c-cpp'); const src = [ @@ -13470,7 +14238,8 @@ describe('C/C++ kernel-port preParse blanks (R7a)', () => { ].join('\n'); const out = blankCStatementMacroCalls(src); expect(out.length).toBe(src.length); - expect(out).not.toContain('list_for_each_entry'); + expect(out).toContain('list_for_each_entry;'); + expect(out).not.toContain('head, member'); expect(out).toContain('use(pos);'); // A real call statement ends with `;` — untouched. expect(out).toContain('use(pos);'); @@ -13677,7 +14446,7 @@ describe('C/C++ kernel-port preParse blanks (R7a)', () => { ].join('\n'); const out = blankCStatementMacroCalls(src); expect(out.length).toBe(src.length); - expect(out).not.toContain('hlist_for_each_entry_rcu'); + expect(out).toContain('hlist_for_each_entry_rcu;'); expect(out).not.toContain('lockdep_is_held'); expect(out).toContain('use(p);'); // A wrapped REAL call ends in `;` — untouched. diff --git a/__tests__/file-cycles.test.ts b/__tests__/file-cycles.test.ts new file mode 100644 index 0000000000..7c23d228af --- /dev/null +++ b/__tests__/file-cycles.test.ts @@ -0,0 +1,146 @@ +/** + * What counts as a circular import: the strongly connected components of the + * file import graph, and nothing else the graph knows about two files. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { CodeGraph } from '../src'; +import { findFileCycles } from '../src/graph/file-cycles'; + +const pairs = (...edges: string[]) => + edges.map((edge) => { + const [source, target] = edge.split('>') as [string, string]; + return { source, target }; + }); + +describe('findFileCycles', () => { + it('finds nothing in a graph with no loop, a diamond included', () => { + expect(findFileCycles(pairs('a>b', 'a>c', 'b>d', 'c>d'))).toEqual([]); + expect(findFileCycles([])).toEqual([]); + }); + + it('reports a knot once, with every file in it', () => { + // a>b>a is a loop and so is a>c>b>a. A walk meets b first and has nothing + // left to do at c>b, so it reported a>b and left c out. + expect(findFileCycles(pairs('a>b', 'b>a', 'a>c', 'c>b'))).toEqual([['a', 'b', 'c']]); + // Two loops through one file are one group, however many ways round there are. + expect(findFileCycles(pairs('a>b', 'b>a', 'a>c', 'c>a'))).toEqual([['a', 'b', 'c']]); + }); + + it('keeps separate knots apart, smallest first, and leaves out the files between them', () => { + expect(findFileCycles(pairs('p>q', 'q>r', 'r>p', 'q>x', 'x>a', 'a>b', 'b>a'))).toEqual([ + ['a', 'b'], + ['p', 'q', 'r'], + ]); + }); + + it('gives the same report for the pairs in any order', () => { + const edges = pairs('a>b', 'b>c', 'c>a', 'd>e', 'e>d', 'c>d'); + expect(findFileCycles([...edges].reverse())).toEqual(findFileCycles(edges)); + }); + + it('walks a chain deeper than the call stack', () => { + const length = 200_000; + const chain = Array.from({ length }, (_, i) => ({ source: `f${i}`, target: `f${i + 1}` })); + chain.push({ source: `f${length}`, target: 'f0' }); + const cycles = findFileCycles(chain); + expect(cycles).toHaveLength(1); + expect(cycles[0]).toHaveLength(length + 1); + }); +}); + +describe('findCircularDependencies', () => { + let dir: string; + let cg: CodeGraph | null = null; + + async function index(files: Record): Promise { + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(dir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + } + cg = CodeGraph.initSync(dir); + await cg.indexAll(); + return cg; + } + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-file-cycles-')); + }); + afterEach(() => { + cg?.close(); + cg = null; + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('finds a loop that runs through a barrel', async () => { + const graph = await index({ + 'src/index.ts': "export * from './a';\n", + 'src/a.ts': "import { helper } from './helper';\nexport function a() { return helper(); }\n", + 'src/helper.ts': "import { a } from './index';\nexport function helper() { return a; }\n", + }); + expect(graph.findCircularDependencies()).toEqual([['src/a.ts', 'src/helper.ts', 'src/index.ts']]); + }); + + it('finds a loop in Python', async () => { + const graph = await index({ + 'a.py': 'from b import bb\n\ndef aa():\n return bb()\n', + 'b.py': 'from a import aa\n\ndef bb():\n return aa()\n', + }); + expect(graph.findCircularDependencies()).toEqual([['a.py', 'b.py']]); + }); + + it('does not take an interface call reaching a test fake for an import', async () => { + // Production code imports nothing from the test. The interface's call fans + // out to every implementer, the fake included, so the file edge list holds + // the loop src>test>src that no import makes. + const graph = await index({ + 'src/pool.ts': [ + 'export interface Worker { post(task: string): void }', + 'export function run(worker: Worker) { worker.post("x"); }', + '', + ].join('\n'), + 'src/pool.test.ts': [ + "import { run, Worker } from './pool';", + 'class FakeWorker implements Worker { post(task: string): void { void task; } }', + 'export function check() { run(new FakeWorker()); }', + '', + ].join('\n'), + }); + const edgeList = graph.getFileDependencyPairs(); + expect(edgeList).toContainEqual({ source: 'src/pool.ts', target: 'src/pool.test.ts' }); + expect(edgeList).toContainEqual({ source: 'src/pool.test.ts', target: 'src/pool.ts' }); + expect(graph.findCircularDependencies()).toEqual([]); + }); + + it('does not take two pages that link to each other for an import', async () => { + const graph = await index({ + 'package.json': JSON.stringify({ name: 'site', dependencies: { next: '15.0.0', react: '19.0.0' } }), + 'app/page.tsx': [ + "import Link from 'next/link';", + 'export default function Home() { return About; }', + '', + ].join('\n'), + 'app/about/page.tsx': [ + "import Link from 'next/link';", + 'export default function About() { return Home; }', + '', + ].join('\n'), + }); + const edgeList = graph.getFileDependencyPairs(); + expect(edgeList).toContainEqual({ source: 'app/page.tsx', target: 'app/about/page.tsx' }); + expect(edgeList).toContainEqual({ source: 'app/about/page.tsx', target: 'app/page.tsx' }); + expect(graph.findCircularDependencies()).toEqual([]); + }); + + it('does not take the files of one Go package calling each other for an import', async () => { + const graph = await index({ + 'go.mod': 'module example.com/m\n\ngo 1.21\n', + 'one.go': 'package m\n\nfunc One() { Two() }\n', + 'two.go': 'package m\n\nfunc Two() { One() }\n', + }); + expect(graph.findCircularDependencies()).toEqual([]); + }); +}); diff --git a/__tests__/filelock-stale-takeover.test.ts b/__tests__/filelock-stale-takeover.test.ts new file mode 100644 index 0000000000..b3c55cdf13 --- /dev/null +++ b/__tests__/filelock-stale-takeover.test.ts @@ -0,0 +1,212 @@ +/** + * Taking over a stale `codegraph.lock` must leave exactly one holder (#2979). + * + * `FileLock.acquire()` used to read the lock, see a dead PID, unlink the file + * and create its own. Two processes that both read the same dead PID could + * then both succeed: the first replaced the stale lock, and the second, acting + * on what it had read a moment earlier, unlinked the first one's live lock and + * created its own. Both then wrote to the index as its only writer. + * + * The two-process cases pin that interleaving with a barrier rather than + * timing: the slow contender stops right after reading the stale lock and + * resumes only once the fast one holds it (see fixtures/filelock-contender.cjs). + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { spawn, type ChildProcess } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { FileLock } from '../src/utils'; + +const UTILS = path.resolve(__dirname, '../dist/utils.js'); +const CONTENDER = path.resolve(__dirname, 'fixtures/filelock-contender.cjs'); +/** Above every platform's PID range (and odd, which no Windows PID is): never a live process. */ +const DEAD_PID = '99999999'; + +interface Contender { + child: ChildProcess; + name: string; + stderr: () => string; + exited: Promise; +} + +describe('FileLock stale takeover (#2979)', () => { + let dir: string; + let lockPath: string; + let markers: string; + let contenders: Contender[] = []; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-filelock-')); + lockPath = path.join(dir, 'codegraph.lock'); + markers = path.join(dir, 'markers'); + fs.mkdirSync(markers); + }); + + afterEach(async () => { + for (const { child, exited } of contenders) { + if (child.exitCode === null && child.signalCode === null) child.kill(); + await exited; + } + contenders = []; + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function track(name: string, args: string[]): Contender { + const child = spawn(process.execPath, args, { + // Off the fixture, so a child still exiting cannot hold it (Windows EBUSY). + cwd: os.tmpdir(), + stdio: ['ignore', 'ignore', 'pipe'], + windowsHide: true, + }); + let stderr = ''; + child.stderr!.on('data', (chunk) => { stderr += String(chunk); }); + const exited = new Promise((resolve) => child.once('exit', (code) => resolve(code))); + const contender = { child, name, stderr: () => stderr, exited }; + contenders.push(contender); + return contender; + } + + const start = (name: string, mode: 'pause' | 'run') => + track(name, [CONTENDER, UTILS, lockPath, markers, name, mode]); + /** A live process that never touches the lock: its PID names a holder that is alive. */ + const idle = () => track('idle', ['-e', 'setInterval(() => {}, 1000)']); + + /** The first of `names` (marker files) to appear, and its content. */ + async function firstMarker(names: string[], from: Contender[]): Promise<{ name: string; text: string }> { + const deadline = Date.now() + 20_000; + for (;;) { + for (const name of names) { + const file = path.join(markers, name); + if (fs.existsSync(file)) return { name, text: fs.readFileSync(file, 'utf8') }; + } + if (Date.now() > deadline) { + const detail = from.map((c) => `${c.name} (exit ${c.child.exitCode}): ${c.stderr()}`).join('\n'); + throw new Error(`none of ${names.join(', ')} appeared\n${detail}`); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } + + const release = (who: Contender) => fs.writeFileSync(path.join(markers, `${who.name}.release`), ''); + const go = () => fs.writeFileSync(path.join(markers, 'go'), ''); + + it('lets only one of two processes take over a stale lock', async () => { + fs.writeFileSync(lockPath, DEAD_PID); + + // The slow process reads the dead PID, then stalls before acting on it. + const slow = start('slow', 'pause'); + const read = await firstMarker(['slow.read'], [slow]); + expect(read.text.trim()).toBe(DEAD_PID); + + // Meanwhile another process takes the stale lock over, and holds it. + const fast = start('fast', 'run'); + expect((await firstMarker(['fast.acquired', 'fast.refused'], [fast])).name).toBe('fast.acquired'); + + // The slow one resumes with the dead PID it read: it must not replace a live lock. + go(); + const slowOutcome = await firstMarker(['slow.acquired', 'slow.refused'], [slow]); + expect(slowOutcome.name, 'both processes held the lock at once').toBe('slow.refused'); + expect(slowOutcome.text).toMatch(/locked by another process/); + expect(fs.readFileSync(lockPath, 'utf8').trim()).toBe(String(fast.child.pid)); + + release(fast); + expect(await fast.exited).toBe(0); + expect(await slow.exited).toBe(0); + expect(fs.existsSync(lockPath)).toBe(false); + }, 60_000); + + it('still lets the slow process in once the other one has released the lock', async () => { + fs.writeFileSync(lockPath, DEAD_PID); + + const slow = start('slow', 'pause'); + await firstMarker(['slow.read'], [slow]); + const fast = start('fast', 'run'); + expect((await firstMarker(['fast.acquired', 'fast.refused'], [fast])).name).toBe('fast.acquired'); + release(fast); + await firstMarker(['fast.released'], [fast]); + expect(await fast.exited).toBe(0); + + // The lock is free now: what the slow one read is out of date in the other direction. + go(); + expect((await firstMarker(['slow.acquired', 'slow.refused'], [slow])).name).toBe('slow.acquired'); + expect(fs.readFileSync(lockPath, 'utf8').trim()).toBe(String(slow.child.pid)); + release(slow); + expect(await slow.exited).toBe(0); + expect(fs.existsSync(lockPath)).toBe(false); + }, 60_000); + + it('leaves a stale lock to the process already taking it over', async () => { + // A live process holds the takeover sentinel: it is replacing the stale lock right now. + const breaker = idle(); + fs.writeFileSync(lockPath, DEAD_PID); + fs.writeFileSync(`${lockPath}.break`, String(breaker.child.pid)); + + const lock = new FileLock(lockPath); + expect(() => lock.acquire()).toThrow(new RegExp(`locked by another process \\(PID ${breaker.child.pid}\\)`)); + expect(fs.readFileSync(lockPath, 'utf8')).toBe(DEAD_PID); + expect(fs.readFileSync(`${lockPath}.break`, 'utf8')).toBe(String(breaker.child.pid)); + }); + + it('takes over a stale lock whose takeover sentinel a dead process left behind', () => { + fs.writeFileSync(lockPath, DEAD_PID); + fs.writeFileSync(`${lockPath}.break`, DEAD_PID); + + const lock = new FileLock(lockPath); + lock.acquire(); + try { + expect(fs.readFileSync(lockPath, 'utf8')).toBe(String(process.pid)); + expect(fs.readdirSync(dir).sort()).toEqual(['codegraph.lock', 'markers']); + } finally { + lock.release(); + } + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it('treats an empty lock file its creator may still be writing as held, and an old one as stale', () => { + // A filesystem without hard links creates the lock first and writes the PID after. + fs.writeFileSync(lockPath, ''); + const lock = new FileLock(lockPath); + expect(() => lock.acquire()).toThrow(/locked by another process/); + expect(fs.readFileSync(lockPath, 'utf8')).toBe(''); + + // Left empty by a crash long ago: stale. + const old = new Date(Date.now() - 60_000); + fs.utimesSync(lockPath, old, old); + lock.acquire(); + expect(fs.readFileSync(lockPath, 'utf8')).toBe(String(process.pid)); + lock.release(); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it('treats a PID it may not signal (EPERM) as a live holder', () => { + // Another user's process, or an elevated one on Windows: alive, just not ours to signal. + const foreign = 4242; + fs.writeFileSync(lockPath, String(foreign)); + const kill = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => { + if (pid === foreign && signal === 0) throw Object.assign(new Error('kill EPERM'), { code: 'EPERM' }); + return true; + }) as typeof process.kill); + try { + expect(() => new FileLock(lockPath).acquire()).toThrow(/locked by another process \(PID 4242\)/); + expect(fs.readFileSync(lockPath, 'utf8')).toBe(String(foreign)); + } finally { + kill.mockRestore(); + } + }); + + it.runIf(process.platform === 'win32')('keeps a lock whose holder Windows will not let us open, like an elevated process', () => { + // PID 4 is always the System process: opening it to signal is refused (EPERM), as for an elevated one. + fs.writeFileSync(lockPath, '4'); + expect(() => new FileLock(lockPath).acquire()).toThrow(/locked by another process \(PID 4\)/); + expect(fs.readFileSync(lockPath, 'utf8')).toBe('4'); + }); + + it('leaves nothing but the lock file while held, and nothing after release', () => { + const lock = new FileLock(lockPath); + lock.acquire(); + expect(fs.readdirSync(dir).sort()).toEqual(['codegraph.lock', 'markers']); + lock.release(); + expect(fs.readdirSync(dir)).toEqual(['markers']); + }); +}); diff --git a/__tests__/fixtures/filelock-contender.cjs b/__tests__/fixtures/filelock-contender.cjs new file mode 100644 index 0000000000..cfd68ac8f7 --- /dev/null +++ b/__tests__/fixtures/filelock-contender.cjs @@ -0,0 +1,57 @@ +/** + * One contender for a `FileLock`, run as its own process by + * filelock-stale-takeover.test.ts (#2979). + * + * node filelock-contender.cjs + * + * It takes the lock with the built `FileLock`, writes `.acquired` (and + * holds the lock until `.release` appears) or `.refused`, then + * exits. With `` = `pause`, it stops right after its first read of the + * lock file, with whatever that read returned in hand, writes `.read`, + * and waits for `go`: the moment a slow process has decided a lock is stale + * but has not acted on it yet. Every wait is on a marker file the test writes, + * so the interleaving is the same on every run. A contender gives up after + * 30 s of waiting, so none outlives a test that stopped driving it. + */ +const fs = require('fs'); +const path = require('path'); + +const [utilsJs, lockPath, markers, name, pause] = process.argv.slice(2); +const marker = (suffix) => path.join(markers, `${name}.${suffix}`); +const mark = (suffix, text = String(process.pid)) => fs.writeFileSync(marker(suffix), text); +const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +const waitFor = (file) => { + const deadline = Date.now() + 30_000; + while (!fs.existsSync(file)) { + if (Date.now() > deadline) process.exit(3); + sleep(5); + } +}; + +if (pause === 'pause') { + const target = path.resolve(lockPath); + const read = fs.readFileSync; + let paused = false; + fs.readFileSync = function readThenPause(file, ...rest) { + const content = read.call(this, file, ...rest); + if (!paused && typeof file === 'string' && path.resolve(file) === target) { + paused = true; + mark('read', String(content)); + waitFor(path.join(markers, 'go')); + } + return content; + }; +} + +const { FileLock } = require(utilsJs); +const lock = new FileLock(lockPath); +try { + lock.acquire(); +} catch (err) { + mark('refused', err instanceof Error ? err.message : String(err)); + process.exit(0); +} +mark('acquired'); +waitFor(marker('release')); +lock.release(); +mark('released'); diff --git a/__tests__/fixtures/kernel-parity/Torture.java b/__tests__/fixtures/kernel-parity/Torture.java index d1e71cbd20..84ab79b1f6 100644 --- a/__tests__/fixtures/kernel-parity/Torture.java +++ b/__tests__/fixtures/kernel-parity/Torture.java @@ -31,6 +31,7 @@ public void run() { } }; private final Runnable fieldMethodRef = TortureService::compute; + private static final List FIELD_TABLE = List.of(TortureService::compute); /** Ctor javadoc. */ public TortureService(List names) { @@ -130,3 +131,18 @@ interface Shape extends Comparable, Cloneable { @interface Marker { String value() default ""; } + +class DottedAnon { + Api.Listener listener = new Api.Listener() { + public void on() {} + }; + Object plain = new java.util.concurrent.Callable() { + public Integer call() { return 1; } + }; +} +interface TortureConsts { + int MAX = 10; + Helper HELPER = new Helper(); + String NAME = "x", OTHER = "y"; + Runnable TASK = new Runnable() { public void run() { go(); } }; +} diff --git a/__tests__/fixtures/kernel-parity/TortureRecord.java b/__tests__/fixtures/kernel-parity/TortureRecord.java new file mode 100644 index 0000000000..5107fa57ca --- /dev/null +++ b/__tests__/fixtures/kernel-parity/TortureRecord.java @@ -0,0 +1,49 @@ +package com.example.records; + +import java.util.List; +import lombok.Builder; + +/** A 2-D point. */ +public record Point(int x, int y) implements Comparable, Shape { + public static final Point ORIGIN = new Point(0, 0); + private static int created; + + public Point { + if (x < 0) throw new IllegalArgumentException("x"); + created++; + } + + public Point(int v) { this(v, v); } + + /** Explicit accessor — no implicit one is minted. */ + public int x() { return x; } + + public double norm() { return Math.sqrt(x * x + y * y); } + + @Override + public int compareTo(Point o) { return Double.compare(norm(), o.norm()); } + + record Inner(String name, List tags) { + String first() { return tags.get(0); } + } +} + +record Pair(A first, B second) { + A left() { return first; } +} + +@Builder +record Config(@Deprecated String host, int[] ports, java.util.Map limits, String... rest) {} + +interface Shape { double norm(); } + +class Client { + private Point.Inner inner; + + String use(Point p, Pair pr) { + record Local(int n) { int twice() { return n * 2; } } + Local l = new Local(3); + Config c = Config.builder().build(); + return inner.first() + p.y() + pr.left() + l.twice() + c.host() + Point.ORIGIN.norm(); + } +} diff --git a/__tests__/fixtures/kernel-parity/TortureTry.swift b/__tests__/fixtures/kernel-parity/TortureTry.swift new file mode 100644 index 0000000000..aaafa38f74 --- /dev/null +++ b/__tests__/fixtures/kernel-parity/TortureTry.swift @@ -0,0 +1,56 @@ +// Calls written after `try!`, `try?` and `try`. `try!` is the shape the +// Windows (MSVC) build of the kernel's grammar used to misparse: its scanner +// read the `!` as a logical not, so `try! Message2(data: data)` came out as a +// call of `!Message2` (a reference named `! Message2`, at the `!`). +import Foundation + +let globalStore = try! Store(path: "global") +let globalLoader = try? Loader.make() + +final class Repository { + static let shared = try! Repository(path: "shared") + let decoder = try! JSONDecoder.make(strategy: .iso8601) + lazy var cache: Cache = try! Cache.open(named: "repo") + var current: Message2? = try? Message2(data: Data()) + + init(path: String) throws {} + + func load(data: Data?) throws -> Message2 { + let msg2 = try! Message2(data: data!) + let msg3 = try? Message2(data: data!) + let msg4 = try Message2(data: data!) + let user = try! JSONDecoder().decode(User.self, from: data!) + let parsed = try! parser.parse(data!) + let settings = try! Settings.Defaults.load(from: "defaults") + let realm = try! Realm() + _ = (msg3, msg4, user, parsed, settings, realm) + try! FileManager.default.removeItem(atPath: "/tmp/x") + try! self.flush() + try! validate(msg2) + try !isReady() + use(try! Foo(), try? Bar.make(), try Baz.build()) + if let cached = try? Cache.open(named: "x") { + cached.touch() + } + guard let fresh = try? decodeFresh(data) else { + return try! Message2(data: Data()) + } + _ = fresh + return try! Message2(data: data!) + } + + func refresh() async throws { + let page = try! await client.fetch(page: 1) + let next = try? await Client.shared.fetch(page: 2) + try await store.save(page, next) + let items = try! items(in: page).map { try! Item(raw: $0) } + _ = items + } + + func flush() throws {} + func validate(_ m: Message2) throws {} + func isReady() throws -> Bool { true } + func use(_ a: Any?, _ b: Any?, _ c: Any?) {} + func decodeFresh(_ d: Data?) throws -> Message2 { try Message2(data: d!) } + func items(in page: Page) throws -> [String] { [] } +} diff --git a/__tests__/fixtures/kernel-parity/TortureTypedefs.dart b/__tests__/fixtures/kernel-parity/TortureTypedefs.dart new file mode 100644 index 0000000000..f152ad8f35 --- /dev/null +++ b/__tests__/fixtures/kernel-parity/TortureTypedefs.dart @@ -0,0 +1,78 @@ +// Typedefs reference the types they alias (#2327): every new-style and legacy +// shape, type parameters (the typedef's own and a generic function type's), +// import prefixes, annotations, comments and wrapped declarations. +import 'package:meta/meta.dart'; +import 'model.dart' as m; +import 'model.dart' as M; + +class Report { + final int score; + const Report(this.score); +} + +class T {} + +void tag(Object o) {} + +// New style. +typedef ReportCallback = void Function(Report r); +typedef ReportMaker = Report Function(); +typedef Reports = List; +typedef ReportsByName = Map; +typedef Results = Future?> Function(int page); +typedef Opt = Report?; +typedef MaybeMaker = Report Function()?; +typedef Pair = (Report, int); +typedef NamedPair = ({Report first, m.Report second}); +typedef Named = void Function({required Report report, int? n}); +typedef Positional = void Function([Report? report]); +typedef Unnamed = void Function(Report, m.Report); +typedef Nested = void Function(void Function(Report) inner); +typedef Maker = void Function(Report Function() make); +typedef Dynamic = dynamic Function(Object? o); +typedef Void = void; +typedef Fn = Function; +typedef Prefixed = m.Report; +typedef UpperPrefixed = M.Report; +typedef PrefixedGeneric = m.Box; + +// Type parameters: the typedef's own, a bound, a generic function type's. +typedef Mapper = T Function(Report r); +typedef Bounded = List; +typedef SelfBounded> = int Function(C a, C b); +typedef Picker = T Function(T a, Report r); +typedef Outer = S Function(U u, S s, T t); +typedef Shadow = m.T Function(T t); +typedef TwoParams = Map>; + +// Legacy. +typedef void LegacyCallback(Report r); +typedef Report LegacyMaker(int n); +typedef List LegacyLoader([Report? seed]); +typedef m.Report PrefixedLegacy(m.Report r); +typedef Report? LegacyOptional({Report? fallback}); +typedef void Function(Report) LegacyFnReturn(int n); +typedef T LegacyPicker(T a, Report r); +typedef void LegacyFnParam(Report make(int n)); +typedef void LegacyGenericFnParam(S make(S s, Report r)); +typedef void LegacyCovariant(covariant Report r, final Report s); + +// Annotated, documented, commented, wrapped. +/// Documented. +@Deprecated('Use ReportCallback') +typedef OldCallback = void Function(Report r); +@visibleForTesting +@Tag(tag) +typedef Tagged = Report Function(/* the report */ Report r); +typedef Commented = // why + Report Function(Report r); +typedef Wrapped = + Map Function( + Report first, + Report second, + ); + +class Tag { + const Tag(this.f); + final Object f; +} diff --git a/__tests__/fixtures/kernel-parity/TortureWrappedCalls.swift b/__tests__/fixtures/kernel-parity/TortureWrappedCalls.swift new file mode 100644 index 0000000000..00f50406cb --- /dev/null +++ b/__tests__/fixtures/kernel-parity/TortureWrappedCalls.swift @@ -0,0 +1,49 @@ +// Calls whose callee or receiver the grammar wraps. tree-sitter-swift binds a +// prefix operator to the leftmost operand, so `!isReady()` parses as +// `(!isReady)()` and `!store.contains(x)` as `(!store).contains(x)`; `try` and +// `await` before a chain bind to its first call, so `try! Decoder().decode(d)` +// parses as `(try! Decoder()).decode(d)`. None of them changes what is called. +// An operator applied to a parenthesized value parses as a call of the +// operator: `!(x)` calls `!` (a project may declare `prefix func !`), and +// `-(x)` has no callee node at all. +import Foundation + +let globalFlag = !Config.isEnabled() + +final class Screen { + static let ready = !Config.shared.isReady() + let store = Store() + var negated: Bool = !Store.isEmpty() + + func update(items: [Item], data: Data, id: Int) async throws -> Bool { + let a = !isReady() + let b = !store.contains(id) + let c = !Config.check() + let d = -abs(id) + let e = !Settings.Defaults.isOn() + let f = !Settings.Defaults.shared.isOn() + let g = !self.isCancelled + let h = !Config.shared.isEnabled + let i = ~Mask.all.rawValue + let j = try! Decoder().decode(Item.self, from: data) + let k = try Parser.make().parse(data) + let l = try! Parser.make().tokens().first() + let m = try await Client.shared.fetch(id).items() + let n = await loader.load().items() + let o = try !isValid(data) + let p: Result = .success(id) + let q = !(items.isEmpty) + let r = -(id + 1) + let s = -(id) + if !items.contains(where: { $0.id == id }) { + return !store.remove(id) + } + guard !Task.isCancelled, !isReady() else { return false } + _ = (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) + return !Config.check() && -Offset.zero() < 0 + } + + func isReady() -> Bool { true } + func isValid(_ d: Data) throws -> Bool { true } + var isCancelled: Bool { false } +} diff --git a/__tests__/fixtures/kernel-parity/torture.c b/__tests__/fixtures/kernel-parity/torture.c index a785980512..7f808d333d 100644 --- a/__tests__/fixtures/kernel-parity/torture.c +++ b/__tests__/fixtures/kernel-parity/torture.c @@ -188,3 +188,50 @@ static int declared_types_local(void) { static struct { int hits; } local_stats; return ++local_stats.hits + current_mode; } + +/* members: the types each is declared with are references from its holder, or + from the alias they are named through */ +typedef struct member_s member_t; +typedef enum { PAUSE_OFF, PAUSE_ON } pause_mode; +typedef union { int i; float f; } member_value; +typedef void (*member_cb)(member_t *m, word_t w); +typedef word_t *word_ptr; + +struct member_s { + member_t *next; + struct member_s *prev; + word_t by_value; + word_t by_array[4]; + const word_t *by_pointer; + word_t **by_pointers; + struct parse_state *tagged, second, third[2]; + union packet_hdr header; + enum hdr_kind kind; + pause_mode mode; + member_value (*make)(member_t *from, const struct parse_state *state); + void *(*table[2])(word_t w); + unsigned long mask; + int bits : 3; +#ifdef MEMBER_EXTRA + word_t extra; +#elif defined(MEMBER_OTHER) + member_value other; +#else + pause_mode fallback; +#endif + union { word_t in_anonymous; int raw; }; + struct { member_value in_unnamed; } named_member; + struct member_inner { word_t deep; } nested; + enum { MEMBER_ON, MEMBER_OFF } state; + enum member_level { MEMBER_LOW, MEMBER_HIGH } level; +}; + +typedef struct self_list { + struct self_list *next; + word_t value; +} self_list; + +typedef union { + word_t a; + struct { self_list *b; } c; +} wrapped_member; diff --git a/__tests__/fixtures/kernel-parity/torture.cpp b/__tests__/fixtures/kernel-parity/torture.cpp index b2ba1ea089..b33843ac6b 100644 --- a/__tests__/fixtures/kernel-parity/torture.cpp +++ b/__tests__/fixtures/kernel-parity/torture.cpp @@ -155,3 +155,51 @@ float drive() { (void)m; return r + f + flags + leg; } +// A function named past its noexcept/throw/requires clauses. +namespace clauses { +template