diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index e66a214fb7..ed22465bca 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -68,6 +68,7 @@ import { type TestProviderAdapterHarness, } from "./TestProviderAdapter.integration.ts"; import { deriveServerPaths, ServerConfig } from "../src/config.ts"; +import { WorkspaceEntriesLive } from "../src/project/Layers/WorkspaceEntries.ts"; function runGit(cwd: string, args: ReadonlyArray) { return execFileSync("git", args, { @@ -317,6 +318,12 @@ export const makeOrchestrationIntegrationHarness = ( ); const checkpointReactorLayer = CheckpointReactorLive.pipe( Layer.provideMerge(runtimeServicesLayer), + Layer.provideMerge( + WorkspaceEntriesLive.pipe( + Layer.provideMerge(gitCoreLayer), + Layer.provide(NodeServices.layer), + ), + ), ); const orchestrationReactorLayer = OrchestrationReactorLive.pipe( Layer.provideMerge(runtimeIngestionLayer), diff --git a/apps/server/src/git/Layers/GitCore.test.ts b/apps/server/src/git/Layers/GitCore.test.ts index 547a69e7e1..0d34589d68 100644 --- a/apps/server/src/git/Layers/GitCore.test.ts +++ b/apps/server/src/git/Layers/GitCore.test.ts @@ -138,6 +138,13 @@ function buildLargeText(lineCount = 20_000): string { .concat("\n"); } +function splitNullSeparatedPaths(input: string): string[] { + return input + .split("\0") + .map((value) => value.trim()) + .filter((value) => value.length > 0); +} + // ── Tests ── it.layer(TestLayer)("git integration", (it) => { @@ -181,6 +188,55 @@ it.layer(TestLayer)("git integration", (it) => { ); }); + describe("workspace helpers", () => { + it.effect("filterIgnoredPaths chunks large path lists and preserves kept paths", () => + Effect.gen(function* () { + const cwd = "/virtual/repo"; + const relativePaths = Array.from({ length: 340 }, (_, index) => { + const prefix = index % 3 === 0 ? "ignored" : "kept"; + return `${prefix}/segment-${String(index).padStart(4, "0")}/${"x".repeat(900)}.ts`; + }); + const expectedPaths = relativePaths.filter( + (relativePath) => !relativePath.startsWith("ignored/"), + ); + + const seenChunks: string[][] = []; + const core = yield* makeIsolatedGitCore((input) => { + if (input.args.join(" ") !== "check-ignore --no-index -z --stdin") { + return Effect.fail( + new GitCommandError({ + operation: input.operation, + command: `git ${input.args.join(" ")}`, + cwd: input.cwd, + detail: "unexpected git command in chunking test", + }), + ); + } + + const chunkPaths = splitNullSeparatedPaths(input.stdin ?? ""); + seenChunks.push(chunkPaths); + const ignoredPaths = chunkPaths.filter((relativePath) => + relativePath.startsWith("ignored/"), + ); + + return Effect.succeed({ + code: ignoredPaths.length > 0 ? 0 : 1, + stdout: ignoredPaths.length > 0 ? `${ignoredPaths.join("\0")}\0` : "", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }); + }); + + const result = yield* core.filterIgnoredPaths(cwd, relativePaths); + + expect(seenChunks.length).toBeGreaterThan(1); + expect(seenChunks.flat()).toEqual(relativePaths); + expect(result).toEqual(expectedPaths); + }), + ); + }); + // ── listGitBranches ── describe("listGitBranches", () => { @@ -541,7 +597,13 @@ it.layer(TestLayer)("git integration", (it) => { const core = yield* makeIsolatedGitCore((input) => { if (input.args[0] === "fetch") { fetchArgs = [...input.args]; - return Effect.succeed({ code: 0, stdout: "", stderr: "" }); + return Effect.succeed({ + code: 0, + stdout: "", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }); } return realGitCore.execute(input); }); @@ -594,7 +656,13 @@ it.layer(TestLayer)("git integration", (it) => { if (input.args[0] === "fetch") { fetchStarted = true; return Effect.promise(() => - waitForReleasePromise.then(() => ({ code: 0, stdout: "", stderr: "" })), + waitForReleasePromise.then(() => ({ + code: 0, + stdout: "", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + })), ); } return realGitCore.execute(input); diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.ts index 64ed409508..81d6cbb549 100644 --- a/apps/server/src/git/Layers/GitCore.ts +++ b/apps/server/src/git/Layers/GitCore.ts @@ -37,6 +37,8 @@ const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000; const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000; const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000; const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000; +const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; +const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15); const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5); const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048; @@ -55,6 +57,7 @@ class StatusUpstreamRefreshCacheKey extends Data.Class<{ }> {} interface ExecuteGitOptions { + stdin?: string | undefined; timeoutMs?: number | undefined; allowNonZeroExit?: boolean | undefined; fallbackErrorMessage?: string | undefined; @@ -96,6 +99,47 @@ function parseNumstatEntries( return entries; } +function splitNullSeparatedPaths(input: string, truncated: boolean): string[] { + const parts = input.split("\0"); + if (parts.length === 0) return []; + + if (truncated && parts[parts.length - 1]?.length) { + parts.pop(); + } + + return parts.filter((value) => value.length > 0); +} + +function chunkPathsForGitCheckIgnore(relativePaths: readonly string[]): string[][] { + const chunks: string[][] = []; + let chunk: string[] = []; + let chunkBytes = 0; + + for (const relativePath of relativePaths) { + const relativePathBytes = Buffer.byteLength(relativePath) + 1; + if (chunk.length > 0 && chunkBytes + relativePathBytes > GIT_CHECK_IGNORE_MAX_STDIN_BYTES) { + chunks.push(chunk); + chunk = []; + chunkBytes = 0; + } + + chunk.push(relativePath); + chunkBytes += relativePathBytes; + + if (chunkBytes >= GIT_CHECK_IGNORE_MAX_STDIN_BYTES) { + chunks.push(chunk); + chunk = []; + chunkBytes = 0; + } + } + + if (chunk.length > 0) { + chunks.push(chunk); + } + + return chunks; +} + function parsePorcelainPath(line: string): string | null { if (line.startsWith("? ") || line.startsWith("! ")) { const simple = line.slice(2).trim(); @@ -445,7 +489,7 @@ const collectOutput = Effect.fn("collectOutput")(function* ( maxOutputBytes: number, truncateOutputAtMaxBytes: boolean, onLine: ((line: string) => Effect.Effect) | undefined, -): Effect.fn.Return { +): Effect.fn.Return<{ readonly text: string; readonly truncated: boolean }, GitCommandError> { const decoder = new TextDecoder(); let bytes = 0; let text = ""; @@ -507,7 +551,10 @@ const collectOutput = Effect.fn("collectOutput")(function* ( text += remainder; lineBuffer += remainder; yield* emitCompleteLines(true); - return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text; + return { + text, + truncated, + }; }); export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { @@ -571,13 +618,18 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { Effect.map((value) => Number(value)), Effect.mapError(toGitCommandError(commandInput, "failed to report exit code.")), ), + input.stdin === undefined + ? Effect.void + : Stream.run(Stream.encodeText(Stream.make(input.stdin)), child.stdin).pipe( + Effect.mapError(toGitCommandError(commandInput, "failed to write stdin.")), + ), ], { concurrency: "unbounded" }, - ); + ).pipe(Effect.map(([stdout, stderr, exitCode]) => [stdout, stderr, exitCode] as const)); yield* trace2Monitor.flush; if (!input.allowNonZeroExit && exitCode !== 0) { - const trimmedStderr = stderr.trim(); + const trimmedStderr = stderr.text.trim(); return yield* new GitCommandError({ operation: commandInput.operation, command: quoteGitCommand(commandInput.args), @@ -589,7 +641,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { }); } - return { code: exitCode, stdout, stderr } satisfies ExecuteGitResult; + return { + code: exitCode, + stdout: stdout.text, + stderr: stderr.text, + stdoutTruncated: stdout.truncated, + stderrTruncated: stderr.truncated, + } satisfies ExecuteGitResult; }); return yield* runGitCommand().pipe( @@ -618,11 +676,12 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { cwd: string, args: readonly string[], options: ExecuteGitOptions = {}, - ): Effect.Effect<{ code: number; stdout: string; stderr: string }, GitCommandError> => + ): Effect.Effect => execute({ operation, cwd, args, + ...(options.stdin !== undefined ? { stdin: options.stdin } : {}), allowNonZeroExit: true, ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}), @@ -679,7 +738,11 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { args: readonly string[], options: ExecuteGitOptions = {}, ): Effect.Effect => - executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout)); + executeGit(operation, cwd, args, options).pipe( + Effect.map((result) => + result.stdoutTruncated ? `${result.stdout}${OUTPUT_TRUNCATED_MARKER}` : result.stdout, + ), + ); const branchExists = (cwd: string, branch: string): Effect.Effect => executeGit( @@ -1416,6 +1479,86 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { Effect.map((trimmed) => (trimmed.length > 0 ? trimmed : null)), ); + const isInsideWorkTree: GitCoreShape["isInsideWorkTree"] = (cwd) => + executeGit("GitCore.isInsideWorkTree", cwd, ["rev-parse", "--is-inside-work-tree"], { + allowNonZeroExit: true, + timeoutMs: 5_000, + maxOutputBytes: 4_096, + }).pipe(Effect.map((result) => result.code === 0 && result.stdout.trim() === "true")); + + const listWorkspaceFiles: GitCoreShape["listWorkspaceFiles"] = (cwd) => + executeGit( + "GitCore.listWorkspaceFiles", + cwd, + ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], + { + allowNonZeroExit: true, + timeoutMs: 20_000, + maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES, + truncateOutputAtMaxBytes: true, + }, + ).pipe( + Effect.flatMap((result) => + result.code === 0 + ? Effect.succeed({ + paths: splitNullSeparatedPaths(result.stdout, result.stdoutTruncated), + truncated: result.stdoutTruncated, + }) + : Effect.fail( + createGitCommandError( + "GitCore.listWorkspaceFiles", + cwd, + ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], + result.stderr.trim().length > 0 ? result.stderr.trim() : "git ls-files failed", + ), + ), + ), + ); + + const filterIgnoredPaths: GitCoreShape["filterIgnoredPaths"] = (cwd, relativePaths) => + Effect.gen(function* () { + if (relativePaths.length === 0) { + return relativePaths; + } + + const ignoredPaths = new Set(); + const chunks = chunkPathsForGitCheckIgnore(relativePaths); + + for (const chunk of chunks) { + const result = yield* executeGit( + "GitCore.filterIgnoredPaths", + cwd, + ["check-ignore", "--no-index", "-z", "--stdin"], + { + stdin: `${chunk.join("\0")}\0`, + allowNonZeroExit: true, + timeoutMs: 20_000, + maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES, + truncateOutputAtMaxBytes: true, + }, + ); + + if (result.code !== 0 && result.code !== 1) { + return yield* createGitCommandError( + "GitCore.filterIgnoredPaths", + cwd, + ["check-ignore", "--no-index", "-z", "--stdin"], + result.stderr.trim().length > 0 ? result.stderr.trim() : "git check-ignore failed", + ); + } + + for (const ignoredPath of splitNullSeparatedPaths(result.stdout, result.stdoutTruncated)) { + ignoredPaths.add(ignoredPath); + } + } + + if (ignoredPaths.size === 0) { + return relativePaths; + } + + return relativePaths.filter((relativePath) => !ignoredPaths.has(relativePath)); + }); + const listBranches: GitCoreShape["listBranches"] = Effect.fn("listBranches")(function* (input) { const branchRecencyPromise = readBranchRecency(input.cwd).pipe( Effect.catch(() => Effect.succeed(new Map())), @@ -1834,6 +1977,9 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: { pullCurrentBranch, readRangeContext, readConfigValue, + isInsideWorkTree, + listWorkspaceFiles, + filterIgnoredPaths, listBranches, createWorktree, fetchPullRequestBranch, diff --git a/apps/server/src/git/Services/GitCore.ts b/apps/server/src/git/Services/GitCore.ts index f1a4e065cd..3c30f17121 100644 --- a/apps/server/src/git/Services/GitCore.ts +++ b/apps/server/src/git/Services/GitCore.ts @@ -28,6 +28,7 @@ export interface ExecuteGitInput { readonly operation: string; readonly cwd: string; readonly args: ReadonlyArray; + readonly stdin?: string; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -40,6 +41,8 @@ export interface ExecuteGitResult { readonly code: number; readonly stdout: string; readonly stderr: string; + readonly stdoutTruncated: boolean; + readonly stderrTruncated: boolean; } export interface GitStatusDetails extends Omit { @@ -93,6 +96,11 @@ export interface GitRangeContext { diffPatch: string; } +export interface GitListWorkspaceFilesResult { + readonly paths: ReadonlyArray; + readonly truncated: boolean; +} + export interface GitRenameBranchInput { cwd: string; oldBranch: string; @@ -190,6 +198,26 @@ export interface GitCoreShape { key: string, ) => Effect.Effect; + /** + * Determine whether the provided cwd is inside a git work tree. + */ + readonly isInsideWorkTree: (cwd: string) => Effect.Effect; + + /** + * List tracked and untracked workspace file paths relative to cwd. + */ + readonly listWorkspaceFiles: ( + cwd: string, + ) => Effect.Effect; + + /** + * Remove gitignored paths from a relative path list. + */ + readonly filterIgnoredPaths: ( + cwd: string, + relativePaths: ReadonlyArray, + ) => Effect.Effect, GitCommandError>; + /** * List local + remote branches and branch metadata. */ diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts index 075f62f889..1cc5e28e52 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.test.ts @@ -38,6 +38,7 @@ import { } from "../../provider/Services/ProviderService.ts"; import { checkpointRefForThreadTurn } from "../../checkpointing/Utils.ts"; import { ServerConfig } from "../../config.ts"; +import { WorkspaceEntriesLive } from "../../project/Layers/WorkspaceEntries.ts"; const asProjectId = (value: string): ProjectId => ProjectId.makeUnsafe(value); const asTurnId = (value: string): TurnId => TurnId.makeUnsafe(value); @@ -261,7 +262,9 @@ describe("CheckpointReactor", () => { Layer.provideMerge(orchestrationLayer), Layer.provideMerge(RuntimeReceiptBusLive), Layer.provideMerge(Layer.succeed(ProviderService, provider.service)), - Layer.provideMerge(CheckpointStoreLive.pipe(Layer.provide(GitCoreLive))), + Layer.provideMerge(CheckpointStoreLive), + Layer.provideMerge(WorkspaceEntriesLive), + Layer.provideMerge(GitCoreLive), Layer.provideMerge(ServerConfigLayer), Layer.provideMerge(NodeServices.layer), ); diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.ts index 561626b8de..d016b785db 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.ts @@ -16,7 +16,6 @@ import { checkpointRefForThreadTurn, resolveThreadWorkspaceCwd, } from "../../checkpointing/Utils.ts"; -import { clearWorkspaceIndexCache } from "../../workspaceEntries.ts"; import { CheckpointStore } from "../../checkpointing/Services/CheckpointStore.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { CheckpointReactor, type CheckpointReactorShape } from "../Services/CheckpointReactor.ts"; @@ -25,6 +24,7 @@ import { RuntimeReceiptBus } from "../Services/RuntimeReceiptBus.ts"; import { CheckpointStoreError } from "../../checkpointing/Errors.ts"; import { OrchestrationDispatchError } from "../Errors.ts"; import { isGitRepository } from "../../git/Utils.ts"; +import { WorkspaceEntries } from "../../project/Services/WorkspaceEntries.ts"; type ReactorInput = | { @@ -68,6 +68,7 @@ const make = Effect.gen(function* () { const providerService = yield* ProviderService; const checkpointStore = yield* CheckpointStore; const receiptBus = yield* RuntimeReceiptBus; + const workspaceEntries = yield* WorkspaceEntries; const appendRevertFailureActivity = (input: { readonly threadId: ThreadId; @@ -226,7 +227,7 @@ const make = Effect.gen(function* () { // Invalidate the workspace entry cache so the @-mention file picker // reflects files created or deleted during this turn. - clearWorkspaceIndexCache(input.cwd); + yield* workspaceEntries.invalidate(input.cwd); const files = yield* checkpointStore .diffCheckpoints({ @@ -642,7 +643,7 @@ const make = Effect.gen(function* () { // Invalidate the workspace entry cache so the @-mention file picker // reflects the reverted filesystem state. - clearWorkspaceIndexCache(sessionRuntime.value.cwd); + yield* workspaceEntries.invalidate(sessionRuntime.value.cwd); const rolledBackTurns = Math.max(0, currentTurnCount - event.payload.turnCount); if (rolledBackTurns > 0) { diff --git a/apps/server/src/project/Layers/WorkspaceEntries.test.ts b/apps/server/src/project/Layers/WorkspaceEntries.test.ts new file mode 100644 index 0000000000..1d1eb4f0e2 --- /dev/null +++ b/apps/server/src/project/Layers/WorkspaceEntries.test.ts @@ -0,0 +1,264 @@ +import fsPromises from "node:fs/promises"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it, afterEach, describe, expect, vi } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Path, PlatformError } from "effect"; + +import { ServerConfig } from "../../config.ts"; +import { GitCoreLive } from "../../git/Layers/GitCore.ts"; +import { GitCore } from "../../git/Services/GitCore.ts"; +import { WorkspaceEntries } from "../Services/WorkspaceEntries.ts"; +import { WorkspaceEntriesLive } from "./WorkspaceEntries.ts"; + +const TestLayer = Layer.empty.pipe( + Layer.provideMerge(WorkspaceEntriesLive), + Layer.provideMerge(GitCoreLive), + Layer.provide( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3-workspace-entries-test-", + }), + ), + Layer.provideMerge(NodeServices.layer), +); + +const makeTempDir = Effect.fn(function* (opts?: { prefix?: string; git?: boolean }) { + const fileSystem = yield* FileSystem.FileSystem; + const gitCore = yield* GitCore; + const dir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: opts?.prefix ?? "t3code-workspace-entries-", + }); + if (opts?.git) { + yield* gitCore.initRepo({ cwd: dir }); + } + return dir; +}); + +function writeTextFile( + cwd: string, + relativePath: string, + contents = "", +): Effect.Effect { + return Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const absolutePath = path.join(cwd, relativePath); + yield* fileSystem.makeDirectory(path.dirname(absolutePath), { recursive: true }); + yield* fileSystem.writeFileString(absolutePath, contents); + }); +} + +const git = (cwd: string, args: ReadonlyArray, env?: NodeJS.ProcessEnv) => + Effect.gen(function* () { + const gitCore = yield* GitCore; + const result = yield* gitCore.execute({ + operation: "WorkspaceEntries.test.git", + cwd, + args, + ...(env ? { env } : {}), + timeoutMs: 10_000, + }); + return result.stdout.trim(); + }); + +const searchWorkspaceEntries = (input: { cwd: string; query: string; limit: number }) => + Effect.gen(function* () { + const workspaceEntries = yield* WorkspaceEntries; + return yield* workspaceEntries.search(input); + }); + +it.layer(TestLayer)("WorkspaceEntriesLive", (it) => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("search", () => { + it.effect("returns files and directories relative to cwd", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir(); + yield* writeTextFile(cwd, "src/components/Composer.tsx"); + yield* writeTextFile(cwd, "src/index.ts"); + yield* writeTextFile(cwd, "README.md"); + yield* writeTextFile(cwd, ".git/HEAD"); + yield* writeTextFile(cwd, "node_modules/pkg/index.js"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "", limit: 100 }); + const paths = result.entries.map((entry) => entry.path); + + expect(paths).toContain("src"); + expect(paths).toContain("src/components"); + expect(paths).toContain("src/components/Composer.tsx"); + expect(paths).toContain("README.md"); + expect(paths.some((entryPath) => entryPath.startsWith(".git"))).toBe(false); + expect(paths.some((entryPath) => entryPath.startsWith("node_modules"))).toBe(false); + expect(result.truncated).toBe(false); + }), + ); + + it.effect("filters and ranks entries by query", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-query-" }); + yield* writeTextFile(cwd, "src/components/Composer.tsx"); + yield* writeTextFile(cwd, "src/components/composePrompt.ts"); + yield* writeTextFile(cwd, "docs/composition.md"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "compo", limit: 5 }); + + expect(result.entries.length).toBeGreaterThan(0); + expect(result.entries.some((entry) => entry.path === "src/components")).toBe(true); + expect(result.entries.every((entry) => entry.path.toLowerCase().includes("compo"))).toBe( + true, + ); + }), + ); + + it.effect("supports fuzzy subsequence queries for composer path search", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-fuzzy-query-" }); + yield* writeTextFile(cwd, "src/components/Composer.tsx"); + yield* writeTextFile(cwd, "src/components/composePrompt.ts"); + yield* writeTextFile(cwd, "docs/composition.md"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "cmp", limit: 10 }); + const paths = result.entries.map((entry) => entry.path); + + expect(result.entries.length).toBeGreaterThan(0); + expect(paths).toContain("src/components"); + expect(paths).toContain("src/components/Composer.tsx"); + }), + ); + + it.effect("tracks truncation without sorting every fuzzy match", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-fuzzy-limit-" }); + yield* writeTextFile(cwd, "src/components/Composer.tsx"); + yield* writeTextFile(cwd, "src/components/composePrompt.ts"); + yield* writeTextFile(cwd, "docs/composition.md"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "cmp", limit: 1 }); + + expect(result.entries).toHaveLength(1); + expect(result.truncated).toBe(true); + }), + ); + + it.effect("excludes gitignored paths for git repositories", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-gitignore-", git: true }); + yield* writeTextFile(cwd, ".gitignore", ".convex/\nconvex/\nignored.txt\n"); + yield* writeTextFile(cwd, "src/keep.ts", "export {};"); + yield* writeTextFile(cwd, "ignored.txt", "ignore me"); + yield* writeTextFile(cwd, ".convex/local-storage/data.json", "{}"); + yield* writeTextFile(cwd, "convex/UOoS-l/convex_local_storage/modules/data.json", "{}"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "", limit: 100 }); + const paths = result.entries.map((entry) => entry.path); + + expect(paths).toContain("src"); + expect(paths).toContain("src/keep.ts"); + expect(paths).not.toContain("ignored.txt"); + expect(paths.some((entryPath) => entryPath.startsWith(".convex/"))).toBe(false); + expect(paths.some((entryPath) => entryPath.startsWith("convex/"))).toBe(false); + }), + ); + + it.effect("excludes tracked paths that match ignore rules", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ + prefix: "t3code-workspace-tracked-gitignore-", + git: true, + }); + yield* writeTextFile(cwd, ".convex/local-storage/data.json", "{}"); + yield* writeTextFile(cwd, "src/keep.ts", "export {};"); + yield* git(cwd, ["add", ".convex/local-storage/data.json", "src/keep.ts"]); + yield* writeTextFile(cwd, ".gitignore", ".convex/\n"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "", limit: 100 }); + const paths = result.entries.map((entry) => entry.path); + + expect(paths).toContain("src"); + expect(paths).toContain("src/keep.ts"); + expect(paths.some((entryPath) => entryPath.startsWith(".convex/"))).toBe(false); + }), + ); + + it.effect("excludes .convex in non-git workspaces", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-non-git-convex-" }); + yield* writeTextFile(cwd, ".convex/local-storage/data.json", "{}"); + yield* writeTextFile(cwd, "src/keep.ts", "export {};"); + + const result = yield* searchWorkspaceEntries({ cwd, query: "", limit: 100 }); + const paths = result.entries.map((entry) => entry.path); + + expect(paths).toContain("src"); + expect(paths).toContain("src/keep.ts"); + expect(paths.some((entryPath) => entryPath.startsWith(".convex/"))).toBe(false); + }), + ); + + it.effect("deduplicates concurrent index builds for the same cwd", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-concurrent-build-" }); + yield* writeTextFile(cwd, "src/components/Composer.tsx"); + + let rootReadCount = 0; + const originalReaddir = fsPromises.readdir.bind(fsPromises); + vi.spyOn(fsPromises, "readdir").mockImplementation((async ( + ...args: Parameters + ) => { + if (args[0] === cwd) { + rootReadCount += 1; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return originalReaddir(...args); + }) as typeof fsPromises.readdir); + + yield* Effect.all( + [ + searchWorkspaceEntries({ cwd, query: "", limit: 100 }), + searchWorkspaceEntries({ cwd, query: "comp", limit: 100 }), + searchWorkspaceEntries({ cwd, query: "src", limit: 100 }), + ], + { concurrency: "unbounded" }, + ); + + expect(rootReadCount).toBe(1); + }), + ); + + it.effect("limits concurrent directory reads while walking the filesystem", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-read-concurrency-" }); + yield* Effect.forEach( + Array.from({ length: 80 }, (_, index) => index), + (index) => writeTextFile(cwd, `group-${index}/entry-${index}.ts`, "export {};"), + { discard: true }, + ); + + let activeReads = 0; + let peakReads = 0; + const originalReaddir = fsPromises.readdir.bind(fsPromises); + vi.spyOn(fsPromises, "readdir").mockImplementation((async ( + ...args: Parameters + ) => { + const target = args[0]; + if (typeof target === "string" && target.startsWith(cwd)) { + activeReads += 1; + peakReads = Math.max(peakReads, activeReads); + await new Promise((resolve) => setTimeout(resolve, 4)); + try { + return await originalReaddir(...args); + } finally { + activeReads -= 1; + } + } + return originalReaddir(...args); + }) as typeof fsPromises.readdir); + + yield* searchWorkspaceEntries({ cwd, query: "", limit: 200 }); + + expect(peakReads).toBeLessThanOrEqual(32); + }), + ); + }); +}); diff --git a/apps/server/src/project/Layers/WorkspaceEntries.ts b/apps/server/src/project/Layers/WorkspaceEntries.ts new file mode 100644 index 0000000000..d8a2368df7 --- /dev/null +++ b/apps/server/src/project/Layers/WorkspaceEntries.ts @@ -0,0 +1,461 @@ +import fsPromises from "node:fs/promises"; +import type { Dirent } from "node:fs"; + +import { Cache, Duration, Effect, Exit, Layer, Path } from "effect"; + +import { + type ProjectEntry, + type ProjectSearchEntriesInput, + type ProjectSearchEntriesResult, +} from "@t3tools/contracts"; + +import { GitCore } from "../../git/Services/GitCore.ts"; +import { + WorkspaceEntries, + WorkspaceEntriesError, + type WorkspaceEntriesShape, +} from "../Services/WorkspaceEntries.ts"; + +const WORKSPACE_CACHE_TTL_MS = 15_000; +const WORKSPACE_CACHE_MAX_KEYS = 4; +const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; +const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32; +const IGNORED_DIRECTORY_NAMES = new Set([ + ".git", + ".convex", + "node_modules", + ".next", + ".turbo", + "dist", + "build", + "out", + ".cache", +]); + +interface WorkspaceIndex { + scannedAt: number; + entries: SearchableWorkspaceEntry[]; + truncated: boolean; +} + +interface SearchableWorkspaceEntry extends ProjectEntry { + normalizedPath: string; + normalizedName: string; +} + +interface RankedWorkspaceEntry { + entry: SearchableWorkspaceEntry; + score: number; +} + +function toPosixPath(input: string): string { + return input.replaceAll("\\", "/"); +} + +function parentPathOf(input: string): string | undefined { + const separatorIndex = input.lastIndexOf("/"); + if (separatorIndex === -1) { + return undefined; + } + return input.slice(0, separatorIndex); +} + +function basenameOf(input: string): string { + const separatorIndex = input.lastIndexOf("/"); + if (separatorIndex === -1) { + return input; + } + return input.slice(separatorIndex + 1); +} + +function toSearchableWorkspaceEntry(entry: ProjectEntry): SearchableWorkspaceEntry { + const normalizedPath = entry.path.toLowerCase(); + return { + ...entry, + normalizedPath, + normalizedName: basenameOf(normalizedPath), + }; +} + +function normalizeQuery(input: string): string { + return input + .trim() + .replace(/^[@./]+/, "") + .toLowerCase(); +} + +function scoreSubsequenceMatch(value: string, query: string): number | null { + if (!query) return 0; + + let queryIndex = 0; + let firstMatchIndex = -1; + let previousMatchIndex = -1; + let gapPenalty = 0; + + for (let valueIndex = 0; valueIndex < value.length; valueIndex += 1) { + if (value[valueIndex] !== query[queryIndex]) { + continue; + } + + if (firstMatchIndex === -1) { + firstMatchIndex = valueIndex; + } + if (previousMatchIndex !== -1) { + gapPenalty += valueIndex - previousMatchIndex - 1; + } + + previousMatchIndex = valueIndex; + queryIndex += 1; + if (queryIndex === query.length) { + const spanPenalty = valueIndex - firstMatchIndex + 1 - query.length; + const lengthPenalty = Math.min(64, value.length - query.length); + return firstMatchIndex * 2 + gapPenalty * 3 + spanPenalty + lengthPenalty; + } + } + + return null; +} + +function scoreEntry(entry: SearchableWorkspaceEntry, query: string): number | null { + if (!query) { + return entry.kind === "directory" ? 0 : 1; + } + + const { normalizedPath, normalizedName } = entry; + + if (normalizedName === query) return 0; + if (normalizedPath === query) return 1; + if (normalizedName.startsWith(query)) return 2; + if (normalizedPath.startsWith(query)) return 3; + if (normalizedPath.includes(`/${query}`)) return 4; + if (normalizedName.includes(query)) return 5; + if (normalizedPath.includes(query)) return 6; + + const nameFuzzyScore = scoreSubsequenceMatch(normalizedName, query); + if (nameFuzzyScore !== null) { + return 100 + nameFuzzyScore; + } + + const pathFuzzyScore = scoreSubsequenceMatch(normalizedPath, query); + if (pathFuzzyScore !== null) { + return 200 + pathFuzzyScore; + } + + return null; +} + +function compareRankedWorkspaceEntries( + left: RankedWorkspaceEntry, + right: RankedWorkspaceEntry, +): number { + const scoreDelta = left.score - right.score; + if (scoreDelta !== 0) return scoreDelta; + return left.entry.path.localeCompare(right.entry.path); +} + +function findInsertionIndex( + rankedEntries: RankedWorkspaceEntry[], + candidate: RankedWorkspaceEntry, +): number { + let low = 0; + let high = rankedEntries.length; + + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const current = rankedEntries[middle]; + if (!current) { + break; + } + + if (compareRankedWorkspaceEntries(candidate, current) < 0) { + high = middle; + } else { + low = middle + 1; + } + } + + return low; +} + +function insertRankedEntry( + rankedEntries: RankedWorkspaceEntry[], + candidate: RankedWorkspaceEntry, + limit: number, +): void { + if (limit <= 0) { + return; + } + + const insertionIndex = findInsertionIndex(rankedEntries, candidate); + if (rankedEntries.length < limit) { + rankedEntries.splice(insertionIndex, 0, candidate); + return; + } + + if (insertionIndex >= limit) { + return; + } + + rankedEntries.splice(insertionIndex, 0, candidate); + rankedEntries.pop(); +} + +function isPathInIgnoredDirectory(relativePath: string): boolean { + const firstSegment = relativePath.split("/")[0]; + if (!firstSegment) return false; + return IGNORED_DIRECTORY_NAMES.has(firstSegment); +} + +function directoryAncestorsOf(relativePath: string): string[] { + const segments = relativePath.split("/").filter((segment) => segment.length > 0); + if (segments.length <= 1) return []; + + const directories: string[] = []; + for (let index = 1; index < segments.length; index += 1) { + directories.push(segments.slice(0, index).join("/")); + } + return directories; +} + +const processErrorDetail = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +export const makeWorkspaceEntries = Effect.gen(function* () { + const path = yield* Path.Path; + const git = yield* GitCore; + + const isInsideGitWorkTree = (cwd: string): Effect.Effect => + git.isInsideWorkTree(cwd).pipe(Effect.catch(() => Effect.succeed(false))); + + const filterGitIgnoredPaths = ( + cwd: string, + relativePaths: string[], + ): Effect.Effect => + git.filterIgnoredPaths(cwd, relativePaths).pipe( + Effect.map((paths) => [...paths]), + Effect.catch(() => Effect.succeed(relativePaths)), + ); + + const buildWorkspaceIndexFromGit = (cwd: string): Effect.Effect => + Effect.gen(function* () { + if (!(yield* isInsideGitWorkTree(cwd))) { + return null; + } + + const listedFiles = yield* git + .listWorkspaceFiles(cwd) + .pipe(Effect.catch(() => Effect.succeed(null))); + + if (!listedFiles) { + return null; + } + + const listedPaths = [...listedFiles.paths] + .map((entry) => toPosixPath(entry)) + .filter((entry) => entry.length > 0 && !isPathInIgnoredDirectory(entry)); + const filePaths = yield* filterGitIgnoredPaths(cwd, listedPaths); + + const directorySet = new Set(); + for (const filePath of filePaths) { + for (const directoryPath of directoryAncestorsOf(filePath)) { + if (!isPathInIgnoredDirectory(directoryPath)) { + directorySet.add(directoryPath); + } + } + } + + const directoryEntries = [...directorySet] + .toSorted((left, right) => left.localeCompare(right)) + .map( + (directoryPath): ProjectEntry => ({ + path: directoryPath, + kind: "directory", + parentPath: parentPathOf(directoryPath), + }), + ) + .map(toSearchableWorkspaceEntry); + const fileEntries = [...new Set(filePaths)] + .toSorted((left, right) => left.localeCompare(right)) + .map( + (filePath): ProjectEntry => ({ + path: filePath, + kind: "file", + parentPath: parentPathOf(filePath), + }), + ) + .map(toSearchableWorkspaceEntry); + + const entries = [...directoryEntries, ...fileEntries]; + return { + scannedAt: Date.now(), + entries: entries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES), + truncated: listedFiles.truncated || entries.length > WORKSPACE_INDEX_MAX_ENTRIES, + }; + }); + + const readDirectoryEntries = ( + cwd: string, + relativeDir: string, + ): Effect.Effect< + { readonly relativeDir: string; readonly dirents: Dirent[] | null }, + WorkspaceEntriesError + > => + Effect.tryPromise({ + try: async () => { + const absoluteDir = relativeDir ? path.join(cwd, relativeDir) : cwd; + const dirents = await fsPromises.readdir(absoluteDir, { withFileTypes: true }); + return { relativeDir, dirents }; + }, + catch: (cause) => + new WorkspaceEntriesError({ + cwd, + operation: "workspaceEntries.readDirectoryEntries", + detail: processErrorDetail(cause), + cause, + }), + }).pipe( + Effect.catchIf( + () => relativeDir.length > 0, + () => Effect.succeed({ relativeDir, dirents: null }), + ), + ); + + const buildWorkspaceIndexFromFilesystem = ( + cwd: string, + ): Effect.Effect => + Effect.gen(function* () { + const shouldFilterWithGitIgnore = yield* isInsideGitWorkTree(cwd); + + let pendingDirectories: string[] = [""]; + const entries: SearchableWorkspaceEntry[] = []; + let truncated = false; + + while (pendingDirectories.length > 0 && !truncated) { + const currentDirectories = pendingDirectories; + pendingDirectories = []; + + const directoryEntries = yield* Effect.forEach( + currentDirectories, + (relativeDir) => readDirectoryEntries(cwd, relativeDir), + { concurrency: WORKSPACE_SCAN_READDIR_CONCURRENCY }, + ); + + const candidateEntriesByDirectory = directoryEntries.map((directoryEntry) => { + const { relativeDir, dirents } = directoryEntry; + if (!dirents) return [] as Array<{ dirent: Dirent; relativePath: string }>; + + dirents.sort((left, right) => left.name.localeCompare(right.name)); + const candidates: Array<{ dirent: Dirent; relativePath: string }> = []; + for (const dirent of dirents) { + if (!dirent.name || dirent.name === "." || dirent.name === "..") { + continue; + } + if (dirent.isDirectory() && IGNORED_DIRECTORY_NAMES.has(dirent.name)) { + continue; + } + if (!dirent.isDirectory() && !dirent.isFile()) { + continue; + } + + const relativePath = toPosixPath( + relativeDir ? path.join(relativeDir, dirent.name) : dirent.name, + ); + if (isPathInIgnoredDirectory(relativePath)) { + continue; + } + candidates.push({ dirent, relativePath }); + } + return candidates; + }); + + const candidatePaths = candidateEntriesByDirectory.flatMap((candidateEntries) => + candidateEntries.map((entry) => entry.relativePath), + ); + const allowedPathSet = shouldFilterWithGitIgnore + ? new Set(yield* filterGitIgnoredPaths(cwd, candidatePaths)) + : null; + + for (const candidateEntries of candidateEntriesByDirectory) { + for (const candidate of candidateEntries) { + if (allowedPathSet && !allowedPathSet.has(candidate.relativePath)) { + continue; + } + + const entry = toSearchableWorkspaceEntry({ + path: candidate.relativePath, + kind: candidate.dirent.isDirectory() ? "directory" : "file", + parentPath: parentPathOf(candidate.relativePath), + }); + entries.push(entry); + + if (candidate.dirent.isDirectory()) { + pendingDirectories.push(candidate.relativePath); + } + + if (entries.length >= WORKSPACE_INDEX_MAX_ENTRIES) { + truncated = true; + break; + } + } + + if (truncated) { + break; + } + } + } + + return { + scannedAt: Date.now(), + entries, + truncated, + }; + }); + + const buildWorkspaceIndex = (cwd: string): Effect.Effect => + Effect.gen(function* () { + const gitIndexed = yield* buildWorkspaceIndexFromGit(cwd); + if (gitIndexed) { + return gitIndexed; + } + return yield* buildWorkspaceIndexFromFilesystem(cwd); + }); + + const workspaceIndexCache = yield* Cache.makeWith({ + capacity: WORKSPACE_CACHE_MAX_KEYS, + lookup: buildWorkspaceIndex, + timeToLive: (exit) => + Exit.isSuccess(exit) ? Duration.millis(WORKSPACE_CACHE_TTL_MS) : Duration.zero, + }); + + return { + invalidate: (cwd: string) => Cache.invalidate(workspaceIndexCache, cwd), + search: ( + input: ProjectSearchEntriesInput, + ): Effect.Effect => + Cache.get(workspaceIndexCache, input.cwd).pipe( + Effect.map((index) => { + const normalizedQuery = normalizeQuery(input.query); + const limit = Math.max(0, Math.floor(input.limit)); + const rankedEntries: RankedWorkspaceEntry[] = []; + let matchedEntryCount = 0; + + for (const entry of index.entries) { + const score = scoreEntry(entry, normalizedQuery); + if (score === null) { + continue; + } + + matchedEntryCount += 1; + insertRankedEntry(rankedEntries, { entry, score }, limit); + } + + return { + entries: rankedEntries.map((candidate) => candidate.entry), + truncated: index.truncated || matchedEntryCount > limit, + }; + }), + ), + } satisfies WorkspaceEntriesShape; +}); + +export const WorkspaceEntriesLive = Layer.effect(WorkspaceEntries, makeWorkspaceEntries); diff --git a/apps/server/src/project/Services/WorkspaceEntries.ts b/apps/server/src/project/Services/WorkspaceEntries.ts new file mode 100644 index 0000000000..8d6d4839d5 --- /dev/null +++ b/apps/server/src/project/Services/WorkspaceEntries.ts @@ -0,0 +1,25 @@ +import { Schema, ServiceMap } from "effect"; +import type { Effect } from "effect"; + +import type { ProjectSearchEntriesInput, ProjectSearchEntriesResult } from "@t3tools/contracts"; + +export class WorkspaceEntriesError extends Schema.TaggedErrorClass()( + "WorkspaceEntriesError", + { + cwd: Schema.String, + operation: Schema.String, + detail: Schema.String, + cause: Schema.optional(Schema.Defect), + }, +) {} + +export interface WorkspaceEntriesShape { + readonly search: ( + input: ProjectSearchEntriesInput, + ) => Effect.Effect; + readonly invalidate: (cwd: string) => Effect.Effect; +} + +export class WorkspaceEntries extends ServiceMap.Service()( + "t3/project/Services/WorkspaceEntries", +) {} diff --git a/apps/server/src/serverLayers.ts b/apps/server/src/serverLayers.ts index a8c1a13f7f..ce94c15798 100644 --- a/apps/server/src/serverLayers.ts +++ b/apps/server/src/serverLayers.ts @@ -34,6 +34,7 @@ import { GitHubCliLive } from "./git/Layers/GitHubCli"; import { RoutingTextGenerationLive } from "./git/Layers/RoutingTextGeneration"; import { PtyAdapter } from "./terminal/Services/PTY"; import { AnalyticsService } from "./telemetry/Services/AnalyticsService"; +import { WorkspaceEntriesLive } from "./project/Layers/WorkspaceEntries.ts"; type RuntimePtyAdapterLoader = { layer: Layer.Layer; @@ -91,7 +92,8 @@ export function makeServerProviderLayer(): Layer.Layer< export function makeServerRuntimeServicesLayer() { const textGenerationLayer = RoutingTextGenerationLive; - const checkpointStoreLayer = CheckpointStoreLive.pipe(Layer.provide(GitCoreLive)); + const gitCoreLayer = GitCoreLive; + const checkpointStoreLayer = CheckpointStoreLive; const orchestrationLayer = OrchestrationEngineLive.pipe( Layer.provide(OrchestrationProjectionPipelineLive), @@ -116,11 +118,11 @@ export function makeServerRuntimeServicesLayer() { ); const providerCommandReactorLayer = ProviderCommandReactorLive.pipe( Layer.provideMerge(runtimeServicesLayer), - Layer.provideMerge(GitCoreLive), Layer.provideMerge(textGenerationLayer), ); const checkpointReactorLayer = CheckpointReactorLive.pipe( Layer.provideMerge(runtimeServicesLayer), + Layer.provideMerge(WorkspaceEntriesLive), ); const orchestrationReactorLayer = OrchestrationReactorLive.pipe( Layer.provideMerge(runtimeIngestionLayer), @@ -131,16 +133,17 @@ export function makeServerRuntimeServicesLayer() { const terminalLayer = TerminalManagerLive.pipe(Layer.provide(makeRuntimePtyAdapterLayer())); const gitManagerLayer = GitManagerLive.pipe( - Layer.provideMerge(GitCoreLive), Layer.provideMerge(GitHubCliLive), Layer.provideMerge(textGenerationLayer), ); + const workspaceEntriesLayer = WorkspaceEntriesLive; + return Layer.mergeAll( orchestrationReactorLayer, - GitCoreLive, + workspaceEntriesLayer, gitManagerLayer, terminalLayer, KeybindingsLive, - ).pipe(Layer.provideMerge(NodeServices.layer)); + ).pipe(Layer.provideMerge(gitCoreLayer), Layer.provideMerge(NodeServices.layer)); } diff --git a/apps/server/src/workspaceEntries.chunking.test.ts b/apps/server/src/workspaceEntries.chunking.test.ts deleted file mode 100644 index f978c5233e..0000000000 --- a/apps/server/src/workspaceEntries.chunking.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { assert, beforeEach, describe, it, vi } from "vitest"; -import type { ProcessRunOptions, ProcessRunResult } from "./processRunner"; - -const { runProcessMock } = vi.hoisted(() => ({ - runProcessMock: - vi.fn< - ( - command: string, - args: readonly string[], - options?: ProcessRunOptions, - ) => Promise - >(), -})); - -vi.mock("./processRunner", () => ({ - runProcess: runProcessMock, -})); - -function processResult( - overrides: Partial & Pick, -): ProcessRunResult { - return { - stdout: overrides.stdout, - code: overrides.code, - stderr: overrides.stderr ?? "", - signal: overrides.signal ?? null, - timedOut: overrides.timedOut ?? false, - stdoutTruncated: overrides.stdoutTruncated ?? false, - stderrTruncated: overrides.stderrTruncated ?? false, - }; -} - -describe("searchWorkspaceEntries git-ignore chunking", () => { - beforeEach(() => { - runProcessMock.mockReset(); - vi.resetModules(); - }); - - it("chunks git check-ignore stdin to avoid building giant strings", async () => { - const ignoredPaths = Array.from( - { length: 5000 }, - (_, index) => `ignored/${index.toString().padStart(5, "0")}/${"x".repeat(80)}.ts`, - ); - const keptPaths = ["src/keep.ts", "docs/readme.md"]; - const listedPaths = [...ignoredPaths, ...keptPaths]; - let checkIgnoreCalls = 0; - - runProcessMock.mockImplementation(async (_command, args, options) => { - if (args[0] === "rev-parse") { - return processResult({ code: 0, stdout: "true\n" }); - } - - if (args[0] === "ls-files") { - return processResult({ code: 0, stdout: `${listedPaths.join("\0")}\0` }); - } - - if (args[0] === "check-ignore") { - checkIgnoreCalls += 1; - const chunkPaths = (options?.stdin ?? "").split("\0").filter((value) => value.length > 0); - const chunkIgnored = chunkPaths.filter((value) => value.startsWith("ignored/")); - return processResult({ - code: chunkIgnored.length > 0 ? 0 : 1, - stdout: chunkIgnored.length > 0 ? `${chunkIgnored.join("\0")}\0` : "", - }); - } - - throw new Error(`Unexpected command: git ${args.join(" ")}`); - }); - - const { searchWorkspaceEntries } = await import("./workspaceEntries"); - const result = await searchWorkspaceEntries({ - cwd: "/virtual/workspace", - query: "", - limit: 100, - }); - - assert.isAbove(checkIgnoreCalls, 1); - assert.isFalse(result.entries.some((entry) => entry.path.startsWith("ignored/"))); - assert.isTrue(result.entries.some((entry) => entry.path === "src/keep.ts")); - }); -}); diff --git a/apps/server/src/workspaceEntries.test.ts b/apps/server/src/workspaceEntries.test.ts deleted file mode 100644 index d867ad910d..0000000000 --- a/apps/server/src/workspaceEntries.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -import fs from "node:fs"; -import fsPromises from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { spawnSync } from "node:child_process"; - -import { afterEach, assert, describe, it, vi } from "vitest"; - -import { searchWorkspaceEntries } from "./workspaceEntries"; - -const tempDirs: string[] = []; - -function makeTempDir(prefix: string): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); - tempDirs.push(dir); - return dir; -} - -function writeFile(cwd: string, relativePath: string, contents = ""): void { - const absolutePath = path.join(cwd, relativePath); - fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); - fs.writeFileSync(absolutePath, contents, "utf8"); -} - -function runGit(cwd: string, args: string[]): void { - const result = spawnSync("git", args, { cwd, encoding: "utf8" }); - if (result.status !== 0) { - throw new Error(result.stderr || `git ${args.join(" ")} failed`); - } -} - -describe("searchWorkspaceEntries", () => { - afterEach(() => { - vi.restoreAllMocks(); - for (const dir of tempDirs.splice(0, tempDirs.length)) { - fs.rmSync(dir, { recursive: true, force: true }); - } - }); - - it("returns files and directories relative to cwd", async () => { - const cwd = makeTempDir("t3code-workspace-entries-"); - writeFile(cwd, "src/components/Composer.tsx"); - writeFile(cwd, "src/index.ts"); - writeFile(cwd, "README.md"); - writeFile(cwd, ".git/HEAD"); - writeFile(cwd, "node_modules/pkg/index.js"); - - const result = await searchWorkspaceEntries({ cwd, query: "", limit: 100 }); - const paths = result.entries.map((entry) => entry.path); - - assert.include(paths, "src"); - assert.include(paths, "src/components"); - assert.include(paths, "src/components/Composer.tsx"); - assert.include(paths, "README.md"); - assert.isFalse(paths.some((entryPath) => entryPath.startsWith(".git"))); - assert.isFalse(paths.some((entryPath) => entryPath.startsWith("node_modules"))); - assert.isFalse(result.truncated); - }); - - it("filters and ranks entries by query", async () => { - const cwd = makeTempDir("t3code-workspace-query-"); - writeFile(cwd, "src/components/Composer.tsx"); - writeFile(cwd, "src/components/composePrompt.ts"); - writeFile(cwd, "docs/composition.md"); - - const result = await searchWorkspaceEntries({ cwd, query: "compo", limit: 5 }); - - assert.isAbove(result.entries.length, 0); - assert.isTrue(result.entries.some((entry) => entry.path === "src/components")); - assert.isTrue(result.entries.every((entry) => entry.path.toLowerCase().includes("compo"))); - }); - - it("supports fuzzy subsequence queries for composer path search", async () => { - const cwd = makeTempDir("t3code-workspace-fuzzy-query-"); - writeFile(cwd, "src/components/Composer.tsx"); - writeFile(cwd, "src/components/composePrompt.ts"); - writeFile(cwd, "docs/composition.md"); - - const result = await searchWorkspaceEntries({ cwd, query: "cmp", limit: 10 }); - const paths = result.entries.map((entry) => entry.path); - - assert.isAbove(result.entries.length, 0); - assert.include(paths, "src/components"); - assert.include(paths, "src/components/Composer.tsx"); - }); - - it("tracks truncation without sorting every fuzzy match", async () => { - const cwd = makeTempDir("t3code-workspace-fuzzy-limit-"); - writeFile(cwd, "src/components/Composer.tsx"); - writeFile(cwd, "src/components/composePrompt.ts"); - writeFile(cwd, "docs/composition.md"); - - const result = await searchWorkspaceEntries({ cwd, query: "cmp", limit: 1 }); - - assert.lengthOf(result.entries, 1); - assert.isTrue(result.truncated); - }); - - it("excludes gitignored paths for git repositories", async () => { - const cwd = makeTempDir("t3code-workspace-gitignore-"); - runGit(cwd, ["init"]); - writeFile(cwd, ".gitignore", ".convex/\nconvex/\nignored.txt\n"); - writeFile(cwd, "src/keep.ts", "export {};"); - writeFile(cwd, "ignored.txt", "ignore me"); - writeFile(cwd, ".convex/local-storage/data.json", "{}"); - writeFile(cwd, "convex/UOoS-l/convex_local_storage/modules/data.json", "{}"); - - const result = await searchWorkspaceEntries({ cwd, query: "", limit: 100 }); - const paths = result.entries.map((entry) => entry.path); - - assert.include(paths, "src"); - assert.include(paths, "src/keep.ts"); - assert.notInclude(paths, "ignored.txt"); - assert.isFalse(paths.some((entryPath) => entryPath.startsWith(".convex/"))); - assert.isFalse(paths.some((entryPath) => entryPath.startsWith("convex/"))); - }); - - it("excludes tracked paths that match ignore rules", async () => { - const cwd = makeTempDir("t3code-workspace-tracked-gitignore-"); - runGit(cwd, ["init"]); - writeFile(cwd, ".convex/local-storage/data.json", "{}"); - writeFile(cwd, "src/keep.ts", "export {};"); - runGit(cwd, ["add", ".convex/local-storage/data.json", "src/keep.ts"]); - writeFile(cwd, ".gitignore", ".convex/\n"); - - const result = await searchWorkspaceEntries({ cwd, query: "", limit: 100 }); - const paths = result.entries.map((entry) => entry.path); - - assert.include(paths, "src"); - assert.include(paths, "src/keep.ts"); - assert.isFalse(paths.some((entryPath) => entryPath.startsWith(".convex/"))); - }); - - it("excludes .convex in non-git workspaces", async () => { - const cwd = makeTempDir("t3code-workspace-non-git-convex-"); - writeFile(cwd, ".convex/local-storage/data.json", "{}"); - writeFile(cwd, "src/keep.ts", "export {};"); - - const result = await searchWorkspaceEntries({ cwd, query: "", limit: 100 }); - const paths = result.entries.map((entry) => entry.path); - - assert.include(paths, "src"); - assert.include(paths, "src/keep.ts"); - assert.isFalse(paths.some((entryPath) => entryPath.startsWith(".convex/"))); - }); - - it("deduplicates concurrent index builds for the same cwd", async () => { - const cwd = makeTempDir("t3code-workspace-concurrent-build-"); - writeFile(cwd, "src/components/Composer.tsx"); - - let rootReadCount = 0; - const originalReaddir = fsPromises.readdir.bind(fsPromises); - vi.spyOn(fsPromises, "readdir").mockImplementation((async ( - ...args: Parameters - ) => { - if (args[0] === cwd) { - rootReadCount += 1; - await new Promise((resolve) => setTimeout(resolve, 20)); - } - return originalReaddir(...args); - }) as typeof fsPromises.readdir); - - await Promise.all([ - searchWorkspaceEntries({ cwd, query: "", limit: 100 }), - searchWorkspaceEntries({ cwd, query: "comp", limit: 100 }), - searchWorkspaceEntries({ cwd, query: "src", limit: 100 }), - ]); - - assert.equal(rootReadCount, 1); - }); - - it("limits concurrent directory reads while walking the filesystem", async () => { - const cwd = makeTempDir("t3code-workspace-read-concurrency-"); - for (let index = 0; index < 80; index += 1) { - writeFile(cwd, `group-${index}/entry-${index}.ts`, "export {};"); - } - - let activeReads = 0; - let peakReads = 0; - const originalReaddir = fsPromises.readdir.bind(fsPromises); - vi.spyOn(fsPromises, "readdir").mockImplementation((async ( - ...args: Parameters - ) => { - const target = args[0]; - if (typeof target === "string" && target.startsWith(cwd)) { - activeReads += 1; - peakReads = Math.max(peakReads, activeReads); - await new Promise((resolve) => setTimeout(resolve, 4)); - try { - return await originalReaddir(...args); - } finally { - activeReads -= 1; - } - } - return originalReaddir(...args); - }) as typeof fsPromises.readdir); - - await searchWorkspaceEntries({ cwd, query: "", limit: 200 }); - - assert.isAtMost(peakReads, 32); - }); -}); diff --git a/apps/server/src/workspaceEntries.ts b/apps/server/src/workspaceEntries.ts deleted file mode 100644 index 684b005e83..0000000000 --- a/apps/server/src/workspaceEntries.ts +++ /dev/null @@ -1,565 +0,0 @@ -import fs from "node:fs/promises"; -import type { Dirent } from "node:fs"; -import path from "node:path"; -import { runProcess } from "./processRunner"; - -import { - ProjectEntry, - ProjectSearchEntriesInput, - ProjectSearchEntriesResult, -} from "@t3tools/contracts"; - -const WORKSPACE_CACHE_TTL_MS = 15_000; -const WORKSPACE_CACHE_MAX_KEYS = 4; -const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; -const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32; -const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; -const IGNORED_DIRECTORY_NAMES = new Set([ - ".git", - ".convex", - "node_modules", - ".next", - ".turbo", - "dist", - "build", - "out", - ".cache", -]); - -interface WorkspaceIndex { - scannedAt: number; - entries: SearchableWorkspaceEntry[]; - truncated: boolean; -} - -interface SearchableWorkspaceEntry extends ProjectEntry { - normalizedPath: string; - normalizedName: string; -} - -interface RankedWorkspaceEntry { - entry: SearchableWorkspaceEntry; - score: number; -} - -const workspaceIndexCache = new Map(); -const inFlightWorkspaceIndexBuilds = new Map>(); - -function toPosixPath(input: string): string { - return input.split(path.sep).join("/"); -} - -function parentPathOf(input: string): string | undefined { - const separatorIndex = input.lastIndexOf("/"); - if (separatorIndex === -1) { - return undefined; - } - return input.slice(0, separatorIndex); -} - -function basenameOf(input: string): string { - const separatorIndex = input.lastIndexOf("/"); - if (separatorIndex === -1) { - return input; - } - return input.slice(separatorIndex + 1); -} - -function toSearchableWorkspaceEntry(entry: ProjectEntry): SearchableWorkspaceEntry { - const normalizedPath = entry.path.toLowerCase(); - return { - ...entry, - normalizedPath, - normalizedName: basenameOf(normalizedPath), - }; -} - -function normalizeQuery(input: string): string { - return input - .trim() - .replace(/^[@./]+/, "") - .toLowerCase(); -} - -function scoreSubsequenceMatch(value: string, query: string): number | null { - if (!query) return 0; - - let queryIndex = 0; - let firstMatchIndex = -1; - let previousMatchIndex = -1; - let gapPenalty = 0; - - for (let valueIndex = 0; valueIndex < value.length; valueIndex += 1) { - if (value[valueIndex] !== query[queryIndex]) { - continue; - } - - if (firstMatchIndex === -1) { - firstMatchIndex = valueIndex; - } - if (previousMatchIndex !== -1) { - gapPenalty += valueIndex - previousMatchIndex - 1; - } - - previousMatchIndex = valueIndex; - queryIndex += 1; - if (queryIndex === query.length) { - const spanPenalty = valueIndex - firstMatchIndex + 1 - query.length; - const lengthPenalty = Math.min(64, value.length - query.length); - return firstMatchIndex * 2 + gapPenalty * 3 + spanPenalty + lengthPenalty; - } - } - - return null; -} - -function scoreEntry(entry: SearchableWorkspaceEntry, query: string): number | null { - if (!query) { - return entry.kind === "directory" ? 0 : 1; - } - - const { normalizedPath, normalizedName } = entry; - - if (normalizedName === query) return 0; - if (normalizedPath === query) return 1; - if (normalizedName.startsWith(query)) return 2; - if (normalizedPath.startsWith(query)) return 3; - if (normalizedPath.includes(`/${query}`)) return 4; - if (normalizedName.includes(query)) return 5; - if (normalizedPath.includes(query)) return 6; - - const nameFuzzyScore = scoreSubsequenceMatch(normalizedName, query); - if (nameFuzzyScore !== null) { - return 100 + nameFuzzyScore; - } - - const pathFuzzyScore = scoreSubsequenceMatch(normalizedPath, query); - if (pathFuzzyScore !== null) { - return 200 + pathFuzzyScore; - } - - return null; -} - -function compareRankedWorkspaceEntries( - left: RankedWorkspaceEntry, - right: RankedWorkspaceEntry, -): number { - const scoreDelta = left.score - right.score; - if (scoreDelta !== 0) return scoreDelta; - return left.entry.path.localeCompare(right.entry.path); -} - -function findInsertionIndex( - rankedEntries: RankedWorkspaceEntry[], - candidate: RankedWorkspaceEntry, -): number { - let low = 0; - let high = rankedEntries.length; - - while (low < high) { - const middle = low + Math.floor((high - low) / 2); - const current = rankedEntries[middle]; - if (!current) { - break; - } - - if (compareRankedWorkspaceEntries(candidate, current) < 0) { - high = middle; - } else { - low = middle + 1; - } - } - - return low; -} - -function insertRankedEntry( - rankedEntries: RankedWorkspaceEntry[], - candidate: RankedWorkspaceEntry, - limit: number, -): void { - if (limit <= 0) { - return; - } - - const insertionIndex = findInsertionIndex(rankedEntries, candidate); - if (rankedEntries.length < limit) { - rankedEntries.splice(insertionIndex, 0, candidate); - return; - } - - if (insertionIndex >= limit) { - return; - } - - rankedEntries.splice(insertionIndex, 0, candidate); - rankedEntries.pop(); -} - -function isPathInIgnoredDirectory(relativePath: string): boolean { - const firstSegment = relativePath.split("/")[0]; - if (!firstSegment) return false; - return IGNORED_DIRECTORY_NAMES.has(firstSegment); -} - -function splitNullSeparatedPaths(input: string, truncated: boolean): string[] { - const parts = input.split("\0"); - if (parts.length === 0) return []; - - // If output was truncated, the final token can be partial. - if (truncated && parts[parts.length - 1]?.length) { - parts.pop(); - } - - return parts.filter((value) => value.length > 0); -} - -function directoryAncestorsOf(relativePath: string): string[] { - const segments = relativePath.split("/").filter((segment) => segment.length > 0); - if (segments.length <= 1) return []; - const directories: string[] = []; - for (let index = 1; index < segments.length; index += 1) { - directories.push(segments.slice(0, index).join("/")); - } - return directories; -} - -async function mapWithConcurrency( - items: readonly TInput[], - concurrency: number, - mapper: (item: TInput, index: number) => Promise, -): Promise { - if (items.length === 0) { - return []; - } - - const boundedConcurrency = Math.max(1, Math.min(concurrency, items.length)); - const results = Array.from({ length: items.length }) as TOutput[]; - let nextIndex = 0; - - const workers = Array.from({ length: boundedConcurrency }, async () => { - while (nextIndex < items.length) { - const currentIndex = nextIndex; - nextIndex += 1; - results[currentIndex] = await mapper(items[currentIndex] as TInput, currentIndex); - } - }); - - await Promise.all(workers); - return results; -} - -async function isInsideGitWorkTree(cwd: string): Promise { - const insideWorkTree = await runProcess("git", ["rev-parse", "--is-inside-work-tree"], { - cwd, - allowNonZeroExit: true, - timeoutMs: 5_000, - maxBufferBytes: 4_096, - }).catch(() => null); - return Boolean( - insideWorkTree && insideWorkTree.code === 0 && insideWorkTree.stdout.trim() === "true", - ); -} - -async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Promise { - if (relativePaths.length === 0) { - return relativePaths; - } - - const ignoredPaths = new Set(); - let chunk: string[] = []; - let chunkBytes = 0; - - const flushChunk = async (): Promise => { - if (chunk.length === 0) { - return true; - } - - const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], { - cwd, - allowNonZeroExit: true, - timeoutMs: 20_000, - maxBufferBytes: 16 * 1024 * 1024, - outputMode: "truncate", - stdin: `${chunk.join("\0")}\0`, - }).catch(() => null); - chunk = []; - chunkBytes = 0; - - if (!checkIgnore) { - return false; - } - - // git-check-ignore exits with 1 when no paths match. - if (checkIgnore.code !== 0 && checkIgnore.code !== 1) { - return false; - } - - const matchedIgnoredPaths = splitNullSeparatedPaths( - checkIgnore.stdout, - Boolean(checkIgnore.stdoutTruncated), - ); - for (const ignoredPath of matchedIgnoredPaths) { - ignoredPaths.add(ignoredPath); - } - return true; - }; - - for (const relativePath of relativePaths) { - const relativePathBytes = Buffer.byteLength(relativePath) + 1; - if ( - chunk.length > 0 && - chunkBytes + relativePathBytes > GIT_CHECK_IGNORE_MAX_STDIN_BYTES && - !(await flushChunk()) - ) { - return relativePaths; - } - - chunk.push(relativePath); - chunkBytes += relativePathBytes; - - if (chunkBytes >= GIT_CHECK_IGNORE_MAX_STDIN_BYTES && !(await flushChunk())) { - return relativePaths; - } - } - - if (!(await flushChunk())) { - return relativePaths; - } - - if (ignoredPaths.size === 0) { - return relativePaths; - } - - return relativePaths.filter((relativePath) => !ignoredPaths.has(relativePath)); -} - -async function buildWorkspaceIndexFromGit(cwd: string): Promise { - if (!(await isInsideGitWorkTree(cwd))) { - return null; - } - - const listedFiles = await runProcess( - "git", - ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], - { - cwd, - allowNonZeroExit: true, - timeoutMs: 20_000, - maxBufferBytes: 16 * 1024 * 1024, - outputMode: "truncate", - }, - ).catch(() => null); - if (!listedFiles || listedFiles.code !== 0) { - return null; - } - - const listedPaths = splitNullSeparatedPaths( - listedFiles.stdout, - Boolean(listedFiles.stdoutTruncated), - ) - .map((entry) => toPosixPath(entry)) - .filter((entry) => entry.length > 0 && !isPathInIgnoredDirectory(entry)); - const filePaths = await filterGitIgnoredPaths(cwd, listedPaths); - - const directorySet = new Set(); - for (const filePath of filePaths) { - for (const directoryPath of directoryAncestorsOf(filePath)) { - if (!isPathInIgnoredDirectory(directoryPath)) { - directorySet.add(directoryPath); - } - } - } - - const directoryEntries = [...directorySet] - .toSorted((left, right) => left.localeCompare(right)) - .map( - (directoryPath): ProjectEntry => ({ - path: directoryPath, - kind: "directory", - parentPath: parentPathOf(directoryPath), - }), - ) - .map(toSearchableWorkspaceEntry); - const fileEntries = [...new Set(filePaths)] - .toSorted((left, right) => left.localeCompare(right)) - .map( - (filePath): ProjectEntry => ({ - path: filePath, - kind: "file", - parentPath: parentPathOf(filePath), - }), - ) - .map(toSearchableWorkspaceEntry); - - const entries = [...directoryEntries, ...fileEntries]; - return { - scannedAt: Date.now(), - entries: entries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES), - truncated: Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES, - }; -} - -async function buildWorkspaceIndex(cwd: string): Promise { - const gitIndexed = await buildWorkspaceIndexFromGit(cwd); - if (gitIndexed) { - return gitIndexed; - } - const shouldFilterWithGitIgnore = await isInsideGitWorkTree(cwd); - - let pendingDirectories: string[] = [""]; - const entries: SearchableWorkspaceEntry[] = []; - let truncated = false; - - while (pendingDirectories.length > 0 && !truncated) { - const currentDirectories = pendingDirectories; - pendingDirectories = []; - const directoryEntries = await mapWithConcurrency( - currentDirectories, - WORKSPACE_SCAN_READDIR_CONCURRENCY, - async (relativeDir) => { - const absoluteDir = relativeDir ? path.join(cwd, relativeDir) : cwd; - try { - const dirents = await fs.readdir(absoluteDir, { withFileTypes: true }); - return { relativeDir, dirents }; - } catch (error) { - if (!relativeDir) { - throw new Error( - `Unable to scan workspace entries at '${cwd}': ${error instanceof Error ? error.message : "unknown error"}`, - { cause: error }, - ); - } - return { relativeDir, dirents: null }; - } - }, - ); - - const candidateEntriesByDirectory = directoryEntries.map((directoryEntry) => { - const { relativeDir, dirents } = directoryEntry; - if (!dirents) return [] as Array<{ dirent: Dirent; relativePath: string }>; - - dirents.sort((left, right) => left.name.localeCompare(right.name)); - const candidates: Array<{ dirent: Dirent; relativePath: string }> = []; - for (const dirent of dirents) { - if (!dirent.name || dirent.name === "." || dirent.name === "..") { - continue; - } - if (dirent.isDirectory() && IGNORED_DIRECTORY_NAMES.has(dirent.name)) { - continue; - } - if (!dirent.isDirectory() && !dirent.isFile()) { - continue; - } - - const relativePath = toPosixPath( - relativeDir ? path.join(relativeDir, dirent.name) : dirent.name, - ); - if (isPathInIgnoredDirectory(relativePath)) { - continue; - } - candidates.push({ dirent, relativePath }); - } - return candidates; - }); - - const candidatePaths = candidateEntriesByDirectory.flatMap((candidateEntries) => - candidateEntries.map((entry) => entry.relativePath), - ); - const allowedPathSet = shouldFilterWithGitIgnore - ? new Set(await filterGitIgnoredPaths(cwd, candidatePaths)) - : null; - - for (const candidateEntries of candidateEntriesByDirectory) { - for (const candidate of candidateEntries) { - if (allowedPathSet && !allowedPathSet.has(candidate.relativePath)) { - continue; - } - - const entry = toSearchableWorkspaceEntry({ - path: candidate.relativePath, - kind: candidate.dirent.isDirectory() ? "directory" : "file", - parentPath: parentPathOf(candidate.relativePath), - }); - entries.push(entry); - - if (candidate.dirent.isDirectory()) { - pendingDirectories.push(candidate.relativePath); - } - - if (entries.length >= WORKSPACE_INDEX_MAX_ENTRIES) { - truncated = true; - break; - } - } - - if (truncated) { - break; - } - } - } - - return { - scannedAt: Date.now(), - entries, - truncated, - }; -} - -async function getWorkspaceIndex(cwd: string): Promise { - const cached = workspaceIndexCache.get(cwd); - if (cached && Date.now() - cached.scannedAt < WORKSPACE_CACHE_TTL_MS) { - return cached; - } - - const inFlight = inFlightWorkspaceIndexBuilds.get(cwd); - if (inFlight) { - return inFlight; - } - - const nextPromise = buildWorkspaceIndex(cwd) - .then((next) => { - workspaceIndexCache.set(cwd, next); - while (workspaceIndexCache.size > WORKSPACE_CACHE_MAX_KEYS) { - const oldestKey = workspaceIndexCache.keys().next().value; - if (!oldestKey) break; - workspaceIndexCache.delete(oldestKey); - } - return next; - }) - .finally(() => { - inFlightWorkspaceIndexBuilds.delete(cwd); - }); - inFlightWorkspaceIndexBuilds.set(cwd, nextPromise); - return nextPromise; -} - -export function clearWorkspaceIndexCache(cwd: string): void { - workspaceIndexCache.delete(cwd); - inFlightWorkspaceIndexBuilds.delete(cwd); -} - -export async function searchWorkspaceEntries( - input: ProjectSearchEntriesInput, -): Promise { - const index = await getWorkspaceIndex(input.cwd); - const normalizedQuery = normalizeQuery(input.query); - const limit = Math.max(0, Math.floor(input.limit)); - const rankedEntries: RankedWorkspaceEntry[] = []; - let matchedEntryCount = 0; - - for (const entry of index.entries) { - const score = scoreEntry(entry, normalizedQuery); - if (score === null) { - continue; - } - - matchedEntryCount += 1; - insertRankedEntry(rankedEntries, { entry, score }, limit); - } - - return { - entries: rankedEntries.map((candidate) => candidate.entry), - truncated: index.truncated || matchedEntryCount > limit, - }; -} diff --git a/apps/server/src/wsServer.test.ts b/apps/server/src/wsServer.test.ts index 826b9ad6fd..165d1e6eae 100644 --- a/apps/server/src/wsServer.test.ts +++ b/apps/server/src/wsServer.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Effect, Exit, Layer, PlatformError, PubSub, Scope, Stream } from "effect"; +import { Effect, Exit, Layer, ManagedRuntime, PlatformError, PubSub, Scope, Stream } from "effect"; import { describe, expect, it, afterEach, vi } from "vitest"; import { createServer } from "./wsServer"; import WebSocket from "ws"; @@ -474,6 +474,7 @@ function deriveServerPathsSync(baseDir: string, devUrl: URL | undefined) { describe("WebSocket Server", () => { let server: Http.Server | null = null; let serverScope: Scope.Closeable | null = null; + let disposeServerRuntime: (() => Promise) | null = null; const connections: WebSocket[] = []; const tempDirs: string[] = []; @@ -520,6 +521,12 @@ describe("WebSocket Server", () => { options.providerRegistry ?? defaultProviderRegistryService, ); const openLayer = Layer.succeed(Open, options.open ?? defaultOpenService); + const nodeServicesLayer = NodeServices.layer; + const serverSettingsLayer = ServerSettingsService.layerTest(options.serverSettings); + const serverSettingsRuntimeLayer = serverSettingsLayer.pipe( + Layer.provideMerge(nodeServicesLayer), + ); + const analyticsLayer = AnalyticsService.layerTest; const serverConfigLayer = Layer.succeed(ServerConfig, { mode: "web", port: 0, @@ -535,6 +542,11 @@ describe("WebSocket Server", () => { logWebSocketEvents: options.logWebSocketEvents ?? Boolean(options.devUrl), } satisfies ServerConfigShape); const infrastructureLayer = providerLayer.pipe(Layer.provideMerge(persistenceLayer)); + const providerRuntimeLayer = infrastructureLayer.pipe( + Layer.provideMerge(serverConfigLayer), + Layer.provideMerge(serverSettingsRuntimeLayer), + Layer.provideMerge(analyticsLayer), + ); const runtimeOverrides = Layer.mergeAll( options.gitManager ? Layer.succeed(GitManager, options.gitManager) : Layer.empty, options.gitCore @@ -547,41 +559,49 @@ describe("WebSocket Server", () => { const runtimeLayer = Layer.merge( Layer.merge( - makeServerRuntimeServicesLayer().pipe(Layer.provide(infrastructureLayer)), - infrastructureLayer, + makeServerRuntimeServicesLayer().pipe( + Layer.provideMerge(providerRuntimeLayer), + Layer.provideMerge(serverConfigLayer), + Layer.provideMerge(serverSettingsRuntimeLayer), + Layer.provideMerge(analyticsLayer), + Layer.provideMerge(nodeServicesLayer), + ), + Layer.mergeAll(providerRuntimeLayer, serverSettingsRuntimeLayer, analyticsLayer), ), runtimeOverrides, ); - const dependenciesLayer = Layer.empty.pipe( - Layer.provideMerge(runtimeLayer), - Layer.provideMerge(providerRegistryLayer), - Layer.provideMerge(openLayer), - Layer.provideMerge(ServerSettingsService.layerTest(options.serverSettings)), - Layer.provideMerge(serverConfigLayer), - Layer.provideMerge(AnalyticsService.layerTest), - Layer.provideMerge(NodeServices.layer), - ); - const runtimeServices = await Effect.runPromise( - Layer.build(dependenciesLayer).pipe(Scope.provide(scope)), + const dependenciesLayer = Layer.mergeAll( + runtimeLayer, + providerRegistryLayer, + openLayer, + serverConfigLayer, + nodeServicesLayer, ); - + const runtime = ManagedRuntime.make(dependenciesLayer); try { - const runtime = await Effect.runPromise( - createServer().pipe(Effect.provide(runtimeServices), Scope.provide(scope)), - ); + const httpServer = await runtime.runPromise(createServer().pipe(Scope.provide(scope))); + disposeServerRuntime = () => runtime.dispose(); serverScope = scope; - return runtime; + return httpServer; } catch (error) { + await runtime.dispose(); await Effect.runPromise(Scope.close(scope, Exit.void)); throw error; } } async function closeTestServer() { - if (!serverScope) return; + if (!serverScope && !disposeServerRuntime) return; const scope = serverScope; + const disposeRuntime = disposeServerRuntime; serverScope = null; - await Effect.runPromise(Scope.close(scope, Exit.void)); + disposeServerRuntime = null; + if (scope) { + await Effect.runPromise(Scope.close(scope, Exit.void)); + } + if (disposeRuntime) { + await disposeRuntime(); + } } afterEach(async () => { @@ -1658,6 +1678,50 @@ describe("WebSocket Server", () => { ); }); + it("invalidates workspace entry search cache after projects.writeFile", async () => { + const workspace = makeTempDir("t3code-ws-write-file-invalidate-"); + fs.mkdirSync(path.join(workspace, "src"), { recursive: true }); + fs.writeFileSync(path.join(workspace, "src", "existing.ts"), "export {};\n", "utf8"); + + server = await createTestServer({ cwd: "/test" }); + const addr = server.address(); + const port = typeof addr === "object" && addr !== null ? addr.port : 0; + + const [ws] = await connectAndAwaitWelcome(port); + connections.push(ws); + + const beforeWrite = await sendRequest(ws, WS_METHODS.projectsSearchEntries, { + cwd: workspace, + query: "rpc", + limit: 10, + }); + expect(beforeWrite.error).toBeUndefined(); + expect(beforeWrite.result).toEqual({ + entries: [], + truncated: false, + }); + + const writeResponse = await sendRequest(ws, WS_METHODS.projectsWriteFile, { + cwd: workspace, + relativePath: "plans/effect-rpc.md", + contents: "# Plan\n", + }); + expect(writeResponse.error).toBeUndefined(); + + const afterWrite = await sendRequest(ws, WS_METHODS.projectsSearchEntries, { + cwd: workspace, + query: "rpc", + limit: 10, + }); + expect(afterWrite.error).toBeUndefined(); + expect(afterWrite.result).toEqual({ + entries: expect.arrayContaining([ + expect.objectContaining({ path: "plans/effect-rpc.md", kind: "file" }), + ]), + truncated: false, + }); + }); + it("rejects projects.writeFile paths outside the workspace root", async () => { const workspace = makeTempDir("t3code-ws-write-file-reject-"); diff --git a/apps/server/src/wsServer.ts b/apps/server/src/wsServer.ts index c04d913d52..73f7b09fca 100644 --- a/apps/server/src/wsServer.ts +++ b/apps/server/src/wsServer.ts @@ -50,7 +50,6 @@ import { GitManager } from "./git/Services/GitManager.ts"; import { TerminalManager } from "./terminal/Services/Manager.ts"; import { Keybindings } from "./keybindings"; import { ServerSettingsService } from "./serverSettings"; -import { searchWorkspaceEntries } from "./workspaceEntries"; import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine"; import { ProjectionSnapshotQuery } from "./orchestration/Services/ProjectionSnapshotQuery"; import { OrchestrationReactor } from "./orchestration/Services/OrchestrationReactor"; @@ -79,6 +78,7 @@ import { expandHomePath } from "./os-jank.ts"; import { makeServerPushBus } from "./wsServer/pushBus.ts"; import { makeServerReadiness } from "./wsServer/readiness.ts"; import { decodeJsonResult, formatSchemaError } from "@t3tools/shared/schemaJson"; +import { WorkspaceEntries } from "./project/Services/WorkspaceEntries.ts"; /** * ServerShape - Service API for server lifecycle control. @@ -218,6 +218,7 @@ export type ServerRuntimeServices = | TerminalManager | Keybindings | ServerSettingsService + | WorkspaceEntries | Open | AnalyticsService; @@ -263,6 +264,7 @@ export const createServer = Effect.fn(function* (): Effect.fn.Return< const serverSettingsManager = yield* ServerSettingsService; const providerRegistry = yield* ProviderRegistry; const git = yield* GitCore; + const workspaceEntries = yield* WorkspaceEntries; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -768,13 +770,14 @@ export const createServer = Effect.fn(function* (): Effect.fn.Return< case WS_METHODS.projectsSearchEntries: { const body = stripRequestTag(request.body); - return yield* Effect.tryPromise({ - try: () => searchWorkspaceEntries(body), - catch: (cause) => - new RouteRequestError({ - message: `Failed to search workspace entries: ${String(cause)}`, - }), - }); + return yield* workspaceEntries.search(body).pipe( + Effect.mapError( + (cause) => + new RouteRequestError({ + message: `Failed to search workspace entries: ${cause.detail}`, + }), + ), + ); } case WS_METHODS.projectsWriteFile: { @@ -802,6 +805,7 @@ export const createServer = Effect.fn(function* (): Effect.fn.Return< }), ), ); + yield* workspaceEntries.invalidate(body.cwd); return { relativePath: target.relativePath }; }