diff --git a/README.md b/README.md index 1aaa92f5..39aa1465 100644 --- a/README.md +++ b/README.md @@ -57,11 +57,16 @@ This repo keeps test fixtures out of default Codegraph scans with `codegraph.con { "discovery": { "ignoreGlobs": ["tests/samples/**", "tests/languages/samples/**"] + }, + "languages": { + "extensions": { + ".tpl": "php" + } } } ``` -Use this pattern in other repos when large fixture, generated, or vendored trees should not participate in search, unresolved-import checks, graphing, indexing, inspect, impact, or review runs. Config globs are project-root-relative. CLI `--include-glob` and `--ignore-glob` stay relative to each active scan root. +Use discovery globs in other repos when large fixture, generated, or vendored trees should not participate in search, unresolved-import checks, graphing, indexing, inspect, impact, or review runs. Config globs are project-root-relative, while CLI `--include-glob` and `--ignore-glob` stay relative to each active scan root. `languages.extensions` maps additional or built-in file suffixes to supported language IDs; longer suffixes win, and built-ins remain active unless explicitly remapped. ## Quick start diff --git a/codegraph-skill/codegraph/SKILL.md b/codegraph-skill/codegraph/SKILL.md index ae3d2c7c..3993b734 100644 --- a/codegraph-skill/codegraph/SKILL.md +++ b/codegraph-skill/codegraph/SKILL.md @@ -80,7 +80,8 @@ Fall back to CLI when MCP is unavailable. ## Discovery -Durable repo-local ignores belong in `codegraph.config.json`. +Durable repo-local ignores and custom language suffixes belong in `codegraph.config.json`. +Use `languages.extensions` for suffix-to-language mappings such as `.tpl` to `php`; keys start with `.`, values are supported language IDs, and the longest suffix wins. One-off CLI filters use scan-root-relative `--include-glob` and `--ignore-glob`. Use `--no-gitignore` only when ignored files are intentionally in scope. diff --git a/docs/cli.md b/docs/cli.md index 1df78910..4460bd05 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -36,18 +36,27 @@ Commands that scan a project read `codegraph.config.json` from `--root` when it "includeGlobs": ["src/**/*.ts"], "ignoreGlobs": ["tests/samples/**", "tests/languages/samples/**"], "useGitignore": true + }, + "languages": { + "extensions": { + ".tpl": "php", + ".inc.php": "php", + ".build.ts": "ts" + } } } ``` - `discovery.includeGlobs` and `discovery.ignoreGlobs` are project-root-relative, even when a command scans child include roots. - `discovery.ignoreGlobs` is for large fixture, generated, or vendored folders that should not be indexed. +- `languages.extensions` maps additional or built-in suffixes to supported language IDs; keys must start with `.`, values must name a supported language, and the longest suffix wins. +- Built-in suffixes remain active unless explicitly remapped by `languages.extensions`. - CLI `--include-glob` and `--ignore-glob` values are one-off additions relative to each scanned root. - `inspect` follow-up commands preserve the selected `--root` and include roots. - `--no-gitignore` overrides `useGitignore`. Config globs and one-off CLI globs apply at different layers. `codegraph.config.json` globs are durable and project-root-relative. CLI scan-root globs are additive for a single command and are evaluated relative to each active scan root. `--no-gitignore` disables `.gitignore` filtering for that command only; it does not change config. -Cache and manifest reuse is rooted at `--root`. Reusing a project root lets commands share compatible index and graph entries when the file signatures, config, graph options, and relevant build options still match. Changing `--root`, changing discovery config, or changing graph options creates a different reuse boundary. Child include-root scans can reuse project-root cache entries, but command summaries and follow-up commands stay scoped to the selected include roots. +Configured language extensions automatically extend discovery for matching files and participate in cache compatibility checks. Reusing a project root lets commands share compatible index and graph entries when the file signatures, config, graph options, and relevant build options still match. Changing `--root`, changing discovery or language-extension config, or changing graph options creates a different reuse boundary. Child include-root scans can reuse project-root cache entries, but command summaries and follow-up commands stay scoped to the selected include roots. ## Core commands diff --git a/src/cli.ts b/src/cli.ts index be288e7c..ba1cda77 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -191,6 +191,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { return { ...(progressHandler ? { onProgress: progressHandler } : {}), discovery: discoveryOptions, + ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), ...(cache !== undefined ? { cache } : {}), ...(hasFlag("--cache-strict") ? { cacheStrict: true } : {}), ...(hasFlag("--cache-verify") ? { cacheVerify: true } : {}), @@ -756,6 +757,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { discoveryOptions, nativeMode, workerOpts, + languageExtensions: config.languages?.extensions, progressHandler, graphOptions: hasGraphOverrides ? buildGraphOptions() : undefined, reportEnabled, @@ -785,6 +787,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { onProgress: progressHandler, discovery: discoveryOptions, ...(nativeMode !== "auto" ? { native: nativeMode } : {}), + ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), ...workerOpts, }, writeJSONLine, @@ -807,6 +810,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { discovery: discoveryOptions, ...(hasGraphOverrides ? { graph: buildGraphOptions() } : {}), ...(nativeMode !== "auto" ? { native: nativeMode } : {}), + ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), ...workerOpts, }, writeJSONLine, @@ -956,6 +960,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { discovery: discoveryOptions, ...(graphOptions ? { graph: graphOptions } : {}), ...(nativeMode !== "auto" ? { native: nativeMode } : {}), + ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), ...workerOpts, }); @@ -1008,6 +1013,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { projectRootFs, includeRootsAbs, discoveryOptions, + languageExtensions: config.languages?.extensions, graphOptions: hasGraphOverrides || nativeMode !== "auto" ? buildGraphOptions() : undefined, nativeMode, workerOpts, @@ -1027,6 +1033,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { projectRootFs, includeRootsAbs, discoveryOptions, + languageExtensions: config.languages?.extensions, graphOptions: hasGraphOverrides || nativeMode !== "auto" ? buildGraphOptions() : undefined, nativeMode, workerOpts, @@ -1058,6 +1065,7 @@ async function runCliWithActiveRuntime(rawArgs: string[]) { onProgress: progressHandler, discovery: discoveryOptions, ...(nativeMode !== "auto" ? { native: nativeMode } : {}), + ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), ...workerOpts, }, }); diff --git a/src/cli/index.ts b/src/cli/index.ts index 453689ec..5d630cdb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -26,6 +26,7 @@ export type IndexCommandContext = { discoveryOptions: ProjectFileDiscoveryOptions; nativeMode: NativeRuntimeMode; workerOpts: { useNativeWorkers: true } | Record; + languageExtensions: BuildOptions["languageExtensions"]; progressHandler: ((update: { current: number; total: number }) => void) | undefined; graphOptions: GraphBuildOptions | undefined; reportEnabled: boolean; @@ -66,6 +67,7 @@ export async function handleIndexCommand(context: IndexCommandContext): Promise< threads, discovery: context.discoveryOptions, ...(context.nativeMode !== "auto" ? { native: context.nativeMode } : {}), + ...(context.languageExtensions ? { languageExtensions: context.languageExtensions } : {}), ...context.workerOpts, ...(cache !== undefined ? { cache } : {}), cacheStrict, diff --git a/src/cli/inspect.ts b/src/cli/inspect.ts index b335b586..20c547da 100644 --- a/src/cli/inspect.ts +++ b/src/cli/inspect.ts @@ -1,23 +1,25 @@ import fs from "node:fs"; import path from "node:path"; -import { findDuplicates, type DuplicateConfidence, type DuplicateGroup } from "../duplicates.js"; +import { findDuplicates } from "../duplicates.js"; +import type { DuplicateConfidence, DuplicateGroup } from "../duplicates.js"; import { collectGraph } from "../graph-builder.js"; import { findDetailedCycles, getUnresolvedImports } from "../graphs/queries.js"; import { getHotspots } from "../graphs/hotspots.js"; -import { type GraphBuildOptions } from "../graphs/types.js"; +import type { GraphBuildOptions } from "../graphs/types.js"; import { buildProjectIndexIncremental } from "../indexer/build-index.js"; -import { type BuildReport } from "../indexer/types.js"; +import type { BuildReport } from "../indexer/types.js"; import { getNativeTreeSitterLoadError, getNativeTreeSitterSupportedLanguageIds, isNativeTreeSitterAvailable, - type NativeRuntimeMode, } from "../native/treeSitterNative.js"; +import type { NativeRuntimeMode } from "../native/treeSitterNative.js"; import type { Graph } from "../types.js"; import { restrictGraphToIncludeRoots } from "../util/includeRoots.js"; import { supportForFile } from "../languages.js"; +import type { LanguageExtensionMap } from "../languages.js"; import { toProjectDisplayPath } from "../util/paths.js"; -import { type ProjectFileDiscoveryOptions } from "../util/projectFiles.js"; +import type { ProjectFileDiscoveryOptions } from "../util/projectFiles.js"; import { parseCacheModeOption, parsePositiveIntegerOption } from "./options.js"; type CacheMode = "off" | "memory" | "disk"; @@ -94,6 +96,7 @@ export type InspectCommandContext = { projectRootFs: string; includeRootsAbs: string[]; discoveryOptions: ProjectFileDiscoveryOptions; + languageExtensions: LanguageExtensionMap | undefined; graphOptions: GraphBuildOptions | undefined; nativeMode: NativeRuntimeMode; workerOpts: { useNativeWorkers: true } | Record; @@ -149,6 +152,7 @@ async function buildScopedReportGraph( opts: { cache?: CacheMode; discovery?: ProjectFileDiscoveryOptions; + languageExtensions?: LanguageExtensionMap; graphOptions?: GraphBuildOptions; nativeMode?: NativeRuntimeMode; workerOpts?: { useNativeWorkers: true } | Record; @@ -165,6 +169,7 @@ async function buildScopedReportGraph( files, cache: "disk", ...(opts.discovery ? { discovery: opts.discovery } : {}), + ...(opts.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), ...(opts.progressHandler ? { onProgress: opts.progressHandler } : {}), ...(opts.nativeMode && opts.nativeMode !== "auto" ? { native: opts.nativeMode } : {}), ...(opts.workerOpts ?? {}), @@ -179,6 +184,7 @@ async function buildScopedReportGraph( const sourceGraph = await collectGraph(projectRoot, files, { ...(opts.graphOptions ?? {}), + ...(opts.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), ...(opts.report ? { report: opts.report } : {}), }); return { @@ -208,10 +214,13 @@ function summarizeDuplicateGroup(group: DuplicateGroup): DuplicateOpportunitySum }; } -function countFilesByLanguage(files: Iterable): Record { +function countFilesByLanguage( + files: Iterable, + languageExtensions?: LanguageExtensionMap, +): Record { const byLanguage: Record = {}; for (const file of files) { - const languageId = supportForFile(file)?.id ?? "other"; + const languageId = supportForFile(file, languageExtensions)?.id ?? "other"; byLanguage[languageId] = (byLanguage[languageId] ?? 0) + 1; } return byLanguage; @@ -248,6 +257,7 @@ async function buildInspectReport( files: string[], discovery: ProjectFileDiscoveryOptions, graphOptions: GraphBuildOptions | undefined, + languageExtensions: LanguageExtensionMap | undefined, cache: CacheMode | undefined, nativeMode: NativeRuntimeMode, workerOpts: { useNativeWorkers: true } | Record, @@ -264,6 +274,7 @@ async function buildInspectReport( files, cache: cache ?? "disk", discovery, + ...(languageExtensions ? { languageExtensions } : {}), ...(progressHandler ? { onProgress: progressHandler } : {}), ...(nativeMode !== "auto" ? { native: nativeMode } : {}), ...workerOpts, @@ -297,7 +308,7 @@ async function buildInspectReport( }, files: { total: files.length, - byLanguage: countFilesByLanguage(files), + byLanguage: countFilesByLanguage(files, languageExtensions), }, hotspots, unresolved: { @@ -343,6 +354,7 @@ export async function handleInspectCommand(context: InspectCommandContext): Prom files, context.discoveryOptions, context.graphOptions, + context.languageExtensions, cache, context.nativeMode, context.workerOpts, @@ -361,6 +373,7 @@ export async function handleHotspotsCommand(context: InspectCommandContext): Pro const { graph } = await buildScopedReportGraph(context.projectRootFs, context.includeRootsAbs, files, { ...(cache ? { cache } : {}), discovery: context.discoveryOptions, + ...(context.languageExtensions ? { languageExtensions: context.languageExtensions } : {}), ...(context.graphOptions ? { graphOptions: context.graphOptions } : {}), nativeMode: context.nativeMode, workerOpts: context.workerOpts, diff --git a/src/config.ts b/src/config.ts index 9b671345..65c270ed 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,12 +1,16 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { z } from "zod"; -import { type ProjectFileDiscoveryOptions } from "./util/projectFiles.js"; +import { supportById } from "./languages.js"; +import type { LanguageExtensionMap } from "./indexer/types.js"; +import type { ProjectFileDiscoveryOptions } from "./util/projectFiles.js"; export const CODEGRAPH_CONFIG_FILE = "codegraph.config.json"; const stringArraySchema = z.array(z.string().trim().min(1)); +const languageExtensionsSchema = z.record(z.string().trim().min(1), z.string().trim().min(1)); + const codegraphConfigSchema = z .object({ discovery: z @@ -17,6 +21,12 @@ const codegraphConfigSchema = z }) .strict() .optional(), + languages: z + .object({ + extensions: languageExtensionsSchema.optional(), + }) + .strict() + .optional(), }) .strict(); @@ -24,6 +34,9 @@ type ParsedCodegraphConfig = z.infer; export type CodegraphConfig = { discovery?: ProjectFileDiscoveryOptions; + languages?: { + extensions?: LanguageExtensionMap; + }; }; function uniq(values: readonly string[]): string[] { @@ -78,6 +91,33 @@ function normalizeDiscoveryConfig( return hasDiscoveryOptions(normalized) ? normalized : undefined; } +function normalizeLanguageExtensions(extensions: Record | undefined): LanguageExtensionMap | undefined { + if (!extensions) return undefined; + const normalized: LanguageExtensionMap = {}; + for (const [rawKey, rawLanguageId] of Object.entries(extensions)) { + const key = rawKey.trim().toLowerCase(); + const languageId = rawLanguageId.trim(); + if (!key.startsWith(".")) { + throw new Error(`Invalid ${CODEGRAPH_CONFIG_FILE}: languages.extensions key "${rawKey}" must start with ".".`); + } + if (!supportById(languageId)) { + throw new Error( + `Invalid ${CODEGRAPH_CONFIG_FILE}: languages.extensions["${rawKey}"] references unknown language "${languageId}".`, + ); + } + normalized[key] = languageId; + } + return Object.keys(normalized).length ? normalized : undefined; +} + +function languageExtensionIncludeGlobs(languageExtensions: LanguageExtensionMap | undefined): string[] { + return Object.keys(languageExtensions ?? {}) + .map((extension) => extension.trim().toLowerCase()) + .filter((extension) => extension.startsWith(".")) + .sort() + .map((extension) => `**/*${extension}`); +} + export async function loadCodegraphConfig(projectRoot: string): Promise { const configPath = path.join(projectRoot, CODEGRAPH_CONFIG_FILE); let raw: string; @@ -102,8 +142,13 @@ export async function loadCodegraphConfig(projectRoot: string): Promise; logLevel?: LogLevel; allFiles?: string[]; + languageExtensions?: LanguageExtensionMap; }, ): Promise { const normalizePath = (file: string) => file.replace(/\\/g, "/"); @@ -146,6 +150,7 @@ export async function collectGraph( ...(opts?.report ? { report: opts.report } : {}), allFiles: normalizedAllFiles, ...(sqlFactCache ? { sqlFactCache } : {}), + ...(opts?.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), }); addEdgeTargetsToGraph(edges); return edges; diff --git a/src/graph-edge-collector.ts b/src/graph-edge-collector.ts index 64c81818..6c15b0d0 100644 --- a/src/graph-edge-collector.ts +++ b/src/graph-edge-collector.ts @@ -1,30 +1,29 @@ import path from "node:path"; -import { type ParserLanguage } from "./parserBackend.js"; +import type { ParserLanguage } from "./parserBackend.js"; import { prepareSourceInput } from "./languages/filePrep.js"; -import { type LanguageSupport } from "./languages.js"; +import type { LanguageExtensionMap, LanguageSupport } from "./languages.js"; import type { Edge } from "./types.js"; import { loadNearestTsconfigFor } from "./util/resolution.js"; -import { type WorkspaceConfig } from "./util/workspace.js"; +import type { WorkspaceConfig } from "./util/workspace.js"; import { extractJsTsDynamicSpecifiers } from "./util/specifiers.js"; -import { logWithLevel, type LogLevel } from "./logging.js"; +import { logWithLevel } from "./logging.js"; +import type { LogLevel } from "./logging.js"; import { graphOnlyLanguageSupportsImportAliases, graphOnlySpecifierNeedsResolutionConfig, isGraphOnlyLanguage, } from "./documentLinks.js"; -import { - getCompactImportsExecution, - type NativeRuntimeMode, - type CompactQueryResults, - type NativeQueryResults, -} from "./native/treeSitterNative.js"; +import { getCompactImportsExecution } from "./native/treeSitterNative.js"; +import type { NativeRuntimeMode, CompactQueryResults, NativeQueryResults } from "./native/treeSitterNative.js"; import { recordNativeExecutionOutcome } from "./native/nativeBackendReport.js"; -import { collectModuleSpecifiersFromSource, type FallbackImportExtractionEvent } from "./graphs/specifiers.js"; +import { collectModuleSpecifiersFromSource } from "./graphs/specifiers.js"; +import type { FallbackImportExtractionEvent } from "./graphs/specifiers.js"; import { collectPhpComposerImplicitEdges, resolveModuleSpecifierEdges } from "./graphs/edgeResolution.js"; import type { GraphCacheEntry } from "./graphs/types.js"; import type { BuildReport } from "./indexer/types.js"; import type { SyntaxTreeLike } from "./languages/types.js"; -import { collectSqlEdgesForFile, type SqlFactCache } from "./sql/sourceGraph.js"; +import { collectSqlEdgesForFile } from "./sql/sourceGraph.js"; +import type { SqlFactCache } from "./sql/sourceGraph.js"; const cloneEdge = (edge: Edge): Edge => ({ ...edge, @@ -52,6 +51,7 @@ export async function collectEdgesForFile( fileSignature?: { sig: string; gitSig?: string; cacheSig?: string }; sqlCorpusSig?: string; cachedFileEdges?: GraphCacheEntry; + languageExtensions?: LanguageExtensionMap; onFileEdges?: (file: string, entry: GraphCacheEntry) => void; onFallbackImportExtraction?: (event: FallbackImportExtractionEvent) => void; report?: BuildReport; @@ -96,7 +96,7 @@ export async function collectEdgesForFile( let compactNativeImports: CompactQueryResults | null = null; let graphOnlyLanguage = sup ? isGraphOnlyLanguage(sup.id) : false; if (!sup || src === undefined) { - const prep = await prepareSourceInput(file); + const prep = await prepareSourceInput(file, { languageExtensions: opts.languageExtensions }); sup = prep.sup; src = prep.source; graphOnlyLanguage = isGraphOnlyLanguage(sup.id); diff --git a/src/indexer/build-cache/module-cache.ts b/src/indexer/build-cache/module-cache.ts index e3a513f9..ea95f418 100644 --- a/src/indexer/build-cache/module-cache.ts +++ b/src/indexer/build-cache/module-cache.ts @@ -208,10 +208,11 @@ export async function cacheSignatureForFile(file: string, sigInfo: FileSignature export async function buildBloomFilterForFile( file: string, + opts?: Pick, ): Promise { try { const source = await fsp.readFile(file, "utf8"); - const support = supportForFile(file); + const support = supportForFile(file, opts?.languageExtensions); if (!support) return null; return buildBloomFilterFromSource(source, support.id); } catch { diff --git a/src/indexer/build-cache/options.ts b/src/indexer/build-cache/options.ts index 2956bb85..bea6c5d1 100644 --- a/src/indexer/build-cache/options.ts +++ b/src/indexer/build-cache/options.ts @@ -1,7 +1,7 @@ import path from "node:path"; import type { GraphBuildOptions } from "../../graphs/types.js"; import { normalizePath, normalizeResolutionHints } from "../../util/paths.js"; -import { type ProjectFileDiscoveryOptions } from "../../util/projectFiles.js"; +import type { ProjectFileDiscoveryOptions } from "../../util/projectFiles.js"; import type { BuildOptions } from "../types.js"; export type ManifestBuildOptions = { @@ -17,9 +17,11 @@ export type ManifestBuildOptions = { gitignoreRoot?: string; useGitignore: boolean; }; + languageExtensions?: Record; }; function normalizeManifestBuildOptions(opts?: ManifestBuildOptions): ManifestBuildOptions { + const languageExtensions = normalizeLanguageExtensions(opts?.languageExtensions); return { cache: opts?.cache ?? "off", cacheStrict: opts?.cacheStrict ?? true, @@ -27,6 +29,7 @@ function normalizeManifestBuildOptions(opts?: ManifestBuildOptions): ManifestBui preset: opts?.preset, incrementalStrict: opts?.incrementalStrict ?? false, ...(opts?.discovery ? { discovery: opts.discovery } : {}), + ...(languageExtensions ? { languageExtensions } : {}), }; } @@ -50,8 +53,18 @@ function normalizeDiscoveryOptions(discovery?: ProjectFileDiscoveryOptions): Man }; } +function normalizeLanguageExtensions(extensions?: Record): Record | undefined { + const entries = Object.entries(extensions ?? {}) + .map(([key, value]) => [key.trim().toLowerCase(), value.trim()] as const) + .filter(([key, value]) => key && value) + .sort((left, right) => left[0].localeCompare(right[0])); + if (!entries.length) return undefined; + return Object.fromEntries(entries); +} + function normalizeBuildOptions(opts?: BuildOptions): ManifestBuildOptions { const discovery = normalizeDiscoveryOptions(opts?.discovery); + const languageExtensions = normalizeLanguageExtensions(opts?.languageExtensions); return { cache: opts?.cache ?? "off", cacheStrict: opts?.cacheStrict ?? true, @@ -59,6 +72,7 @@ function normalizeBuildOptions(opts?: BuildOptions): ManifestBuildOptions { preset: opts?.preset, incrementalStrict: opts?.incrementalStrict ?? false, ...(discovery ? { discovery } : {}), + ...(languageExtensions ? { languageExtensions } : {}), }; } @@ -101,6 +115,19 @@ function normalizedDiscoveryOptionsEqual( return true; } +function normalizedLanguageExtensionsEqual( + a: Record | undefined, + b: Record | undefined, +): boolean { + const normalizedA = normalizeLanguageExtensions(a) ?? {}; + const normalizedB = normalizeLanguageExtensions(b) ?? {}; + const keys = Array.from(new Set([...Object.keys(normalizedA), ...Object.keys(normalizedB)])).sort(); + for (const key of keys) { + if (normalizedA[key] !== normalizedB[key]) return false; + } + return true; +} + export function diffBuildOptions( manifestOpts: ManifestBuildOptions | undefined, currentOpts: BuildOptions | undefined, @@ -123,6 +150,9 @@ export function diffBuildOptions( if (!normalizedDiscoveryOptionsEqual(normalizedManifest.discovery, normalizedCurrent.discovery)) { diffs.push("discovery"); } + if (!normalizedLanguageExtensionsEqual(normalizedManifest.languageExtensions, normalizedCurrent.languageExtensions)) { + diffs.push("languageExtensions"); + } return diffs; } diff --git a/src/indexer/build-index.ts b/src/indexer/build-index.ts index 02bc00a6..749951eb 100644 --- a/src/indexer/build-index.ts +++ b/src/indexer/build-index.ts @@ -3,7 +3,12 @@ import path from "node:path"; import { performance } from "node:perf_hooks"; import { supportForFile, type LanguageSupport } from "../languages.js"; import { loadWorkspaceConfig, resolveWorkspacePackage } from "../util/workspace.js"; -import { discoverProjectFiles, listProjectFiles, type ProjectFileInfo } from "../util/projectFiles.js"; +import { + DEFAULT_PROJECT_PATTERNS, + discoverProjectFiles, + listProjectFiles, + type ProjectFileInfo, +} from "../util/projectFiles.js"; import { getGitHead, isGitRepo, getGitBlobHashes, listChangedFiles } from "../util/git.js"; import { clearImportResolutionCaches, resolveSpecifier } from "../util/resolution.js"; import { assertFilePathWithinRoot, normalizePath } from "../util/paths.js"; @@ -239,6 +244,7 @@ async function buildIndexedModuleForFile(args: { graphOptions: args.graphOptions, ...(args.opts?.native ? { native: args.opts.native } : {}), ...(args.opts?.logLevel ? { logLevel: args.opts.logLevel } : {}), + ...(args.opts?.languageExtensions ? { languageExtensions: args.opts.languageExtensions } : {}), ...(args.onFallbackImportExtraction ? { onFallbackImportExtraction: args.onFallbackImportExtraction } : {}), }); collectJsonDependencies(imports, args.jsonDependencies); @@ -268,7 +274,9 @@ async function buildIndexedModuleForFile(args: { const sigInfo = args.fileSignatures.get(args.file); if (sigInfo) { - const cacheSig = args.cacheEnabled ? await cacheSignatureForFile(args.file, sigInfo) : sigInfo.cacheSig; + const cacheSig = args.cacheEnabled + ? await moduleCacheSignatureForFile(args.file, sigInfo, args.opts) + : sigInfo.cacheSig; writeToCache(args.projectRoot, args.file, cacheSig, mod, args.opts); } @@ -320,7 +328,28 @@ function graphEdgeKey(edge: Edge): string { return `${edge.from}::${target}::${edge.raw ?? ""}::${edge.typeOnly ? 1 : 0}`; } -function expandStarImports(modules: Map): void { +async function moduleCacheSignatureForFile(file: string, sigInfo: FileSignature, opts?: BuildOptions): Promise { + const baseSignature = await cacheSignatureForFile(file, sigInfo); + const languageExtensions = opts?.languageExtensions; + if (!languageExtensions || !Object.keys(languageExtensions).length) return baseSignature; + const normalizedExtensions = Object.entries(languageExtensions) + .map(([key, value]) => [key.trim().toLowerCase(), value.trim()] as const) + .filter(([key, value]) => key && value) + .sort((left, right) => left[0].localeCompare(right[0])); + return `${baseSignature}\0languageExtensions:${JSON.stringify(normalizedExtensions)}`; +} + +function projectPatternsForLanguageExtensions(opts?: BuildOptions): string[] | undefined { + const extensions = Object.keys(opts?.languageExtensions ?? {}) + .map((extension) => extension.trim().toLowerCase()) + .filter((extension) => extension.startsWith(".")) + .sort(); + if (!extensions.length) return undefined; + const customPatterns = extensions.map((extension) => `**/*${extension}`); + return [...DEFAULT_PROJECT_PATTERNS, ...customPatterns]; +} + +function expandStarImports(modules: Map, opts?: BuildOptions): void { const importAlreadyPresent = (imports: ImportBinding[], candidate: ImportBinding): boolean => imports.some((existing) => { if (existing.kind !== candidate.kind) return false; @@ -343,7 +372,7 @@ function expandStarImports(modules: Map): void { if (imp.kind !== "star" || typeof imp.resolved !== "string") continue; const target = modules.get(imp.resolved); if (!target) continue; - const targetSupport = supportForFile(imp.resolved); + const targetSupport = supportForFile(imp.resolved, opts?.languageExtensions); const exportedSymbols = target.exports.filter((entry) => entry.type === "local").length ? target.exports .filter((entry): entry is Extract => entry.type === "local") @@ -576,7 +605,7 @@ async function buildIndexFromFileListShared( fileSignatures.set(file, sigInfo); } manifestEntriesForIndex.set(file, toProjectIndexManifestEntry(sigInfo)); - const cacheSig = cacheEnabled ? await cacheSignatureForFile(file, sigInfo) : sigInfo.cacheSig; + const cacheSig = cacheEnabled ? await moduleCacheSignatureForFile(file, sigInfo, opts) : sigInfo.cacheSig; let mod: ModuleIndex | null = cacheEnabled ? tryLoadFromCache(projectRoot, file, cacheSig, opts, report) : null; if (mod && fileReport) { fileReport.cached = (fileReport.cached ?? 0) + 1; @@ -599,6 +628,7 @@ async function buildIndexFromFileListShared( resolveNodeModules: !!graphOptions.resolveNodeModules, dynamicImportHeuristics: !!graphOptions.dynamicImportHeuristics, ...(opts?.native ? { native: opts.native } : {}), + ...(opts?.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), ...(opts?.logLevel ? { logLevel: opts.logLevel } : {}), ...(graphOptions.resolutionHints ? { resolutionHints: graphOptions.resolutionHints } : {}), fileSignature: sigInfo, @@ -610,13 +640,13 @@ async function buildIndexFromFileListShared( ...(sqlFactCache ? { sqlFactCache } : {}), }); if (bloomFilterCache) { - const filter = await buildBloomFilterForFile(file); + const filter = await buildBloomFilterForFile(file, opts); if (filter) bloomFilterCache.set(file, filter); } return [file, mod, edges] as const; } if (fileReport) fileReport.parsed = (fileReport.parsed ?? 0) + 1; - const support = supportForFile(file); + const support = supportForFile(file, opts?.languageExtensions); if (!support) return [file, createEmptyModuleIndex(file), []] as const; let graphContext: IndexedFileGraphContext | undefined; if (!mod) { @@ -651,6 +681,7 @@ async function buildIndexFromFileListShared( resolveNodeModules: !!graphOptions.resolveNodeModules, dynamicImportHeuristics: !!graphOptions.dynamicImportHeuristics, ...(opts?.native ? { native: opts.native } : {}), + ...(opts?.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), ...(opts?.logLevel ? { logLevel: opts.logLevel } : {}), ...(graphOptions.resolutionHints ? { resolutionHints: graphOptions.resolutionHints } : {}), fileSignature: sigInfo, @@ -709,7 +740,7 @@ async function buildIndexFromFileListShared( for (const jsonPath of jsonDependencies) { ensureJsonModule(modules, jsonPath); } - expandStarImports(modules); + expandStarImports(modules, opts); if (manifestEntries) { await writeIndexManifestSnapshot({ projectRoot, @@ -750,7 +781,7 @@ async function buildProjectIndexWithManifestOptions( ): Promise { try { const [files, projectFiles] = await Promise.all([ - listProjectFiles(projectRoot, undefined, { + listProjectFiles(projectRoot, projectPatternsForLanguageExtensions(opts), { ...opts?.discovery, ...(opts?.logLevel ? { logLevel: opts.logLevel } : {}), }), @@ -1043,14 +1074,14 @@ export async function buildProjectIndexIncremental( for (const file of allFiles) { if (changedFiles.has(file)) continue; const sigInfo = fileSignatures.get(file)!; - const cacheSig = cacheEnabled ? await cacheSignatureForFile(file, sigInfo) : sigInfo.cacheSig; + const cacheSig = cacheEnabled ? await moduleCacheSignatureForFile(file, sigInfo, opts) : sigInfo.cacheSig; const cached = cacheEnabled ? tryLoadFromCache(projectRoot, file, cacheSig, opts, report) : null; if (cached) { if (fileReport) fileReport.cached = (fileReport.cached ?? 0) + 1; modules.set(file, cached); collectJsonDependencies(cached.imports, jsonDependencies); if (bloomFilterCache) { - const filter = await buildBloomFilterForFile(file); + const filter = await buildBloomFilterForFile(file, opts); if (filter) bloomFilterCache.set(file, filter); } } else { @@ -1067,7 +1098,7 @@ export async function buildProjectIndexIncremental( const fileResults = await mapLimit(changedList, conc, async (file) => { try { if (fileReport) fileReport.parsed = (fileReport.parsed ?? 0) + 1; - const support = supportForFile(file); + const support = supportForFile(file, opts?.languageExtensions); if (!support) return [file, createEmptyModuleIndex(file)] as const; const built = await buildIndexedModuleForFile({ file, @@ -1114,7 +1145,7 @@ export async function buildProjectIndexIncremental( for (const jsonPath of jsonDependencies) { ensureJsonModule(modules, jsonPath); } - expandStarImports(modules); + expandStarImports(modules, opts); const retainedTrackedEntries = Object.entries(trackedEntries).filter(([file]) => !deletedTrackedFiles.has(file)); const cachedGraphEntries = new Map(retainedTrackedEntries); const manifestEntries = new Map(cachedGraphEntries); @@ -1145,6 +1176,7 @@ export async function buildProjectIndexIncremental( resolveNodeModules: !!graphOptions.resolveNodeModules, dynamicImportHeuristics: !!graphOptions.dynamicImportHeuristics, ...(opts?.native ? { native: opts.native } : {}), + ...(opts?.languageExtensions ? { languageExtensions: opts.languageExtensions } : {}), ...(opts?.logLevel ? { logLevel: opts.logLevel } : {}), ...(graphOptions.resolutionHints ? { resolutionHints: graphOptions.resolutionHints } : {}), allFiles: Array.from(allFiles), diff --git a/src/indexer/build-workers.ts b/src/indexer/build-workers.ts index 73e69696..37bbfe3a 100644 --- a/src/indexer/build-workers.ts +++ b/src/indexer/build-workers.ts @@ -126,10 +126,10 @@ export async function prepareFileContextForBuild( }); } } - prepared = await prepareFileForIndexing(file, opts?.native); + prepared = await prepareFileForIndexing(file, opts?.native, opts?.languageExtensions); } } else { - prepared = await prepareFileForIndexing(file, opts?.native); + prepared = await prepareFileForIndexing(file, opts?.native, opts?.languageExtensions); } if (isGraphOnlyLanguage(prepared.sup.id)) { return prepared; diff --git a/src/indexer/imports.ts b/src/indexer/imports.ts index c2c56bd4..f294300b 100644 --- a/src/indexer/imports.ts +++ b/src/indexer/imports.ts @@ -1,9 +1,10 @@ import { prepareSourceInput } from "../languages/filePrep.js"; import { loadNearestTsconfigFor, resolveImportSpecifier } from "../util/resolution.js"; import { loadWorkspaceConfig } from "../util/workspace.js"; -import { type LogLevel } from "../logging.js"; -import { type FallbackImportExtractionEvent, type FallbackImportExtractionReason } from "../graphs/specifiers.js"; +import type { LogLevel } from "../logging.js"; +import type { FallbackImportExtractionEvent, FallbackImportExtractionReason } from "../graphs/specifiers.js"; import type { GraphBuildOptions } from "../graphs/types.js"; +import type { LanguageExtensionMap } from "../languages.js"; import { isGraphOnlyLanguage } from "../documentLinks.js"; import { stripJsLikeComments } from "../util/comments.js"; import { @@ -12,10 +13,8 @@ import { isNativeBindingLoadedForLanguage, isNativeRequiredUnavailableError, isNativeQueryAuthoritative, - type NativeQueryExecution, - type NativeQueryResults, - type NativeRuntimeMode, } from "../native/treeSitterNative.js"; +import type { NativeQueryExecution, NativeQueryResults, NativeRuntimeMode } from "../native/treeSitterNative.js"; import type { ResolvedImportTarget } from "./imports/context.js"; import { collectGraphOnlyImports } from "./imports/graphOnly.js"; import { collectJsTextImports, collectJsTextValueRequireImports } from "./imports/jsTextImports.js"; @@ -43,13 +42,19 @@ export async function collectImportsForFile( native?: NativeRuntimeMode; onFallbackImportExtraction?: (event: FallbackImportExtractionEvent) => void; logLevel?: LogLevel; + languageExtensions?: LanguageExtensionMap; }, ): Promise { let source = opts?.source; let sup = opts?.sup; if (!source || !sup) { - const prep = await prepareSourceInput(file, source !== undefined ? { source } : undefined); + const prep = await prepareSourceInput( + file, + source !== undefined + ? { source, languageExtensions: opts?.languageExtensions } + : { languageExtensions: opts?.languageExtensions }, + ); source = prep.source; sup = prep.sup; } diff --git a/src/indexer/parse-context.ts b/src/indexer/parse-context.ts index b9bde573..d7b57735 100644 --- a/src/indexer/parse-context.ts +++ b/src/indexer/parse-context.ts @@ -1,4 +1,5 @@ import { isGraphOnlyLanguage } from "../documentLinks.js"; +import type { LanguageExtensionMap } from "../languages.js"; import { prepareSourceInput } from "../languages/filePrep.js"; import { getNativeQueryExecution, @@ -115,8 +116,12 @@ export function parsePreparedFileContext(context: PreparedFileContext): ParsedFi throw new Error(`Failed to reconstruct syntax tree for ${context.file}`); } -export async function prepareFileForIndexing(file: string, native?: NativeRuntimeMode): Promise { - const prep = await prepareSourceInput(file); +export async function prepareFileForIndexing( + file: string, + native?: NativeRuntimeMode, + languageExtensions?: LanguageExtensionMap, +): Promise { + const prep = await prepareSourceInput(file, { languageExtensions }); if (isGraphOnlyLanguage(prep.sup.id)) { return { file, diff --git a/src/indexer/types.ts b/src/indexer/types.ts index 25ecb0b3..2cbb0e24 100644 --- a/src/indexer/types.ts +++ b/src/indexer/types.ts @@ -6,8 +6,8 @@ import type { NativeFallbackReason, NativeRuntimeMode } from "../native/contract import type { ScopeIndex } from "./scope-types.js"; import type { ReferenceCandidateIndex } from "./reference-candidate-types.js"; import type { ParsedFileContext } from "./parse-context.js"; -import type { Edge, FileId, Graph, Range } from "../types.js"; -import { type ProjectFileDiscoveryOptions, type ProjectFileInfo } from "../util/projectFiles.js"; +import type { Edge, FileId, Graph, ProgressUpdate, Range } from "../types.js"; +import type { ProjectFileDiscoveryOptions, ProjectFileInfo } from "../util/projectFiles.js"; import type { ImportBinding } from "./import-types.js"; export type { ImportBinding } from "./import-types.js"; @@ -128,8 +128,10 @@ export type ProjectIndex = { * optional `discovery` globs, and a `report` object when the caller wants * timings/backend diagnostics alongside the resulting index. */ +export type LanguageExtensionMap = Record; + export type BuildOptions = { - onProgress?: ((progress: import("../types.js").ProgressUpdate) => void) | undefined; + onProgress?: ((progress: ProgressUpdate) => void) | undefined; threads?: number; cache?: "off" | "memory" | "disk"; cacheDir?: string; @@ -147,6 +149,7 @@ export type BuildOptions = { useNativeWorkers?: boolean; nativeThreads?: number; discovery?: ProjectFileDiscoveryOptions; + languageExtensions?: LanguageExtensionMap; }; /** diff --git a/src/languages.ts b/src/languages.ts index 57fe8a4b..4f141d2a 100644 --- a/src/languages.ts +++ b/src/languages.ts @@ -73,7 +73,30 @@ export const SQL_SUPPORT = adaptDefinition(getLanguageById("sql")!); export const LANGUAGE_SUPPORTS: LanguageSupport[] = getAllLanguages().map(adaptDefinition); -export function supportForFile(filename: string): LanguageSupport | undefined { +export type LanguageExtensionMap = Record; + +function mappedSupportForFile( + filename: string, + extensionMap: LanguageExtensionMap | undefined, +): LanguageSupport | undefined { + const mappings = Object.entries(extensionMap ?? {}) + .map(([extension, languageId]) => [extension.trim().toLowerCase(), languageId.trim()] as const) + .filter(([extension, languageId]) => extension && languageId) + .sort((left, right) => right[0].length - left[0].length || left[0].localeCompare(right[0])); + const lowerFilename = filename.toLowerCase(); + for (const [extension, languageId] of mappings) { + if (!lowerFilename.endsWith(extension)) continue; + return supportById(languageId); + } + return undefined; +} + +export function supportForFile( + filename: string, + extensionMap?: LanguageExtensionMap | undefined, +): LanguageSupport | undefined { + const mapped = mappedSupportForFile(filename, extensionMap); + if (mapped) return mapped; const ext = path.extname(filename).toLowerCase(); if (ext === ".h") { const sample = readFileSample(filename); @@ -82,8 +105,8 @@ export function supportForFile(filename: string): LanguageSupport | undefined { } return LANGUAGE_SUPPORTS.find((s) => s.matchExts.includes(ext)); } -export function languageForFile(filename: string): ParserLanguage { - const sup = supportForFile(filename); +export function languageForFile(filename: string, extensionMap?: LanguageExtensionMap | undefined): ParserLanguage { + const sup = supportForFile(filename, extensionMap); if (!sup) throw new Error(`Unsupported file extension: ${filename}`); return sup.language(filename); } diff --git a/src/languages/filePrep.ts b/src/languages/filePrep.ts index 5e55b262..de90d0f4 100644 --- a/src/languages/filePrep.ts +++ b/src/languages/filePrep.ts @@ -1,15 +1,10 @@ import fsp from "node:fs/promises"; import type { ParserLanguage } from "../languages/types.js"; -import { - JS_SUPPORT, - TS_SUPPORT, - TSX_SUPPORT, - supportForFile, - supportById, - type LanguageSupport, -} from "../languages.js"; -import { prepareSFCScriptSource, detectSFCFramework, type SFCFramework } from "./sfc.js"; +import { JS_SUPPORT, TS_SUPPORT, TSX_SUPPORT, supportForFile, supportById } from "../languages.js"; +import type { LanguageExtensionMap, LanguageSupport } from "../languages.js"; +import { prepareSFCScriptSource, detectSFCFramework } from "./sfc.js"; +import type { SFCFramework } from "./sfc.js"; interface ParserInput { source: string; @@ -38,7 +33,10 @@ const SCRIPT_SUPPORT_MAP: Record = { tsx: TSX_SUPPORT, }; -export async function prepareParserInput(file: string, opts?: { source?: string }): Promise { +export async function prepareParserInput( + file: string, + opts?: { source?: string | undefined; languageExtensions?: LanguageExtensionMap | undefined }, +): Promise { const prepared = await prepareSourceInput(file, opts); return { ...prepared, @@ -46,14 +44,17 @@ export async function prepareParserInput(file: string, opts?: { source?: string }; } -export async function prepareSourceInput(file: string, opts?: { source?: string }): Promise { +export async function prepareSourceInput( + file: string, + opts?: { source?: string | undefined; languageExtensions?: LanguageExtensionMap | undefined }, +): Promise { const framework = detectSFCFramework(file); if (framework) { const rawSource = opts?.source ?? (await fsp.readFile(file, "utf8")); return prepareSFCSourceInput(rawSource, framework); } - const sup = supportForFile(file); + const sup = supportForFile(file, opts?.languageExtensions); if (!sup) throw new UnsupportedParserInputError(file); const rawSource = opts?.source ?? (await fsp.readFile(file, "utf8")); return { diff --git a/src/session.ts b/src/session.ts index fe43f52d..f3691245 100644 --- a/src/session.ts +++ b/src/session.ts @@ -110,6 +110,7 @@ function normalizeBuildOptions(options?: BuildOptions): Record gitignoreRoot: options.discovery.gitignoreRoot ? path.resolve(options.discovery.gitignoreRoot) : undefined, } : undefined, + languageExtensions: options.languageExtensions ? { ...options.languageExtensions } : undefined, }; } @@ -241,10 +242,16 @@ export class CodeReviewSession implements ICodeReviewSession { private async currentBuildOptions(): Promise { const config = await loadCodegraphConfig(this.root); const discovery = mergeDiscoveryOptions(config.discovery, this.buildOptions?.discovery); - if (!hasDiscoveryOptions(discovery)) { + const hasDiscovery = hasDiscoveryOptions(discovery); + const languageExtensions = this.buildOptions?.languageExtensions ?? config.languages?.extensions; + if (!hasDiscovery && !languageExtensions) { return this.buildOptions; } - return { ...this.buildOptions, discovery }; + return { + ...this.buildOptions, + ...(hasDiscovery ? { discovery } : {}), + ...(languageExtensions ? { languageExtensions } : {}), + }; } private async buildIndex(options: { forceFull?: boolean } = {}): Promise<{ diff --git a/tests/codegraph-config.test.ts b/tests/codegraph-config.test.ts index ab001e8d..221b0b81 100644 --- a/tests/codegraph-config.test.ts +++ b/tests/codegraph-config.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { hasDiscoveryOptions, loadCodegraphConfig, mergeDiscoveryOptions } from "../src/config.js"; import { searchCodegraph } from "../src/agent/search.js"; +import { buildProjectIndex } from "../src/indexer/build-index.js"; async function mkRepo(): Promise { const root = await fs.mkdtemp(path.join(os.tmpdir(), "cg-config-")); @@ -14,6 +15,36 @@ async function mkRepo(): Promise { return root; } +async function writeConfig(root: string, config: unknown): Promise { + await fs.writeFile(path.join(root, "codegraph.config.json"), JSON.stringify(config), "utf8"); +} + +function normalizedFile(root: string, relativePath: string): string { + return path.join(root, relativePath).replace(/\\/g, "/"); +} + +async function buildWithProjectConfig(root: string) { + const config = await loadCodegraphConfig(root); + return await buildProjectIndex(root, { + cache: "disk", + ...(config.languages?.extensions ? { languageExtensions: config.languages.extensions } : {}), + }); +} + +function localExportNames( + index: Awaited>, + root: string, + relativePath: string, +): string[] { + const moduleIndex = index.byFile.get(normalizedFile(root, relativePath)); + return ( + moduleIndex?.exports + .filter((entry) => entry.type === "local") + .map((entry) => entry.exportedAs) + .sort() ?? [] + ); +} + describe("codegraph config", () => { it("loads discovery settings from codegraph.config.json", async () => { const root = await mkRepo(); @@ -119,4 +150,139 @@ describe("codegraph config", () => { expect(kept.results.map((result) => result.file)).toContain("src/kept.ts"); expect(ignored.results).toEqual([]); }); + + it("loads language extension mappings from codegraph.config.json", async () => { + const root = await mkRepo(); + await writeConfig(root, { + languages: { + extensions: { + ".TPL": "php", + }, + }, + }); + + const config = await loadCodegraphConfig(root); + + expect(config.languages?.extensions).toEqual({ ".tpl": "php" }); + expect(config.discovery?.includeGlobs).toEqual(["**/*.tpl"]); + }); + + it("indexes configured nonstandard extensions with the mapped language", async () => { + const root = await mkRepo(); + await fs.writeFile( + path.join(root, "src", "template.tpl"), + " { + const root = await mkRepo(); + await fs.writeFile( + path.join(root, "src", "view.inc.php"), + " { + const root = await mkRepo(); + await fs.writeFile(path.join(root, "src", "builtin.ts"), "export const builtinValue = 1;\n", "utf8"); + await fs.writeFile(path.join(root, "src", "remapped.php"), " { + const root = await mkRepo(); + await fs.writeFile( + path.join(root, "src", "cached.tpl"), + " { + const root = await mkRepo(); + await writeConfig(root, { + languages: { + extensions: { + tpl: "php", + }, + }, + }); + + await expect(loadCodegraphConfig(root)).rejects.toThrow('languages.extensions key "tpl" must start with "."'); + }); + + it("rejects language extension mappings to unknown languages", async () => { + const root = await mkRepo(); + await writeConfig(root, { + languages: { + extensions: { + ".tpl": "wat", + }, + }, + }); + + await expect(loadCodegraphConfig(root)).rejects.toThrow( + 'languages.extensions[".tpl"] references unknown language "wat"', + ); + }); });