From 739f544d5360a562d30d859396b8ca97a21eb07f Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Thu, 22 Oct 2026 09:57:24 +0700 Subject: [PATCH 1/2] analyzers: add tree-sitter based Go analyzer Functions, methods, structs, interfaces (with their method signatures), and other defined types (type Kind int) become components; call, field-type, embedding, composite-literal, signature/local-var type, and interface-implementation edges connect them. Methods are keyed by their receiver type (Buffer.Push, pointer or value receiver alike) across all files of the package. _test.go files and vendor/ are skipped. A Go package is a directory, so names not defined in the file are emitted as package references ("sub/*.go::Name") and resolved by CallGraphAnalyzer inside that package, the way Dart resolution is scoped. Imports of the repo's own packages map to directories through the module path in go.mod (and local replace directives); standard library and third-party calls are dropped. Types are linked to the interfaces of their own package whose methods they declare. Wires .go through the call-graph dispatch, supported-language lists, extension maps, CLI validators, and MCP change detection. The grammar comes from tree-sitter-language-pack, so no new dependency. --- CHANGELOG.md | 12 + README.md | 2 +- codewiki/cli/main.py | 2 +- codewiki/cli/utils/repo_validator.py | 3 +- codewiki/cli/utils/validation.py | 1 + codewiki/mcp/tools/analysis.py | 1 + .../analysis/analysis_service.py | 3 +- .../analysis/call_graph_analyzer.py | 72 +- .../be/dependency_analyzer/analyzers/go.py | 1273 +++++++++++++++++ .../src/be/dependency_analyzer/ast_parser.py | 1 + .../be/dependency_analyzer/utils/patterns.py | 7 +- codewiki/src/be/prompt_template.py | 1 + guides/development.md | 7 +- tests/test_go_analyzer.py | 421 ++++++ 14 files changed, 1793 insertions(+), 13 deletions(-) create mode 100644 codewiki/src/be/dependency_analyzer/analyzers/go.py create mode 100644 tests/test_go_analyzer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c3bb5f..074ab2be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,18 @@ uses [Semantic Versioning](https://semver.org/). providers (including `@riverpod` codegen), and GoRouter routes link to their screens. Generated `*.g.dart`/`*.freezed.dart` files and `.dart_tool/` are ignored, and Dart modules get Flutter-specific documentation guidance. +- **Go analyzer.** Functions, methods, structs, interfaces (and their method + signatures), and other defined types (`type Kind int`) become components. + Methods are keyed by their receiver type (`Buffer.Push`, pointer or value + receiver alike), whichever file of the package declares them. Edges cover + calls (plain, `pkg.F()`, and method calls on receivers, parameters and + locals whose type can be inferred), struct field and embedded types, + composite literals, types named in signatures and local `var`s, and types + that implement an interface of their own package. Names resolve inside the + package (directory) they name; imports of the repository's own packages map + to directories through the module path in `go.mod` (and local `replace` + directives), and standard-library or third-party calls are dropped. + `_test.go` files and `vendor/` are skipped. ### Changed diff --git a/README.md b/README.md index 95a6edfa..cfc8318b 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ Full list in the [CHANGELOG](./CHANGELOG.md). ## Features **Languages.** Python, Java, JavaScript, TypeScript, C, C++, C#, Kotlin, -PHP, Ruby, Scala, Rust, Dart (Flutter-aware). +PHP, Ruby, Scala, Rust, Dart (Flutter-aware), Go. **Hierarchical decomposition.** The dependency graph is clustered into a module tree, recursively, so a 1.4M-line repository gets the same treatment diff --git a/codewiki/cli/main.py b/codewiki/cli/main.py index 254c360d..57f5391b 100644 --- a/codewiki/cli/main.py +++ b/codewiki/cli/main.py @@ -20,7 +20,7 @@ def cli(ctx): AI agents write the documentation from your dependency graph. Supports Python, Java, JavaScript, TypeScript, C, C++, C#, Kotlin, PHP, Ruby, - Scala, Rust, and Dart. + Scala, Rust, Dart, and Go. """ # Ensure context object exists ctx.ensure_object(dict) diff --git a/codewiki/cli/utils/repo_validator.py b/codewiki/cli/utils/repo_validator.py index 63da2646..e32dc75c 100644 --- a/codewiki/cli/utils/repo_validator.py +++ b/codewiki/cli/utils/repo_validator.py @@ -35,6 +35,7 @@ ".sc", # Scala Scripts ".rs", # Rust ".dart", # Dart + ".go", # Go } @@ -66,7 +67,7 @@ def validate_repository(repo_path: Path) -> tuple[Path, list[tuple[str, int]]]: raise RepositoryError( f"No supported code files found in {repo_path}\n\n" "CodeWiki supports: Python, Java, JavaScript, TypeScript, C, C++, C#, Kotlin, PHP, " - "Ruby, Scala, Rust, Dart\n\n" + "Ruby, Scala, Rust, Dart, Go\n\n" "Please navigate to a code repository or specify a custom directory:\n" " cd /path/to/your/project\n" " codewiki generate" diff --git a/codewiki/cli/utils/validation.py b/codewiki/cli/utils/validation.py index 315aa9b2..87777cc2 100644 --- a/codewiki/cli/utils/validation.py +++ b/codewiki/cli/utils/validation.py @@ -169,6 +169,7 @@ def detect_supported_languages(directory: Path) -> list[tuple[str, int]]: "Scala": [".scala", ".sc"], "Rust": [".rs"], "Dart": [".dart"], + "Go": [".go"], } # Directories to exclude from counting diff --git a/codewiki/mcp/tools/analysis.py b/codewiki/mcp/tools/analysis.py index 3a4f8df5..1b2b2162 100644 --- a/codewiki/mcp/tools/analysis.py +++ b/codewiki/mcp/tools/analysis.py @@ -230,6 +230,7 @@ def _detect_via_mtime( ".sc", ".rs", ".dart", + ".go", } changed: list[str] = [] diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index 861ab97b..413aef01 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -340,7 +340,7 @@ def _filter_supported_languages(self, code_files: list[dict]) -> list[dict]: Filter code files to only include supported languages. Supports Python, JavaScript, TypeScript, Java, C#, C, C++, PHP, Ruby, Kotlin, - Scala, Rust, and Dart. + Scala, Rust, Dart, and Go. """ supported_languages = { "python", @@ -381,6 +381,7 @@ def _get_supported_languages(self) -> list[str]: "scala", "rust", "dart", + "go", ] def _cleanup_repository(self, temp_dir: str): diff --git a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index b95c93f8..b6bf63c8 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -73,6 +73,8 @@ def __init__(self): self._dart_visible: dict[str, set[str]] = {} self._dart_library_members: dict[str, set[str]] = {} self._dart_unscoped = False + self._go_visible: dict[str, set[str]] = {} + self._go_index = None self.call_relationships: list[CallRelationship] = [] self._python_project_modules: set = set() self._python_external_import_roots: set = set() @@ -100,6 +102,8 @@ def analyze_code_files(self, code_files: list[dict], base_dir: str) -> dict: self._dart_visible = {} self._dart_library_members = {} self._dart_unscoped = False + self._go_visible = {} + self._go_index = None files_analyzed = 0 files_failed = 0 @@ -281,6 +285,8 @@ def _analyze_code_file(self, repo_dir: str, file_info: dict): self._analyze_rust_file(file_path, content, repo_dir) elif language == "dart": self._analyze_dart_file(file_path, content, repo_dir) + elif language == "go": + self._analyze_go_file(file_path, content, repo_dir) # else: # logger.warning( # f"Unsupported language for call graph analysis: {language} for file {file_path}" @@ -606,6 +612,47 @@ def _analyze_dart_file(self, file_path: str, content: str, repo_dir: str): except Exception: logger.exception(f"Failed to analyze Dart file {file_path}") + def _analyze_go_file(self, file_path: str, content: str, repo_dir: str): + """ + Analyze Go file using tree-sitter based analyzer. + + Besides components and relationships, records the package directories + the file can see (its own and the repository packages it imports) for + Go package-scoped resolution. + + Args: + file_path: Path to the Go file + content: File content string + repo_dir: Repository base directory + """ + from codewiki.src.be.dependency_analyzer.analyzers.go import TreeSitterGoAnalyzer + + try: + analyzer = TreeSitterGoAnalyzer(str(file_path), content, repo_path=repo_dir) + + for func in analyzer.nodes: + func_id = func.id if func.id else f"{file_path}:{func.name}" + self.functions[func_id] = func + + self.call_relationships.extend(analyzer.call_relationships) + self._go_visible[analyzer.relative_path] = analyzer.visible_dirs + except Exception: + logger.exception(f"Failed to analyze Go file {file_path}") + + def _resolve_go_callee(self, callee: str, caller: Node) -> str | None: + """Go names resolve inside the package they name (a directory), or, + for a method on a value of unknown type, among the packages the + caller's file can see.""" + from codewiki.src.be.dependency_analyzer.analyzers.go import ( + GoPackageIndex, + resolve_go_callee, + ) + + if self._go_index is None: + self._go_index = GoPackageIndex(self.functions) + caller_file = Path(caller.relative_path).as_posix() + return resolve_go_callee(callee, caller, self._go_index, self._go_visible.get(caller_file)) + def _build_dart_scopes(self, base_dir: str) -> None: """Resolve recorded Dart directives into per-file visibility scopes.""" from codewiki.src.be.dependency_analyzer.analyzers.dart_imports import ( @@ -723,6 +770,7 @@ def _resolve_call_relationships(self): func_info.language = CODE_EXTENSIONS.get(file_ext) indexes = self._build_resolution_indexes() + self._go_index = None resolved_count = 0 for relationship in self.call_relationships: @@ -740,7 +788,7 @@ def _resolve_call_relationships(self): relationship for relationship in self.call_relationships if relationship.is_resolved - or not self._is_unresolvable_dart(relationship) + or not self._has_authoritative_resolution(relationship) and not self._is_external_callee( self._caller_language(relationship.caller), relationship.callee, @@ -748,12 +796,19 @@ def _resolve_call_relationships(self): ) ] - def _is_unresolvable_dart(self, relationship: CallRelationship) -> bool: - """A Dart caller's callee that Dart resolution rejected is external or - invisible; drop it so ast_parser's name-only fallback cannot bind it - to an unrelated component.""" + if any(func.language == "go" for func in self.functions.values()): + from codewiki.src.be.dependency_analyzer.analyzers.go import link_go_interfaces + + # Go types implement interfaces implicitly: link them once every + # method of the package is known. + self.call_relationships.extend(link_go_interfaces(self.functions)) + + def _has_authoritative_resolution(self, relationship: CallRelationship) -> bool: + """A Dart or Go caller's callee that the language's scoped resolution + rejected is external or invisible; drop it so ast_parser's name-only + fallback cannot bind it to an unrelated component.""" caller = self.functions.get(relationship.caller) - return caller is not None and caller.language == "dart" + return caller is not None and caller.language in ("dart", "go") def _dotted_project_packages(self) -> dict[str, set]: """Project packages/namespaces, partitioned by language. Java and C# @@ -893,6 +948,9 @@ def _resolve_callee( # Dart resolution is authoritative: it never falls back to name # guessing in other languages or invisible libraries. return self._resolve_dart_callee(relationship.callee, caller, lang_indexes) + if caller_language == "go" and caller is not None: + # So is Go's: a name resolves only in the package it names. + return self._resolve_go_callee(relationship.callee, caller) if lang_indexes: match = self._resolve_callee_in( relationship, lang_indexes["exact"], lang_indexes["simple"] @@ -1034,6 +1092,8 @@ def _generate_visualization_data(self) -> dict: node_classes.append("lang-rust") elif file_ext == ".dart": node_classes.append("lang-dart") + elif file_ext == ".go": + node_classes.append("lang-go") cytoscape_elements.append( { diff --git a/codewiki/src/be/dependency_analyzer/analyzers/go.py b/codewiki/src/be/dependency_analyzer/analyzers/go.py new file mode 100644 index 00000000..e09a1388 --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/analyzers/go.py @@ -0,0 +1,1273 @@ +"""Tree-sitter based Go analyzer. + +Extracts functions, methods, structs, interfaces (with their method +signatures), and other defined types (`type Kind int`) as documentable +components, plus embedding, field-type, composite-literal instantiation, +call, and interface-implementation dependency edges. + +Component types follow the same two-level `component_type` / `node_type` +split as the Rust analyzer: `component_type` is the coarse type the +pipeline gates leaf selection on (structs stay "struct", interfaces +"interface", other defined types map to "class"), while `node_type` and +`display_name` preserve the faithful Go construct. + +Methods are keyed by their receiver's base type (`Buffer.Push` for both +`func (b *Buffer) Push` and `func (b Buffer) Push`), no matter which file +of the package declares them. Owners are joined with `.` rather than `::` +because component ids already use `::` as the path separator. + +A Go package is a directory, so a name that is not defined in the file +itself is emitted as a *package reference*, `"/*.go::"` (the +repository root is `"*.go::"`): "the declaration `Name` in some file +of the package in ``". Imports of the repository's own packages are +mapped to directories through the module path in `go.mod` (and local +`replace` directives); calls into the standard library or third-party +modules are dropped. `CallGraphAnalyzer` resolves package references +against the components of that directory (`resolve_go_callee`) and links +types to the interfaces of their package that they implement +(`link_go_interfaces`). + +Known limitations: +- `_test.go` files and anything under `vendor/` produce no components, the + way the Rust analyzer skips `#[cfg(test)]` modules. +- Local variable types are inferred from declarations, composite literals, + `new(T)`, and same-file function results only; a method call on a value + of unknown type falls back to its method name, which resolves only when + exactly one method of that name exists in the caller's package or the + repository packages it imports. +- Methods promoted through embedding resolve for embedded types of the + same package only; dot imports are not followed. +- Type aliases (`type A = B`) are not components. +""" + +import logging +import os +import posixpath +import re +from collections import defaultdict +from functools import lru_cache +from pathlib import Path + +from tree_sitter_language_pack import get_parser + +from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node + +logger = logging.getLogger(__name__) + +# `component_type` / `node_type` per underlying type of a `type_spec`. +_TYPE_MAPPING = { + "struct_type": ("struct", "struct"), + "interface_type": ("interface", "interface"), +} +_DEFINED_TYPE = ("class", "type") + +_FUNCTION_SCOPES = ("function_declaration", "method_declaration") + +# Predeclared types (and `any`/`comparable`) never become edges. +GO_PRIMITIVE_TYPES = frozenset( + { + "bool", + "byte", + "rune", + "string", + "error", + "any", + "comparable", + "int", + "int8", + "int16", + "int32", + "int64", + "uint", + "uint8", + "uint16", + "uint32", + "uint64", + "uintptr", + "float32", + "float64", + "complex64", + "complex128", + } +) + +# Predeclared functions. +GO_BUILTIN_FUNCS = frozenset( + { + "append", + "cap", + "clear", + "close", + "complex", + "copy", + "delete", + "imag", + "len", + "make", + "max", + "min", + "new", + "panic", + "print", + "println", + "real", + "recover", + } +) + +# Standard-library method names. A call whose receiver type is unknown and +# whose only signal is one of these names is dropped instead of being +# matched by name against the repository's methods. +GO_COMMON_METHODS = frozenset( + { + "Error", + "String", + "GoString", + "Format", + "Unwrap", + "Is", + "As", + "Len", + "Cap", + "Less", + "Swap", + "Close", + "Read", + "ReadByte", + "ReadRune", + "ReadString", + "ReadBytes", + "ReadFrom", + "Write", + "WriteByte", + "WriteRune", + "WriteString", + "WriteTo", + "Seek", + "Sync", + "Flush", + "Bytes", + "Reset", + "Grow", + "Truncate", + "Lock", + "Unlock", + "RLock", + "RUnlock", + "TryLock", + "Add", + "Done", + "Wait", + "Load", + "Store", + "CompareAndSwap", + "Err", + "Value", + "Deadline", + "Printf", + "Println", + "Print", + "Sprintf", + "Fprintf", + "Errorf", + "Fatal", + "Fatalf", + "Helper", + "Log", + "Logf", + "Scan", + "Text", + "Next", + "Name", + "Size", + "Stat", + "Kind", + "Type", + "Elem", + "Interface", + "Int", + "Uint", + "Float", + "Bool", + } +) + +_PACKAGE_REF_FILES = "*.go" +_MODULE_RE = re.compile(r"^\s*module\s+\"?([^\s\"]+)\"?", re.MULTILINE) +_REPLACE_RE = re.compile(r"^\s*(?:replace\s+)?\"?([^\s\"(]+)\"?(?:\s+\S+)?\s*=>\s*\"?([^\s\"]+)") +_PACKAGE_CLAUSE_RE = re.compile(r"^\s*package\s+([A-Za-z_]\w*)", re.MULTILINE) +_MAJOR_VERSION_RE = re.compile(r"^v\d+$") + + +# ---------------------------------------------------------------------- +# Package references +# ---------------------------------------------------------------------- + + +def go_package_ref(package_dir: str, name: str) -> str: + """`"sub/*.go::Name"`: declaration `Name` in the package at repo dir `sub`.""" + prefix = f"{package_dir}/" if package_dir else "" + return f"{prefix}{_PACKAGE_REF_FILES}::{name}" + + +def split_go_package_ref(callee: str) -> tuple[str, str] | None: + head, sep, name = callee.partition("::") + if not sep or not name: + return None + if head == _PACKAGE_REF_FILES: + return "", name + suffix = f"/{_PACKAGE_REF_FILES}" + if head.endswith(suffix): + return head[: -len(suffix)], name + return None + + +def is_skipped_go_file(relative_path: str) -> bool: + """`_test.go` files and vendored dependencies yield no components.""" + posix = Path(relative_path).as_posix() + return posix.endswith("_test.go") or "vendor" in posix.split("/")[:-1] + + +def package_dir_of(relative_path: str) -> str: + return posixpath.dirname(Path(relative_path).as_posix()) + + +# ---------------------------------------------------------------------- +# go.mod +# ---------------------------------------------------------------------- + + +@lru_cache(maxsize=256) +def _parse_go_mod(path: str, mtime_ns: int) -> tuple[str | None, tuple[tuple[str, str], ...]]: + """Module path and local `replace` targets of a go.mod file.""" + del mtime_ns # cache key only + try: + with open(path, encoding="utf-8", errors="replace") as f: + text = f.read() + except OSError: + return None, () + match = _MODULE_RE.search(text) + module = match.group(1) if match else None + replaces = [] + in_block = False + for raw in text.splitlines(): + line = raw.split("//", 1)[0].strip() + if line.startswith("replace") and line.rstrip().endswith("("): + in_block = True + continue + if in_block and line == ")": + in_block = False + continue + if not (in_block or line.startswith("replace")): + continue + m = _REPLACE_RE.match(line) + if m and m.group(2).startswith(("./", "../")): + replaces.append((m.group(1), m.group(2))) + return module, tuple(replaces) + + +def _read_go_mod(path: str): + try: + mtime = os.stat(path).st_mtime_ns + except OSError: + return None, () + return _parse_go_mod(path, mtime) + + +def _normalize_dir(path: str) -> str | None: + norm = posixpath.normpath(path) + if norm == ".": + return "" + if norm.startswith(".."): + return None + return norm + + +@lru_cache(maxsize=1024) +def _package_name_in_dir(abs_dir: str) -> str | None: + """The package clause of the first non-test Go file in a directory.""" + try: + names = sorted(os.listdir(abs_dir)) + except OSError: + return None + for name in names: + if not name.endswith(".go") or name.endswith("_test.go"): + continue + try: + with open(os.path.join(abs_dir, name), encoding="utf-8", errors="replace") as f: + head = f.read(8192) + except OSError: + continue + match = _PACKAGE_CLAUSE_RE.search(re.sub(r"//[^\n]*", "", head)) + if match: + return match.group(1) + return None + + +def _default_import_names(import_path: str) -> set[str]: + """Likely package names of an import path without an explicit alias.""" + parts = [p for p in import_path.split("/") if p] + if not parts: + return set() + last = parts[-1] + if _MAJOR_VERSION_RE.match(last) and len(parts) > 1: + last = parts[-2] + last = re.sub(r"\.v\d+$", "", last) + names = {last} + for prefix in ("go-", "go."): + if last.startswith(prefix): + names.add(last[len(prefix) :]) + for suffix in ("-go", ".go"): + if last.endswith(suffix): + names.add(last[: -len(suffix)]) + names |= {n.replace("-", "") for n in names} | {n.replace("-", "_") for n in names} + return {n for n in names if n.isidentifier()} + + +# ---------------------------------------------------------------------- +# Analyzer +# ---------------------------------------------------------------------- + + +@lru_cache(maxsize=1) +def _go_parser(): + return get_parser("go") + + +class TreeSitterGoAnalyzer: + def __init__(self, file_path: str, content: str, repo_path: str | None = None): + self.file_path = Path(file_path) + self.content = content + self.repo_path = repo_path or "" + self.nodes: list[Node] = [] + self.call_relationships: list[CallRelationship] = [] + # Same-file symbol table keyed by logical name ("Foo", "Foo.Bar"). + self.top_level_nodes: dict = {} + self.seen_relationships: set = set() + self.relative_path = Path(self._get_relative_path()).as_posix() + self.package_dir = package_dir_of(self.relative_path) + # Import name -> repository dir of the package, or None if external. + self.imports: dict[str, str | None] = {} + # The caller's own package plus the repository packages it imports. + self.visible_dirs: set[str] = {self.package_dir} + self._struct_fields: dict[str, dict] = {} + self._func_results: dict[str, object] = {} + self._package_vars: dict[str, tuple[str | None, str] | None] = {} + self._bindings_cache: dict = {} + if is_skipped_go_file(self.relative_path): + return + self._analyze() + + # ------------------------------------------------------------------ + # Setup + # ------------------------------------------------------------------ + + def _get_relative_path(self) -> str: + if self.repo_path: + try: + return os.path.relpath(str(self.file_path), self.repo_path) + except ValueError: + return str(self.file_path) + return str(self.file_path) + + def _get_component_id(self, logical_name: str) -> str: + return f"{self._get_relative_path()}::{logical_name}" + + def _analyze(self): + try: + tree = _go_parser().parse(bytes(self.content, "utf8")) + root = tree.root_node + lines = self.content.splitlines() + + self._collect_imports(root) + self._extract_nodes(root, lines) + self._collect_package_vars(root) + self._extract_relationships(root) + except Exception as e: # noqa: BLE001 — a broken file must not abort the sweep + logger.error(f"Error parsing Go file {self.file_path}: {e}") + + @staticmethod + def _text(node) -> str: + return node.text.decode("utf8", errors="replace") if node is not None else "" + + def _field_text(self, node, field_name: str) -> str | None: + child = node.child_by_field_name(field_name) + return self._text(child) if child is not None else None + + # ------------------------------------------------------------------ + # Imports + # ------------------------------------------------------------------ + + def _collect_imports(self, root): + for child in root.children: + if child.type == "import_declaration": + for spec in self._descendants(child, "import_spec"): + self._register_import(spec) + + def _register_import(self, spec): + path_node = spec.child_by_field_name("path") + if path_node is None: + return + import_path = self._text(path_node).strip('"`') + alias_node = spec.child_by_field_name("name") + alias = self._text(alias_node) if alias_node is not None else None + if alias in ("_", "."): + return + repo_dir = self._resolve_import_dir(import_path) + if repo_dir is not None: + self.visible_dirs.add(repo_dir) + if alias: + names = {alias} + else: + names = _default_import_names(import_path) + if repo_dir is not None and self.repo_path: + actual = _package_name_in_dir(os.path.join(self.repo_path, repo_dir)) + if actual: + names = {actual} + for name in names: + # A repository package wins over an external one of the same name. + if self.imports.get(name) is None: + self.imports[name] = repo_dir + + def _resolve_import_dir(self, import_path: str) -> str | None: + """Repository dir of an imported package, or None when external.""" + if not self.repo_path: + return None + directory = self.package_dir + while True: + go_mod = os.path.join(self.repo_path, directory, "go.mod") + if os.path.isfile(go_mod): + return self._import_dir_from_go_mod(import_path, directory, go_mod) + if not directory: + break + directory = posixpath.dirname(directory) + return self._import_dir_without_go_mod(import_path) + + @staticmethod + def _import_dir_from_go_mod(import_path: str, mod_dir: str, go_mod: str) -> str | None: + module, replaces = _read_go_mod(go_mod) + targets = [(module, "")] if module else [] + targets += [(old, new) for old, new in replaces] + for prefix, local in targets: + if import_path == prefix or import_path.startswith(prefix + "/"): + rest = import_path[len(prefix) :].lstrip("/") + return _normalize_dir(posixpath.join(mod_dir, local, rest)) + return None + + def _import_dir_without_go_mod(self, import_path: str) -> str | None: + """No go.mod above the file (GOPATH layout or a sub-tree was + analyzed): match the longest import-path suffix that is a repo dir. + Standard-library paths (no dot in the first element) never match.""" + parts = import_path.split("/") + if len(parts) < 2 or "." not in parts[0]: + return None + for start in range(1, len(parts)): + candidate = "/".join(parts[start:]) + if os.path.isdir(os.path.join(self.repo_path, candidate)): + return candidate + return None + + # ------------------------------------------------------------------ + # Pass 1: components + # ------------------------------------------------------------------ + + def _extract_nodes(self, root, lines): + for child in root.children: + if child.type == "function_declaration": + self._add_function(child, lines) + elif child.type == "method_declaration": + self._add_method(child, lines) + elif child.type == "type_declaration": + specs = [c for c in child.named_children if c.type == "type_spec"] + grouped = any(c.type == "(" for c in child.children) + for spec in specs: + self._add_type(spec, child if not grouped else spec, lines) + + def _add_function(self, node, lines): + name = self._field_text(node, "name") + if not name or name == "_": + return + result = node.child_by_field_name("result") + if result is not None: + self._func_results.setdefault(name, result) + self._add_node( + node, + name, + "function", + "function", + lines, + parameters=self._extract_parameters(node), + ) + + def _add_method(self, node, lines): + name = self._field_text(node, "name") + owner = self._receiver_type_name(node) + if not name or not owner or name == "_": + return + logical = f"{owner}.{name}" + result = node.child_by_field_name("result") + if result is not None: + self._func_results.setdefault(logical, result) + self._add_node( + node, + logical, + "method", + "method", + lines, + class_name=owner, + parameters=self._extract_parameters(node), + ) + + def _receiver_type_name(self, method_node) -> str | None: + receiver = method_node.child_by_field_name("receiver") + if receiver is None: + return None + for param in receiver.named_children: + if param.type == "parameter_declaration": + ref = self._primary_type_ref(param.child_by_field_name("type")) + if ref and ref[0] is None: + return ref[1] + return None + + def _add_type(self, spec, outer, lines): + name = self._field_text(spec, "name") + type_node = spec.child_by_field_name("type") + if not name or name == "_" or type_node is None: + return + component_type, node_type = _TYPE_MAPPING.get(type_node.type, _DEFINED_TYPE) + node_obj = self._add_node(outer, name, component_type, node_type, lines) + if node_obj is None: + return + + bases: list[str] = [] + if type_node.type == "struct_type": + fields: dict = {} + for decl in self._descendants(type_node, "field_declaration", stop_at_nested=True): + type_child = decl.child_by_field_name("type") + names = decl.children_by_field_name("name") + if not names: + embedded = self._primary_type_ref(type_child) + if embedded: + qualifier, base = embedded + bases.append(f"{qualifier}.{base}" if qualifier else base) + fields.setdefault(base, type_child) + for field_name in names: + fields[self._text(field_name)] = type_child + self._struct_fields[name] = fields + elif type_node.type == "interface_type": + for elem in type_node.named_children: + if elem.type == "type_elem": + for ref in self._collect_type_refs(elem): + qualifier, base = ref + bases.append(f"{qualifier}.{base}" if qualifier else base) + elif elem.type in ("method_elem", "method_spec"): + method = self._field_text(elem, "name") + if method: + self._add_node( + elem, + f"{name}.{method}", + "method", + "method", + lines, + class_name=name, + parameters=self._extract_parameters(elem), + ) + if bases: + node_obj.base_classes = bases + + def _extract_parameters(self, func_node) -> list[str] | None: + params_node = func_node.child_by_field_name("parameters") + if params_node is None: + return None + params = [ + self._text(child).strip() + for child in params_node.named_children + if child.type in ("parameter_declaration", "variadic_parameter_declaration") + ] + return params or None + + def _extract_docstring(self, node) -> str: + """Contiguous comment lines directly above a declaration.""" + comments = [] + expected_line = node.start_point[0] - 1 + sibling = node.prev_sibling + while ( + sibling is not None + and sibling.type == "comment" + and sibling.end_point[0] == expected_line + ): + comments.append(self._text(sibling).strip()) + expected_line = sibling.start_point[0] - 1 + sibling = sibling.prev_sibling + return "\n".join(reversed(comments)) + + def _add_node( + self, + node, + logical_name: str, + component_type: str, + node_type: str, + lines, + class_name: str | None = None, + parameters: list[str] | None = None, + ) -> Node | None: + if logical_name in self.top_level_nodes: + # A second `func init()` in the same file: keep the first. + return None + component_id = self._get_component_id(logical_name) + relative_path = self._get_relative_path() + + docstring = self._extract_docstring(node) + + start_line_idx = node.start_point[0] + end_line_idx = node.end_point[0] + 1 + code_snippet = ( + "\n".join(lines[start_line_idx:end_line_idx]) if start_line_idx < len(lines) else "" + ) + + node_obj = Node( + id=component_id, + name=logical_name, + component_type=component_type, + file_path=str(self.file_path), + relative_path=relative_path, + source_code=code_snippet, + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, + has_docstring=bool(docstring), + docstring=docstring, + parameters=parameters, + node_type=node_type, + class_name=class_name, + display_name=f"{node_type} {logical_name}", + component_id=component_id, + language="go", + ) + self.nodes.append(node_obj) + self.top_level_nodes[logical_name] = node_obj + return node_obj + + def _collect_package_vars(self, root): + """Types of package-level `var`s declared in this file.""" + for child in root.children: + if child.type != "var_declaration": + continue + for spec in self._descendants(child, "var_spec"): + for name, ref in self._var_spec_bindings(spec): + self._package_vars[name] = ref + + # ------------------------------------------------------------------ + # Pass 2: relationships + # ------------------------------------------------------------------ + + def _extract_relationships(self, node): + if node.type == "type_spec" and node.parent is not None: + if node.parent.parent is not None and node.parent.parent.type == "source_file": + self._emit_type_spec_edges(node) + elif node.type == "composite_literal": + self._emit_instantiation_edge(node) + elif node.type == "call_expression": + self._emit_call_edge(node) + elif node.type == "type_conversion_expression": + self._emit_generic_call_edge(node) + elif node.type in ("parameter_declaration", "variadic_parameter_declaration"): + if not self._is_receiver(node): + self._emit_type_use_edges(node, node.child_by_field_name("type")) + elif node.type in ("var_spec", "type_assertion_expression"): + self._emit_type_use_edges(node, node.child_by_field_name("type")) + elif node.type in ("function_declaration", "method_declaration", "func_literal"): + result = node.child_by_field_name("result") + if result is not None and result.type != "parameter_list": + self._emit_type_use_edges(result, result) + + for child in node.children: + self._extract_relationships(child) + + @staticmethod + def _descendants(node, node_type: str, stop_at_nested: bool = False): + """Descendants of a given type; with `stop_at_nested`, do not look + inside a match (fields of an anonymous nested struct stay with it).""" + found = [] + stack = list(reversed(node.children)) + while stack: + n = stack.pop() + if n.type == node_type: + found.append(n) + if stop_at_nested: + continue + stack.extend(reversed(n.children)) + return found + + def _primary_type_ref(self, type_node) -> tuple[str | None, str] | None: + """`(package alias or None, type name)` of a type, stripping + pointers, parentheses, and type arguments. Slices, arrays, and maps + give their element type, so `xs[i].M()` and `range xs` values type + their method calls.""" + while type_node is not None and type_node.type in ( + "pointer_type", + "parenthesized_type", + "generic_type", + "type_elem", + "slice_type", + "array_type", + "map_type", + ): + if type_node.type == "generic_type": + type_node = type_node.child_by_field_name("type") + elif type_node.type in ("slice_type", "array_type"): + type_node = type_node.child_by_field_name("element") + elif type_node.type == "map_type": + type_node = type_node.child_by_field_name("value") + else: + type_node = type_node.named_children[0] if type_node.named_children else None + if type_node is None: + return None + if type_node.type == "type_identifier": + return None, self._text(type_node) + if type_node.type == "qualified_type": + package = self._field_text(type_node, "package") + name = self._field_text(type_node, "name") + return (package, name) if package and name else None + return None + + def _collect_type_refs(self, type_node) -> list[tuple[str | None, str]]: + """Every named type a type expression mentions (`map[string]*Event` + and `[]sub.Item` still yield `Event` and `sub.Item`), minus + predeclared types and type parameters in scope.""" + refs: list[tuple[str | None, str]] = [] + if type_node is None: + return refs + generics = self._type_params_in_scope(type_node) + stack = [type_node] + while stack: + n = stack.pop() + ref = None + if n.type == "qualified_type": + ref = self._primary_type_ref(n) + elif n.type == "type_identifier": + name = self._text(n) + if name not in GO_PRIMITIVE_TYPES and name not in generics: + ref = (None, name) + else: + stack.extend(reversed(n.children)) + if ref and ref not in refs: + refs.append(ref) + return refs + + def _type_params_in_scope(self, node) -> set[str]: + """Type parameter names of the declarations enclosing a node, + including those a generic receiver (`func (b *Buffer[T])`) binds.""" + params: set[str] = set() + current = node + while current is not None: + type_params = current.child_by_field_name("type_parameters") + if type_params is not None: + for decl in type_params.named_children: + for name in decl.children_by_field_name("name"): + params.add(self._text(name)) + if current.type == "method_declaration": + receiver = current.child_by_field_name("receiver") + for generic in self._descendants(receiver, "generic_type") if receiver else []: + args = generic.child_by_field_name("type_arguments") + for ident in self._descendants(args, "type_identifier") if args else []: + params.add(self._text(ident)) + current = current.parent + return params + + def _type_ref_target(self, ref) -> tuple[str, bool] | None: + """`(callee, resolved)` for a type reference, or None when it points + outside the repository.""" + qualifier, name = ref + if qualifier is None: + node_obj = self.top_level_nodes.get(name) + if node_obj is not None: + return node_obj.id, True + if name in GO_PRIMITIVE_TYPES: + return None + return go_package_ref(self.package_dir, name), False + repo_dir = self.imports.get(qualifier) + if repo_dir is None: + return None + return go_package_ref(repo_dir, name), False + + def _add_type_ref_relationship(self, caller_id: str, ref, line: int): + target = self._type_ref_target(ref) + if target: + self._add_relationship_raw(caller_id, target[0], line, target[1]) + + def _emit_type_spec_edges(self, spec): + name = self._field_text(spec, "name") + type_node = spec.child_by_field_name("type") + owner = self.top_level_nodes.get(name) if name else None + if owner is None or type_node is None: + return + if type_node.type == "struct_type": + for decl in self._descendants(type_node, "field_declaration"): + for ref in self._collect_type_refs(decl.child_by_field_name("type")): + self._add_type_ref_relationship(owner.id, ref, decl.start_point[0] + 1) + elif type_node.type == "interface_type": + for elem in type_node.named_children: + if elem.type == "type_elem": + for ref in self._collect_type_refs(elem): + self._add_type_ref_relationship(owner.id, ref, elem.start_point[0] + 1) + else: + for ref in self._collect_type_refs(type_node): + self._add_type_ref_relationship(owner.id, ref, spec.start_point[0] + 1) + + # -- callers -------------------------------------------------------- + + def _find_containing_function(self, node): + current = node.parent + while current is not None: + if current.type in _FUNCTION_SCOPES: + return current + current = current.parent + return None + + def _function_logical_name(self, func_node) -> str | None: + name = self._field_text(func_node, "name") + if not name: + return None + if func_node.type == "method_declaration": + owner = self._receiver_type_name(func_node) + return f"{owner}.{name}" if owner else None + return name + + def _resolve_caller(self, node) -> str | None: + func_node = self._find_containing_function(node) + if func_node is None: + return None + logical = self._function_logical_name(func_node) + node_obj = self.top_level_nodes.get(logical) if logical else None + return node_obj.id if node_obj is not None else None + + # -- type uses ------------------------------------------------------ + + @staticmethod + def _is_receiver(param) -> bool: + params = param.parent + owner = params.parent if params is not None else None + if owner is None or owner.type != "method_declaration": + return False + receiver = owner.child_by_field_name("receiver") + return receiver is not None and receiver.start_byte == params.start_byte + + def _emit_type_use_edges(self, anchor, type_node): + """Types a function names in its signature, closures, local `var`s, + and type assertions: it cannot be ported before them.""" + caller_id = self._resolve_caller(anchor) + if not caller_id or type_node is None: + return + for ref in self._collect_type_refs(type_node): + self._add_type_ref_relationship(caller_id, ref, anchor.start_point[0] + 1) + + # -- instantiation -------------------------------------------------- + + def _emit_instantiation_edge(self, literal): + caller_id = self._resolve_caller(literal) + if not caller_id: + return + type_node = literal.child_by_field_name("type") + ref = self._primary_type_ref(type_node) + # `[]Event{...}` / `map[string]Event{...}` instantiate their element types. + refs = [ref] if ref else self._collect_type_refs(type_node) + generics = self._type_params_in_scope(literal) + for ref in refs: + if ref[1] not in generics: + self._add_type_ref_relationship(caller_id, ref, literal.start_point[0] + 1) + + # -- calls ---------------------------------------------------------- + + def _emit_call_edge(self, call_node): + caller_id = self._resolve_caller(call_node) + if not caller_id: + return + func_node = call_node.child_by_field_name("function") + if func_node is None: + return + line = call_node.start_point[0] + 1 + if func_node.type == "identifier": + self._emit_bare_call_edge(caller_id, self._text(func_node), call_node, line) + elif func_node.type == "selector_expression": + self._emit_selector_call_edge(caller_id, func_node, call_node, line) + elif func_node.type == "parenthesized_expression": + # Conversion to a pointer type: `(*stringHeader)(unsafe.Pointer(&s))`. + inner = func_node.named_children[0] if func_node.named_children else None + if inner is not None and inner.type == "unary_expression": + inner = inner.child_by_field_name("operand") + if inner is not None and inner.type == "identifier": + target = self.top_level_nodes.get(self._text(inner)) + if target is not None and target.component_type != "function": + self._add_relationship_raw(caller_id, target.id, line, True) + elif func_node.type == "index_expression": + # Explicitly instantiated generic function: `Map[int](xs)`. + operand = func_node.child_by_field_name("operand") + if operand is not None and operand.type == "identifier": + self._emit_bare_call_edge(caller_id, self._text(operand), call_node, line) + + def _emit_generic_call_edge(self, node): + """`f[int](x)` parses as a conversion to a generic type.""" + caller_id = self._resolve_caller(node) + type_node = node.child_by_field_name("type") + if not caller_id or type_node is None: + return + ref = self._primary_type_ref(type_node) + if not ref: + return + line = node.start_point[0] + 1 + if ref[0] is None: + self._emit_bare_call_edge(caller_id, ref[1], node, line) + else: + self._add_type_ref_relationship(caller_id, ref, line) + + def _emit_bare_call_edge(self, caller_id: str, name: str, call_node, line: int): + found, _ = self._local_binding(call_node, name) + if found: + return # a function value held in a local or parameter + node_obj = self.top_level_nodes.get(name) + if node_obj is not None: + self._add_relationship_raw(caller_id, node_obj.id, line, True) + return + if name in GO_BUILTIN_FUNCS: + if name in ("new", "make"): + args = call_node.child_by_field_name("arguments") + first = args.named_children[0] if args is not None and args.named_children else None + for ref in self._collect_type_refs(first): + self._add_type_ref_relationship(caller_id, ref, line) + return + if name in GO_PRIMITIVE_TYPES or name in self._type_params_in_scope(call_node): + return + if name in self._package_vars: + return # a package-level function variable + self._add_relationship_raw(caller_id, go_package_ref(self.package_dir, name), line, False) + + def _emit_selector_call_edge(self, caller_id: str, func_node, call_node, line: int): + operand = func_node.child_by_field_name("operand") + field = func_node.child_by_field_name("field") + if operand is None or field is None: + return + method = self._text(field) + + if operand.type == "identifier": + ident = self._text(operand) + found, ref = self._local_binding(call_node, ident) + if found: + self._emit_typed_method_edge(caller_id, ref, method, line) + return + if ident in self.imports: + repo_dir = self.imports[ident] + if repo_dir is not None: + self._add_relationship_raw( + caller_id, go_package_ref(repo_dir, method), line, False + ) + return + if ident in self._package_vars: + self._emit_typed_method_edge(caller_id, self._package_vars[ident], method, line) + return + if ident in self.top_level_nodes or ident[:1].isupper(): + # Method expression `Buffer.Push(b)`. + self._emit_typed_method_edge(caller_id, (None, ident), method, line) + return + self._emit_typed_method_edge(caller_id, None, method, line) + return + + self._emit_typed_method_edge(caller_id, self._expression_type(operand), method, line) + + def _emit_typed_method_edge(self, caller_id: str, ref, method: str, line: int): + if ref is None: + if method not in GO_COMMON_METHODS: + self._add_relationship_raw(caller_id, method, line, False) + return + qualifier, type_name = ref + logical = f"{type_name}.{method}" + if qualifier is None: + node_obj = self.top_level_nodes.get(logical) + if node_obj is not None: + self._add_relationship_raw(caller_id, node_obj.id, line, True) + return + if type_name in GO_PRIMITIVE_TYPES: + return + self._add_relationship_raw( + caller_id, go_package_ref(self.package_dir, logical), line, False + ) + return + repo_dir = self.imports.get(qualifier) + if repo_dir is not None: + self._add_relationship_raw(caller_id, go_package_ref(repo_dir, logical), line, False) + + # -- local types ---------------------------------------------------- + + def _expression_type(self, expr) -> tuple[str | None, str] | None: + """Best-effort type of an expression used as a method receiver.""" + if expr is None: + return None + if expr.type == "parenthesized_expression": + return self._expression_type(expr.named_children[0] if expr.named_children else None) + if expr.type in ("unary_expression", "index_expression", "slice_expression"): + return self._expression_type(expr.child_by_field_name("operand")) + if expr.type == "identifier": + found, ref = self._local_binding(expr, self._text(expr)) + if found: + return ref + return self._package_vars.get(self._text(expr)) + if expr.type in ("composite_literal", "type_assertion_expression"): + return self._primary_type_ref(expr.child_by_field_name("type")) + if expr.type == "selector_expression": + return self._field_type(expr) + if expr.type == "call_expression": + return self._call_result_type(expr) + return None + + def _field_type(self, selector) -> tuple[str | None, str] | None: + """Type of `x.field` when `x`'s struct is declared in this file.""" + owner = self._expression_type(selector.child_by_field_name("operand")) + field = selector.child_by_field_name("field") + if owner is None or owner[0] is not None or field is None: + return None + type_node = self._struct_fields.get(owner[1], {}).get(self._text(field)) + return self._primary_type_ref(type_node) + + def _call_result_type(self, call, index: int = 0) -> tuple[str | None, str] | None: + """Type of the `index`-th result of a call to a same-file function + or method (or of `new(T)` / a conversion `Kind(x)`).""" + func = call.child_by_field_name("function") + if func is None: + return None + if func.type == "identifier": + name = self._text(func) + if name == "new": + args = call.child_by_field_name("arguments") + first = args.named_children[0] if args is not None and args.named_children else None + return self._primary_type_ref(first) + if name in self.top_level_nodes and self.top_level_nodes[name].node_type != "function": + return None, name # conversion `Kind(x)` + return self._result_ref(self._func_results.get(name), index) + if func.type == "selector_expression": + receiver = self._expression_type(func.child_by_field_name("operand")) + field = func.child_by_field_name("field") + if receiver and receiver[0] is None and field is not None: + return self._result_ref( + self._func_results.get(f"{receiver[1]}.{self._text(field)}"), index + ) + return None + + def _result_ref(self, result, index: int = 0) -> tuple[str | None, str] | None: + if result is None: + return None + if result.type != "parameter_list": + return self._primary_type_ref(result) if index == 0 else None + # `(int, Result, bool)` or `(n, m int, r Result)`: one slot per name. + slots = [] + for decl in result.named_children: + if decl.type == "parameter_declaration": + type_node = decl.child_by_field_name("type") + slots += [type_node] * max(1, len(decl.children_by_field_name("name"))) + return self._primary_type_ref(slots[index]) if index < len(slots) else None + + def _var_spec_bindings(self, spec) -> list[tuple[str, tuple[str | None, str] | None]]: + names = [self._text(n) for n in spec.children_by_field_name("name")] + declared = self._primary_type_ref(spec.child_by_field_name("type")) + if declared or spec.child_by_field_name("type") is not None: + return [(n, declared) for n in names] + value = spec.child_by_field_name("value") + values = value.named_children if value is not None else [] + return [ + (n, self._expression_type(values[i]) if len(values) == len(names) else None) + for i, n in enumerate(names) + ] + + def _function_bindings(self, func_node) -> dict[str, list[tuple[int, object]]]: + """Every name a function binds (receiver, parameters, `var`, + `:=`, range and type-switch variables, closure parameters), with the + byte offset where its scope starts (the end of the declaring + statement, so `res := res.Get(k)` still sees the outer `res`) and its + inferred type (or None).""" + key = (func_node.start_byte, func_node.end_byte) + cached = self._bindings_cache.get(key) + if cached is not None: + return cached + bindings: dict[str, list[tuple[int, object]]] = defaultdict(list) + self._bindings_cache[key] = bindings + + # Document order, so a binding's initializer can see earlier ones. + stack = [func_node] + while stack: + n = stack.pop() + stack.extend(reversed(n.children)) + if n.type in ("parameter_declaration", "variadic_parameter_declaration"): + ref = self._primary_type_ref(n.child_by_field_name("type")) + for name in n.children_by_field_name("name"): + bindings[self._text(name)].append((n.end_byte, ref)) + elif n.type in ("var_spec", "const_spec"): + for name, ref in self._var_spec_bindings(n): + bindings[name].append((n.end_byte, ref)) + elif n.type == "short_var_declaration": + left = n.child_by_field_name("left") + right = n.child_by_field_name("right") + names = [c for c in left.named_children] if left is not None else [] + values = right.named_children if right is not None else [] + for i, name in enumerate(names): + ref = None + if len(values) == len(names): + ref = self._expression_type(values[i]) + elif len(values) == 1 and values[0].type == "call_expression": + ref = self._call_result_type(values[0], i) + bindings[self._text(name)].append((n.end_byte, ref)) + elif n.type == "range_clause": + left = n.child_by_field_name("left") + names = left.named_children if left is not None else [] + for i, name in enumerate(names): + # `for _, v := range xs`: v is an element of xs. + ref = None + if i == 1: + ref = self._expression_type(n.child_by_field_name("right")) + bindings[self._text(name)].append((n.end_byte, ref)) + elif n.type == "type_switch_statement": + alias = n.child_by_field_name("alias") + for name in alias.named_children if alias is not None else []: + bindings[self._text(name)].append((alias.end_byte, None)) + return bindings + + def _local_binding(self, use_node, name: str) -> tuple[bool, tuple[str | None, str] | None]: + """Whether `name` is a local of the enclosing function at `use_node`, + and its inferred type: the closest binding that precedes the use.""" + func_node = self._find_containing_function(use_node) + if func_node is None: + return False, None + best = None + for start, ref in self._function_bindings(func_node).get(name, []): + if start <= use_node.start_byte and (best is None or start > best[0]): + best = (start, ref) + if best is None: + return False, None + return True, best[1] + + # ------------------------------------------------------------------ + # Bookkeeping + # ------------------------------------------------------------------ + + def _add_relationship_raw( + self, caller: str, callee: str, call_line: int | None, resolved: bool + ): + key = (caller, callee, call_line) + if caller == callee or key in self.seen_relationships: + return + self.seen_relationships.add(key) + self.call_relationships.append( + CallRelationship( + caller=caller, callee=callee, call_line=call_line, is_resolved=resolved + ) + ) + + +def analyze_go_file( + file_path: str, content: str, repo_path: str | None = None +) -> tuple[list[Node], list[CallRelationship]]: + analyzer = TreeSitterGoAnalyzer(file_path, content, repo_path) + return analyzer.nodes, analyzer.call_relationships + + +# ---------------------------------------------------------------------- +# Repository-wide resolution (used by CallGraphAnalyzer) +# ---------------------------------------------------------------------- + + +class GoPackageIndex: + """Go components grouped by package directory.""" + + def __init__(self, functions: dict[str, Node]): + self.functions = functions + self.by_name: dict[tuple[str, str], list[str]] = defaultdict(list) + self.methods: dict[tuple[str, str], list[str]] = defaultdict(list) + for func_id, node in functions.items(): + if node.language != "go": + continue + directory = package_dir_of(node.relative_path) + self.by_name[(directory, node.name)].append(func_id) + if node.component_type == "method" and "." in node.name: + self.methods[(directory, node.name.rsplit(".", 1)[1])].append(func_id) + for ids in (*self.by_name.values(), *self.methods.values()): + ids.sort() + + def lookup(self, directory: str, name: str, depth: int = 0) -> str | None: + """A declaration of the package. Several hits can only come from + build-constrained files (`x_linux.go`, `x_windows.go`): take the + first. `Type.Method` falls back to methods promoted from embedded + types of the same package.""" + ids = self.by_name.get((directory, name)) + if ids: + return ids[0] + if "." not in name or depth >= 3: + return None + owner, method = name.split(".", 1) + for owner_id in self.by_name.get((directory, owner), []): + for base in self.functions[owner_id].base_classes or []: + if "." in base: + continue + found = self.lookup(directory, f"{base}.{method}", depth + 1) + if found: + return found + return None + + +def resolve_go_callee( + callee: str, caller: Node, index: GoPackageIndex, visible_dirs: set[str] | None +) -> str | None: + """Resolve a Go analyzer callee: a package reference names its package; + a bare method name (receiver type unknown) resolves when exactly one + method of that name exists in the packages the caller can see.""" + ref = split_go_package_ref(callee) + if ref is not None: + return index.lookup(*ref) + if "::" in callee or "." in callee: + return None + dirs = visible_dirs or {package_dir_of(caller.relative_path)} + candidates = sorted({fid for d in dirs for fid in index.methods.get((d, callee), [])}) + return candidates[0] if len(candidates) == 1 else None + + +def link_go_interfaces(functions: dict[str, Node]) -> list[CallRelationship]: + """Link each named type to the interfaces of its own package whose + methods it declares (Go implements interfaces implicitly). The + interface is also recorded as a base class of the type.""" + types: dict[str, dict[str, Node]] = defaultdict(dict) + method_sets: dict[tuple[str, str], set[str]] = defaultdict(set) + for node in functions.values(): + if node.language != "go": + continue + directory = package_dir_of(node.relative_path) + if node.component_type == "method" and node.class_name: + method_sets[(directory, node.class_name)].add(node.name.rsplit(".", 1)[-1]) + elif node.node_type in ("struct", "interface", "type"): + types[directory][node.name] = node + + relationships: list[CallRelationship] = [] + for directory, by_name in types.items(): + interfaces = [ + (iface, method_sets.get((directory, iface.name))) + for iface in by_name.values() + if iface.node_type == "interface" + ] + interfaces = [(iface, methods) for iface, methods in interfaces if methods] + if not interfaces: + continue + for type_node in by_name.values(): + if type_node.node_type == "interface": + continue + own = method_sets.get((directory, type_node.name)) + if not own: + continue + for iface, methods in interfaces: + if methods <= own: + bases = type_node.base_classes or [] + if iface.name not in bases: + type_node.base_classes = [*bases, iface.name] + relationships.append( + CallRelationship( + caller=type_node.id, + callee=iface.id, + call_line=type_node.start_line, + is_resolved=True, + ) + ) + return relationships diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index e3f67a2d..6d8df152 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -191,6 +191,7 @@ def _file_to_module_path(self, file_path: str) -> str: ".sc", ".rs", ".dart", + ".go", ] for ext in extensions: if path.endswith(ext): diff --git a/codewiki/src/be/dependency_analyzer/utils/patterns.py b/codewiki/src/be/dependency_analyzer/utils/patterns.py index ce48c541..cf4b0dcc 100644 --- a/codewiki/src/be/dependency_analyzer/utils/patterns.py +++ b/codewiki/src/be/dependency_analyzer/utils/patterns.py @@ -516,7 +516,12 @@ "python": ["def {name}"], "javascript": ["function {name}", "const {name}", "export {name}"], "typescript": ["function {name}", "const {name}", "export {name}"], - "go": ["func {name}"], + "go": [ + "func {name}", + "type {name} struct", + "type {name} interface", + "type {name}", + ], "rust": [ "fn {name}", "pub fn {name}", diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index 662ec3e2..0a715bc1 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -386,6 +386,7 @@ ".sc": "scala", ".rs": "rust", ".dart": "dart", + ".go": "go", ".php": "php", ".phtml": "php", ".inc": "php", diff --git a/guides/development.md b/guides/development.md index e20e8d8e..648cb681 100644 --- a/guides/development.md +++ b/guides/development.md @@ -119,7 +119,9 @@ Analyzers are standalone classes, one file per language. Follow `analyzers/scala.py` (the newest one) as a template. 1. **Grammar.** Add the `tree-sitter-` package to `pyproject.toml` - and `requirements.txt`. + and `requirements.txt`, or take the grammar from + `tree-sitter-language-pack` (already a dependency) with + `get_parser("")`, as `analyzers/dart.py` and `analyzers/go.py` do. 2. **Analyzer.** Create `analyzers/.py` with a `TreeSitterAnalyzer` class that extracts components and relationships, plus a module-level `analyze__file(...)` function. @@ -134,7 +136,8 @@ Analyzers are standalone classes, one file per language. Follow 5. **Tests.** Add `tests/test__analyzer.py` with small source snippets that check component and edge extraction. See `tests/test_scala_analyzer.py`, `tests/test_ruby_analyzer.py`, - `tests/test_rust_analyzer.py`, and `tests/test_dart_analyzer.py`. + `tests/test_rust_analyzer.py`, `tests/test_dart_analyzer.py`, and + `tests/test_go_analyzer.py`. 6. **Docs.** Add the language to the README list. ## Tests and lint diff --git a/tests/test_go_analyzer.py b/tests/test_go_analyzer.py new file mode 100644 index 00000000..81779cd8 --- /dev/null +++ b/tests/test_go_analyzer.py @@ -0,0 +1,421 @@ +"""Tests for the tree-sitter based Go analyzer.""" + +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_language_pack") + +from codewiki.src.be.dependency_analyzer.analyzers.artifact import ( + ArtifactOptions, + classify_artifact, +) +from codewiki.src.be.dependency_analyzer.analyzers.go import ( + analyze_go_file, + go_package_ref, + split_go_package_ref, +) +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser + +SAMPLE = """\ +package buffer + +import ( + "fmt" + "strings" +) + +// Flusher is implemented by buffers that can be flushed. +type Flusher interface { + Flush() error +} + +// Logger writes structured messages. +type Logger struct { + prefix string +} + +func NewLogger() *Logger { + return &Logger{prefix: "buf"} +} + +// Warn logs a warning. +func (l *Logger) Warn(message string) { + fmt.Println(l.prefix, message) +} + +type Event struct { + ID uint64 +} + +// Kind tells events apart. +type Kind int + +func (k Kind) Label() string { return "kind" } + +type ( + // Signal carries an event downstream. + Signal struct { + event *Event + kind Kind + } + Handler func(ev Event) error +) + +// Buffer holds events before flushing them. +type Buffer[T any] struct { + Logger + capacity int + items []T + index map[uint64]*Event + sink Flusher +} + +func NewBuffer[T any](capacity int) *Buffer[T] { + return &Buffer[T]{capacity: capacity, Logger: *NewLogger()} +} + +func (b *Buffer[T]) Push(item T) { + b.validate() + b.Warn("pushed") + local := NewLogger() + local.Warn("local") + var event Event + record(&event) + b.items = append(b.items, item) + var sb strings.Builder + sb.WriteString("x") + k := Kind(1) + k.Label() + b.sink.Flush() + callback := func() { helper() } + callback() +} + +func (b Buffer[T]) validate() bool { + return len(b.items) < b.capacity +} + +func (b *Buffer[T]) Flush() error { + b.items = nil + return nil +} + +func record(ev *Event) {} +""" + + +def _analyze(tmp_path: Path, source: str = SAMPLE, name: str = "buffer.go"): + file_path = tmp_path / name + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(source, encoding="utf-8") + return analyze_go_file(str(file_path), source, repo_path=str(tmp_path)) + + +def _edges(relationships) -> set[tuple[str, str]]: + return {(r.caller, r.callee) for r in relationships} + + +def test_extracts_types_functions_and_methods(tmp_path: Path) -> None: + nodes, _ = _analyze(tmp_path) + by_name = {node.name: node for node in nodes} + + assert by_name["Buffer"].component_type == "struct" + assert by_name["Flusher"].component_type == "interface" + assert by_name["Kind"].component_type == "class" + assert by_name["Kind"].node_type == "type" + assert by_name["Handler"].node_type == "type" + assert by_name["Signal"].component_type == "struct" + + # Pointer and value receivers both key the method by the base type. + assert by_name["Buffer.Push"].component_type == "method" + assert by_name["Buffer.Push"].class_name == "Buffer" + assert by_name["Buffer.validate"].class_name == "Buffer" + assert by_name["Kind.Label"].class_name == "Kind" + # Interface method signatures are owned by the interface. + assert by_name["Flusher.Flush"].component_type == "method" + + assert by_name["NewBuffer"].component_type == "function" + assert by_name["Buffer"].id == "buffer.go::Buffer" + assert by_name["Buffer.Push"].id == "buffer.go::Buffer.Push" + assert all(node.language == "go" for node in nodes) + + +def test_docstrings_and_parameters(tmp_path: Path) -> None: + nodes, _ = _analyze(tmp_path) + by_name = {node.name: node for node in nodes} + + assert by_name["Buffer"].docstring == "// Buffer holds events before flushing them." + # A spec inside a grouped `type (...)` takes the comment above the spec. + assert by_name["Signal"].docstring == "// Signal carries an event downstream." + assert not by_name["Event"].has_docstring + assert by_name["Logger.Warn"].parameters == ["message string"] + + +def test_embedding_and_interfaces_become_base_classes(tmp_path: Path) -> None: + nodes, relationships = _analyze(tmp_path) + buffer = next(n for n in nodes if n.name == "Buffer") + assert buffer.base_classes == ["Logger"] + assert ("buffer.go::Buffer", "buffer.go::Logger") in _edges(relationships) + + +def test_type_edges(tmp_path: Path) -> None: + _, relationships = _analyze(tmp_path) + edges = _edges(relationships) + + # Field types through pointers, maps, and named types. + assert ("buffer.go::Buffer", "buffer.go::Event") in edges + assert ("buffer.go::Buffer", "buffer.go::Flusher") in edges + assert ("buffer.go::Signal", "buffer.go::Event") in edges + assert ("buffer.go::Signal", "buffer.go::Kind") in edges + # The underlying type of a defined type. + assert ("buffer.go::Handler", "buffer.go::Event") in edges + # Type parameters and predeclared types never become edges. + callees = {callee for _, callee in edges} + assert not any(c.endswith(("::T", "::int", "::uint64", "::any")) for c in callees) + + +def test_call_and_literal_edges(tmp_path: Path) -> None: + _, relationships = _analyze(tmp_path) + edges = _edges(relationships) + + # Composite literals, through `&` and type arguments. + assert ("buffer.go::NewLogger", "buffer.go::Logger") in edges + assert ("buffer.go::NewBuffer", "buffer.go::Buffer") in edges + assert ("buffer.go::NewBuffer", "buffer.go::NewLogger") in edges + # Receiver method calls (value and pointer receivers). + assert ("buffer.go::Buffer.Push", "buffer.go::Buffer.validate") in edges + # A local whose type comes from a same-file constructor's result. + assert ("buffer.go::Buffer.Push", "buffer.go::Logger.Warn") in edges + # A conversion `Kind(1)` types the local. + assert ("buffer.go::Buffer.Push", "buffer.go::Kind.Label") in edges + # A field's interface type resolves the method on the interface. + assert ("buffer.go::Buffer.Push", "buffer.go::Flusher.Flush") in edges + assert ("buffer.go::Buffer.Push", "buffer.go::record") in edges + # Types named in signatures and local `var`s (not the receiver). + assert ("buffer.go::record", "buffer.go::Event") in edges + assert ("buffer.go::NewLogger", "buffer.go::Logger") in edges + assert ("buffer.go::Buffer.Push", "buffer.go::Event") in edges + assert ("buffer.go::Buffer.validate", "buffer.go::Buffer") not in edges + + +def test_promoted_method_is_left_to_package_resolution(tmp_path: Path) -> None: + _, relationships = _analyze(tmp_path) + # `b.Warn()` is promoted from the embedded Logger: the file analyzer + # names `Buffer.Warn` in the package, CallGraphAnalyzer follows the + # embedding. + assert ("buffer.go::Buffer.Push", go_package_ref("", "Buffer.Warn")) in _edges(relationships) + + +def test_external_and_builtin_calls_are_dropped(tmp_path: Path) -> None: + _, relationships = _analyze(tmp_path) + callees = {r.callee for r in relationships} + + for noise in ("Println", "WriteString", "append", "len", "fmt", "strings"): + assert noise not in callees + assert not any("Println" in c or "WriteString" in c for c in callees) + # A closure held in a local is not a call to a package function, but + # the closure body's call is attributed to the enclosing function. + assert go_package_ref("", "callback") not in callees + assert go_package_ref("", "helper") in callees + + +def test_package_ref_round_trip() -> None: + assert go_package_ref("", "Get") == "*.go::Get" + assert go_package_ref("sub/pkg", "T.M") == "sub/pkg/*.go::T.M" + assert split_go_package_ref("sub/pkg/*.go::T.M") == ("sub/pkg", "T.M") + assert split_go_package_ref("*.go::Get") == ("", "Get") + assert split_go_package_ref("buffer.go::Get") is None + assert split_go_package_ref("Get") is None + + +def test_test_files_and_vendor_are_skipped(tmp_path: Path) -> None: + source = "package buffer\n\nfunc TestPush(t *testing.T) { record(nil) }\n" + assert _analyze(tmp_path, source, "buffer_test.go") == ([], []) + vendored = "package dep\n\nfunc Helper() {}\n" + assert _analyze(tmp_path, vendored, "vendor/github.com/x/dep/dep.go") == ([], []) + + +def test_empty_and_broken_files_return_no_components(tmp_path: Path) -> None: + assert _analyze(tmp_path, "", "empty.go") == ([], []) + nodes, _ = _analyze(tmp_path, "package x\nfunc broken( {{{ type", "broken.go") + assert isinstance(nodes, list) + + +def test_go_mod_classifies_as_manifest() -> None: + opts = ArtifactOptions() + assert classify_artifact("go.mod", "go.mod", 100, opts) == "manifest" + assert classify_artifact("main.go", "main.go", 100, opts) is None + + +def _write(root: Path, rel: str, text: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def test_dependency_parser_end_to_end(tmp_path: Path) -> None: + _write(tmp_path, "go.mod", "module github.com/acme/shop\n\ngo 1.22\n") + _write( + tmp_path, + "store/store.go", + "package store\n\n" + "type Config struct { Name string }\n\n" + "type Store struct { cfg Config }\n\n" + "func Open(cfg Config) *Store { return &Store{cfg: cfg} }\n\n" + "func (s *Store) Save(key string) error { return nil }\n\n" + "func (s *Store) Load(key string) string { return key }\n", + ) + # Two files of one package: names cross files without imports. + _write( + tmp_path, + "cart/cart.go", + "package cart\n\n" + "import (\n" + '\t"fmt"\n\n' + '\tdb "github.com/acme/shop/store"\n' + ")\n\n" + "type Base struct{}\n\n" + "func (Base) Describe() string { return describe() }\n\n" + "type Cart struct {\n" + "\tBase\n" + "\tstore *db.Store\n" + "\tlines []Line\n" + "}\n\n" + "type Saver interface { Save(key string) error }\n\n" + "func New() *Cart {\n" + '\ts := db.Open(db.Config{Name: "carts"})\n' + "\treturn &Cart{store: s}\n" + "}\n\n" + "func (c *Cart) Save(key string) error {\n" + "\tc.store.Save(key)\n" + "\tfmt.Println(c.Describe())\n" + "\treturn c.store.Load(key)\n" + "}\n", + ) + _write( + tmp_path, + "cart/line.go", + "package cart\n\ntype Line struct{ Qty int }\n\nfunc describe() string { return total() }\n", + ) + _write(tmp_path, "cart/total.go", "package cart\n\nfunc total() string { return \"\" }\n") + _write( + tmp_path, + "cart/cart_test.go", + "package cart\n\nimport \"testing\"\n\nfunc TestNew(t *testing.T) { New() }\n", + ) + + components = DependencyParser(str(tmp_path)).parse_repository() + + assert "cart/cart.go::Cart" in components + assert "cart/cart.go::Cart.Save" in components + assert "store/store.go::Store.Save" in components + # _test.go files contribute nothing. + assert not any(cid.startswith("cart/cart_test.go") for cid in components) + + cart = components["cart/cart.go::Cart"].depends_on + # Cross-file field type in the same package, and a type of another + # package imported under an alias through go.mod's module path. + assert "cart/line.go::Line" in cart + assert "store/store.go::Store" in cart + # Embedded struct, and the interface of its package that Cart implements. + assert "cart/cart.go::Base" in cart + assert "cart/cart.go::Saver" in cart + assert "Saver" in components["cart/cart.go::Cart"].base_classes + + new = components["cart/cart.go::New"].depends_on + assert "store/store.go::Open" in new + assert "store/store.go::Config" in new + assert "cart/cart.go::Cart" in new + + save = components["cart/cart.go::Cart.Save"].depends_on + # Method calls through a field of another package's type. + assert "store/store.go::Store.Save" in save + assert "store/store.go::Store.Load" in save + # A method promoted from the embedded Base. + assert "cart/cart.go::Base.Describe" in save + # fmt is external: nothing resolves to it. + assert all(dep in components for dep in save) + + # Same-package call to a function in another file. + assert "cart/line.go::describe" in components["cart/cart.go::Base.Describe"].depends_on + assert "cart/total.go::total" in components["cart/line.go::describe"].depends_on + # Store does not have Save via Cart's Saver: interfaces only link + # within their own package. + assert "cart/cart.go::Saver" not in components["store/store.go::Store"].depends_on + + +def test_unknown_receiver_resolves_only_when_unique(tmp_path: Path) -> None: + _write(tmp_path, "go.mod", "module example.com/m\n") + _write( + tmp_path, + "a.go", + "package m\n\n" + "type Parser struct{}\n\n" + "func (p *Parser) ParseAll() {}\n\n" + "type Lexer struct{}\n\n" + "func (l *Lexer) Step() {}\n\n" + "type Walker struct{}\n\n" + "func (w *Walker) Step() {}\n\n" + "func Run(next func() *Parser, x interface{ Step() }) {\n" + "\tnext().ParseAll()\n" + "\tx.Step()\n" + "}\n", + ) + + components = DependencyParser(str(tmp_path)).parse_repository() + run = components["a.go::Run"].depends_on + assert "a.go::Parser.ParseAll" in run + # Two `Step` methods in scope: ambiguous, no guess. + assert not any(dep.endswith(".Step") for dep in run) + + +def test_local_types_follow_scope_results_and_elements(tmp_path: Path) -> None: + source = """\ +package m + +type Result struct{ Raw string } + +func (r Result) Get(key string) Result { return r } + +func (r Result) Exists() bool { return true } + +func parse(s string) (int, Result, bool) { return 0, Result{}, true } + +func Walk(all []Result, byKey map[string]*Result) { +\t_, res, ok := parse("x") +\tif ok { +\t\tres := res.Get("a") +\t\tres.Exists() +\t} +\tfor _, r := range all { +\t\tr.Exists() +\t} +\tbyKey["k"].Get("b") +} +""" + _, relationships = _analyze(tmp_path, source, "walk.go") + edges = _edges(relationships) + assert ("walk.go::Walk", "walk.go::Result.Get") in edges + assert ("walk.go::Walk", "walk.go::Result.Exists") in edges + assert all(r.is_resolved for r in relationships) + + +def test_local_replace_directive_maps_another_module(tmp_path: Path) -> None: + _write( + tmp_path, + "go.mod", + "module example.com/app\n\n" + "require example.com/lib v0.0.0\n\n" + "replace (\n\texample.com/lib v0.0.0 => ./lib // local copy\n)\n", + ) + _write(tmp_path, "lib/go.mod", "module example.com/lib\n") + _write(tmp_path, "lib/util/util.go", "package util\n\nfunc Trim(s string) string { return s }\n") + _write( + tmp_path, + "main.go", + 'package main\n\nimport "example.com/lib/util"\n\nfunc main() { util.Trim("x") }\n', + ) + + components = DependencyParser(str(tmp_path)).parse_repository() + assert "lib/util/util.go::Trim" in components["main.go::main"].depends_on From 74b51dfef4e692bc0114b7de91bf106ee16d576c Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Thu, 22 Oct 2026 09:59:24 +0700 Subject: [PATCH 2/2] tests: ruff format test_go_analyzer.py --- tests/test_go_analyzer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_go_analyzer.py b/tests/test_go_analyzer.py index 81779cd8..db8849f8 100644 --- a/tests/test_go_analyzer.py +++ b/tests/test_go_analyzer.py @@ -298,11 +298,11 @@ def test_dependency_parser_end_to_end(tmp_path: Path) -> None: "cart/line.go", "package cart\n\ntype Line struct{ Qty int }\n\nfunc describe() string { return total() }\n", ) - _write(tmp_path, "cart/total.go", "package cart\n\nfunc total() string { return \"\" }\n") + _write(tmp_path, "cart/total.go", 'package cart\n\nfunc total() string { return "" }\n') _write( tmp_path, "cart/cart_test.go", - "package cart\n\nimport \"testing\"\n\nfunc TestNew(t *testing.T) { New() }\n", + 'package cart\n\nimport "testing"\n\nfunc TestNew(t *testing.T) { New() }\n', ) components = DependencyParser(str(tmp_path)).parse_repository() @@ -410,7 +410,9 @@ def test_local_replace_directive_maps_another_module(tmp_path: Path) -> None: "replace (\n\texample.com/lib v0.0.0 => ./lib // local copy\n)\n", ) _write(tmp_path, "lib/go.mod", "module example.com/lib\n") - _write(tmp_path, "lib/util/util.go", "package util\n\nfunc Trim(s string) string { return s }\n") + _write( + tmp_path, "lib/util/util.go", "package util\n\nfunc Trim(s string) string { return s }\n" + ) _write( tmp_path, "main.go",