diff --git a/.ci/scripts/check_arm_buck_declarations.py b/.ci/scripts/check_arm_buck_declarations.py new file mode 100644 index 00000000000..5421131551a --- /dev/null +++ b/.ci/scripts/check_arm_buck_declarations.py @@ -0,0 +1,951 @@ +#!/usr/bin/env python3 +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +"""Check regressions in Arm test-library Buck declarations, without executing Buck. + +This is a static declaration lint, not a replacement for Meta's packaged tests. +It models literal Python library declarations and internal (is_oss=False) branches. +Unsupported declarations under backends/arm fail as 'cannot validate'; limitations +elsewhere are warnings. Existing findings are compared by identity and severity, +not line number, against an explicit base. +""" +from __future__ import annotations + +import argparse +import ast +import fnmatch +import functools +import subprocess +import sys +import textwrap +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +SCOPE = "backends/arm/test" +TEST_MACRO = SCOPE + "/targets.bzl" + + +class Unsupported(ValueError): + pass + + +class Snapshot: + def __init__(self, files: dict[str, str]): + self.paths = set(files) + self._files = files + + def read(self, path: str) -> str: + return self._files[path] + + +class GitSnapshot(Snapshot): + def __init__(self, repo: Path, revision: str | None): + super().__init__({}) + self.repo = repo + self.revision = revision + self._files: dict[str, str] = {} + if revision is None: + command = ["ls-files", "--cached", "--others", "--exclude-standard", "-z"] + else: + revision = self.git("rev-parse", "--verify", revision + "^{commit}").strip() + self.revision = revision + command = ["ls-tree", "-r", "--name-only", "-z", revision] + self.paths = set(self.git(*command).split("\0")) - {""} + if revision is None: + self.paths = {p for p in self.paths if (repo / p).is_file()} + + def git(self, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(self.repo), *args], text=True) + + def read(self, path: str) -> str: + if path not in self.paths: + raise KeyError(path) + if path not in self._files: + self._files[path] = ( + self.git("show", f"{self.revision}:{path}") + if self.revision + else (self.repo / path).read_text(encoding="utf-8") + ) + return self._files[path] + + +@functools.lru_cache(maxsize=None) +def glob_match(path: str, pattern: str) -> bool: + """Buck-style path glob: '*' does not cross '/', '**' can.""" + parts, patterns = tuple(path.split("/")), tuple(pattern.split("/")) + + @functools.lru_cache(maxsize=None) + def match(i: int, j: int) -> bool: + if j == len(patterns): + return i == len(parts) + if patterns[j] == "**": + return match(i, j + 1) or (i < len(parts) and match(i + 1, j)) + return ( + i < len(parts) + and fnmatch.fnmatchcase(parts[i], patterns[j]) + and match(i + 1, j + 1) + ) + + return match(0, 0) + + +def dotted(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return dotted(node.value) + "." + node.attr + return "" + + +def label_for(package: str, name: str) -> str: + return "//executorch/" + package + ":" + name + + +def is_scoped_label(label: str) -> bool: + return label.startswith((label_for(SCOPE, ""), "//executorch/" + SCOPE + "/")) + + +def normalize(label: str, package: str) -> str | None: + if label.startswith(":"): + return label_for(package, label[1:]) + if label.startswith("//executorch/"): + return label + # Dependencies outside the in-repo executorch namespace are out of scope. + return None + + +@dataclass(frozen=True, order=True) +class Issue: + # Messages may gain provider suggestions without creating a new violation. + kind: str = field(compare=False) + owner: str + source: str + detail: str = field(compare=False) + subject: str = "" + category: str = "declaration" + severity: str = "error" + + def __str__(self) -> str: + return f"{self.kind}: {self.owner}: {self.source}: {self.detail}" + + +def validation_severity(source: str) -> str: + return ( + "error" + if source == "backends/arm" or source.startswith("backends/arm/") + else "warning" + ) + + +@dataclass +class Rule: + label: str + kind: str + declaration_file: str + sources: set[str] = field(default_factory=set) + deps: set[str] = field(default_factory=set) + errors: list[str] = field(default_factory=list) + + +class Declarations: + def __init__(self, snapshot: Snapshot): + self.snapshot = snapshot + self.rules: dict[str, Rule] = {} + self.loaded: set[str] = set() + self.called_helpers: set[tuple[str, str, str]] = set() + self.package_errors: dict[str, list[tuple[str, str]]] = {} + + def package_error(self, package: str, filename: str, error: str) -> None: + self.package_errors.setdefault(package, []).append((filename, error)) + + def load_scope(self) -> None: + self.load_package(SCOPE) + for filename in sorted(self.snapshot.paths): + package, _, name = filename.rpartition("/") + if name in {"BUCK", "TARGETS"} and ( + package == SCOPE or package.startswith(SCOPE + "/") + ): + self.load_package(package) + + def package_files(self, package: str) -> list[str]: + prefix = package + "/" if package else "" + result = [] + for path in self.snapshot.paths: + if not path.startswith(prefix): + continue + relative = path[len(prefix) :] + # A glob cannot reach into another Buck package. + parents = list(PurePosixPath(relative).parents)[:-1] + if any( + prefix + str(p) + "/BUCK" in self.snapshot.paths + or prefix + str(p) + "/TARGETS" in self.snapshot.paths + for p in parents + ): + continue + result.append(relative) + return sorted(result) + + # Each branch corresponds to one explicitly supported AST form. + # pylint: disable-next=too-many-return-statements + def value(self, node: ast.AST, env: dict[str, object], package: str): # noqa: C901 + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, (ast.List, ast.Tuple)): + return [self.value(x, env, package) for x in node.elts] + if isinstance(node, ast.Name) and node.id in env: + return env[node.id] + if dotted(node) == "runtime.is_oss": + return env.get("__is_oss__", False) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not self.value(node.operand, env, package) + if isinstance(node, ast.BoolOp): + for operand in node.values: + value = self.value(operand, env, package) + if bool(value) == isinstance(node.op, ast.Or): + return value + return value + if isinstance(node, ast.IfExp): + branch = node.body if self.value(node.test, env, package) else node.orelse + return self.value(branch, env, package) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return self.value(node.left, env, package) + self.value( + node.right, env, package + ) + if isinstance(node, ast.Call) and dotted(node.func) in {"glob", "native.glob"}: + patterns = self.value(node.args[0], env, package) + kwargs = {k.arg: self.value(k.value, env, package) for k in node.keywords} + unknown = set(kwargs) - {"exclude", "exclude_directories"} + if unknown: + raise Unsupported( + "unsupported glob arguments: " + ", ".join(sorted(unknown)) + ) + excludes = kwargs.get("exclude", []) + return [ + p + for p in self.package_files(package) + if any(glob_match(p, pattern) for pattern in patterns) + and not any(glob_match(p, pattern) for pattern in excludes) + ] + raise Unsupported(ast.unparse(node)) + + def strings(self, node: ast.AST, env: dict[str, object], package: str) -> list[str]: + value = self.value(node, env, package) + if not isinstance(value, list) or not all(isinstance(x, str) for x in value): + raise Unsupported("expected a static list of strings: " + ast.unparse(node)) + return value + + def parse_rule( # noqa: C901 + self, call: ast.Call, env: dict[str, object], package: str, filename: str + ) -> None: + kwargs = {k.arg: k.value for k in call.keywords} + function = dotted(call.func) + if function == "non_fbcode_target": + return + kind = dotted(kwargs.get("_kind", call.func)).split(".")[-1] + if kind not in { + "python_library", + "python_binary", + "python_test", + "python_pytest", + "python_unittest", + "cxx_library", + "cxx_python_extension", + "cxx_test", + "export_file", + "filegroup", + }: + if "name" in kwargs and "srcs" in kwargs: + self.package_error(package, filename, "unsupported rule kind: " + kind) + return + try: + name = self.value(kwargs["name"], env, package) + if not isinstance(name, str): + raise Unsupported("non-string target name") + except (KeyError, Unsupported) as error: + self.package_error(package, filename, f"target name: {error}") + return + rule = Rule(label_for(package, name), kind, filename) + for attribute in ("srcs", "deps", "exported_deps", "fbcode_deps"): + if attribute not in kwargs: + continue + try: + values = self.strings(kwargs[attribute], env, package) + if attribute == "srcs": + for source in values: + if source.startswith((":", "//", "@")): + if kind == "python_library": + raise Unsupported("source target reference " + source) + elif source.endswith(".py"): + rule.sources.add(package + "/" + source) + else: + rule.deps.update( + target + for value in values + if (target := normalize(value, package)) is not None + ) + except Unsupported as error: + rule.errors.append(f"{attribute}: {error}") + if rule.label in self.rules: + rule.errors.append("multiple declarations for the same target") + self.rules[rule.label] = rule + + def load_package(self, package: str) -> None: + if package in self.loaded: + return + self.loaded.add(package) + filename = next( + ( + package + "/" + name + for name in ("BUCK", "TARGETS") + if package + "/" + name in self.snapshot.paths + ), + None, + ) + if filename is None: + self.package_error(package, package + "/BUCK", "no in-repo build file") + return + self.parse_file(filename, package, set()) + + # Keep declaration traversal and its environment in a single scope. + # pylint: disable-next=too-many-locals,too-many-statements + def parse_file( # noqa: C901 + self, filename: str, package: str, seen: set, selected: set[str] | None = None + ) -> None: + key = (filename, tuple(sorted(selected or []))) + if key in seen: + return + seen.add(key) + try: + tree = ast.parse(self.snapshot.read(filename), filename=filename) + except (KeyError, SyntaxError) as error: + self.package_error(package, filename, f"cannot read {filename}: {error}") + return + env: dict[str, object] = {} + functions = { + node.name: node for node in tree.body if isinstance(node, ast.FunctionDef) + } + imports: dict[str, tuple[str, str]] = {} + for node in tree.body: + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Call) + and dotted(node.value.func) == "load" + ): + args = node.value.args + if ( + args + and isinstance(args[0], ast.Constant) + and isinstance(args[0].value, str) + ): + source = args[0].value + # Only local declaration helpers; never load external macros. + if source.startswith(":") or not source.startswith(("@", "//")): + helper = str( + PurePosixPath(filename).parent / source.lstrip(":") + ) + if helper in self.snapshot.paths: + imports.update( + (arg.value, (helper, arg.value)) + for arg in args[1:] + if isinstance(arg, ast.Constant) + and isinstance(arg.value, str) + ) + imports.update( + (k.arg, (helper, k.value.value)) + for k in node.value.keywords + if isinstance(k.value, ast.Constant) + and isinstance(k.value.value, str) + ) + + called: set[str] = set() + visiting: set[str] = set() + + def call_name(call: ast.Call) -> str: + return dotted( + next((k.value for k in call.keywords if k.arg == "_kind"), call.func) + ) + + def visit_helper(name: str) -> None: + function = functions.get(name) + if function is None: + return + args = function.args + if ( + args.posonlyargs + or args.args + or args.kwonlyargs + or args.vararg + or args.kwarg + ): + self.package_error( + package, filename, "parameterized declaration helper: " + name + ) + elif name in visiting: + self.package_error( + package, filename, "recursive declaration helper: " + name + ) + else: + self.called_helpers.add((package, filename, name)) + visiting.add(name) + visit(function.body, dict(env)) + visiting.remove(name) + + def visit(statements: list[ast.stmt], local_env: dict[str, object]) -> None: + for node in statements: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + try: + local_env[node.targets[0].id] = self.value( + node.value, local_env, package + ) + except Unsupported: + local_env.pop(node.targets[0].id, None) + elif isinstance(node, ast.AugAssign): + target = node.target + try: + if not isinstance(target, ast.Name): + raise Unsupported("non-name assignment target") + local_env[target.id] = self.value( + ast.BinOp(left=target, op=node.op, right=node.value), + local_env, + package, + ) + except (Unsupported, TypeError) as error: + while isinstance(target, (ast.Attribute, ast.Subscript)): + target = target.value + if isinstance(target, ast.Name): + local_env.pop(target.id, None) + self.package_error( + package, + filename, + f"unsupported augmented assignment: {ast.unparse(node)} ({error})", + ) + elif isinstance(node, ast.If): + try: + branch = ( + node.body + if self.value(node.test, local_env, package) + else node.orelse + ) + visit(branch, local_env) + except Unsupported as error: + if any( + isinstance(x, ast.Call) + and dotted(x.func) != "non_fbcode_target" + and ( + call_name(x) in functions + or call_name(x) in imports + or call_name(x).split(".")[-1] == "python_library" + ) + for x in ast.walk(node) + ): + self.package_error( + package, filename, f"conditional declaration: {error}" + ) + elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + call = node.value + if dotted(call.func) == "non_fbcode_target": + continue + name = call_name(call) + if name in functions or name in imports: + if call.args or any(k.arg != "_kind" for k in call.keywords): + self.package_error( + package, + filename, + "parameterized declaration helper: " + name, + ) + elif name in functions: + visit_helper(name) + else: + called.add(name) + else: + self.parse_rule(call, local_env, package, filename) + elif isinstance(node, ast.For): + kinds = { + dotted( + next( + (k.value for k in call.keywords if k.arg == "_kind"), + call.func, + ) + ).split(".")[-1] + for call in ast.walk(node) + if isinstance(call, ast.Call) + } + known_test_loop = ( + isinstance(node.target, ast.Name) + and node.target.id == "test_file" + and isinstance(node.iter, ast.Name) + and node.iter.id == "test_files" + and "python_pytest" in kinds + and "python_library" not in kinds + ) + if not known_test_loop: + self.package_error( + package, + filename, + "unsupported declaration loop: " + + ast.unparse(node.target) + + " in " + + ast.unparse(node.iter), + ) + + visit(tree.body, env) + for name in sorted(selected or set()): + visit_helper(name) + needed: dict[str, set[str]] = {} + for name in sorted(called): + helper, original = imports[name] + needed.setdefault(helper, set()).add(original) + for helper, names in needed.items(): + self.parse_file(helper, package, seen, names) + + def get(self, label: str) -> Rule | None: + package = label.removeprefix("//executorch/").split(":", 1)[0] + self.load_package(package) + return self.rules.get(label) + + def owners(self, source: str) -> set[str]: + for parent in PurePosixPath(source).parents: + package = str(parent) + if package == ".": + continue + if any( + package + "/" + name in self.snapshot.paths + for name in ("BUCK", "TARGETS") + ): + self.load_package(package) + return { + r.label + for r in self.rules.values() + if r.kind == "python_library" and source in r.sources + } + return set() + + def reachable(self, start: Rule) -> tuple[set[str], set[Issue]]: + visited, pending, errors = set(), [start.label], set() + while pending: + label = pending.pop() + if label in visited: + continue + visited.add(label) + rule = self.get(label) + if rule is None: + package = label.removeprefix("//executorch/").split(":", 1)[0] + source = package + "/BUCK" + errors.add( + Issue( + "cannot validate", + label, + source, + ( + "unresolved scoped dependency: " + label + if is_scoped_label(label) + else "no supported declaration for " + label + ), + label, + severity=validation_severity(source), + ) + ) + continue + if rule.kind != "python_library": + continue + errors.update( + Issue( + "cannot validate", + label, + rule.declaration_file, + error, + error.split(":", 1)[0], + severity=validation_severity(rule.declaration_file), + ) + for error in rule.errors + ) + pending.extend(rule.deps) + return visited, errors + + +def python_imports( # noqa: C901 + source: str, filename: str, snapshot: Snapshot +) -> set[tuple[str, str | None]]: + results: set[tuple[str, str | None]] = set() + + def resolve(module: str) -> str | None: + if not module.startswith("executorch."): + return None + path = module.removeprefix("executorch.").replace(".", "/") + return next( + ( + candidate + for candidate in (path + ".py", path + "/__init__.py") + if candidate in snapshot.paths + ), + None, + ) + + class Imports(ast.NodeVisitor): + # ast.NodeVisitor dispatch requires these method names. + # pylint: disable=invalid-name + def visit_If(self, node: ast.If) -> None: + if dotted(node.test) in {"TYPE_CHECKING", "typing.TYPE_CHECKING"}: + for child in node.orelse: + self.visit(child) + else: + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + if alias.name.startswith("executorch."): + results.add((alias.name, resolve(alias.name))) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + module = node.module or "" + if node.level: + package = ["executorch"] + list(PurePosixPath(filename).parent.parts) + module = ".".join( + package[: len(package) - node.level + 1] + + ([module] if module else []) + ) + if module != "executorch" and not module.startswith("executorch."): + return + for alias in node.names: + child = module + "." + alias.name + child_file = resolve(child) + results.add( + (child, child_file) if child_file else (module, resolve(module)) + ) + + Imports().visit(ast.parse(source, filename=filename)) + return results + + +def is_test_file(source: str, filename: str) -> bool: + tree = ast.parse(source, filename=filename) + function_types = (ast.FunctionDef, ast.AsyncFunctionDef) + for node in tree.body: + if isinstance(node, function_types) and node.name.startswith("test_"): + if not any( + dotted(d.func if isinstance(d, ast.Call) else d).endswith("fixture") + for d in node.decorator_list + ): + return True + if isinstance(node, ast.ClassDef) and ( + node.name.startswith("Test") + or any(dotted(base).endswith("TestCase") for base in node.bases) + ): + if any( + isinstance(child, function_types) and child.name.startswith("test") + for child in node.body + ): + return True + return False + + +def registered_tests( # noqa: C901 + declarations: Declarations, +) -> tuple[set[str], list[str]]: + if (SCOPE, TEST_MACRO, "define_arm_tests") not in declarations.called_helpers: + return set(), [] + try: + tree = ast.parse(declarations.snapshot.read(TEST_MACRO), filename=TEST_MACRO) + except (KeyError, SyntaxError) as error: + return set(), [str(error)] + errors = [] + registrations: set[str] = set() + + def visit(statements: list[ast.stmt], env: dict[str, object]) -> None: + for node in statements: + if isinstance(node, ast.FunctionDef) and node.name == "define_arm_tests": + visit(node.body, env) + elif ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + try: + env[node.targets[0].id] = declarations.value(node.value, env, SCOPE) + except Unsupported as error: + env.pop(node.targets[0].id, None) + if node.targets[0].id == "test_files": + errors.append(str(error)) + elif ( + isinstance(node, ast.AugAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "test_files" + ): + try: + if not isinstance(node.op, ast.Add): + raise Unsupported(ast.unparse(node)) + env["test_files"] = env.get( + "test_files", [] + ) + declarations.strings(node.value, env, SCOPE) + except Unsupported as error: + errors.append(str(error)) + elif isinstance(node, ast.If): + try: + visit( + ( + node.body + if declarations.value(node.test, env, SCOPE) + else node.orelse + ), + env, + ) + except Unsupported as error: + if any( + isinstance(x, ast.Name) and x.id == "test_files" + for x in ast.walk(node) + ): + errors.append(str(error)) + elif isinstance(node, ast.For): + if any( + isinstance(x, ast.Name) + and x.id == "test_files" + and isinstance(x.ctx, ast.Store) + for x in ast.walk(node) + ): + errors.append("loop-modified test_files") + + # A test may intentionally be registered only for one build environment. + for is_oss in (False, True): + env = {"__is_oss__": is_oss} + visit(tree.body, env) + registrations.update(SCOPE + "/" + p for p in env.get("test_files", [])) + return registrations, errors + + +# pylint: disable-next=too-many-locals +def analyze(snapshot: Snapshot) -> set[Issue]: # noqa: C901 + declarations = Declarations(snapshot) + declarations.load_scope() + issues: set[Issue] = set() + libraries = [ + rule + for rule in declarations.rules.values() + if rule.kind == "python_library" and is_scoped_label(rule.label) + ] + for library in libraries: + reachable, incomplete = declarations.reachable(library) + issues.update(incomplete) + for filename in sorted(library.sources): + try: + imports = python_imports(snapshot.read(filename), filename, snapshot) + except (KeyError, SyntaxError) as error: + issues.add( + Issue("cannot validate", library.label, filename, str(error)) + ) + continue + for module, imported_file in imports: + if imported_file in library.sources: + continue + if imported_file is None: + issues.add( + Issue( + "cannot validate", + library.label, + filename, + "no in-repo source for " + module, + module, + "dependency", + validation_severity( + module.removeprefix("executorch.").replace(".", "/") + ), + ) + ) + continue + owners = declarations.owners(imported_file) + if owners & reachable: + continue + if not owners: + issues.add( + Issue( + "cannot validate", + library.label, + filename, + "no supported python_library provider for " + module, + module, + "dependency", + validation_severity(imported_file), + ) + ) + continue + detail = module + " requires " + " or ".join(sorted(owners)) + severity = "error" + if incomplete: + detail += "; unresolved dependency declarations: " + ", ".join( + f"{issue.owner} ({issue.source}: {issue.detail})" + for issue in sorted(incomplete) + ) + kind = "cannot validate" + if all(issue.severity == "warning" for issue in incomplete): + severity = "warning" + else: + kind = "missing dependency" + issues.add( + Issue( + kind, + library.label, + filename, + detail, + module, + "dependency", + severity, + ) + ) + registrations, errors = registered_tests(declarations) + for error in errors: + issues.add( + Issue( + "cannot validate", + "Arm test registration", + TEST_MACRO, + error, + error, + "registration", + ) + ) + for filename in sorted(snapshot.paths): + if filename.startswith(SCOPE + "/") and filename.endswith(".py"): + try: + actual_test = is_test_file(snapshot.read(filename), filename) + except SyntaxError as error: + issues.add( + Issue( + "cannot validate", "Arm test registration", filename, str(error) + ) + ) + continue + if actual_test and filename not in registrations: + # A nested package may declare its own ordinary Python test. + declarations.owners(filename) # Load its nearest build package. + explicitly_registered = any( + rule.kind in {"python_test", "python_pytest", "python_unittest"} + and filename in rule.sources + for rule in declarations.rules.values() + ) + if not explicitly_registered: + issues.add( + Issue( + "unregistered test", + "Arm test registration", + filename, + "not in define_arm_tests test_files or a supported explicit Python test target", + "", + "registration", + ) + ) + # Dependency traversal and owner discovery load packages lazily. + issues.update( + Issue( + "cannot validate", + package, + filename, + error, + error, + severity=validation_severity(filename), + ) + for package, errors in declarations.package_errors.items() + for filename, error in errors + ) + issues.update( + Issue( + "cannot validate", + rule.label, + rule.declaration_file, + error, + error.split(":", 1)[0], + severity=validation_severity(rule.declaration_file), + ) + for rule in declarations.rules.values() + if rule.kind == "python_library" + for error in rule.errors + ) + return issues + + +def compare(base: Snapshot, head: Snapshot) -> tuple[set[Issue], int]: + baseline, current = analyze(base), analyze(head) + previous = Declarations(base) + previous.load_scope() + present = Declarations(head) + present.load_scope() + for rule in previous.rules.values(): + if rule.kind != "python_library" or not is_scoped_label(rule.label): + continue + for source in rule.sources & head.paths: + present.owners(source) + if not any( + source in target.sources + and target.kind + in {"python_library", "python_test", "python_pytest", "python_unittest"} + for target in present.rules.values() + ): + current.add( + Issue( + "cannot validate", + rule.label, + source, + "previously declared Python source has no supported owner in the head snapshot", + "source ownership", + "ownership", + ) + ) + return current - baseline, len(current & baseline) + + +def print_report(regressions: set[Issue], unchanged: int) -> None: + errors = sum(issue.severity == "error" for issue in regressions) + status = "FAIL" if errors else "PASS" + print(f"Arm Python Buck declaration check: {status}") + print(f" New errors: {errors}") + print(f" New warnings: {len(regressions) - errors}") + print(f" Existing issues unchanged: {unchanged}") + wrapper = textwrap.TextWrapper( + width=100, break_long_words=False, break_on_hyphens=False + ) + for index, issue in enumerate(sorted(regressions), start=1): + print( + f"\n[{index}/{len(regressions)}] {issue.severity.upper()}: {issue.kind.capitalize()}" + ) + print(f" File: {issue.source}") + owner_label = "Target" if issue.owner.startswith("//") else "Scope" + print(f" {owner_label}: {issue.owner}") + detail, _, unresolved = issue.detail.partition( + "; unresolved dependency declarations: " + ) + wrapper.initial_indent = " Detail: " + wrapper.subsequent_indent = " " + print(wrapper.fill(detail)) + if unresolved: + print(" Unresolved dependency declarations:") + wrapper.initial_indent = " " + wrapper.subsequent_indent = " " + print(wrapper.fill(unresolved)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument( + "--base", required=True, help="explicit pre-change Git revision" + ) + parser.add_argument("--head", help="Git revision; omit to inspect the working tree") + args = parser.parse_args(argv) + try: + regressions, unchanged = compare( + GitSnapshot(args.repo.resolve(), args.base), + GitSnapshot(args.repo.resolve(), args.head), + ) + except (OSError, subprocess.CalledProcessError, ValueError) as error: + print(f"Cannot validate Arm Python Buck declarations: {error}", file=sys.stderr) + return 2 + print_report(regressions, unchanged) + return 1 if any(issue.severity == "error" for issue in regressions) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.ci/scripts/tests/test_arm_buck_declarations.py b/.ci/scripts/tests/test_arm_buck_declarations.py new file mode 100644 index 00000000000..e967d2aa758 --- /dev/null +++ b/.ci/scripts/tests/test_arm_buck_declarations.py @@ -0,0 +1,1138 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path +from unittest import mock + + +def load_checker(): + path = Path(__file__).resolve().parents[1] / "check_arm_buck_declarations.py" + spec = importlib.util.spec_from_file_location("check_arm_buck_declarations", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +checker = load_checker() +Declarations = checker.Declarations +Snapshot = checker.Snapshot +analyze = checker.analyze +compare = checker.compare +glob_match = checker.glob_match + +SCOPE = "backends/arm/test" +TESTER = SCOPE + "/tester/runner.py" +BUCK = SCOPE + "/BUCK" +TARGETS = SCOPE + "/targets.bzl" +ARM_TEST_REGISTRATION = ( + 'load(":targets.bzl", "define_arm_tests")\n' + "fbcode_target(_kind=define_arm_tests)\n" +) + + +def library(name, sources, deps="[]"): + return f"fbcode_target(_kind=runtime.python_library, name={name!r}, srcs={sources}, deps={deps})\n" + + +def fixture(): + return { + BUCK: library( + "arm_tester_lib", + 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', + '["//executorch/exir:umbrella"]', + ) + + ARM_TEST_REGISTRATION, + TESTER: "from executorch.exir.schema import Tensor\n", + TARGETS: 'def define_arm_tests():\n test_files = []\n test_files += native.glob(["passes/test_*.py"])\n', + "exir/BUCK": library("umbrella", "[]", '[":schema"]') + + library("schema", '["schema.py"]'), + "exir/schema.py": "class Tensor: pass\n", + "devtools/pte_tool/BUCK": library( + "diff_pte_lib", '["diff_pte.py"]', '["//executorch/exir:schema"]' + ) + + 'runtime.python_binary(name="diff_pte", srcs=["diff_pte.py"])\n', + "devtools/pte_tool/diff_pte.py": "class PTEDiffResult: pass\n", + } + + +def regressions(before, after): + return compare(Snapshot(before), Snapshot(after))[0] + + +# Each test covers an independent declaration regression. +# pylint: disable-next=too-many-public-methods +class DeclarationTests(unittest.TestCase): + def test_known_missing_import_and_registration_then_metadata_fix(self): + before = fixture() + after = dict(before) + after[ + TESTER + ] += "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + after[SCOPE + "/tester/pte_diff_utils.py"] = ( + "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + "from executorch.exir.schema import Tensor\n" + ) + test = SCOPE + "/misc/test_diff_pte.py" + after[test] = "def test_diff_pte():\n assert True\n" + issues = regressions(before, after) + self.assertEqual(3, len(issues)) + self.assertEqual(2, sum(i.kind == "missing dependency" for i in issues)) + self.assertTrue( + any(i.kind == "unregistered test" and i.source == test for i in issues) + ) + self.assertTrue( + all( + "diff_pte_lib" in i.detail + for i in issues + if i.kind == "missing dependency" + ) + ) + after[BUCK] = ( + library( + "arm_tester_lib", + 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', + '["//executorch/exir:umbrella", "//executorch/devtools/pte_tool:diff_pte_lib"]', + ) + + ARM_TEST_REGISTRATION + ) + after[TARGETS] += ' test_files += ["misc/test_diff_pte.py"]\n' + self.assertEqual(set(), regressions(before, after)) + + def test_transitive_dependency_and_cycle_are_valid(self): + files = fixture() + files["exir/BUCK"] += library("cycle", "[]", '[":umbrella"]') + files["exir/BUCK"] = files["exir/BUCK"].replace( + 'deps=[":schema"]', 'deps=[":schema", ":cycle"]' + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_dependency_deletion_without_python_change_fails(self): + before = fixture() + after = dict(before) + after[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + self.assertIn("//executorch/exir:schema", next(iter(issues)).detail) + + def test_existing_gaps_do_not_mask_different_new_gap(self): + before = fixture() + before[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + after = dict(before) + after[ + TESTER + ] += "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + self.assertIn("diff_pte_lib", next(iter(issues)).detail) + + def test_python_binary_is_not_an_import_provider(self): + files = fixture() + files[TESTER] = ( + "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + ) + files["devtools/pte_tool/BUCK"] = ( + 'runtime.python_binary(name="diff_pte", srcs=["diff_pte.py"])\n' + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.kind == "cannot validate" + and "no supported python_library provider" in i.detail + for i in issues + ) + ) + + def test_internal_branch_is_not_union_of_incompatible_deps(self): + files = fixture() + files[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/*.py"])', + '["//executorch/exir:umbrella"] if runtime.is_oss else []', + ) + issues = analyze(Snapshot(files)) + self.assertTrue(any(i.kind == "missing dependency" for i in issues)) + files[BUCK] = files[BUCK].replace( + "if runtime.is_oss else", "if not runtime.is_oss else" + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_unknown_deps_fail_instead_of_becoming_empty(self): + before = fixture() + after = dict(before) + after[BUCK] = library( + "arm_tester_lib", 'glob(["tester/*.py"])', "choose_dependencies()" + ) + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" and "choose_dependencies" in i.detail + for i in issues + ) + ) + + def test_unknown_sources_fail_instead_of_disappearing(self): + before = fixture() + after = dict(before) + after[BUCK] = library("arm_tester_lib", "choose_sources()") + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" and "choose_sources" in i.detail + for i in issues + ) + ) + + def test_unknown_conditional_rule_fails(self): + before = fixture() + after = dict(before) + after[BUCK] += ( + 'if native.read_config("mode", "value"):\n' + ' runtime.python_library(name="conditional", srcs=["extra.py"])\n' + ) + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" and "conditional declaration" in i.detail + for i in issues + ) + ) + + def test_helpers_are_not_registered_by_filename_alone(self): + before = fixture() + after = dict(before) + after[SCOPE + "/misc/test_helpers.py"] = "def make_payload():\n return b''\n" + self.assertEqual(set(), regressions(before, after)) + after[ + SCOPE + "/misc/test_helpers.py" + ] += "class TestPayload:\n def test_payload(self):\n pass\n" + self.assertTrue( + any(i.kind == "unregistered test" for i in regressions(before, after)) + ) + + def test_unittest_class_and_glob_registered_pytest_function(self): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_operation():\n pass\n" + ) + files[SCOPE + "/misc/unusual_name.py"] = ( + "import unittest\nclass Case(unittest.TestCase):\n def test_stuff(self):\n pass\n" + ) + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertEqual(SCOPE + "/misc/unusual_name.py", next(iter(issues)).source) + + def test_type_checking_only_import_is_not_runtime_requirement(self): + files = fixture() + files[ + TESTER + ] += "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from executorch.missing import Whatever\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_glob_exclude_and_relative_intralibrary_import(self): + files = fixture() + files[TESTER] += "from .helper import answer\n" + files[SCOPE + "/tester/helper.py"] = "answer = 42\n" + files[SCOPE + "/tester/excluded.py"] = "from executorch.missing import Nope\n" + self.assertEqual(set(), analyze(Snapshot(files))) + self.assertFalse(glob_match("tester/sub/deep.py", "tester/*.py")) + self.assertTrue(glob_match("tester/helper.py", "tester/**/*.py")) + self.assertTrue(glob_match("tester/sub/deep.py", "tester/**/*.py")) + + def test_from_package_submodule_import(self): + files = fixture() + files[TESTER] = "from executorch.exir import schema\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_only_called_local_declaration_helpers_create_providers(self): + files = fixture() + files["exir/BUCK"] = ( + 'load(":targets.bzl", "define_common_targets")\ndefine_common_targets()\n' + ) + files["exir/targets.bzl"] = ( + "def define_common_targets():\n " + + library("umbrella", "[]", '[":schema"]') + + " " + + library("schema", '["schema.py"]') + + "def define_unused():\n " + + library("ghost", '["ghost.py"]') + ) + files["exir/ghost.py"] = "x = 1\n" + declarations = Declarations(Snapshot(files)) + self.assertIsNone(declarations.get("//executorch/exir:ghost")) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_boolean_expressions_short_circuit_and_preserve_operand_values(self): + declarations = Declarations(Snapshot({})) + for expression, expected in ( + ("runtime.is_oss and unsupported()", False), + ("not runtime.is_oss or unsupported()", True), + ('[] or ["schema.py"]', ["schema.py"]), + ('["schema.py"] and []', []), + ('[] or False or ["schema.py"]', ["schema.py"]), + ): + with self.subTest(expression=expression): + self.assertEqual( + expected, + declarations.value( + checker.ast.parse(expression, mode="eval").body, {}, SCOPE + ), + ) + for expression in ("False or unsupported()", "True and unsupported()"): + with self.subTest(expression=expression): + with self.assertRaisesRegex(checker.Unsupported, "unsupported"): + declarations.value( + checker.ast.parse(expression, mode="eval").body, {}, SCOPE + ) + + def test_short_circuit_condition_does_not_reject_valid_declarations(self): + files = fixture() + files[BUCK] = ( + 'if not runtime.is_oss or native.read_config("mode", "value"):\n' + + " " + + files[BUCK] + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_only_active_local_or_imported_helpers_create_providers(self): + for imported in (False, True): + for condition, active in ( + ("runtime.is_oss", False), + ("not runtime.is_oss", True), + ): + with self.subTest(imported=imported, condition=condition): + files = fixture() + helper = "def define_schema():\n " + library( + "schema", '["schema.py"]' + ) + if imported: + files["exir/BUCK"] = ( + 'load(":targets.bzl", declare="define_schema")\n' + ) + files["exir/targets.bzl"] = helper + call = "declare()" + else: + files["exir/BUCK"] = helper + call = "define_schema()" + files["exir/BUCK"] += ( + library("umbrella", "[]", '[":schema"]') + + f"if {condition}:\n {call}\n" + ) + declarations = Declarations(Snapshot(files)) + self.assertEqual( + active, declarations.get("//executorch/exir:schema") is not None + ) + issues = analyze(Snapshot(files)) + if active: + self.assertEqual(set(), issues) + else: + self.assertTrue( + any( + "no supported python_library provider" in issue.detail + for issue in issues + ) + ) + + def test_inactive_helpers_do_not_add_spurious_declaration_errors(self): + for imported in (False, True): + with self.subTest(imported=imported): + files = fixture() + helper = "def define_oss():\n " + library( + "oss", "unsupported_sources()" + ) + if imported: + files[BUCK] += 'load(":helpers.bzl", "define_oss")\n' + files[SCOPE + "/helpers.bzl"] = helper + else: + files[BUCK] += helper + files[BUCK] += "if runtime.is_oss:\n define_oss()\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_nested_local_helper_calls_preserve_internal_branch(self): + files = fixture() + files["exir/BUCK"] = ( + "def define_schema():\n " + + library("schema", '["schema.py"]') + + "def define_all():\n " + + library("umbrella", "[]", '[":schema"]') + + " if not runtime.is_oss:\n define_schema()\n" + + "fbcode_target(_kind=define_all)\n" + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_parameterized_helpers_fail_clearly_only_when_called(self): + for imported in (False, True): + for condition, active in ( + ("runtime.is_oss", False), + ("not runtime.is_oss", True), + ): + with self.subTest(imported=imported, condition=condition): + before = fixture() + after = dict(before) + helper = ( + "def declare(name, srcs):\n" + + " runtime.python_library(name=name, srcs=srcs)\n" + ) + if imported: + after[BUCK] += 'load(":helpers.bzl", "declare")\n' + after[SCOPE + "/helpers.bzl"] = helper + else: + after[BUCK] += helper + after[ + BUCK + ] += f'if {condition}:\n declare("extra", ["extra.py"])\n' + after[SCOPE + "/extra.py"] = ( + "from executorch.missing import value\n" + ) + issues = regressions(before, after) + if active: + self.assertTrue( + any( + issue.kind == "cannot validate" + and "parameterized declaration helper: declare" + in issue.detail + for issue in issues + ) + ) + else: + self.assertEqual(set(), issues) + + def test_internal_wrapper_selection_applies_to_helpers(self): + for imported in (False, True): + for wrapper, active in ( + ("fbcode_target", True), + ("non_fbcode_target", False), + ): + with self.subTest(imported=imported, wrapper=wrapper): + files = fixture() + helper = "def define_extra():\n " + library( + "extra", '["extra.py"]' + ) + if imported: + files[BUCK] += 'load(":helpers.bzl", "define_extra")\n' + files[SCOPE + "/helpers.bzl"] = helper + else: + files[BUCK] += helper + files[BUCK] += f"{wrapper}(_kind=define_extra)\n" + declarations = Declarations(Snapshot(files)) + self.assertEqual( + active, + declarations.get("//executorch/" + SCOPE + ":extra") + is not None, + ) + + def test_parameterized_helper_definition_is_rejected_without_arguments(self): + for imported in (False, True): + for parameters in ('name="extra"', "*names", "**options"): + with self.subTest(imported=imported, parameters=parameters): + files = fixture() + helper = f"def define_extra({parameters}):\n " + library( + "extra", '["extra.py"]' + ) + if imported: + files[BUCK] += 'load(":helpers.bzl", "define_extra")\n' + files[SCOPE + "/helpers.bzl"] = helper + else: + files[BUCK] += helper + files[BUCK] += "define_extra()\n" + self.assertTrue( + any( + issue.kind == "cannot validate" + and "parameterized declaration helper: define_extra" + in issue.detail + for issue in analyze(Snapshot(files)) + ) + ) + + def test_recursive_helper_fails_clearly(self): + files = fixture() + files[BUCK] += "def define_extra():\n define_extra()\ndefine_extra()\n" + self.assertTrue( + any( + issue.kind == "cannot validate" + and "recursive declaration helper: define_extra" in issue.detail + for issue in analyze(Snapshot(files)) + ) + ) + + def test_unknown_helper_condition_fails_instead_of_inventing_provider(self): + files = fixture() + files[BUCK] += ( + "def define_extra():\n " + + library("extra", '["extra.py"]') + + 'if native.read_config("mode", "value"):\n define_extra()\n' + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + issue.kind == "cannot validate" + and "conditional declaration" in issue.detail + for issue in issues + ) + ) + + def test_unresolvable_new_inrepo_module_fails_clearly(self): + before = fixture() + after = dict(before) + after[TESTER] += "from executorch.generated_unknown import value\n" + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + self.assertEqual("cannot validate", next(iter(issues)).kind) + + def test_glob_does_not_cross_buck_package(self): + files = fixture() + files[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/**/*.py"])', + '["//executorch/exir:umbrella"]', + ) + files[SCOPE + "/tester/nested/BUCK"] = library("other", '["module.py"]') + nested_source = SCOPE + "/tester/nested/module.py" + files[nested_source] = "from executorch.missing import Nope\n" + declarations = Declarations(Snapshot(files)) + root = declarations.get("//executorch/" + SCOPE + ":arm_tester_lib") + self.assertNotIn(nested_source, root.sources) + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertEqual( + "//executorch/" + SCOPE + "/tester/nested:other", next(iter(issues)).owner + ) + + def test_standalone_nested_library_dependency_is_checked(self): + for build_file in ("BUCK", "TARGETS"): + with self.subTest(build_file=build_file): + before = fixture() + after = dict(before) + package = SCOPE + "/standalone" + source = package + "/module.py" + after[package + "/" + build_file] = library("lib", '["module.py"]') + after[source] = "from executorch.exir.schema import Tensor\n" + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("missing dependency", issue.kind) + self.assertEqual("//executorch/" + package + ":lib", issue.owner) + self.assertEqual(source, issue.source) + after[package + "/" + build_file] = library( + "lib", '["module.py"]', '["//executorch/exir:schema"]' + ) + self.assertEqual(set(), regressions(before, after)) + + def test_nested_helper_libraries_keep_relative_package_dependencies(self): + files = fixture() + package = SCOPE + "/standalone/nested" + files[package + "/BUCK"] = ( + 'load(":targets.bzl", "define_libs")\ndefine_libs()\n' + ) + files[package + "/targets.bzl"] = ( + "def define_libs():\n " + + library("lib", '["module.py"]', '[":schema"]') + + " " + + library("schema", '["schema.py"]') + ) + files[package + "/module.py"] = "from .schema import Tensor\n" + files[package + "/schema.py"] = "class Tensor: pass\n" + self.assertEqual(set(), analyze(Snapshot(files))) + files[package + "/targets.bzl"] = files[package + "/targets.bzl"].replace( + 'deps=[":schema"]', "deps=[]" + ) + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertIn("//executorch/" + package + ":schema", next(iter(issues)).detail) + + def test_nested_declaration_errors_identify_the_owning_file(self): + declarations = ( + (library("lib", "choose_sources()"), "srcs: choose_sources"), + ( + 'def declare(name):\n pass\ndeclare("lib")\n', + "parameterized declaration helper", + ), + ("runtime.python_library(", "cannot read"), + ) + for build_file in ("BUCK", "TARGETS"): + for definition, expected in declarations: + with self.subTest(build_file=build_file, expected=expected): + files = fixture() + filename = SCOPE + "/standalone/" + build_file + files[filename] = definition + issues = analyze(Snapshot(files)) + relevant = [issue for issue in issues if expected in issue.detail] + self.assertTrue(relevant) + self.assertTrue(all(issue.source == filename for issue in relevant)) + + def test_nested_helper_errors_identify_the_helper_file(self): + files = fixture() + package = SCOPE + "/standalone" + files[package + "/TARGETS"] = ( + 'load(":helpers.bzl", "define_libs")\ndefine_libs()\n' + ) + helper = package + "/helpers.bzl" + files[helper] = "def define_libs():\n " + library("lib", "choose_sources()") + issues = analyze(Snapshot(files)) + self.assertTrue(issues) + self.assertTrue(all(issue.source == helper for issue in issues)) + files[helper] = "def define_libs():\n declare(\n" + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertEqual(helper, next(iter(issues)).source) + + def test_similar_sibling_package_is_outside_arm_test_scope(self): + files = fixture() + package = SCOPE + "_other" + files[package + "/BUCK"] = library("lib", "choose_sources()") + files[package + "/module.py"] = "from executorch.missing import value\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_nested_baseline_gap_does_not_hide_a_different_new_gap(self): + before = fixture() + package = SCOPE + "/standalone" + before[package + "/BUCK"] = library("lib", '["module.py"]') + before[package + "/module.py"] = "from executorch.exir.schema import Tensor\n" + self.assertEqual((set(), 1), compare(Snapshot(before), Snapshot(before))) + after = dict(before) + after[ + package + "/module.py" + ] += "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + issues, unchanged = compare(Snapshot(before), Snapshot(after)) + self.assertEqual(1, unchanged) + self.assertEqual(1, len(issues)) + self.assertIn("diff_pte_lib", next(iter(issues)).detail) + + def test_deleted_nested_owner_with_retained_source_fails(self): + before = fixture() + package = SCOPE + "/standalone" + source = package + "/module.py" + before[package + "/TARGETS"] = library("lib", '["module.py"]') + before[source] = "value = 42\n" + after = dict(before) + del after[package + "/TARGETS"] + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("ownership", issue.category) + self.assertEqual(source, issue.source) + del after[source] + self.assertEqual(set(), regressions(before, after)) + + def test_loop_generated_library_is_not_silently_ignored(self): + before = fixture() + after = dict(before) + after[ + BUCK + ] += 'for name in ["extra"]:\n runtime.python_library(name=name, srcs=["extra.py"])\n' + after[SCOPE + "/extra.py"] = "from executorch.missing import value\n" + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" + and "unsupported declaration loop" in i.detail + for i in issues + ) + ) + + def test_known_pytest_generation_loop_is_handled_by_registration(self): + files = fixture() + files[ + TARGETS + ] += " for test_file in test_files:\n python_pytest(name=test_file, srcs=[test_file])\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_oss_only_test_registration_is_valid(self): + before = fixture() + after = dict(before) + after[SCOPE + "/misc/test_oss.py"] = "def test_oss():\n pass\n" + after[ + TARGETS + ] += ' if runtime.is_oss:\n test_files += ["misc/test_oss.py"]\n' + self.assertEqual(set(), regressions(before, after)) + + def test_provider_suggestion_change_is_not_new_violation(self): + before = fixture() + before[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + after = dict(before) + after["exir/BUCK"] += library("schema_alternative", '["schema.py"]') + self.assertNotEqual( + {i.detail for i in analyze(Snapshot(before))}, + {i.detail for i in analyze(Snapshot(after))}, + ) + self.assertEqual(set(), regressions(before, after)) + + def test_diagnostic_category_change_only_adds_external_warnings(self): + before = fixture() + before[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + after = dict(before) + after[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/*.py"])', + '["//executorch/internal:unavailable"]', + ) + issues = regressions(before, after) + self.assertTrue(issues) + self.assertTrue(all(i.severity == "warning" for i in issues)) + + def test_lazy_external_package_errors_keep_the_original_cause(self): + declarations = ( + ( + 'for name in ["schema"]:\n' + ' runtime.python_library(name=name, srcs=["schema.py"])\n', + "unsupported declaration loop", + ), + ( + "def declare(name):\n" + ' runtime.python_library(name=name, srcs=["schema.py"])\n' + 'declare("schema")\n', + "parameterized", + ), + ( + "if configured():\n" + ' runtime.python_library(name="schema", srcs=["schema.py"])\n', + "conditional", + ), + ) + for declaration, cause in declarations: + for through_dependency in (False, True): + with self.subTest(cause=cause, through_dependency=through_dependency): + files = fixture() + files["exir/BUCK"] = declaration + files[BUCK] = library( + "arm_tester_lib", + '["tester/runner.py"]', + '["//executorch/exir:schema"]' if through_dependency else "[]", + ) + issues = analyze(Snapshot(files)) + self.assertTrue(all(i.severity == "warning" for i in issues)) + self.assertTrue( + any( + i.source == "exir/BUCK" and cause in i.detail + for i in issues + ), + issues, + ) + + def test_lazy_external_helper_error_names_helper_file(self): + files = fixture() + files["exir/BUCK"] = 'load(":rules.bzl", "declare")\ndeclare()\n' + files["exir/rules.bzl"] = ( + "def declare():\n" + " if configured():\n" + ' runtime.python_library(name="schema", srcs=["schema.py"])\n' + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.source == "exir/rules.bzl" + and "conditional" in i.detail + and i.severity == "warning" + for i in issues + ), + issues, + ) + self.assertTrue(all(i.severity == "warning" for i in issues)) + + def test_computed_external_sources_report_the_original_expression(self): + for through_dependency in (False, True): + with self.subTest(through_dependency=through_dependency): + files = fixture() + files["exir/BUCK"] = library("schema", "choose_sources()") + files[BUCK] = library( + "arm_tester_lib", + '["tester/runner.py"]', + '["//executorch/exir:schema"]' if through_dependency else "[]", + ) + issues = analyze(Snapshot(files)) + self.assertTrue(all(i.severity == "warning" for i in issues)) + self.assertTrue( + any( + i.source == "exir/BUCK" and "choose_sources()" in i.detail + for i in issues + ), + issues, + ) + + def test_computed_external_dependencies_do_not_cascade_to_errors(self): + before = fixture() + after = dict(before) + after["exir/BUCK"] = library("umbrella", "[]", "choose_deps()") + library( + "schema", '["schema.py"]' + ) + issues = regressions(before, after) + self.assertTrue(issues) + self.assertTrue(all(i.severity == "warning" for i in issues)) + self.assertTrue( + any(i.source == "exir/BUCK" and "choose_deps()" in i.detail for i in issues) + ) + self.assertTrue( + any( + i.source == TESTER and "exir/BUCK: deps: choose_deps()" in i.detail + for i in issues + ) + ) + + def test_known_missing_dependency_still_fails_with_external_package_warning(self): + files = fixture() + files[BUCK] = library("arm_tester_lib", '["tester/runner.py"]') + files["exir/BUCK"] += ( + 'for name in ["other"]:\n' + " runtime.python_library(name=name, srcs=[])\n" + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.kind == "missing dependency" and i.severity == "error" for i in issues + ) + ) + self.assertTrue( + any(i.source == "exir/BUCK" and i.severity == "warning" for i in issues) + ) + + def test_unsupported_arm_provider_outside_test_scope_still_fails(self): + files = fixture() + files[TESTER] = "from executorch.backends.arm.provider import value\n" + files["backends/arm/BUCK"] = library("provider", "choose_sources()") + files["backends/arm/provider.py"] = "value = 1\n" + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.source == "backends/arm/BUCK" + and "choose_sources()" in i.detail + and i.severity == "error" + for i in issues + ) + ) + + def test_unrelated_external_packages_are_not_reported(self): + files = fixture() + files["unrelated/BUCK"] = ( + 'for name in ["other"]:\n' + " runtime.python_library(name=name, srcs=[])\n" + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_unchanged_external_warnings_are_baselined(self): + files = fixture() + files["exir/BUCK"] = library("umbrella", "[]", "choose_deps()") + library( + "schema", '["schema.py"]' + ) + new, unchanged = compare(Snapshot(files), Snapshot(files)) + self.assertEqual(set(), new) + self.assertGreater(unchanged, 0) + + def test_warning_becoming_proven_missing_dependency_is_new_error(self): + before = fixture() + before["exir/BUCK"] = library("umbrella", "[]", "choose_deps()") + library( + "schema", '["schema.py"]' + ) + after = dict(before) + after["exir/BUCK"] = library("umbrella", "[]") + library( + "schema", '["schema.py"]' + ) + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("missing dependency", issue.kind) + self.assertEqual("error", issue.severity) + self.assertEqual("executorch.exir.schema", issue.subject) + + def test_cli_reports_warnings_without_failing_and_still_fails_errors(self): + before = fixture() + for add_arm_error in (False, True): + with self.subTest(add_arm_error=add_arm_error): + after = dict(before) + after["exir/BUCK"] = library( + "umbrella", "[]", "choose_deps()" + ) + library("schema", '["schema.py"]') + if add_arm_error: + after[SCOPE + "/misc/test_new.py"] = "def test_new(): pass\n" + output = io.StringIO() + with mock.patch.object( + checker, + "GitSnapshot", + side_effect=[Snapshot(before), Snapshot(after)], + ), contextlib.redirect_stdout(output): + status = checker.main(["--repo", ".", "--base", "base"]) + self.assertEqual(1 if add_arm_error else 0, status) + report = output.getvalue() + self.assertIn("FAIL" if add_arm_error else "PASS", report) + self.assertIn( + "New errors: 1" if add_arm_error else "New errors: 0", report + ) + self.assertIn("New warnings: 2", report) + self.assertIn("WARNING: Cannot validate", report) + self.assertIn("File: exir/BUCK", report) + self.assertIn("choose_deps()", report) + if add_arm_error: + self.assertIn("ERROR: Unregistered test", report) + + def test_removing_macro_invocation_unregisters_existing_suite(self): + before = fixture() + test = SCOPE + "/passes/test_registered.py" + before[test] = "def test_registered(): pass\n" + self.assertEqual(set(), analyze(Snapshot(before))) + after = dict(before) + after[BUCK] = after[BUCK].replace("fbcode_target(_kind=define_arm_tests)\n", "") + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("unregistered test", issue.kind) + self.assertEqual(test, issue.source) + self.assertEqual("error", issue.severity) + + def test_inactive_registration_calls_do_not_register_tests(self): + for invocation in ( + "", + "if False:\n define_arm_tests()\n", + "def unused():\n define_arm_tests()\n", + "non_fbcode_target(_kind=define_arm_tests)\n", + ): + with self.subTest(invocation=invocation): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_registered(): pass\n" + ) + files[BUCK] = files[BUCK].replace( + "fbcode_target(_kind=define_arm_tests)\n", invocation + ) + self.assertTrue( + any(i.kind == "unregistered test" for i in analyze(Snapshot(files))) + ) + + def test_alias_and_called_wrappers_register_tests(self): + for registration in ( + 'load(":targets.bzl", register="define_arm_tests")\nregister()\n', + 'load(":targets.bzl", register="define_arm_tests")\nfbcode_target(_kind=register)\n', + 'load(":targets.bzl", "define_arm_tests")\n' + "def wrapper():\n define_arm_tests()\nwrapper()\n", + ): + with self.subTest(registration=registration): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_registered(): pass\n" + ) + files[BUCK] = files[BUCK].replace(ARM_TEST_REGISTRATION, registration) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_parameterized_registration_is_not_counted(self): + for change_signature in (False, True): + with self.subTest(change_signature=change_signature): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_registered(): pass\n" + ) + if change_signature: + files[TARGETS] = files[TARGETS].replace( + "define_arm_tests()", "define_arm_tests(config)" + ) + else: + files[BUCK] = files[BUCK].replace( + "_kind=define_arm_tests)", + "_kind=define_arm_tests, config=True)", + ) + issues = analyze(Snapshot(files)) + self.assertTrue(any(i.kind == "unregistered test" for i in issues)) + self.assertTrue( + any( + i.kind == "cannot validate" + and "parameterized declaration helper" in i.detail + for i in issues + ) + ) + + def test_dormant_registration_macro_is_not_evaluated(self): + files = fixture() + files[BUCK] = files[BUCK].replace("fbcode_target(_kind=define_arm_tests)\n", "") + files[TARGETS] = "def define_arm_tests():\n test_files = choose_tests()\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_root_package_import_matches_dotted_import_dependencies(self): + for source in ( + "import executorch.exir\n", + "from executorch import exir\n", + "from executorch import exir as export_ir\n", + ): + with self.subTest(source=source): + files = fixture() + files["exir/__init__.py"] = "" + files["exir/BUCK"] += library("lib", '["__init__.py"]') + files[TESTER] = source + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("missing dependency", issue.kind) + self.assertEqual("executorch.exir", issue.subject) + self.assertEqual("error", issue.severity) + files[BUCK] = files[BUCK].replace( + '["//executorch/exir:umbrella"]', + '["//executorch/exir:umbrella", "//executorch/exir:lib"]', + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_root_package_import_checks_each_alias_and_skips_type_only(self): + files = fixture() + files["exir/__init__.py"] = "" + files["exir/BUCK"] += library("lib", '["__init__.py"]') + files["devtools/__init__.py"] = "" + files["devtools/BUCK"] = library("lib", '["__init__.py"]') + files[TESTER] = "from executorch import exir, devtools as tools\n" + issues = analyze(Snapshot(files)) + self.assertEqual( + {"executorch.exir", "executorch.devtools"}, {i.subject for i in issues} + ) + self.assertTrue(all(i.kind == "missing dependency" for i in issues)) + files[TESTER] = "if TYPE_CHECKING:\n from executorch import exir, devtools\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_augmented_sources_and_dependencies_are_evaluated(self): + files = fixture() + helper = SCOPE + "/tester/helper.py" + files[helper] = ( + "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + ) + declarations = ( + 'sources = ["tester/runner.py"]\n' + 'sources += ["tester/helper.py"]\n' + 'deps = ["//executorch/exir:umbrella"]\n' + ) + files[BUCK] = declarations + library("arm_tester_lib", "sources", "deps") + issues = analyze(Snapshot(files)) + self.assertTrue( + any(i.kind == "missing dependency" and i.source == helper for i in issues) + ) + files[BUCK] = ( + declarations + + 'deps += ["//executorch/devtools/pte_tool:diff_pte_lib"]\n' + + library("arm_tester_lib", "sources", "deps") + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_unknown_augmented_values_invalidate_previous_declarations(self): + for attribute, initial in ( + ("srcs", '["tester/runner.py"]'), + ("deps", '["//executorch/exir:umbrella"]'), + ): + with self.subTest(attribute=attribute): + files = fixture() + values = { + "srcs": '["tester/runner.py"]', + "deps": '["//executorch/exir:umbrella"]', + } + values[attribute] = "values" + files[BUCK] = ( + f"values = {initial}\n" + "values += choose_values()\n" + + library("arm_tester_lib", values["srcs"], values["deps"]) + ) + declarations = Declarations(Snapshot(files)) + declarations.load_scope() + rule = declarations.rules["//executorch/" + SCOPE + ":arm_tester_lib"] + self.assertTrue(any(e.startswith(attribute + ":") for e in rule.errors)) + self.assertEqual( + set(), rule.sources if attribute == "srcs" else rule.deps + ) + self.assertTrue( + any( + i.kind == "cannot validate" + and i.severity == "error" + and i.source == BUCK + and "values += choose_values()" in i.detail + for i in analyze(Snapshot(files)) + ) + ) + + def test_unsupported_augmented_operator_invalidates_previous_dependencies(self): + files = fixture() + files[BUCK] = 'deps = ["//executorch/exir:umbrella"]\n' "deps *= 0\n" + library( + "arm_tester_lib", '["tester/runner.py"]', "deps" + ) + declarations = Declarations(Snapshot(files)) + declarations.load_scope() + rule = declarations.rules["//executorch/" + SCOPE + ":arm_tester_lib"] + self.assertEqual(set(), rule.deps) + self.assertTrue( + any( + i.kind == "cannot validate" + and i.source == BUCK + and "deps *= 0" in i.detail + for i in analyze(Snapshot(files)) + ) + ) + + def test_augmented_subscript_and_incompatible_types_are_not_ignored(self): + for mutation in ('deps[0] += ":other"', "deps += 1"): + with self.subTest(mutation=mutation): + files = fixture() + files[BUCK] = ( + 'deps = ["//executorch/exir:umbrella"]\n' + + mutation + + "\n" + + library("arm_tester_lib", '["tester/runner.py"]', "deps") + ) + declarations = Declarations(Snapshot(files)) + declarations.load_scope() + rule = declarations.rules["//executorch/" + SCOPE + ":arm_tester_lib"] + self.assertEqual(set(), rule.deps) + self.assertTrue( + any( + i.kind == "cannot validate" + and i.source == BUCK + and "unsupported augmented assignment" in i.detail + for i in analyze(Snapshot(files)) + ) + ) + + def test_nested_explicit_python_test_registration_is_valid(self): + for kind in ("python_test", "python_pytest", "python_unittest"): + with self.subTest(kind=kind): + before = fixture() + after = dict(before) + after[SCOPE + "/unit/test_unit.py"] = "def test_unit():\n pass\n" + after[SCOPE + "/unit/BUCK"] = ( + f'runtime.{kind}(name="test_unit", srcs=["test_unit.py"], deps=[])\n' + ) + self.assertEqual(set(), regressions(before, after)) + + def test_deleted_owner_with_retained_source_fails(self): + before = fixture() + for retain_aggregator in (False, True): + with self.subTest(retain_aggregator=retain_aggregator): + after = dict(before) + after[BUCK] = ( + library("arm_tester", "[]", '[":arm_tester_lib"]') + if retain_aggregator + else "" + ) + issues = regressions(before, after) + self.assertTrue( + any( + i.category == "ownership" and i.source == TESTER for i in issues + ) + ) + if retain_aggregator: + self.assertTrue( + any("unresolved scoped dependency" in i.detail for i in issues) + ) + + def test_deleted_owner_sources_and_edges_is_valid(self): + before = fixture() + before[BUCK] += library("arm_tester", "[]", '[":arm_tester_lib"]') + after = dict(before) + after[BUCK] = "" + del after[TESTER] + self.assertEqual(set(), regressions(before, after)) + + def test_library_rename_with_retained_sources_is_valid(self): + before = fixture() + after = dict(before) + after[BUCK] = after[BUCK].replace("arm_tester_lib", "renamed_lib") + self.assertEqual(set(), regressions(before, after)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/tests/test_arm_buck_entrypoint.py b/.ci/scripts/tests/test_arm_buck_entrypoint.py new file mode 100644 index 00000000000..d38676ec3bd --- /dev/null +++ b/.ci/scripts/tests/test_arm_buck_entrypoint.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Integration checks for the toolchain-free Arm Buck declaration entrypoint.""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ENTRYPOINT = ( + Path(__file__).resolve().parents[3] / "backends/arm/test/test_arm_backend.sh" +) + + +class ArmBuckEntrypointTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.repo = self.root / "repo with spaces" + self.repo.mkdir() + self.git("init", "-q") + self.git("config", "user.name", "Test") + self.git("config", "user.email", "test@example.com") + self.git("config", "commit.gpgsign", "false") + self.git("config", "core.hooksPath", str(self.root / "no-hooks")) + self.script = self.repo / "backends/arm/test/test_arm_backend.sh" + self.script.parent.mkdir(parents=True) + shutil.copyfile(ENTRYPOINT, self.script) + self.checker_dir = self.repo / ".ci/scripts" + tests = self.checker_dir / "tests" + tests.mkdir(parents=True) + (self.checker_dir / "pytest-parallelism.sh").write_text("") + (tests / "test_arm_buck_declarations.py").write_text( + "import os, unittest\n" + "class CheckerTest(unittest.TestCase):\n" + " def test_checker(self):\n" + " self.assertNotIn('FAIL_CHECKER_SELF_TEST', os.environ)\n" + ) + (self.checker_dir / "check_arm_buck_declarations.py").write_text( + "import json, os, pathlib, sys\n" + "pathlib.Path(os.environ['ENTRYPOINT_RESULT']).write_text(json.dumps({\n" + " 'base': sys.argv[sys.argv.index('--base') + 1],\n" + " 'cwd': os.getcwd(),\n" + "}))\n" + "sys.exit(int(os.environ.get('CHECKER_EXIT_CODE', '0')))\n" + ) + self.initial = self.commit("initial") + self.base = self.commit("base") + self.head = self.commit("head") + self.result_file = self.root / "result.json" + + def git(self, *args): + return subprocess.check_output( + ["git", *args], cwd=self.repo, text=True, stderr=subprocess.PIPE + ).strip() + + def commit(self, message): + (self.repo / "revision").write_text(message) + self.git("add", ".") + self.git("commit", "-qm", message) + return self.git("rev-parse", "HEAD") + + def run_suite(self, *args, **overrides): + env = os.environ.copy() + for key in ("BASE_REVISION", "GITHUB_EVENT_NAME", "PYTHONPATH"): + env.pop(key, None) + env.update( + PYTHON_EXECUTABLE=sys.executable, + ENTRYPOINT_RESULT=str(self.result_file), + ) + env.update(overrides) + if self.result_file.exists(): + self.result_file.unlink() + # A caller may invoke the entrypoint from outside the repository. + result = subprocess.run( + ["bash", str(self.script), *args], + cwd=self.root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=30, + ) + return result + + def assert_base(self, expected, **env): + result = self.run_suite("test_buck_declarations", **env) + self.assertEqual(result.returncode, 0, result.stdout) + actual = json.loads(self.result_file.read_text()) + self.assertEqual(actual["base"], expected) + self.assertEqual(Path(actual["cwd"]).resolve(), self.repo.resolve()) + + def test_local_manual_and_new_branch_compare_latest_commit(self): + for env in ( + {}, + {"GITHUB_EVENT_NAME": "workflow_dispatch"}, + {"GITHUB_EVENT_NAME": "push", "BASE_REVISION": "0" * 40}, + ): + with self.subTest(env=env): + self.assert_base(self.base, **env) + + def test_local_base_override(self): + self.assert_base(self.initial, BASE_REVISION=self.initial) + + def test_push_keeps_event_base(self): + self.assert_base( + self.initial, GITHUB_EVENT_NAME="push", BASE_REVISION=self.initial + ) + + def test_pull_request_uses_merge_base(self): + self.git("checkout", "-qb", "target", self.base) + target = self.commit("target-only") + self.git("checkout", "-q", self.head) + self.assert_base( + self.base, GITHUB_EVENT_NAME="pull_request", BASE_REVISION=target + ) + + def test_fetches_missing_base_commit(self): + remote = self.root / "remote" + subprocess.run(["git", "clone", "-q", str(self.repo), str(remote)], check=True) + for key, value in ( + ("user.name", "Test"), + ("user.email", "test@example.com"), + ("commit.gpgsign", "false"), + ("core.hooksPath", str(self.root / "no-hooks")), + ): + subprocess.run(["git", "-C", str(remote), "config", key, value], check=True) + subprocess.run( + ["git", "-C", str(remote), "checkout", "-qb", "target", self.base], + check=True, + ) + subprocess.run( + ["git", "-C", str(remote), "commit", "-qm", "remote", "--allow-empty"], + check=True, + ) + target = subprocess.check_output( + ["git", "-C", str(remote), "rev-parse", "HEAD"], text=True + ).strip() + self.git("remote", "add", "origin", str(remote)) + self.assert_base( + self.base, GITHUB_EVENT_NAME="pull_request", BASE_REVISION=target + ) + self.git("cat-file", "-e", target) + + def test_declaration_suite_skips_arm_and_pytest_setup(self): + setup = self.repo / "examples/arm/arm-scratch/setup_path.sh" + setup.parent.mkdir(parents=True) + setup.write_text("exit 91\n") + (self.checker_dir / "pytest-parallelism.sh").write_text("exit 92\n") + self.assert_base(self.base) + + def test_other_suite_still_requires_arm_setup(self): + result = self.run_suite("test_pytest_ops_no_target") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Missing", result.stdout) + self.assertIn("setup_path.sh", result.stdout) + self.assertFalse(self.result_file.exists()) + + def test_all_does_not_run_change_specific_declarations(self): + setup = self.repo / "examples/arm/arm-scratch/setup_path.sh" + setup.parent.mkdir(parents=True) + setup.write_text("") + tools = self.root / "bin" + tools.mkdir() + shell = tools / "sh" + shell.write_text('#!/bin/sh\ncat > "$ENTRYPOINT_COMMANDS"\n') + shell.chmod(0o755) + commands = self.root / "all-commands" + result = self.run_suite( + "all", + PATH=str(tools) + os.pathsep + os.environ["PATH"], + ENTRYPOINT_COMMANDS=str(commands), + ) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertNotIn("test_buck_declarations", commands.read_text()) + self.assertIn("test_pytest_ops_no_target", commands.read_text()) + self.assertFalse(self.result_file.exists()) + + def test_help_does_not_require_arm_setup(self): + for args in ((), ("help",)): + with self.subTest(args=args): + result = self.run_suite(*args) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("test_buck_declarations", result.stdout) + + def test_checker_failure_reaches_caller(self): + result = self.run_suite("test_buck_declarations", CHECKER_EXIT_CODE="7") + self.assertEqual(result.returncode, 7, result.stdout) + + def test_self_test_failure_stops_checker(self): + result = self.run_suite("test_buck_declarations", FAIL_CHECKER_SELF_TEST="1") + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.result_file.exists()) + + def test_invalid_base_stops_checker(self): + result = self.run_suite("test_buck_declarations", BASE_REVISION="missing-base") + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.result_file.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/unittest-arm-buck2.sh b/.ci/scripts/unittest-arm-buck2.sh new file mode 100755 index 00000000000..f8943c2b8b1 --- /dev/null +++ b/.ci/scripts/unittest-arm-buck2.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +set -euo pipefail + +# This C++ package does not load the Arm Python tests' internal Buck helpers. +target=//backends/arm/test/vela:vela_external_blocks_test +buck2 test "${target}" +binary=$(buck2 build "${target}" --show-full-simple-output) + +# Run each case in a fresh process, as internal CI does, to expose dependencies +# on initialization performed by an earlier case in the same test binary. +cases=$("${binary}" --gtest_list_tests | awk ' + /^[^ ]/ { suite = $1 } + /^ [^ ]/ { print suite $1 } +') +if [[ -z "${cases}" ]]; then + echo "No Vela parser tests were discovered" >&2 + exit 1 +fi +while IFS= read -r test_case; do + "${binary}" --gtest_filter="${test_case}" +done <<< "${cases}" diff --git a/.ci/scripts/unittest-buck2.sh b/.ci/scripts/unittest-buck2.sh index 9c4c4608694..2231ce4294e 100755 --- a/.ci/scripts/unittest-buck2.sh +++ b/.ci/scripts/unittest-buck2.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -43,6 +44,8 @@ for op in "build" "test"; do //runtime/executor: //runtime/kernel/... //runtime/platform/... done +bash .ci/scripts/unittest-arm-buck2.sh + # Build only without testing buck2 build //codegen/tools/... \ //exir/_serialize/test/... \ diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 52c009b5608..40a19c03717 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1,3 +1,8 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + name: pull on: @@ -994,6 +999,26 @@ jobs: exit 1 fi + test-arm-backend-buck-declarations: + name: test-arm-backend-buck-declarations + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Test declaration entrypoint + run: python -m unittest discover -s .ci/scripts/tests -p test_arm_buck_entrypoint.py + - name: Check changed Arm Python declarations + env: + BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before }} + run: backends/arm/test/test_arm_backend.sh test_buck_declarations + test-arm-backend-no-driver: needs: docker-image name: test-arm-backend-no-driver diff --git a/backends/arm/README.md b/backends/arm/README.md index 73f436962f8..b3abc2890ca 100644 --- a/backends/arm/README.md +++ b/backends/arm/README.md @@ -1,3 +1,10 @@ + + # ExecuTorch Arm® Backend This subtree contains the Arm® Backend implementation for ExecuTorch. @@ -402,6 +409,7 @@ Below is an overview of some of the testing options this script provides: | Command | Description | | ---------------------------------------------------- | ------------------------------------------------------------ | +| `test_arm_backend.sh test_buck_declarations` | Checks changed Arm Python Buck declarations without Arm tool setup. | | `test_arm_backend.sh test_pytest_ops_no_target` | Runs operator unit tests for non-target specific use-cases. | | `test_arm_backend.sh test_pytest_models_no_target` | Runs model unit tests for non-target specific use-cases. | | `test_arm_backend.sh test_pytest_ops_tosa` | Runs operator unit tests for TOSA specific use-cases. | @@ -471,6 +479,54 @@ Installing model test dependencies is a standalone process. The script installs only the dependencies needed for model tests, skipping all other setup procedures. +### Buck2 CI coverage + +The GitHub Buck2 job runs the Vela parser C++ tests together and then runs each +case in a separate process, so a test cannot depend on runtime initialization by +an earlier case. With the revision-pinned Buck2 on your `PATH`, run this check from +the repository root: + +```bash +bash .ci/scripts/unittest-arm-buck2.sh +``` + +The separate **test-arm-backend-buck-declarations** job checks for new dependency +and test-registration gaps in the Arm test package. It uses the same entrypoint as +local runs and requires only Git and Python 3, without Arm tool setup or third-party +Python packages: + +```bash +backends/arm/test/test_arm_backend.sh test_buck_declarations +``` + +By default, local runs compare the working tree against `HEAD^`. To check a change +spanning several commits, set `BASE_REVISION` to the commit before your change: + +```bash +BASE_REVISION= backends/arm/test/test_arm_backend.sh test_buck_declarations +``` + +GitHub pull requests compare against the merge base with the PR's target branch; +pushes use the previous commit from the event. Manual runs and new branches use +`HEAD^`. Declaration checks are excluded from the `all` suite because they need a +change-specific baseline. + +This static check follows supported in-repository Python library declarations, +including transitive dependencies, and compares findings against the base revision. +Existing findings do not fail the job. New confirmed missing dependencies, +unregistered Arm tests, and unsupported declarations under `backends/arm/` fail +the check. Unsupported declarations outside Arm are reported as warnings with +their declaration file and unsupported construct; dependency gaps that cannot be +proven because of those declarations are also warnings. A warning becoming an +error is treated as a new regression. + +Arm backend contributors maintain this checker alongside the Arm Buck declarations +and its regression tests in `.ci/scripts/tests/test_arm_buck_declarations.py`. +Moving the Arm Python tests onto the public runtime wrapper would allow Buck2 +queries to replace this limited static analysis. The checker does not run Meta's +internal Python Buck tests or validate third-party dependencies. Those internal +tests remain necessary for full packaged-test coverage. + ## Using git hooks The repo-wide pre-commit hook (lintrunner + torch_pin sync) is installed automatically diff --git a/backends/arm/runtime/tests/ethos-u/CMakeLists.txt b/backends/arm/runtime/tests/ethos-u/CMakeLists.txt index 5a2ee4240e9..70450f75d25 100644 --- a/backends/arm/runtime/tests/ethos-u/CMakeLists.txt +++ b/backends/arm/runtime/tests/ethos-u/CMakeLists.txt @@ -71,7 +71,7 @@ message( add_executable( vela_external_blocks_test "${EXECUTORCH_ROOT}/backends/arm/runtime/VelaBinStream.cpp" - "${EXECUTORCH_ROOT}/backends/arm/test/vela_external_blocks_test.cpp" + "${EXECUTORCH_ROOT}/backends/arm/test/vela/vela_external_blocks_test.cpp" ) target_link_libraries( vela_external_blocks_test PRIVATE executorch_core GTest::gtest_main diff --git a/backends/arm/test/targets.bzl b/backends/arm/test/targets.bzl index a529e2d9449..15719fa0907 100644 --- a/backends/arm/test/targets.bzl +++ b/backends/arm/test/targets.bzl @@ -1,3 +1,8 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + # load("//caffe2/test/fb:defs.bzl", "define_tests") load("@fbcode_macros//build_defs:python_pytest.bzl", "python_pytest") load("@bazel_skylib//lib:paths.bzl", "paths") @@ -199,9 +204,11 @@ def define_arm_tests(): ] if test_file == "misc/test_diff_pte.py" else []), ) + # Preserve the label used by internal CI to select individual Vela cases. + # OSS CI uses the separate vela package to avoid internal-only Python loads. runtime.cxx_test( name = "vela_external_blocks_test", - srcs = ["vela_external_blocks_test.cpp"], + srcs = ["//executorch/backends/arm/test/vela:vela_external_blocks_test.cpp"], deps = [ "//executorch/backends/arm/runtime:vela_bin_stream", "//executorch/runtime/core:core", diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index e2ff9d85b29..99d3cad2f47 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -10,12 +10,10 @@ script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) vulkan_validation_runner="${script_dir}/run_with_vulkan_validation.sh" # Executorch root -et_root_dir=$(cd ${script_dir}/../../.. && pwd) +et_root_dir=$(cd "${script_dir}/../../.." && pwd) cd "${et_root_dir}" pwd -# Cap pytest-xdist's `auto` workers to the container's CPU quota. -source .ci/scripts/pytest-parallelism.sh scratch_dir=${et_root_dir}/examples/arm/arm-scratch setup_path_script=${scratch_dir}/setup_path.sh _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly install necessary tools." @@ -118,21 +116,7 @@ enable_vgf_vulkan_validation() { echo "VK_INSTANCE_LAYERS=${VK_INSTANCE_LAYERS}" } -TEST_SUITE=$1 - -# Enable Vulkan validation for VGF/VKML runtime tests. - -# Source the tools -# This should be prepared by the setup.sh -[[ -f ${setup_path_script} ]] \ - || { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; } -source ${setup_path_script} - -# Enable Vulkan validation for every VGF/VKML test path, including tests that -# launch scripts/executables directly instead of going through runner_utils.py. -if [[ "${TEST_SUITE}" == *vkml* || "${TEST_SUITE}" == *vgf* ]]; then - enable_vgf_vulkan_validation -fi +TEST_SUITE=${1:-} help() { echo "Usage:" @@ -141,17 +125,35 @@ help() { # This will list all lines in this file that is starting with test_ remove () { and print it as a list. # e,g, "test_pytest() { # Test ops and other things" -> test_pytest # Test ops and other things echo "all # run all tests" - grep "^test_" $0 | sed 's/([^)]*)[[:space:]]*{*//g' + grep "^test_" "${script_dir}/test_arm_backend.sh" | sed 's/([^)]*)[[:space:]]*{*//g' exit } if [[ -z "${TEST_SUITE:-}" ]]; then echo "Missing test suite name, exiting..." help +elif [[ "${TEST_SUITE}" == help ]]; then + help else echo "Run Arm backend test suite ${TEST_SUITE}" fi +# Static Buck declarations need only Git and the Python standard library. +if [[ "${TEST_SUITE}" != test_buck_declarations ]]; then + # Cap pytest-xdist's `auto` workers to the container's CPU quota. + source .ci/scripts/pytest-parallelism.sh + + # This should be prepared by setup.sh. + [[ -f ${setup_path_script} ]] \ + || { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; } + source "${setup_path_script}" + + # Enable validation for VGF/VKML paths after setup_path.sh configures the layers. + if [[ "${TEST_SUITE}" == *vkml* || "${TEST_SUITE}" == *vgf* ]]; then + enable_vgf_vulkan_validation + fi +fi + TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}" EXCLUDE_TARGET_EXPR="(not u55) and (not u65) and (not u85) and (not tosa) and (not _vgf_)" @@ -177,13 +179,33 @@ all() { # Run all tests # ---- # That is executed echo "${TEST_SUITE_NAME}: Run all tests" - grep "^test_" backends/arm/test/test_arm_backend.sh | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh -e + # Declaration checks require a change-specific base; run them explicitly. + grep "^test_" backends/arm/test/test_arm_backend.sh | grep -v "^test_buck_declarations()" | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh -e echo "${TEST_SUITE_NAME}: PASS" } # ------------------------------------------- # -------- Non target-specific tests -------- # ------------------------------------------- +test_buck_declarations() { # Check changed Arm Python Buck declarations without Arm tool setup. + local base_revision="${BASE_REVISION:-}" + local python_executable="${PYTHON_EXECUTABLE:-python3}" + + "${python_executable}" -m unittest discover -s .ci/scripts/tests -p test_arm_buck_declarations.py + + if [[ -z "${base_revision}" || "${base_revision}" == 0000000000000000000000000000000000000000 ]]; then + # Local/manual runs and new branches compare the latest commit. + base_revision=$(git rev-parse HEAD^) + elif ! git cat-file -e "${base_revision}^{commit}" 2>/dev/null; then + git fetch --no-tags origin "${base_revision}" + fi + if [[ "${GITHUB_EVENT_NAME:-}" == pull_request ]]; then + base_revision=$(git merge-base HEAD "${base_revision}") + fi + + "${python_executable}" .ci/scripts/check_arm_buck_declarations.py --repo . --base "${base_revision}" +} + test_pytest_ops_no_target() { echo "${TEST_SUITE_NAME}: Run pytest ops for target-less tests" diff --git a/backends/arm/test/vela/BUCK b/backends/arm/test/vela/BUCK new file mode 100644 index 00000000000..b0d58e297f3 --- /dev/null +++ b/backends/arm/test/vela/BUCK @@ -0,0 +1,10 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +load(":targets.bzl", "define_vela_tests") + +oncall("executorch") + +define_vela_tests() diff --git a/backends/arm/test/vela/targets.bzl b/backends/arm/test/vela/targets.bzl new file mode 100644 index 00000000000..14faf6aaeb7 --- /dev/null +++ b/backends/arm/test/vela/targets.bzl @@ -0,0 +1,25 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +def define_vela_tests(): + runtime.export_file( + name = "vela_external_blocks_test.cpp", + visibility = ["//executorch/backends/arm/test:vela_external_blocks_test"], + ) + + if runtime.is_oss: + runtime.cxx_test( + name = "vela_external_blocks_test", + srcs = ["vela_external_blocks_test.cpp"], + compiler_flags = ["-DEXECUTORCH_ARM_VELA_TEST_NO_GLOBAL_INIT"], + deps = [ + "//executorch/backends/arm/runtime:vela_bin_stream", + "//executorch/runtime/core:core", + "//executorch/runtime/core:named_data_map", + "//executorch/runtime/platform:platform", + ], + ) diff --git a/backends/arm/test/vela_external_blocks_test.cpp b/backends/arm/test/vela/vela_external_blocks_test.cpp similarity index 97% rename from backends/arm/test/vela_external_blocks_test.cpp rename to backends/arm/test/vela/vela_external_blocks_test.cpp index 7250b47c8f8..5405a9134fb 100644 --- a/backends/arm/test/vela_external_blocks_test.cpp +++ b/backends/arm/test/vela/vela_external_blocks_test.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. + * Copyright 2026 Arm Limited and/or its affiliates. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. @@ -36,6 +37,13 @@ using executorch::runtime::TensorLayout; namespace { +class VelaExternalBlocksTest : public ::testing::Test { + protected: + void SetUp() override { + executorch::runtime::runtime_init(); + } +}; + constexpr std::string_view kExternalKey = "g123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; constexpr std::string_view kSecondExternalKey = @@ -131,13 +139,6 @@ class FakeNamedDataMap final : public NamedDataMap { std::vector entries_; }; -class VelaExternalBlocksTest : public ::testing::Test { - protected: - void SetUp() override { - executorch::runtime::runtime_init(); - } -}; - } // namespace TEST_F(VelaExternalBlocksTest, ReadsInlineAndExternalBlocksInOnePass) { @@ -383,3 +384,12 @@ TEST_F(VelaExternalBlocksTest, RejectsIoCountExceedingPayload) { &handles), Error::InvalidProgram); } + +#ifdef EXECUTORCH_ARM_VELA_TEST_NO_GLOBAL_INIT +// Only the OSS isolation target overrides the test main. Leave runtime +// initialization to the fixture so each case proves it initializes the PAL. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +#endif