From 3395723f32672dd7a2e5b12e18fcc5682de6d720 Mon Sep 17 00:00:00 2001 From: Usamah Zaheer Date: Thu, 8 Oct 2026 14:52:09 +0100 Subject: [PATCH 1/2] Add Arm Buck2 coverage and declaration checks to GitHub CI The public Buck2 suite omits Arm test coverage, so it can stay green when isolated Vela tests fail or Arm Python imports lack Buck declarations. Run the existing Vela parser suite and each discovered case in a fresh process. Retain runtime initialization in the test fixture before every case. Enable a plain test main only on the OSS isolation target so global initialization cannot mask missing fixture setup, while preserving internal and CMake test entrypoints. Move the C++ test into a standalone Buck package to avoid the internal Python test helper. Preserve the existing internal target label for individual-case selection, declare the platform dependency and update CMake's source path. Cover all ten Vela tests on upstream/main, including the malformed block and I/O payload cases. Add test-arm-backend-buck-declarations beside the other Arm GitHub jobs. Run it through test_arm_backend.sh test_buck_declarations so developers and CI share a command requiring only Git and Python. Compare against the PR merge base, previous push revision or local HEAD^, with BASE_REVISION available for changes spanning several commits. Keep the declaration check independent of Arm tool setup and the all suite. Check Arm Python test-library dependencies and test registration using supported local Buck declarations and their transitive dependencies. Discover every BUCK/TARGETS package beneath the Arm test tree so standalone nested libraries and removal of their source ownership are checked too. Collect errors after lazy dependency and provider discovery to report their actual build or helper file and unsupported construct. Preserve existing findings as a baseline. Fail on confirmed missing dependencies, unregistered Arm tests and unsupported Arm declarations. Report unsupported declarations outside Arm and dependency gaps that cannot be proven because of them as warnings. Include severity in the baseline identity so a warning becoming an error still fails. Document maintenance by Arm backend contributors and the limited static model. Evaluate Boolean expressions lazily, follow helpers only in active internal branches and reject parameterized or recursive helpers that cannot be validated. Present numbered findings with severity, files, targets and wrapped details. This check supplements Meta's internal packaged Python tests. Validation: 53 checker tests and 12 shell entrypoint integration tests pass, covering late diagnostics, warning policy, severity escalation, baseline selection, setup isolation and failure propagation. The local entrypoint reports zero new errors or warnings against upstream/main. UFMT, Flake8, ClangFormat and git diff --check pass. Focused public C++ compile/link checks cover both main configurations and all ten Vela cases together and separately. Meta's internal build and a complete Buck2 build have not been run for this amendment. Authored with assistance from OpenAI Codex. Signed-off-by: Usamah Zaheer Change-Id: I2cdf857306becd0a1eeb844763b4a96a88d33502 --- .ci/scripts/check_arm_buck_declarations.py | 927 ++++++++++++++++++ .../tests/test_arm_buck_declarations.py | 917 +++++++++++++++++ .ci/scripts/tests/test_arm_buck_entrypoint.py | 212 ++++ .ci/scripts/unittest-arm-buck2.sh | 25 + .ci/scripts/unittest-buck2.sh | 3 + .github/workflows/pull.yml | 25 + backends/arm/README.md | 56 ++ .../arm/runtime/tests/ethos-u/CMakeLists.txt | 2 +- backends/arm/test/targets.bzl | 9 +- backends/arm/test/test_arm_backend.sh | 62 +- backends/arm/test/vela/BUCK | 10 + backends/arm/test/vela/targets.bzl | 25 + .../{ => vela}/vela_external_blocks_test.cpp | 24 +- 13 files changed, 2268 insertions(+), 29 deletions(-) create mode 100644 .ci/scripts/check_arm_buck_declarations.py create mode 100644 .ci/scripts/tests/test_arm_buck_declarations.py create mode 100644 .ci/scripts/tests/test_arm_buck_entrypoint.py create mode 100755 .ci/scripts/unittest-arm-buck2.sh create mode 100644 backends/arm/test/vela/BUCK create mode 100644 backends/arm/test/vela/targets.bzl rename backends/arm/test/{ => vela}/vela_external_blocks_test.cpp (97%) diff --git a/.ci/scripts/check_arm_buck_declarations.py b/.ci/scripts/check_arm_buck_declarations.py new file mode 100644 index 00000000000..146833012c4 --- /dev/null +++ b/.ci/scripts/check_arm_buck_declarations.py @@ -0,0 +1,927 @@ +#!/usr/bin/env python3 +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +"""Check regressions in Arm test-library Buck declarations, without executing Buck. + +This is a static declaration lint, not a replacement for Meta's packaged tests. +It models literal Python library declarations and internal (is_oss=False) branches. +Unsupported declarations under backends/arm fail as 'cannot validate'; limitations +elsewhere are warnings. Existing findings are compared by identity and severity, +not line number, against an explicit base. +""" +from __future__ import annotations + +import argparse +import ast +import fnmatch +import functools +import subprocess +import sys +import textwrap +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +SCOPE = "backends/arm/test" +TEST_MACRO = SCOPE + "/targets.bzl" + + +class Unsupported(ValueError): + pass + + +class Snapshot: + def __init__(self, files: dict[str, str]): + self.paths = set(files) + self._files = files + + def read(self, path: str) -> str: + return self._files[path] + + +class GitSnapshot(Snapshot): + def __init__(self, repo: Path, revision: str | None): + super().__init__({}) + self.repo = repo + self.revision = revision + self._files: dict[str, str] = {} + if revision is None: + command = ["ls-files", "--cached", "--others", "--exclude-standard", "-z"] + else: + revision = self.git("rev-parse", "--verify", revision + "^{commit}").strip() + self.revision = revision + command = ["ls-tree", "-r", "--name-only", "-z", revision] + self.paths = set(self.git(*command).split("\0")) - {""} + if revision is None: + self.paths = {p for p in self.paths if (repo / p).is_file()} + + def git(self, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(self.repo), *args], text=True) + + def read(self, path: str) -> str: + if path not in self.paths: + raise KeyError(path) + if path not in self._files: + self._files[path] = ( + self.git("show", f"{self.revision}:{path}") + if self.revision + else (self.repo / path).read_text(encoding="utf-8") + ) + return self._files[path] + + +@functools.lru_cache(maxsize=None) +def glob_match(path: str, pattern: str) -> bool: + """Buck-style path glob: '*' does not cross '/', '**' can.""" + parts, patterns = tuple(path.split("/")), tuple(pattern.split("/")) + + @functools.lru_cache(maxsize=None) + def match(i: int, j: int) -> bool: + if j == len(patterns): + return i == len(parts) + if patterns[j] == "**": + return match(i, j + 1) or (i < len(parts) and match(i + 1, j)) + return ( + i < len(parts) + and fnmatch.fnmatchcase(parts[i], patterns[j]) + and match(i + 1, j + 1) + ) + + return match(0, 0) + + +def dotted(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + return dotted(node.value) + "." + node.attr + return "" + + +def label_for(package: str, name: str) -> str: + return "//executorch/" + package + ":" + name + + +def is_scoped_label(label: str) -> bool: + return label.startswith((label_for(SCOPE, ""), "//executorch/" + SCOPE + "/")) + + +def normalize(label: str, package: str) -> str | None: + if label.startswith(":"): + return label_for(package, label[1:]) + if label.startswith("//executorch/"): + return label + # Dependencies outside the in-repo executorch namespace are out of scope. + return None + + +@dataclass(frozen=True, order=True) +class Issue: + # Messages may gain provider suggestions without creating a new violation. + kind: str = field(compare=False) + owner: str + source: str + detail: str = field(compare=False) + subject: str = "" + category: str = "declaration" + severity: str = "error" + + def __str__(self) -> str: + return f"{self.kind}: {self.owner}: {self.source}: {self.detail}" + + +def validation_severity(source: str) -> str: + return ( + "error" + if source == "backends/arm" or source.startswith("backends/arm/") + else "warning" + ) + + +@dataclass +class Rule: + label: str + kind: str + declaration_file: str + sources: set[str] = field(default_factory=set) + deps: set[str] = field(default_factory=set) + errors: list[str] = field(default_factory=list) + + +class Declarations: + def __init__(self, snapshot: Snapshot): + self.snapshot = snapshot + self.rules: dict[str, Rule] = {} + self.loaded: set[str] = set() + self.package_errors: dict[str, list[tuple[str, str]]] = {} + + def package_error(self, package: str, filename: str, error: str) -> None: + self.package_errors.setdefault(package, []).append((filename, error)) + + def load_scope(self) -> None: + self.load_package(SCOPE) + for filename in sorted(self.snapshot.paths): + package, _, name = filename.rpartition("/") + if name in {"BUCK", "TARGETS"} and ( + package == SCOPE or package.startswith(SCOPE + "/") + ): + self.load_package(package) + + def package_files(self, package: str) -> list[str]: + prefix = package + "/" if package else "" + result = [] + for path in self.snapshot.paths: + if not path.startswith(prefix): + continue + relative = path[len(prefix) :] + # A glob cannot reach into another Buck package. + parents = list(PurePosixPath(relative).parents)[:-1] + if any( + prefix + str(p) + "/BUCK" in self.snapshot.paths + or prefix + str(p) + "/TARGETS" in self.snapshot.paths + for p in parents + ): + continue + result.append(relative) + return sorted(result) + + # Each branch corresponds to one explicitly supported AST form. + # pylint: disable-next=too-many-return-statements + def value(self, node: ast.AST, env: dict[str, object], package: str): # noqa: C901 + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, (ast.List, ast.Tuple)): + return [self.value(x, env, package) for x in node.elts] + if isinstance(node, ast.Name) and node.id in env: + return env[node.id] + if dotted(node) == "runtime.is_oss": + return env.get("__is_oss__", False) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not self.value(node.operand, env, package) + if isinstance(node, ast.BoolOp): + for operand in node.values: + value = self.value(operand, env, package) + if bool(value) == isinstance(node.op, ast.Or): + return value + return value + if isinstance(node, ast.IfExp): + branch = node.body if self.value(node.test, env, package) else node.orelse + return self.value(branch, env, package) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return self.value(node.left, env, package) + self.value( + node.right, env, package + ) + if isinstance(node, ast.Call) and dotted(node.func) in {"glob", "native.glob"}: + patterns = self.value(node.args[0], env, package) + kwargs = {k.arg: self.value(k.value, env, package) for k in node.keywords} + unknown = set(kwargs) - {"exclude", "exclude_directories"} + if unknown: + raise Unsupported( + "unsupported glob arguments: " + ", ".join(sorted(unknown)) + ) + excludes = kwargs.get("exclude", []) + return [ + p + for p in self.package_files(package) + if any(glob_match(p, pattern) for pattern in patterns) + and not any(glob_match(p, pattern) for pattern in excludes) + ] + raise Unsupported(ast.unparse(node)) + + def strings(self, node: ast.AST, env: dict[str, object], package: str) -> list[str]: + value = self.value(node, env, package) + if not isinstance(value, list) or not all(isinstance(x, str) for x in value): + raise Unsupported("expected a static list of strings: " + ast.unparse(node)) + return value + + def parse_rule( # noqa: C901 + self, call: ast.Call, env: dict[str, object], package: str, filename: str + ) -> None: + kwargs = {k.arg: k.value for k in call.keywords} + function = dotted(call.func) + if function == "non_fbcode_target": + return + kind = dotted(kwargs.get("_kind", call.func)).split(".")[-1] + if kind not in { + "python_library", + "python_binary", + "python_test", + "python_pytest", + "python_unittest", + "cxx_library", + "cxx_python_extension", + "cxx_test", + "export_file", + "filegroup", + }: + if "name" in kwargs and "srcs" in kwargs: + self.package_error(package, filename, "unsupported rule kind: " + kind) + return + try: + name = self.value(kwargs["name"], env, package) + if not isinstance(name, str): + raise Unsupported("non-string target name") + except (KeyError, Unsupported) as error: + self.package_error(package, filename, f"target name: {error}") + return + rule = Rule(label_for(package, name), kind, filename) + for attribute in ("srcs", "deps", "exported_deps", "fbcode_deps"): + if attribute not in kwargs: + continue + try: + values = self.strings(kwargs[attribute], env, package) + if attribute == "srcs": + for source in values: + if source.startswith((":", "//", "@")): + if kind == "python_library": + raise Unsupported("source target reference " + source) + elif source.endswith(".py"): + rule.sources.add(package + "/" + source) + else: + rule.deps.update( + target + for value in values + if (target := normalize(value, package)) is not None + ) + except Unsupported as error: + rule.errors.append(f"{attribute}: {error}") + if rule.label in self.rules: + rule.errors.append("multiple declarations for the same target") + self.rules[rule.label] = rule + + def load_package(self, package: str) -> None: + if package in self.loaded: + return + self.loaded.add(package) + filename = next( + ( + package + "/" + name + for name in ("BUCK", "TARGETS") + if package + "/" + name in self.snapshot.paths + ), + None, + ) + if filename is None: + self.package_error(package, package + "/BUCK", "no in-repo build file") + return + self.parse_file(filename, package, set()) + + # Keep declaration traversal and its environment in a single scope. + # pylint: disable-next=too-many-locals,too-many-statements + def parse_file( # noqa: C901 + self, filename: str, package: str, seen: set, selected: set[str] | None = None + ) -> None: + key = (filename, tuple(sorted(selected or []))) + if key in seen: + return + seen.add(key) + try: + tree = ast.parse(self.snapshot.read(filename), filename=filename) + except (KeyError, SyntaxError) as error: + self.package_error(package, filename, f"cannot read {filename}: {error}") + return + env: dict[str, object] = {} + functions = { + node.name: node for node in tree.body if isinstance(node, ast.FunctionDef) + } + imports: dict[str, tuple[str, str]] = {} + for node in tree.body: + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Call) + and dotted(node.value.func) == "load" + ): + args = node.value.args + if ( + args + and isinstance(args[0], ast.Constant) + and isinstance(args[0].value, str) + ): + source = args[0].value + # Only local declaration helpers; never load external macros. + if source.startswith(":") or not source.startswith(("@", "//")): + helper = str( + PurePosixPath(filename).parent / source.lstrip(":") + ) + if helper in self.snapshot.paths: + imports.update( + (arg.value, (helper, arg.value)) + for arg in args[1:] + if isinstance(arg, ast.Constant) + and isinstance(arg.value, str) + ) + imports.update( + (k.arg, (helper, k.value.value)) + for k in node.value.keywords + if isinstance(k.value, ast.Constant) + and isinstance(k.value.value, str) + ) + + called: set[str] = set() + visiting: set[str] = set() + + def call_name(call: ast.Call) -> str: + return dotted( + next((k.value for k in call.keywords if k.arg == "_kind"), call.func) + ) + + def visit_helper(name: str) -> None: + function = functions.get(name) + if function is None: + return + args = function.args + if ( + args.posonlyargs + or args.args + or args.kwonlyargs + or args.vararg + or args.kwarg + ): + self.package_error( + package, filename, "parameterized declaration helper: " + name + ) + elif name in visiting: + self.package_error( + package, filename, "recursive declaration helper: " + name + ) + else: + visiting.add(name) + visit(function.body, dict(env)) + visiting.remove(name) + + def visit(statements: list[ast.stmt], local_env: dict[str, object]) -> None: + for node in statements: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + try: + local_env[node.targets[0].id] = self.value( + node.value, local_env, package + ) + except Unsupported: + local_env.pop(node.targets[0].id, None) + elif isinstance(node, ast.If): + try: + branch = ( + node.body + if self.value(node.test, local_env, package) + else node.orelse + ) + visit(branch, local_env) + except Unsupported as error: + if any( + isinstance(x, ast.Call) + and dotted(x.func) != "non_fbcode_target" + and ( + call_name(x) in functions + or call_name(x) in imports + or call_name(x).split(".")[-1] == "python_library" + ) + for x in ast.walk(node) + ): + self.package_error( + package, filename, f"conditional declaration: {error}" + ) + elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Call): + call = node.value + if dotted(call.func) == "non_fbcode_target": + continue + name = call_name(call) + if name in functions or name in imports: + if call.args or any(k.arg != "_kind" for k in call.keywords): + self.package_error( + package, + filename, + "parameterized declaration helper: " + name, + ) + elif name in functions: + visit_helper(name) + else: + called.add(name) + else: + self.parse_rule(call, local_env, package, filename) + elif isinstance(node, ast.For): + kinds = { + dotted( + next( + (k.value for k in call.keywords if k.arg == "_kind"), + call.func, + ) + ).split(".")[-1] + for call in ast.walk(node) + if isinstance(call, ast.Call) + } + known_test_loop = ( + isinstance(node.target, ast.Name) + and node.target.id == "test_file" + and isinstance(node.iter, ast.Name) + and node.iter.id == "test_files" + and "python_pytest" in kinds + and "python_library" not in kinds + ) + if not known_test_loop: + self.package_error( + package, + filename, + "unsupported declaration loop: " + + ast.unparse(node.target) + + " in " + + ast.unparse(node.iter), + ) + + visit(tree.body, env) + for name in sorted(selected or set()): + visit_helper(name) + needed: dict[str, set[str]] = {} + for name in sorted(called): + helper, original = imports[name] + needed.setdefault(helper, set()).add(original) + for helper, names in needed.items(): + self.parse_file(helper, package, seen, names) + + def get(self, label: str) -> Rule | None: + package = label.removeprefix("//executorch/").split(":", 1)[0] + self.load_package(package) + return self.rules.get(label) + + def owners(self, source: str) -> set[str]: + for parent in PurePosixPath(source).parents: + package = str(parent) + if package == ".": + continue + if any( + package + "/" + name in self.snapshot.paths + for name in ("BUCK", "TARGETS") + ): + self.load_package(package) + return { + r.label + for r in self.rules.values() + if r.kind == "python_library" and source in r.sources + } + return set() + + def reachable(self, start: Rule) -> tuple[set[str], set[Issue]]: + visited, pending, errors = set(), [start.label], set() + while pending: + label = pending.pop() + if label in visited: + continue + visited.add(label) + rule = self.get(label) + if rule is None: + package = label.removeprefix("//executorch/").split(":", 1)[0] + source = package + "/BUCK" + errors.add( + Issue( + "cannot validate", + label, + source, + ( + "unresolved scoped dependency: " + label + if is_scoped_label(label) + else "no supported declaration for " + label + ), + label, + severity=validation_severity(source), + ) + ) + continue + if rule.kind != "python_library": + continue + errors.update( + Issue( + "cannot validate", + label, + rule.declaration_file, + error, + error.split(":", 1)[0], + severity=validation_severity(rule.declaration_file), + ) + for error in rule.errors + ) + pending.extend(rule.deps) + return visited, errors + + +def python_imports( # noqa: C901 + source: str, filename: str, snapshot: Snapshot +) -> set[tuple[str, str | None]]: + results: set[tuple[str, str | None]] = set() + + def resolve(module: str) -> str | None: + if not module.startswith("executorch."): + return None + path = module.removeprefix("executorch.").replace(".", "/") + return next( + ( + candidate + for candidate in (path + ".py", path + "/__init__.py") + if candidate in snapshot.paths + ), + None, + ) + + class Imports(ast.NodeVisitor): + # ast.NodeVisitor dispatch requires these method names. + # pylint: disable=invalid-name + def visit_If(self, node: ast.If) -> None: + if dotted(node.test) in {"TYPE_CHECKING", "typing.TYPE_CHECKING"}: + for child in node.orelse: + self.visit(child) + else: + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + if alias.name.startswith("executorch."): + results.add((alias.name, resolve(alias.name))) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + module = node.module or "" + if node.level: + package = ["executorch"] + list(PurePosixPath(filename).parent.parts) + module = ".".join( + package[: len(package) - node.level + 1] + + ([module] if module else []) + ) + if not module.startswith("executorch."): + return + for alias in node.names: + child = module + "." + alias.name + child_file = resolve(child) + results.add( + (child, child_file) if child_file else (module, resolve(module)) + ) + + Imports().visit(ast.parse(source, filename=filename)) + return results + + +def is_test_file(source: str, filename: str) -> bool: + tree = ast.parse(source, filename=filename) + function_types = (ast.FunctionDef, ast.AsyncFunctionDef) + for node in tree.body: + if isinstance(node, function_types) and node.name.startswith("test_"): + if not any( + dotted(d.func if isinstance(d, ast.Call) else d).endswith("fixture") + for d in node.decorator_list + ): + return True + if isinstance(node, ast.ClassDef) and ( + node.name.startswith("Test") + or any(dotted(base).endswith("TestCase") for base in node.bases) + ): + if any( + isinstance(child, function_types) and child.name.startswith("test") + for child in node.body + ): + return True + return False + + +def registered_tests( # noqa: C901 + declarations: Declarations, +) -> tuple[set[str], list[str]]: + try: + tree = ast.parse(declarations.snapshot.read(TEST_MACRO), filename=TEST_MACRO) + except (KeyError, SyntaxError) as error: + return set(), [str(error)] + errors = [] + registrations: set[str] = set() + + def visit(statements: list[ast.stmt], env: dict[str, object]) -> None: + for node in statements: + if isinstance(node, ast.FunctionDef) and node.name == "define_arm_tests": + visit(node.body, env) + elif ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + try: + env[node.targets[0].id] = declarations.value(node.value, env, SCOPE) + except Unsupported as error: + env.pop(node.targets[0].id, None) + if node.targets[0].id == "test_files": + errors.append(str(error)) + elif ( + isinstance(node, ast.AugAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "test_files" + ): + try: + if not isinstance(node.op, ast.Add): + raise Unsupported(ast.unparse(node)) + env["test_files"] = env.get( + "test_files", [] + ) + declarations.strings(node.value, env, SCOPE) + except Unsupported as error: + errors.append(str(error)) + elif isinstance(node, ast.If): + try: + visit( + ( + node.body + if declarations.value(node.test, env, SCOPE) + else node.orelse + ), + env, + ) + except Unsupported as error: + if any( + isinstance(x, ast.Name) and x.id == "test_files" + for x in ast.walk(node) + ): + errors.append(str(error)) + elif isinstance(node, ast.For): + if any( + isinstance(x, ast.Name) + and x.id == "test_files" + and isinstance(x.ctx, ast.Store) + for x in ast.walk(node) + ): + errors.append("loop-modified test_files") + + # A test may intentionally be registered only for one build environment. + for is_oss in (False, True): + env = {"__is_oss__": is_oss} + visit(tree.body, env) + registrations.update(SCOPE + "/" + p for p in env.get("test_files", [])) + return registrations, errors + + +# pylint: disable-next=too-many-locals +def analyze(snapshot: Snapshot) -> set[Issue]: # noqa: C901 + declarations = Declarations(snapshot) + declarations.load_scope() + issues: set[Issue] = set() + libraries = [ + rule + for rule in declarations.rules.values() + if rule.kind == "python_library" and is_scoped_label(rule.label) + ] + for library in libraries: + reachable, incomplete = declarations.reachable(library) + issues.update(incomplete) + for filename in sorted(library.sources): + try: + imports = python_imports(snapshot.read(filename), filename, snapshot) + except (KeyError, SyntaxError) as error: + issues.add( + Issue("cannot validate", library.label, filename, str(error)) + ) + continue + for module, imported_file in imports: + if imported_file in library.sources: + continue + if imported_file is None: + issues.add( + Issue( + "cannot validate", + library.label, + filename, + "no in-repo source for " + module, + module, + "dependency", + validation_severity( + module.removeprefix("executorch.").replace(".", "/") + ), + ) + ) + continue + owners = declarations.owners(imported_file) + if owners & reachable: + continue + if not owners: + issues.add( + Issue( + "cannot validate", + library.label, + filename, + "no supported python_library provider for " + module, + module, + "dependency", + validation_severity(imported_file), + ) + ) + continue + detail = module + " requires " + " or ".join(sorted(owners)) + severity = "error" + if incomplete: + detail += "; unresolved dependency declarations: " + ", ".join( + f"{issue.owner} ({issue.source}: {issue.detail})" + for issue in sorted(incomplete) + ) + kind = "cannot validate" + if all(issue.severity == "warning" for issue in incomplete): + severity = "warning" + else: + kind = "missing dependency" + issues.add( + Issue( + kind, + library.label, + filename, + detail, + module, + "dependency", + severity, + ) + ) + registrations, errors = registered_tests(declarations) + for error in errors: + issues.add( + Issue( + "cannot validate", + "Arm test registration", + TEST_MACRO, + error, + error, + "registration", + ) + ) + for filename in sorted(snapshot.paths): + if filename.startswith(SCOPE + "/") and filename.endswith(".py"): + try: + actual_test = is_test_file(snapshot.read(filename), filename) + except SyntaxError as error: + issues.add( + Issue( + "cannot validate", "Arm test registration", filename, str(error) + ) + ) + continue + if actual_test and filename not in registrations: + # A nested package may declare its own ordinary Python test. + declarations.owners(filename) # Load its nearest build package. + explicitly_registered = any( + rule.kind in {"python_test", "python_pytest", "python_unittest"} + and filename in rule.sources + for rule in declarations.rules.values() + ) + if not explicitly_registered: + issues.add( + Issue( + "unregistered test", + "Arm test registration", + filename, + "not in define_arm_tests test_files or a supported explicit Python test target", + "", + "registration", + ) + ) + # Dependency traversal and owner discovery load packages lazily. + issues.update( + Issue( + "cannot validate", + package, + filename, + error, + error, + severity=validation_severity(filename), + ) + for package, errors in declarations.package_errors.items() + for filename, error in errors + ) + issues.update( + Issue( + "cannot validate", + rule.label, + rule.declaration_file, + error, + error.split(":", 1)[0], + severity=validation_severity(rule.declaration_file), + ) + for rule in declarations.rules.values() + if rule.kind == "python_library" + for error in rule.errors + ) + return issues + + +def compare(base: Snapshot, head: Snapshot) -> tuple[set[Issue], int]: + baseline, current = analyze(base), analyze(head) + previous = Declarations(base) + previous.load_scope() + present = Declarations(head) + present.load_scope() + for rule in previous.rules.values(): + if rule.kind != "python_library" or not is_scoped_label(rule.label): + continue + for source in rule.sources & head.paths: + present.owners(source) + if not any( + source in target.sources + and target.kind + in {"python_library", "python_test", "python_pytest", "python_unittest"} + for target in present.rules.values() + ): + current.add( + Issue( + "cannot validate", + rule.label, + source, + "previously declared Python source has no supported owner in the head snapshot", + "source ownership", + "ownership", + ) + ) + return current - baseline, len(current & baseline) + + +def print_report(regressions: set[Issue], unchanged: int) -> None: + errors = sum(issue.severity == "error" for issue in regressions) + status = "FAIL" if errors else "PASS" + print(f"Arm Python Buck declaration check: {status}") + print(f" New errors: {errors}") + print(f" New warnings: {len(regressions) - errors}") + print(f" Existing issues unchanged: {unchanged}") + wrapper = textwrap.TextWrapper( + width=100, break_long_words=False, break_on_hyphens=False + ) + for index, issue in enumerate(sorted(regressions), start=1): + print( + f"\n[{index}/{len(regressions)}] {issue.severity.upper()}: {issue.kind.capitalize()}" + ) + print(f" File: {issue.source}") + owner_label = "Target" if issue.owner.startswith("//") else "Scope" + print(f" {owner_label}: {issue.owner}") + detail, _, unresolved = issue.detail.partition( + "; unresolved dependency declarations: " + ) + wrapper.initial_indent = " Detail: " + wrapper.subsequent_indent = " " + print(wrapper.fill(detail)) + if unresolved: + print(" Unresolved dependency declarations:") + wrapper.initial_indent = " " + wrapper.subsequent_indent = " " + print(wrapper.fill(unresolved)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument( + "--base", required=True, help="explicit pre-change Git revision" + ) + parser.add_argument("--head", help="Git revision; omit to inspect the working tree") + args = parser.parse_args(argv) + try: + regressions, unchanged = compare( + GitSnapshot(args.repo.resolve(), args.base), + GitSnapshot(args.repo.resolve(), args.head), + ) + except (OSError, subprocess.CalledProcessError, ValueError) as error: + print(f"Cannot validate Arm Python Buck declarations: {error}", file=sys.stderr) + return 2 + print_report(regressions, unchanged) + return 1 if any(issue.severity == "error" for issue in regressions) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.ci/scripts/tests/test_arm_buck_declarations.py b/.ci/scripts/tests/test_arm_buck_declarations.py new file mode 100644 index 00000000000..8461d0c3b56 --- /dev/null +++ b/.ci/scripts/tests/test_arm_buck_declarations.py @@ -0,0 +1,917 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path +from unittest import mock + + +def load_checker(): + path = Path(__file__).resolve().parents[1] / "check_arm_buck_declarations.py" + spec = importlib.util.spec_from_file_location("check_arm_buck_declarations", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +checker = load_checker() +Declarations = checker.Declarations +Snapshot = checker.Snapshot +analyze = checker.analyze +compare = checker.compare +glob_match = checker.glob_match + +SCOPE = "backends/arm/test" +TESTER = SCOPE + "/tester/runner.py" +BUCK = SCOPE + "/BUCK" +TARGETS = SCOPE + "/targets.bzl" + + +def library(name, sources, deps="[]"): + return f"fbcode_target(_kind=runtime.python_library, name={name!r}, srcs={sources}, deps={deps})\n" + + +def fixture(): + return { + BUCK: library( + "arm_tester_lib", + 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', + '["//executorch/exir:umbrella"]', + ), + TESTER: "from executorch.exir.schema import Tensor\n", + TARGETS: 'def define_arm_tests():\n test_files = []\n test_files += native.glob(["passes/test_*.py"])\n', + "exir/BUCK": library("umbrella", "[]", '[":schema"]') + + library("schema", '["schema.py"]'), + "exir/schema.py": "class Tensor: pass\n", + "devtools/pte_tool/BUCK": library( + "diff_pte_lib", '["diff_pte.py"]', '["//executorch/exir:schema"]' + ) + + 'runtime.python_binary(name="diff_pte", srcs=["diff_pte.py"])\n', + "devtools/pte_tool/diff_pte.py": "class PTEDiffResult: pass\n", + } + + +def regressions(before, after): + return compare(Snapshot(before), Snapshot(after))[0] + + +# Each test covers an independent declaration regression. +# pylint: disable-next=too-many-public-methods +class DeclarationTests(unittest.TestCase): + def test_known_missing_import_and_registration_then_metadata_fix(self): + before = fixture() + after = dict(before) + after[ + TESTER + ] += "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + after[SCOPE + "/tester/pte_diff_utils.py"] = ( + "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + "from executorch.exir.schema import Tensor\n" + ) + test = SCOPE + "/misc/test_diff_pte.py" + after[test] = "def test_diff_pte():\n assert True\n" + issues = regressions(before, after) + self.assertEqual(3, len(issues)) + self.assertEqual(2, sum(i.kind == "missing dependency" for i in issues)) + self.assertTrue( + any(i.kind == "unregistered test" and i.source == test for i in issues) + ) + self.assertTrue( + all( + "diff_pte_lib" in i.detail + for i in issues + if i.kind == "missing dependency" + ) + ) + after[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', + '["//executorch/exir:umbrella", "//executorch/devtools/pte_tool:diff_pte_lib"]', + ) + after[TARGETS] += ' test_files += ["misc/test_diff_pte.py"]\n' + self.assertEqual(set(), regressions(before, after)) + + def test_transitive_dependency_and_cycle_are_valid(self): + files = fixture() + files["exir/BUCK"] += library("cycle", "[]", '[":umbrella"]') + files["exir/BUCK"] = files["exir/BUCK"].replace( + 'deps=[":schema"]', 'deps=[":schema", ":cycle"]' + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_dependency_deletion_without_python_change_fails(self): + before = fixture() + after = dict(before) + after[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + self.assertIn("//executorch/exir:schema", next(iter(issues)).detail) + + def test_existing_gaps_do_not_mask_different_new_gap(self): + before = fixture() + before[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + after = dict(before) + after[ + TESTER + ] += "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + self.assertIn("diff_pte_lib", next(iter(issues)).detail) + + def test_python_binary_is_not_an_import_provider(self): + files = fixture() + files[TESTER] = ( + "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + ) + files["devtools/pte_tool/BUCK"] = ( + 'runtime.python_binary(name="diff_pte", srcs=["diff_pte.py"])\n' + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.kind == "cannot validate" + and "no supported python_library provider" in i.detail + for i in issues + ) + ) + + def test_internal_branch_is_not_union_of_incompatible_deps(self): + files = fixture() + files[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/*.py"])', + '["//executorch/exir:umbrella"] if runtime.is_oss else []', + ) + issues = analyze(Snapshot(files)) + self.assertTrue(any(i.kind == "missing dependency" for i in issues)) + files[BUCK] = files[BUCK].replace( + "if runtime.is_oss else", "if not runtime.is_oss else" + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_unknown_deps_fail_instead_of_becoming_empty(self): + before = fixture() + after = dict(before) + after[BUCK] = library( + "arm_tester_lib", 'glob(["tester/*.py"])', "choose_dependencies()" + ) + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" and "choose_dependencies" in i.detail + for i in issues + ) + ) + + def test_unknown_sources_fail_instead_of_disappearing(self): + before = fixture() + after = dict(before) + after[BUCK] = library("arm_tester_lib", "choose_sources()") + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" and "choose_sources" in i.detail + for i in issues + ) + ) + + def test_unknown_conditional_rule_fails(self): + before = fixture() + after = dict(before) + after[BUCK] += ( + 'if native.read_config("mode", "value"):\n' + ' runtime.python_library(name="conditional", srcs=["extra.py"])\n' + ) + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" and "conditional declaration" in i.detail + for i in issues + ) + ) + + def test_helpers_are_not_registered_by_filename_alone(self): + before = fixture() + after = dict(before) + after[SCOPE + "/misc/test_helpers.py"] = "def make_payload():\n return b''\n" + self.assertEqual(set(), regressions(before, after)) + after[ + SCOPE + "/misc/test_helpers.py" + ] += "class TestPayload:\n def test_payload(self):\n pass\n" + self.assertTrue( + any(i.kind == "unregistered test" for i in regressions(before, after)) + ) + + def test_unittest_class_and_glob_registered_pytest_function(self): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_operation():\n pass\n" + ) + files[SCOPE + "/misc/unusual_name.py"] = ( + "import unittest\nclass Case(unittest.TestCase):\n def test_stuff(self):\n pass\n" + ) + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertEqual(SCOPE + "/misc/unusual_name.py", next(iter(issues)).source) + + def test_type_checking_only_import_is_not_runtime_requirement(self): + files = fixture() + files[ + TESTER + ] += "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from executorch.missing import Whatever\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_glob_exclude_and_relative_intralibrary_import(self): + files = fixture() + files[TESTER] += "from .helper import answer\n" + files[SCOPE + "/tester/helper.py"] = "answer = 42\n" + files[SCOPE + "/tester/excluded.py"] = "from executorch.missing import Nope\n" + self.assertEqual(set(), analyze(Snapshot(files))) + self.assertFalse(glob_match("tester/sub/deep.py", "tester/*.py")) + self.assertTrue(glob_match("tester/helper.py", "tester/**/*.py")) + self.assertTrue(glob_match("tester/sub/deep.py", "tester/**/*.py")) + + def test_from_package_submodule_import(self): + files = fixture() + files[TESTER] = "from executorch.exir import schema\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_only_called_local_declaration_helpers_create_providers(self): + files = fixture() + files["exir/BUCK"] = ( + 'load(":targets.bzl", "define_common_targets")\ndefine_common_targets()\n' + ) + files["exir/targets.bzl"] = ( + "def define_common_targets():\n " + + library("umbrella", "[]", '[":schema"]') + + " " + + library("schema", '["schema.py"]') + + "def define_unused():\n " + + library("ghost", '["ghost.py"]') + ) + files["exir/ghost.py"] = "x = 1\n" + declarations = Declarations(Snapshot(files)) + self.assertIsNone(declarations.get("//executorch/exir:ghost")) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_boolean_expressions_short_circuit_and_preserve_operand_values(self): + declarations = Declarations(Snapshot({})) + for expression, expected in ( + ("runtime.is_oss and unsupported()", False), + ("not runtime.is_oss or unsupported()", True), + ('[] or ["schema.py"]', ["schema.py"]), + ('["schema.py"] and []', []), + ('[] or False or ["schema.py"]', ["schema.py"]), + ): + with self.subTest(expression=expression): + self.assertEqual( + expected, + declarations.value( + checker.ast.parse(expression, mode="eval").body, {}, SCOPE + ), + ) + for expression in ("False or unsupported()", "True and unsupported()"): + with self.subTest(expression=expression): + with self.assertRaisesRegex(checker.Unsupported, "unsupported"): + declarations.value( + checker.ast.parse(expression, mode="eval").body, {}, SCOPE + ) + + def test_short_circuit_condition_does_not_reject_valid_declarations(self): + files = fixture() + files[BUCK] = ( + 'if not runtime.is_oss or native.read_config("mode", "value"):\n' + + " " + + files[BUCK] + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_only_active_local_or_imported_helpers_create_providers(self): + for imported in (False, True): + for condition, active in ( + ("runtime.is_oss", False), + ("not runtime.is_oss", True), + ): + with self.subTest(imported=imported, condition=condition): + files = fixture() + helper = "def define_schema():\n " + library( + "schema", '["schema.py"]' + ) + if imported: + files["exir/BUCK"] = ( + 'load(":targets.bzl", declare="define_schema")\n' + ) + files["exir/targets.bzl"] = helper + call = "declare()" + else: + files["exir/BUCK"] = helper + call = "define_schema()" + files["exir/BUCK"] += ( + library("umbrella", "[]", '[":schema"]') + + f"if {condition}:\n {call}\n" + ) + declarations = Declarations(Snapshot(files)) + self.assertEqual( + active, declarations.get("//executorch/exir:schema") is not None + ) + issues = analyze(Snapshot(files)) + if active: + self.assertEqual(set(), issues) + else: + self.assertTrue( + any( + "no supported python_library provider" in issue.detail + for issue in issues + ) + ) + + def test_inactive_helpers_do_not_add_spurious_declaration_errors(self): + for imported in (False, True): + with self.subTest(imported=imported): + files = fixture() + helper = "def define_oss():\n " + library( + "oss", "unsupported_sources()" + ) + if imported: + files[BUCK] += 'load(":helpers.bzl", "define_oss")\n' + files[SCOPE + "/helpers.bzl"] = helper + else: + files[BUCK] += helper + files[BUCK] += "if runtime.is_oss:\n define_oss()\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_nested_local_helper_calls_preserve_internal_branch(self): + files = fixture() + files["exir/BUCK"] = ( + "def define_schema():\n " + + library("schema", '["schema.py"]') + + "def define_all():\n " + + library("umbrella", "[]", '[":schema"]') + + " if not runtime.is_oss:\n define_schema()\n" + + "fbcode_target(_kind=define_all)\n" + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_parameterized_helpers_fail_clearly_only_when_called(self): + for imported in (False, True): + for condition, active in ( + ("runtime.is_oss", False), + ("not runtime.is_oss", True), + ): + with self.subTest(imported=imported, condition=condition): + before = fixture() + after = dict(before) + helper = ( + "def declare(name, srcs):\n" + + " runtime.python_library(name=name, srcs=srcs)\n" + ) + if imported: + after[BUCK] += 'load(":helpers.bzl", "declare")\n' + after[SCOPE + "/helpers.bzl"] = helper + else: + after[BUCK] += helper + after[ + BUCK + ] += f'if {condition}:\n declare("extra", ["extra.py"])\n' + after[SCOPE + "/extra.py"] = ( + "from executorch.missing import value\n" + ) + issues = regressions(before, after) + if active: + self.assertTrue( + any( + issue.kind == "cannot validate" + and "parameterized declaration helper: declare" + in issue.detail + for issue in issues + ) + ) + else: + self.assertEqual(set(), issues) + + def test_internal_wrapper_selection_applies_to_helpers(self): + for imported in (False, True): + for wrapper, active in ( + ("fbcode_target", True), + ("non_fbcode_target", False), + ): + with self.subTest(imported=imported, wrapper=wrapper): + files = fixture() + helper = "def define_extra():\n " + library( + "extra", '["extra.py"]' + ) + if imported: + files[BUCK] += 'load(":helpers.bzl", "define_extra")\n' + files[SCOPE + "/helpers.bzl"] = helper + else: + files[BUCK] += helper + files[BUCK] += f"{wrapper}(_kind=define_extra)\n" + declarations = Declarations(Snapshot(files)) + self.assertEqual( + active, + declarations.get("//executorch/" + SCOPE + ":extra") + is not None, + ) + + def test_parameterized_helper_definition_is_rejected_without_arguments(self): + for imported in (False, True): + for parameters in ('name="extra"', "*names", "**options"): + with self.subTest(imported=imported, parameters=parameters): + files = fixture() + helper = f"def define_extra({parameters}):\n " + library( + "extra", '["extra.py"]' + ) + if imported: + files[BUCK] += 'load(":helpers.bzl", "define_extra")\n' + files[SCOPE + "/helpers.bzl"] = helper + else: + files[BUCK] += helper + files[BUCK] += "define_extra()\n" + self.assertTrue( + any( + issue.kind == "cannot validate" + and "parameterized declaration helper: define_extra" + in issue.detail + for issue in analyze(Snapshot(files)) + ) + ) + + def test_recursive_helper_fails_clearly(self): + files = fixture() + files[BUCK] += "def define_extra():\n define_extra()\ndefine_extra()\n" + self.assertTrue( + any( + issue.kind == "cannot validate" + and "recursive declaration helper: define_extra" in issue.detail + for issue in analyze(Snapshot(files)) + ) + ) + + def test_unknown_helper_condition_fails_instead_of_inventing_provider(self): + files = fixture() + files[BUCK] += ( + "def define_extra():\n " + + library("extra", '["extra.py"]') + + 'if native.read_config("mode", "value"):\n define_extra()\n' + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + issue.kind == "cannot validate" + and "conditional declaration" in issue.detail + for issue in issues + ) + ) + + def test_unresolvable_new_inrepo_module_fails_clearly(self): + before = fixture() + after = dict(before) + after[TESTER] += "from executorch.generated_unknown import value\n" + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + self.assertEqual("cannot validate", next(iter(issues)).kind) + + def test_glob_does_not_cross_buck_package(self): + files = fixture() + files[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/**/*.py"])', + '["//executorch/exir:umbrella"]', + ) + files[SCOPE + "/tester/nested/BUCK"] = library("other", '["module.py"]') + nested_source = SCOPE + "/tester/nested/module.py" + files[nested_source] = "from executorch.missing import Nope\n" + declarations = Declarations(Snapshot(files)) + root = declarations.get("//executorch/" + SCOPE + ":arm_tester_lib") + self.assertNotIn(nested_source, root.sources) + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertEqual( + "//executorch/" + SCOPE + "/tester/nested:other", next(iter(issues)).owner + ) + + def test_standalone_nested_library_dependency_is_checked(self): + for build_file in ("BUCK", "TARGETS"): + with self.subTest(build_file=build_file): + before = fixture() + after = dict(before) + package = SCOPE + "/standalone" + source = package + "/module.py" + after[package + "/" + build_file] = library("lib", '["module.py"]') + after[source] = "from executorch.exir.schema import Tensor\n" + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("missing dependency", issue.kind) + self.assertEqual("//executorch/" + package + ":lib", issue.owner) + self.assertEqual(source, issue.source) + after[package + "/" + build_file] = library( + "lib", '["module.py"]', '["//executorch/exir:schema"]' + ) + self.assertEqual(set(), regressions(before, after)) + + def test_nested_helper_libraries_keep_relative_package_dependencies(self): + files = fixture() + package = SCOPE + "/standalone/nested" + files[package + "/BUCK"] = ( + 'load(":targets.bzl", "define_libs")\ndefine_libs()\n' + ) + files[package + "/targets.bzl"] = ( + "def define_libs():\n " + + library("lib", '["module.py"]', '[":schema"]') + + " " + + library("schema", '["schema.py"]') + ) + files[package + "/module.py"] = "from .schema import Tensor\n" + files[package + "/schema.py"] = "class Tensor: pass\n" + self.assertEqual(set(), analyze(Snapshot(files))) + files[package + "/targets.bzl"] = files[package + "/targets.bzl"].replace( + 'deps=[":schema"]', "deps=[]" + ) + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertIn("//executorch/" + package + ":schema", next(iter(issues)).detail) + + def test_nested_declaration_errors_identify_the_owning_file(self): + declarations = ( + (library("lib", "choose_sources()"), "srcs: choose_sources"), + ( + 'def declare(name):\n pass\ndeclare("lib")\n', + "parameterized declaration helper", + ), + ("runtime.python_library(", "cannot read"), + ) + for build_file in ("BUCK", "TARGETS"): + for definition, expected in declarations: + with self.subTest(build_file=build_file, expected=expected): + files = fixture() + filename = SCOPE + "/standalone/" + build_file + files[filename] = definition + issues = analyze(Snapshot(files)) + relevant = [issue for issue in issues if expected in issue.detail] + self.assertTrue(relevant) + self.assertTrue(all(issue.source == filename for issue in relevant)) + + def test_nested_helper_errors_identify_the_helper_file(self): + files = fixture() + package = SCOPE + "/standalone" + files[package + "/TARGETS"] = ( + 'load(":helpers.bzl", "define_libs")\ndefine_libs()\n' + ) + helper = package + "/helpers.bzl" + files[helper] = "def define_libs():\n " + library("lib", "choose_sources()") + issues = analyze(Snapshot(files)) + self.assertTrue(issues) + self.assertTrue(all(issue.source == helper for issue in issues)) + files[helper] = "def define_libs():\n declare(\n" + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + self.assertEqual(helper, next(iter(issues)).source) + + def test_similar_sibling_package_is_outside_arm_test_scope(self): + files = fixture() + package = SCOPE + "_other" + files[package + "/BUCK"] = library("lib", "choose_sources()") + files[package + "/module.py"] = "from executorch.missing import value\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_nested_baseline_gap_does_not_hide_a_different_new_gap(self): + before = fixture() + package = SCOPE + "/standalone" + before[package + "/BUCK"] = library("lib", '["module.py"]') + before[package + "/module.py"] = "from executorch.exir.schema import Tensor\n" + self.assertEqual((set(), 1), compare(Snapshot(before), Snapshot(before))) + after = dict(before) + after[ + package + "/module.py" + ] += "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + issues, unchanged = compare(Snapshot(before), Snapshot(after)) + self.assertEqual(1, unchanged) + self.assertEqual(1, len(issues)) + self.assertIn("diff_pte_lib", next(iter(issues)).detail) + + def test_deleted_nested_owner_with_retained_source_fails(self): + before = fixture() + package = SCOPE + "/standalone" + source = package + "/module.py" + before[package + "/TARGETS"] = library("lib", '["module.py"]') + before[source] = "value = 42\n" + after = dict(before) + del after[package + "/TARGETS"] + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("ownership", issue.category) + self.assertEqual(source, issue.source) + del after[source] + self.assertEqual(set(), regressions(before, after)) + + def test_loop_generated_library_is_not_silently_ignored(self): + before = fixture() + after = dict(before) + after[ + BUCK + ] += 'for name in ["extra"]:\n runtime.python_library(name=name, srcs=["extra.py"])\n' + after[SCOPE + "/extra.py"] = "from executorch.missing import value\n" + issues = regressions(before, after) + self.assertTrue( + any( + i.kind == "cannot validate" + and "unsupported declaration loop" in i.detail + for i in issues + ) + ) + + def test_known_pytest_generation_loop_is_handled_by_registration(self): + files = fixture() + files[ + BUCK + ] += 'load(":targets.bzl", "define_arm_tests")\nfbcode_target(_kind=define_arm_tests)\n' + files[ + TARGETS + ] += " for test_file in test_files:\n python_pytest(name=test_file, srcs=[test_file])\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_oss_only_test_registration_is_valid(self): + before = fixture() + after = dict(before) + after[SCOPE + "/misc/test_oss.py"] = "def test_oss():\n pass\n" + after[ + TARGETS + ] += ' if runtime.is_oss:\n test_files += ["misc/test_oss.py"]\n' + self.assertEqual(set(), regressions(before, after)) + + def test_provider_suggestion_change_is_not_new_violation(self): + before = fixture() + before[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + after = dict(before) + after["exir/BUCK"] += library("schema_alternative", '["schema.py"]') + self.assertNotEqual( + {i.detail for i in analyze(Snapshot(before))}, + {i.detail for i in analyze(Snapshot(after))}, + ) + self.assertEqual(set(), regressions(before, after)) + + def test_diagnostic_category_change_only_adds_external_warnings(self): + before = fixture() + before[BUCK] = library("arm_tester_lib", 'glob(["tester/*.py"])') + after = dict(before) + after[BUCK] = library( + "arm_tester_lib", + 'glob(["tester/*.py"])', + '["//executorch/internal:unavailable"]', + ) + issues = regressions(before, after) + self.assertTrue(issues) + self.assertTrue(all(i.severity == "warning" for i in issues)) + + def test_lazy_external_package_errors_keep_the_original_cause(self): + declarations = ( + ( + 'for name in ["schema"]:\n' + ' runtime.python_library(name=name, srcs=["schema.py"])\n', + "unsupported declaration loop", + ), + ( + "def declare(name):\n" + ' runtime.python_library(name=name, srcs=["schema.py"])\n' + 'declare("schema")\n', + "parameterized", + ), + ( + "if configured():\n" + ' runtime.python_library(name="schema", srcs=["schema.py"])\n', + "conditional", + ), + ) + for declaration, cause in declarations: + for through_dependency in (False, True): + with self.subTest(cause=cause, through_dependency=through_dependency): + files = fixture() + files["exir/BUCK"] = declaration + files[BUCK] = library( + "arm_tester_lib", + '["tester/runner.py"]', + '["//executorch/exir:schema"]' if through_dependency else "[]", + ) + issues = analyze(Snapshot(files)) + self.assertTrue(all(i.severity == "warning" for i in issues)) + self.assertTrue( + any( + i.source == "exir/BUCK" and cause in i.detail + for i in issues + ), + issues, + ) + + def test_lazy_external_helper_error_names_helper_file(self): + files = fixture() + files["exir/BUCK"] = 'load(":rules.bzl", "declare")\ndeclare()\n' + files["exir/rules.bzl"] = ( + "def declare():\n" + " if configured():\n" + ' runtime.python_library(name="schema", srcs=["schema.py"])\n' + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.source == "exir/rules.bzl" + and "conditional" in i.detail + and i.severity == "warning" + for i in issues + ), + issues, + ) + self.assertTrue(all(i.severity == "warning" for i in issues)) + + def test_computed_external_sources_report_the_original_expression(self): + for through_dependency in (False, True): + with self.subTest(through_dependency=through_dependency): + files = fixture() + files["exir/BUCK"] = library("schema", "choose_sources()") + files[BUCK] = library( + "arm_tester_lib", + '["tester/runner.py"]', + '["//executorch/exir:schema"]' if through_dependency else "[]", + ) + issues = analyze(Snapshot(files)) + self.assertTrue(all(i.severity == "warning" for i in issues)) + self.assertTrue( + any( + i.source == "exir/BUCK" and "choose_sources()" in i.detail + for i in issues + ), + issues, + ) + + def test_computed_external_dependencies_do_not_cascade_to_errors(self): + before = fixture() + after = dict(before) + after["exir/BUCK"] = library("umbrella", "[]", "choose_deps()") + library( + "schema", '["schema.py"]' + ) + issues = regressions(before, after) + self.assertTrue(issues) + self.assertTrue(all(i.severity == "warning" for i in issues)) + self.assertTrue( + any(i.source == "exir/BUCK" and "choose_deps()" in i.detail for i in issues) + ) + self.assertTrue( + any( + i.source == TESTER and "exir/BUCK: deps: choose_deps()" in i.detail + for i in issues + ) + ) + + def test_known_missing_dependency_still_fails_with_external_package_warning(self): + files = fixture() + files[BUCK] = library("arm_tester_lib", '["tester/runner.py"]') + files["exir/BUCK"] += ( + 'for name in ["other"]:\n' + " runtime.python_library(name=name, srcs=[])\n" + ) + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.kind == "missing dependency" and i.severity == "error" for i in issues + ) + ) + self.assertTrue( + any(i.source == "exir/BUCK" and i.severity == "warning" for i in issues) + ) + + def test_unsupported_arm_provider_outside_test_scope_still_fails(self): + files = fixture() + files[TESTER] = "from executorch.backends.arm.provider import value\n" + files["backends/arm/BUCK"] = library("provider", "choose_sources()") + files["backends/arm/provider.py"] = "value = 1\n" + issues = analyze(Snapshot(files)) + self.assertTrue( + any( + i.source == "backends/arm/BUCK" + and "choose_sources()" in i.detail + and i.severity == "error" + for i in issues + ) + ) + + def test_unrelated_external_packages_are_not_reported(self): + files = fixture() + files["unrelated/BUCK"] = ( + 'for name in ["other"]:\n' + " runtime.python_library(name=name, srcs=[])\n" + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_unchanged_external_warnings_are_baselined(self): + files = fixture() + files["exir/BUCK"] = library("umbrella", "[]", "choose_deps()") + library( + "schema", '["schema.py"]' + ) + new, unchanged = compare(Snapshot(files), Snapshot(files)) + self.assertEqual(set(), new) + self.assertGreater(unchanged, 0) + + def test_warning_becoming_proven_missing_dependency_is_new_error(self): + before = fixture() + before["exir/BUCK"] = library("umbrella", "[]", "choose_deps()") + library( + "schema", '["schema.py"]' + ) + after = dict(before) + after["exir/BUCK"] = library("umbrella", "[]") + library( + "schema", '["schema.py"]' + ) + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("missing dependency", issue.kind) + self.assertEqual("error", issue.severity) + self.assertEqual("executorch.exir.schema", issue.subject) + + def test_cli_reports_warnings_without_failing_and_still_fails_errors(self): + before = fixture() + for add_arm_error in (False, True): + with self.subTest(add_arm_error=add_arm_error): + after = dict(before) + after["exir/BUCK"] = library( + "umbrella", "[]", "choose_deps()" + ) + library("schema", '["schema.py"]') + if add_arm_error: + after[SCOPE + "/misc/test_new.py"] = "def test_new(): pass\n" + output = io.StringIO() + with mock.patch.object( + checker, + "GitSnapshot", + side_effect=[Snapshot(before), Snapshot(after)], + ), contextlib.redirect_stdout(output): + status = checker.main(["--repo", ".", "--base", "base"]) + self.assertEqual(1 if add_arm_error else 0, status) + report = output.getvalue() + self.assertIn("FAIL" if add_arm_error else "PASS", report) + self.assertIn( + "New errors: 1" if add_arm_error else "New errors: 0", report + ) + self.assertIn("New warnings: 2", report) + self.assertIn("WARNING: Cannot validate", report) + self.assertIn("File: exir/BUCK", report) + self.assertIn("choose_deps()", report) + if add_arm_error: + self.assertIn("ERROR: Unregistered test", report) + + def test_nested_explicit_python_test_registration_is_valid(self): + for kind in ("python_test", "python_pytest", "python_unittest"): + with self.subTest(kind=kind): + before = fixture() + after = dict(before) + after[SCOPE + "/unit/test_unit.py"] = "def test_unit():\n pass\n" + after[SCOPE + "/unit/BUCK"] = ( + f'runtime.{kind}(name="test_unit", srcs=["test_unit.py"], deps=[])\n' + ) + self.assertEqual(set(), regressions(before, after)) + + def test_deleted_owner_with_retained_source_fails(self): + before = fixture() + for retain_aggregator in (False, True): + with self.subTest(retain_aggregator=retain_aggregator): + after = dict(before) + after[BUCK] = ( + library("arm_tester", "[]", '[":arm_tester_lib"]') + if retain_aggregator + else "" + ) + issues = regressions(before, after) + self.assertTrue( + any( + i.category == "ownership" and i.source == TESTER for i in issues + ) + ) + if retain_aggregator: + self.assertTrue( + any("unresolved scoped dependency" in i.detail for i in issues) + ) + + def test_deleted_owner_sources_and_edges_is_valid(self): + before = fixture() + before[BUCK] += library("arm_tester", "[]", '[":arm_tester_lib"]') + after = dict(before) + after[BUCK] = "" + del after[TESTER] + self.assertEqual(set(), regressions(before, after)) + + def test_library_rename_with_retained_sources_is_valid(self): + before = fixture() + after = dict(before) + after[BUCK] = after[BUCK].replace("arm_tester_lib", "renamed_lib") + self.assertEqual(set(), regressions(before, after)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/tests/test_arm_buck_entrypoint.py b/.ci/scripts/tests/test_arm_buck_entrypoint.py new file mode 100644 index 00000000000..d38676ec3bd --- /dev/null +++ b/.ci/scripts/tests/test_arm_buck_entrypoint.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Integration checks for the toolchain-free Arm Buck declaration entrypoint.""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ENTRYPOINT = ( + Path(__file__).resolve().parents[3] / "backends/arm/test/test_arm_backend.sh" +) + + +class ArmBuckEntrypointTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.repo = self.root / "repo with spaces" + self.repo.mkdir() + self.git("init", "-q") + self.git("config", "user.name", "Test") + self.git("config", "user.email", "test@example.com") + self.git("config", "commit.gpgsign", "false") + self.git("config", "core.hooksPath", str(self.root / "no-hooks")) + self.script = self.repo / "backends/arm/test/test_arm_backend.sh" + self.script.parent.mkdir(parents=True) + shutil.copyfile(ENTRYPOINT, self.script) + self.checker_dir = self.repo / ".ci/scripts" + tests = self.checker_dir / "tests" + tests.mkdir(parents=True) + (self.checker_dir / "pytest-parallelism.sh").write_text("") + (tests / "test_arm_buck_declarations.py").write_text( + "import os, unittest\n" + "class CheckerTest(unittest.TestCase):\n" + " def test_checker(self):\n" + " self.assertNotIn('FAIL_CHECKER_SELF_TEST', os.environ)\n" + ) + (self.checker_dir / "check_arm_buck_declarations.py").write_text( + "import json, os, pathlib, sys\n" + "pathlib.Path(os.environ['ENTRYPOINT_RESULT']).write_text(json.dumps({\n" + " 'base': sys.argv[sys.argv.index('--base') + 1],\n" + " 'cwd': os.getcwd(),\n" + "}))\n" + "sys.exit(int(os.environ.get('CHECKER_EXIT_CODE', '0')))\n" + ) + self.initial = self.commit("initial") + self.base = self.commit("base") + self.head = self.commit("head") + self.result_file = self.root / "result.json" + + def git(self, *args): + return subprocess.check_output( + ["git", *args], cwd=self.repo, text=True, stderr=subprocess.PIPE + ).strip() + + def commit(self, message): + (self.repo / "revision").write_text(message) + self.git("add", ".") + self.git("commit", "-qm", message) + return self.git("rev-parse", "HEAD") + + def run_suite(self, *args, **overrides): + env = os.environ.copy() + for key in ("BASE_REVISION", "GITHUB_EVENT_NAME", "PYTHONPATH"): + env.pop(key, None) + env.update( + PYTHON_EXECUTABLE=sys.executable, + ENTRYPOINT_RESULT=str(self.result_file), + ) + env.update(overrides) + if self.result_file.exists(): + self.result_file.unlink() + # A caller may invoke the entrypoint from outside the repository. + result = subprocess.run( + ["bash", str(self.script), *args], + cwd=self.root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=30, + ) + return result + + def assert_base(self, expected, **env): + result = self.run_suite("test_buck_declarations", **env) + self.assertEqual(result.returncode, 0, result.stdout) + actual = json.loads(self.result_file.read_text()) + self.assertEqual(actual["base"], expected) + self.assertEqual(Path(actual["cwd"]).resolve(), self.repo.resolve()) + + def test_local_manual_and_new_branch_compare_latest_commit(self): + for env in ( + {}, + {"GITHUB_EVENT_NAME": "workflow_dispatch"}, + {"GITHUB_EVENT_NAME": "push", "BASE_REVISION": "0" * 40}, + ): + with self.subTest(env=env): + self.assert_base(self.base, **env) + + def test_local_base_override(self): + self.assert_base(self.initial, BASE_REVISION=self.initial) + + def test_push_keeps_event_base(self): + self.assert_base( + self.initial, GITHUB_EVENT_NAME="push", BASE_REVISION=self.initial + ) + + def test_pull_request_uses_merge_base(self): + self.git("checkout", "-qb", "target", self.base) + target = self.commit("target-only") + self.git("checkout", "-q", self.head) + self.assert_base( + self.base, GITHUB_EVENT_NAME="pull_request", BASE_REVISION=target + ) + + def test_fetches_missing_base_commit(self): + remote = self.root / "remote" + subprocess.run(["git", "clone", "-q", str(self.repo), str(remote)], check=True) + for key, value in ( + ("user.name", "Test"), + ("user.email", "test@example.com"), + ("commit.gpgsign", "false"), + ("core.hooksPath", str(self.root / "no-hooks")), + ): + subprocess.run(["git", "-C", str(remote), "config", key, value], check=True) + subprocess.run( + ["git", "-C", str(remote), "checkout", "-qb", "target", self.base], + check=True, + ) + subprocess.run( + ["git", "-C", str(remote), "commit", "-qm", "remote", "--allow-empty"], + check=True, + ) + target = subprocess.check_output( + ["git", "-C", str(remote), "rev-parse", "HEAD"], text=True + ).strip() + self.git("remote", "add", "origin", str(remote)) + self.assert_base( + self.base, GITHUB_EVENT_NAME="pull_request", BASE_REVISION=target + ) + self.git("cat-file", "-e", target) + + def test_declaration_suite_skips_arm_and_pytest_setup(self): + setup = self.repo / "examples/arm/arm-scratch/setup_path.sh" + setup.parent.mkdir(parents=True) + setup.write_text("exit 91\n") + (self.checker_dir / "pytest-parallelism.sh").write_text("exit 92\n") + self.assert_base(self.base) + + def test_other_suite_still_requires_arm_setup(self): + result = self.run_suite("test_pytest_ops_no_target") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Missing", result.stdout) + self.assertIn("setup_path.sh", result.stdout) + self.assertFalse(self.result_file.exists()) + + def test_all_does_not_run_change_specific_declarations(self): + setup = self.repo / "examples/arm/arm-scratch/setup_path.sh" + setup.parent.mkdir(parents=True) + setup.write_text("") + tools = self.root / "bin" + tools.mkdir() + shell = tools / "sh" + shell.write_text('#!/bin/sh\ncat > "$ENTRYPOINT_COMMANDS"\n') + shell.chmod(0o755) + commands = self.root / "all-commands" + result = self.run_suite( + "all", + PATH=str(tools) + os.pathsep + os.environ["PATH"], + ENTRYPOINT_COMMANDS=str(commands), + ) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertNotIn("test_buck_declarations", commands.read_text()) + self.assertIn("test_pytest_ops_no_target", commands.read_text()) + self.assertFalse(self.result_file.exists()) + + def test_help_does_not_require_arm_setup(self): + for args in ((), ("help",)): + with self.subTest(args=args): + result = self.run_suite(*args) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("test_buck_declarations", result.stdout) + + def test_checker_failure_reaches_caller(self): + result = self.run_suite("test_buck_declarations", CHECKER_EXIT_CODE="7") + self.assertEqual(result.returncode, 7, result.stdout) + + def test_self_test_failure_stops_checker(self): + result = self.run_suite("test_buck_declarations", FAIL_CHECKER_SELF_TEST="1") + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.result_file.exists()) + + def test_invalid_base_stops_checker(self): + result = self.run_suite("test_buck_declarations", BASE_REVISION="missing-base") + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.result_file.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.ci/scripts/unittest-arm-buck2.sh b/.ci/scripts/unittest-arm-buck2.sh new file mode 100755 index 00000000000..f8943c2b8b1 --- /dev/null +++ b/.ci/scripts/unittest-arm-buck2.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +set -euo pipefail + +# This C++ package does not load the Arm Python tests' internal Buck helpers. +target=//backends/arm/test/vela:vela_external_blocks_test +buck2 test "${target}" +binary=$(buck2 build "${target}" --show-full-simple-output) + +# Run each case in a fresh process, as internal CI does, to expose dependencies +# on initialization performed by an earlier case in the same test binary. +cases=$("${binary}" --gtest_list_tests | awk ' + /^[^ ]/ { suite = $1 } + /^ [^ ]/ { print suite $1 } +') +if [[ -z "${cases}" ]]; then + echo "No Vela parser tests were discovered" >&2 + exit 1 +fi +while IFS= read -r test_case; do + "${binary}" --gtest_filter="${test_case}" +done <<< "${cases}" diff --git a/.ci/scripts/unittest-buck2.sh b/.ci/scripts/unittest-buck2.sh index 9c4c4608694..2231ce4294e 100755 --- a/.ci/scripts/unittest-buck2.sh +++ b/.ci/scripts/unittest-buck2.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -43,6 +44,8 @@ for op in "build" "test"; do //runtime/executor: //runtime/kernel/... //runtime/platform/... done +bash .ci/scripts/unittest-arm-buck2.sh + # Build only without testing buck2 build //codegen/tools/... \ //exir/_serialize/test/... \ diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 52c009b5608..40a19c03717 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1,3 +1,8 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + name: pull on: @@ -994,6 +999,26 @@ jobs: exit 1 fi + test-arm-backend-buck-declarations: + name: test-arm-backend-buck-declarations + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Test declaration entrypoint + run: python -m unittest discover -s .ci/scripts/tests -p test_arm_buck_entrypoint.py + - name: Check changed Arm Python declarations + env: + BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before }} + run: backends/arm/test/test_arm_backend.sh test_buck_declarations + test-arm-backend-no-driver: needs: docker-image name: test-arm-backend-no-driver diff --git a/backends/arm/README.md b/backends/arm/README.md index 73f436962f8..b3abc2890ca 100644 --- a/backends/arm/README.md +++ b/backends/arm/README.md @@ -1,3 +1,10 @@ + + # ExecuTorch Arm® Backend This subtree contains the Arm® Backend implementation for ExecuTorch. @@ -402,6 +409,7 @@ Below is an overview of some of the testing options this script provides: | Command | Description | | ---------------------------------------------------- | ------------------------------------------------------------ | +| `test_arm_backend.sh test_buck_declarations` | Checks changed Arm Python Buck declarations without Arm tool setup. | | `test_arm_backend.sh test_pytest_ops_no_target` | Runs operator unit tests for non-target specific use-cases. | | `test_arm_backend.sh test_pytest_models_no_target` | Runs model unit tests for non-target specific use-cases. | | `test_arm_backend.sh test_pytest_ops_tosa` | Runs operator unit tests for TOSA specific use-cases. | @@ -471,6 +479,54 @@ Installing model test dependencies is a standalone process. The script installs only the dependencies needed for model tests, skipping all other setup procedures. +### Buck2 CI coverage + +The GitHub Buck2 job runs the Vela parser C++ tests together and then runs each +case in a separate process, so a test cannot depend on runtime initialization by +an earlier case. With the revision-pinned Buck2 on your `PATH`, run this check from +the repository root: + +```bash +bash .ci/scripts/unittest-arm-buck2.sh +``` + +The separate **test-arm-backend-buck-declarations** job checks for new dependency +and test-registration gaps in the Arm test package. It uses the same entrypoint as +local runs and requires only Git and Python 3, without Arm tool setup or third-party +Python packages: + +```bash +backends/arm/test/test_arm_backend.sh test_buck_declarations +``` + +By default, local runs compare the working tree against `HEAD^`. To check a change +spanning several commits, set `BASE_REVISION` to the commit before your change: + +```bash +BASE_REVISION= backends/arm/test/test_arm_backend.sh test_buck_declarations +``` + +GitHub pull requests compare against the merge base with the PR's target branch; +pushes use the previous commit from the event. Manual runs and new branches use +`HEAD^`. Declaration checks are excluded from the `all` suite because they need a +change-specific baseline. + +This static check follows supported in-repository Python library declarations, +including transitive dependencies, and compares findings against the base revision. +Existing findings do not fail the job. New confirmed missing dependencies, +unregistered Arm tests, and unsupported declarations under `backends/arm/` fail +the check. Unsupported declarations outside Arm are reported as warnings with +their declaration file and unsupported construct; dependency gaps that cannot be +proven because of those declarations are also warnings. A warning becoming an +error is treated as a new regression. + +Arm backend contributors maintain this checker alongside the Arm Buck declarations +and its regression tests in `.ci/scripts/tests/test_arm_buck_declarations.py`. +Moving the Arm Python tests onto the public runtime wrapper would allow Buck2 +queries to replace this limited static analysis. The checker does not run Meta's +internal Python Buck tests or validate third-party dependencies. Those internal +tests remain necessary for full packaged-test coverage. + ## Using git hooks The repo-wide pre-commit hook (lintrunner + torch_pin sync) is installed automatically diff --git a/backends/arm/runtime/tests/ethos-u/CMakeLists.txt b/backends/arm/runtime/tests/ethos-u/CMakeLists.txt index 5a2ee4240e9..70450f75d25 100644 --- a/backends/arm/runtime/tests/ethos-u/CMakeLists.txt +++ b/backends/arm/runtime/tests/ethos-u/CMakeLists.txt @@ -71,7 +71,7 @@ message( add_executable( vela_external_blocks_test "${EXECUTORCH_ROOT}/backends/arm/runtime/VelaBinStream.cpp" - "${EXECUTORCH_ROOT}/backends/arm/test/vela_external_blocks_test.cpp" + "${EXECUTORCH_ROOT}/backends/arm/test/vela/vela_external_blocks_test.cpp" ) target_link_libraries( vela_external_blocks_test PRIVATE executorch_core GTest::gtest_main diff --git a/backends/arm/test/targets.bzl b/backends/arm/test/targets.bzl index a529e2d9449..15719fa0907 100644 --- a/backends/arm/test/targets.bzl +++ b/backends/arm/test/targets.bzl @@ -1,3 +1,8 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + # load("//caffe2/test/fb:defs.bzl", "define_tests") load("@fbcode_macros//build_defs:python_pytest.bzl", "python_pytest") load("@bazel_skylib//lib:paths.bzl", "paths") @@ -199,9 +204,11 @@ def define_arm_tests(): ] if test_file == "misc/test_diff_pte.py" else []), ) + # Preserve the label used by internal CI to select individual Vela cases. + # OSS CI uses the separate vela package to avoid internal-only Python loads. runtime.cxx_test( name = "vela_external_blocks_test", - srcs = ["vela_external_blocks_test.cpp"], + srcs = ["//executorch/backends/arm/test/vela:vela_external_blocks_test.cpp"], deps = [ "//executorch/backends/arm/runtime:vela_bin_stream", "//executorch/runtime/core:core", diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index e2ff9d85b29..99d3cad2f47 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -10,12 +10,10 @@ script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) vulkan_validation_runner="${script_dir}/run_with_vulkan_validation.sh" # Executorch root -et_root_dir=$(cd ${script_dir}/../../.. && pwd) +et_root_dir=$(cd "${script_dir}/../../.." && pwd) cd "${et_root_dir}" pwd -# Cap pytest-xdist's `auto` workers to the container's CPU quota. -source .ci/scripts/pytest-parallelism.sh scratch_dir=${et_root_dir}/examples/arm/arm-scratch setup_path_script=${scratch_dir}/setup_path.sh _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly install necessary tools." @@ -118,21 +116,7 @@ enable_vgf_vulkan_validation() { echo "VK_INSTANCE_LAYERS=${VK_INSTANCE_LAYERS}" } -TEST_SUITE=$1 - -# Enable Vulkan validation for VGF/VKML runtime tests. - -# Source the tools -# This should be prepared by the setup.sh -[[ -f ${setup_path_script} ]] \ - || { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; } -source ${setup_path_script} - -# Enable Vulkan validation for every VGF/VKML test path, including tests that -# launch scripts/executables directly instead of going through runner_utils.py. -if [[ "${TEST_SUITE}" == *vkml* || "${TEST_SUITE}" == *vgf* ]]; then - enable_vgf_vulkan_validation -fi +TEST_SUITE=${1:-} help() { echo "Usage:" @@ -141,17 +125,35 @@ help() { # This will list all lines in this file that is starting with test_ remove () { and print it as a list. # e,g, "test_pytest() { # Test ops and other things" -> test_pytest # Test ops and other things echo "all # run all tests" - grep "^test_" $0 | sed 's/([^)]*)[[:space:]]*{*//g' + grep "^test_" "${script_dir}/test_arm_backend.sh" | sed 's/([^)]*)[[:space:]]*{*//g' exit } if [[ -z "${TEST_SUITE:-}" ]]; then echo "Missing test suite name, exiting..." help +elif [[ "${TEST_SUITE}" == help ]]; then + help else echo "Run Arm backend test suite ${TEST_SUITE}" fi +# Static Buck declarations need only Git and the Python standard library. +if [[ "${TEST_SUITE}" != test_buck_declarations ]]; then + # Cap pytest-xdist's `auto` workers to the container's CPU quota. + source .ci/scripts/pytest-parallelism.sh + + # This should be prepared by setup.sh. + [[ -f ${setup_path_script} ]] \ + || { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; } + source "${setup_path_script}" + + # Enable validation for VGF/VKML paths after setup_path.sh configures the layers. + if [[ "${TEST_SUITE}" == *vkml* || "${TEST_SUITE}" == *vgf* ]]; then + enable_vgf_vulkan_validation + fi +fi + TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}" EXCLUDE_TARGET_EXPR="(not u55) and (not u65) and (not u85) and (not tosa) and (not _vgf_)" @@ -177,13 +179,33 @@ all() { # Run all tests # ---- # That is executed echo "${TEST_SUITE_NAME}: Run all tests" - grep "^test_" backends/arm/test/test_arm_backend.sh | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh -e + # Declaration checks require a change-specific base; run them explicitly. + grep "^test_" backends/arm/test/test_arm_backend.sh | grep -v "^test_buck_declarations()" | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh -e echo "${TEST_SUITE_NAME}: PASS" } # ------------------------------------------- # -------- Non target-specific tests -------- # ------------------------------------------- +test_buck_declarations() { # Check changed Arm Python Buck declarations without Arm tool setup. + local base_revision="${BASE_REVISION:-}" + local python_executable="${PYTHON_EXECUTABLE:-python3}" + + "${python_executable}" -m unittest discover -s .ci/scripts/tests -p test_arm_buck_declarations.py + + if [[ -z "${base_revision}" || "${base_revision}" == 0000000000000000000000000000000000000000 ]]; then + # Local/manual runs and new branches compare the latest commit. + base_revision=$(git rev-parse HEAD^) + elif ! git cat-file -e "${base_revision}^{commit}" 2>/dev/null; then + git fetch --no-tags origin "${base_revision}" + fi + if [[ "${GITHUB_EVENT_NAME:-}" == pull_request ]]; then + base_revision=$(git merge-base HEAD "${base_revision}") + fi + + "${python_executable}" .ci/scripts/check_arm_buck_declarations.py --repo . --base "${base_revision}" +} + test_pytest_ops_no_target() { echo "${TEST_SUITE_NAME}: Run pytest ops for target-less tests" diff --git a/backends/arm/test/vela/BUCK b/backends/arm/test/vela/BUCK new file mode 100644 index 00000000000..b0d58e297f3 --- /dev/null +++ b/backends/arm/test/vela/BUCK @@ -0,0 +1,10 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +load(":targets.bzl", "define_vela_tests") + +oncall("executorch") + +define_vela_tests() diff --git a/backends/arm/test/vela/targets.bzl b/backends/arm/test/vela/targets.bzl new file mode 100644 index 00000000000..14faf6aaeb7 --- /dev/null +++ b/backends/arm/test/vela/targets.bzl @@ -0,0 +1,25 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +def define_vela_tests(): + runtime.export_file( + name = "vela_external_blocks_test.cpp", + visibility = ["//executorch/backends/arm/test:vela_external_blocks_test"], + ) + + if runtime.is_oss: + runtime.cxx_test( + name = "vela_external_blocks_test", + srcs = ["vela_external_blocks_test.cpp"], + compiler_flags = ["-DEXECUTORCH_ARM_VELA_TEST_NO_GLOBAL_INIT"], + deps = [ + "//executorch/backends/arm/runtime:vela_bin_stream", + "//executorch/runtime/core:core", + "//executorch/runtime/core:named_data_map", + "//executorch/runtime/platform:platform", + ], + ) diff --git a/backends/arm/test/vela_external_blocks_test.cpp b/backends/arm/test/vela/vela_external_blocks_test.cpp similarity index 97% rename from backends/arm/test/vela_external_blocks_test.cpp rename to backends/arm/test/vela/vela_external_blocks_test.cpp index 7250b47c8f8..5405a9134fb 100644 --- a/backends/arm/test/vela_external_blocks_test.cpp +++ b/backends/arm/test/vela/vela_external_blocks_test.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. + * Copyright 2026 Arm Limited and/or its affiliates. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. @@ -36,6 +37,13 @@ using executorch::runtime::TensorLayout; namespace { +class VelaExternalBlocksTest : public ::testing::Test { + protected: + void SetUp() override { + executorch::runtime::runtime_init(); + } +}; + constexpr std::string_view kExternalKey = "g123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; constexpr std::string_view kSecondExternalKey = @@ -131,13 +139,6 @@ class FakeNamedDataMap final : public NamedDataMap { std::vector entries_; }; -class VelaExternalBlocksTest : public ::testing::Test { - protected: - void SetUp() override { - executorch::runtime::runtime_init(); - } -}; - } // namespace TEST_F(VelaExternalBlocksTest, ReadsInlineAndExternalBlocksInOnePass) { @@ -383,3 +384,12 @@ TEST_F(VelaExternalBlocksTest, RejectsIoCountExceedingPayload) { &handles), Error::InvalidProgram); } + +#ifdef EXECUTORCH_ARM_VELA_TEST_NO_GLOBAL_INIT +// Only the OSS isolation target overrides the test main. Leave runtime +// initialization to the fixture so each case proves it initializes the PAL. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +#endif From aa6b0891a8238d6a2b03d147b13c72a7341ad495 Mon Sep 17 00:00:00 2001 From: Usamah Zaheer Date: Mon, 2 Nov 2026 14:04:54 +0000 Subject: [PATCH 2/2] Fix missed Arm Buck declaration regressions Count Arm test registrations only when the declaration visitor actually invokes define_arm_tests. Keep the package, source file and original helper name so aliases and called wrappers work while unused, inactive or unsupported calls cannot keep a removed test suite falsely registered. Evaluate supported augmented assignments to source and dependency lists. Invalidate unsupported mutations and report the original expression instead of retaining stale declaration values. Resolve imports from the executorch root package through the same child-module dependency checks as dotted imports. Add regression coverage for all three new review comments on GitHub PR #23888. The 64 checker tests and 12 entrypoint integration tests pass. The local entrypoint reports zero new errors or warnings against the parent patch. Regression tests cover removed and inactive registration calls, augmented assignments and imports from the root package. UFMT, Flake8 and diff whitespace checks pass. Authored with assistance from OpenAI Codex. Signed-off-by: Usamah Zaheer Change-Id: I6944d6481c6231754269716e4db1e3d7dcf22272 --- .ci/scripts/check_arm_buck_declarations.py | 26 +- .../tests/test_arm_buck_declarations.py | 237 +++++++++++++++++- 2 files changed, 254 insertions(+), 9 deletions(-) diff --git a/.ci/scripts/check_arm_buck_declarations.py b/.ci/scripts/check_arm_buck_declarations.py index 146833012c4..5421131551a 100644 --- a/.ci/scripts/check_arm_buck_declarations.py +++ b/.ci/scripts/check_arm_buck_declarations.py @@ -154,6 +154,7 @@ def __init__(self, snapshot: Snapshot): self.snapshot = snapshot self.rules: dict[str, Rule] = {} self.loaded: set[str] = set() + self.called_helpers: set[tuple[str, str, str]] = set() self.package_errors: dict[str, list[tuple[str, str]]] = {} def package_error(self, package: str, filename: str, error: str) -> None: @@ -386,6 +387,7 @@ def visit_helper(name: str) -> None: package, filename, "recursive declaration helper: " + name ) else: + self.called_helpers.add((package, filename, name)) visiting.add(name) visit(function.body, dict(env)) visiting.remove(name) @@ -403,6 +405,26 @@ def visit(statements: list[ast.stmt], local_env: dict[str, object]) -> None: ) except Unsupported: local_env.pop(node.targets[0].id, None) + elif isinstance(node, ast.AugAssign): + target = node.target + try: + if not isinstance(target, ast.Name): + raise Unsupported("non-name assignment target") + local_env[target.id] = self.value( + ast.BinOp(left=target, op=node.op, right=node.value), + local_env, + package, + ) + except (Unsupported, TypeError) as error: + while isinstance(target, (ast.Attribute, ast.Subscript)): + target = target.value + if isinstance(target, ast.Name): + local_env.pop(target.id, None) + self.package_error( + package, + filename, + f"unsupported augmented assignment: {ast.unparse(node)} ({error})", + ) elif isinstance(node, ast.If): try: branch = ( @@ -588,7 +610,7 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None: package[: len(package) - node.level + 1] + ([module] if module else []) ) - if not module.startswith("executorch."): + if module != "executorch" and not module.startswith("executorch."): return for alias in node.names: child = module + "." + alias.name @@ -626,6 +648,8 @@ def is_test_file(source: str, filename: str) -> bool: def registered_tests( # noqa: C901 declarations: Declarations, ) -> tuple[set[str], list[str]]: + if (SCOPE, TEST_MACRO, "define_arm_tests") not in declarations.called_helpers: + return set(), [] try: tree = ast.parse(declarations.snapshot.read(TEST_MACRO), filename=TEST_MACRO) except (KeyError, SyntaxError) as error: diff --git a/.ci/scripts/tests/test_arm_buck_declarations.py b/.ci/scripts/tests/test_arm_buck_declarations.py index 8461d0c3b56..e967d2aa758 100644 --- a/.ci/scripts/tests/test_arm_buck_declarations.py +++ b/.ci/scripts/tests/test_arm_buck_declarations.py @@ -33,6 +33,10 @@ def load_checker(): TESTER = SCOPE + "/tester/runner.py" BUCK = SCOPE + "/BUCK" TARGETS = SCOPE + "/targets.bzl" +ARM_TEST_REGISTRATION = ( + 'load(":targets.bzl", "define_arm_tests")\n' + "fbcode_target(_kind=define_arm_tests)\n" +) def library(name, sources, deps="[]"): @@ -45,7 +49,8 @@ def fixture(): "arm_tester_lib", 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', '["//executorch/exir:umbrella"]', - ), + ) + + ARM_TEST_REGISTRATION, TESTER: "from executorch.exir.schema import Tensor\n", TARGETS: 'def define_arm_tests():\n test_files = []\n test_files += native.glob(["passes/test_*.py"])\n', "exir/BUCK": library("umbrella", "[]", '[":schema"]') @@ -91,10 +96,13 @@ def test_known_missing_import_and_registration_then_metadata_fix(self): if i.kind == "missing dependency" ) ) - after[BUCK] = library( - "arm_tester_lib", - 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', - '["//executorch/exir:umbrella", "//executorch/devtools/pte_tool:diff_pte_lib"]', + after[BUCK] = ( + library( + "arm_tester_lib", + 'glob(["tester/*.py"], exclude=["tester/excluded.py"])', + '["//executorch/exir:umbrella", "//executorch/devtools/pte_tool:diff_pte_lib"]', + ) + + ARM_TEST_REGISTRATION ) after[TARGETS] += ' test_files += ["misc/test_diff_pte.py"]\n' self.assertEqual(set(), regressions(before, after)) @@ -632,9 +640,6 @@ def test_loop_generated_library_is_not_silently_ignored(self): def test_known_pytest_generation_loop_is_handled_by_registration(self): files = fixture() - files[ - BUCK - ] += 'load(":targets.bzl", "define_arm_tests")\nfbcode_target(_kind=define_arm_tests)\n' files[ TARGETS ] += " for test_file in test_files:\n python_pytest(name=test_file, srcs=[test_file])\n" @@ -866,6 +871,222 @@ def test_cli_reports_warnings_without_failing_and_still_fails_errors(self): if add_arm_error: self.assertIn("ERROR: Unregistered test", report) + def test_removing_macro_invocation_unregisters_existing_suite(self): + before = fixture() + test = SCOPE + "/passes/test_registered.py" + before[test] = "def test_registered(): pass\n" + self.assertEqual(set(), analyze(Snapshot(before))) + after = dict(before) + after[BUCK] = after[BUCK].replace("fbcode_target(_kind=define_arm_tests)\n", "") + issues = regressions(before, after) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("unregistered test", issue.kind) + self.assertEqual(test, issue.source) + self.assertEqual("error", issue.severity) + + def test_inactive_registration_calls_do_not_register_tests(self): + for invocation in ( + "", + "if False:\n define_arm_tests()\n", + "def unused():\n define_arm_tests()\n", + "non_fbcode_target(_kind=define_arm_tests)\n", + ): + with self.subTest(invocation=invocation): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_registered(): pass\n" + ) + files[BUCK] = files[BUCK].replace( + "fbcode_target(_kind=define_arm_tests)\n", invocation + ) + self.assertTrue( + any(i.kind == "unregistered test" for i in analyze(Snapshot(files))) + ) + + def test_alias_and_called_wrappers_register_tests(self): + for registration in ( + 'load(":targets.bzl", register="define_arm_tests")\nregister()\n', + 'load(":targets.bzl", register="define_arm_tests")\nfbcode_target(_kind=register)\n', + 'load(":targets.bzl", "define_arm_tests")\n' + "def wrapper():\n define_arm_tests()\nwrapper()\n", + ): + with self.subTest(registration=registration): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_registered(): pass\n" + ) + files[BUCK] = files[BUCK].replace(ARM_TEST_REGISTRATION, registration) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_parameterized_registration_is_not_counted(self): + for change_signature in (False, True): + with self.subTest(change_signature=change_signature): + files = fixture() + files[SCOPE + "/passes/test_registered.py"] = ( + "def test_registered(): pass\n" + ) + if change_signature: + files[TARGETS] = files[TARGETS].replace( + "define_arm_tests()", "define_arm_tests(config)" + ) + else: + files[BUCK] = files[BUCK].replace( + "_kind=define_arm_tests)", + "_kind=define_arm_tests, config=True)", + ) + issues = analyze(Snapshot(files)) + self.assertTrue(any(i.kind == "unregistered test" for i in issues)) + self.assertTrue( + any( + i.kind == "cannot validate" + and "parameterized declaration helper" in i.detail + for i in issues + ) + ) + + def test_dormant_registration_macro_is_not_evaluated(self): + files = fixture() + files[BUCK] = files[BUCK].replace("fbcode_target(_kind=define_arm_tests)\n", "") + files[TARGETS] = "def define_arm_tests():\n test_files = choose_tests()\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_root_package_import_matches_dotted_import_dependencies(self): + for source in ( + "import executorch.exir\n", + "from executorch import exir\n", + "from executorch import exir as export_ir\n", + ): + with self.subTest(source=source): + files = fixture() + files["exir/__init__.py"] = "" + files["exir/BUCK"] += library("lib", '["__init__.py"]') + files[TESTER] = source + issues = analyze(Snapshot(files)) + self.assertEqual(1, len(issues)) + issue = next(iter(issues)) + self.assertEqual("missing dependency", issue.kind) + self.assertEqual("executorch.exir", issue.subject) + self.assertEqual("error", issue.severity) + files[BUCK] = files[BUCK].replace( + '["//executorch/exir:umbrella"]', + '["//executorch/exir:umbrella", "//executorch/exir:lib"]', + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_root_package_import_checks_each_alias_and_skips_type_only(self): + files = fixture() + files["exir/__init__.py"] = "" + files["exir/BUCK"] += library("lib", '["__init__.py"]') + files["devtools/__init__.py"] = "" + files["devtools/BUCK"] = library("lib", '["__init__.py"]') + files[TESTER] = "from executorch import exir, devtools as tools\n" + issues = analyze(Snapshot(files)) + self.assertEqual( + {"executorch.exir", "executorch.devtools"}, {i.subject for i in issues} + ) + self.assertTrue(all(i.kind == "missing dependency" for i in issues)) + files[TESTER] = "if TYPE_CHECKING:\n from executorch import exir, devtools\n" + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_augmented_sources_and_dependencies_are_evaluated(self): + files = fixture() + helper = SCOPE + "/tester/helper.py" + files[helper] = ( + "from executorch.devtools.pte_tool.diff_pte import PTEDiffResult\n" + ) + declarations = ( + 'sources = ["tester/runner.py"]\n' + 'sources += ["tester/helper.py"]\n' + 'deps = ["//executorch/exir:umbrella"]\n' + ) + files[BUCK] = declarations + library("arm_tester_lib", "sources", "deps") + issues = analyze(Snapshot(files)) + self.assertTrue( + any(i.kind == "missing dependency" and i.source == helper for i in issues) + ) + files[BUCK] = ( + declarations + + 'deps += ["//executorch/devtools/pte_tool:diff_pte_lib"]\n' + + library("arm_tester_lib", "sources", "deps") + ) + self.assertEqual(set(), analyze(Snapshot(files))) + + def test_unknown_augmented_values_invalidate_previous_declarations(self): + for attribute, initial in ( + ("srcs", '["tester/runner.py"]'), + ("deps", '["//executorch/exir:umbrella"]'), + ): + with self.subTest(attribute=attribute): + files = fixture() + values = { + "srcs": '["tester/runner.py"]', + "deps": '["//executorch/exir:umbrella"]', + } + values[attribute] = "values" + files[BUCK] = ( + f"values = {initial}\n" + "values += choose_values()\n" + + library("arm_tester_lib", values["srcs"], values["deps"]) + ) + declarations = Declarations(Snapshot(files)) + declarations.load_scope() + rule = declarations.rules["//executorch/" + SCOPE + ":arm_tester_lib"] + self.assertTrue(any(e.startswith(attribute + ":") for e in rule.errors)) + self.assertEqual( + set(), rule.sources if attribute == "srcs" else rule.deps + ) + self.assertTrue( + any( + i.kind == "cannot validate" + and i.severity == "error" + and i.source == BUCK + and "values += choose_values()" in i.detail + for i in analyze(Snapshot(files)) + ) + ) + + def test_unsupported_augmented_operator_invalidates_previous_dependencies(self): + files = fixture() + files[BUCK] = 'deps = ["//executorch/exir:umbrella"]\n' "deps *= 0\n" + library( + "arm_tester_lib", '["tester/runner.py"]', "deps" + ) + declarations = Declarations(Snapshot(files)) + declarations.load_scope() + rule = declarations.rules["//executorch/" + SCOPE + ":arm_tester_lib"] + self.assertEqual(set(), rule.deps) + self.assertTrue( + any( + i.kind == "cannot validate" + and i.source == BUCK + and "deps *= 0" in i.detail + for i in analyze(Snapshot(files)) + ) + ) + + def test_augmented_subscript_and_incompatible_types_are_not_ignored(self): + for mutation in ('deps[0] += ":other"', "deps += 1"): + with self.subTest(mutation=mutation): + files = fixture() + files[BUCK] = ( + 'deps = ["//executorch/exir:umbrella"]\n' + + mutation + + "\n" + + library("arm_tester_lib", '["tester/runner.py"]', "deps") + ) + declarations = Declarations(Snapshot(files)) + declarations.load_scope() + rule = declarations.rules["//executorch/" + SCOPE + ":arm_tester_lib"] + self.assertEqual(set(), rule.deps) + self.assertTrue( + any( + i.kind == "cannot validate" + and i.source == BUCK + and "unsupported augmented assignment" in i.detail + for i in analyze(Snapshot(files)) + ) + ) + def test_nested_explicit_python_test_registration_is_valid(self): for kind in ("python_test", "python_pytest", "python_unittest"): with self.subTest(kind=kind):