diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index 8778d04..651d99f 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - consumer: [baselibs, lifecycle] + consumer: [baselibs, inc_security_crypto, inc_someip_gateway, kyron, lifecycle, logging, persistency, time] steps: - name: Checkout coverage_tool uses: actions/checkout@v7.0.1 @@ -49,14 +49,18 @@ jobs: with: disk-cache-key: compatibility-${{ matrix.consumer }} - - name: Run the consumer's coverage workflow - run: bazel test --lockfile_mode=error //tests/compatibility:${{ matrix.consumer }} + - name: Run consumer coverage tests + run: bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=${{ matrix.consumer }} - - name: Publish coverage summary + - name: Publish test and coverage summaries if: always() env: - REPORT_ARCHIVE: bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.outputs/coverage-report.zip + RESULTS_DIR: bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }} run: | + if [ -f "$RESULTS_DIR/compatibility_summary.md" ]; then + cat "$RESULTS_DIR/compatibility_summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + REPORT_ARCHIVE="$RESULTS_DIR/coverage-report.zip" if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY" fi @@ -67,7 +71,8 @@ jobs: with: name: compatibility-${{ matrix.consumer }}-results path: | - bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.xml - bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.outputs/coverage-report.zip + bazel-testlogs/tests/compatibility/consumers/test.xml + bazel-testlogs/tests/compatibility/consumers/test.log + bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/ if-no-files-found: ignore retention-days: 3 diff --git a/README.md b/README.md index 3833450..7347313 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,6 @@ from the main repository's, so nested Bazel commands do not wait on the outer test's lock. Run the end-to-end commands sequentially because they share the consumer's coverage report; the pytest CLI case deletes an existing report to exercise the missing-report diagnostic. + +Compatibility tests run coverage tests against real consumer repositories. Select one +with `bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=kyron`. diff --git a/tests/compatibility/BUILD b/tests/compatibility/BUILD index e17d07e..b190dce 100644 --- a/tests/compatibility/BUILD +++ b/tests/compatibility/BUILD @@ -11,36 +11,33 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +load("@rules_python//python:defs.bzl", "py_library") load("@score_tools//score_pytest:pytest.bzl", "score_pytest") +# Unit tests run the command helper with a fake Bazel executable to cover +# malformed output and failure reporting without downloading consumer toolchains. +py_library( + name = "consumer_workspace", + srcs = ["_consumer_workspace.py"], + imports = ["."], + visibility = ["//tests/unit/tools:__pkg__"], +) + # Each test runs the consumer's full coverage pipeline, which can take many # minutes on a cold CI runner. The tests need network access to clone current # consumer main branches and run Bazel in those nested workspaces. # -# Keep each consumer scenario in its own target so CI and JUnit reports show -# exactly which real downstream workflow failed. -score_pytest( - name = "baselibs", - size = "large", - timeout = "eternal", - srcs = ["test_baselibs.py"], - data = ["_consumer_workspace.py"], - imports = ["."], - tags = [ - "integration", - "local", - ], -) - +# The parametrized test runs all consumers by default. CI selects one with -k +# so each job reports its own result; report archives use consumer subdirectories. score_pytest( - name = "lifecycle", + name = "consumers", size = "large", timeout = "eternal", - srcs = ["test_lifecycle.py"], - data = ["_consumer_workspace.py"], - imports = ["."], + srcs = ["test_consumers.py"], + args = ["--tb=short"], tags = [ "integration", "local", ], + deps = [":consumer_workspace"], ) diff --git a/tests/compatibility/_consumer_workspace.py b/tests/compatibility/_consumer_workspace.py index 5752890..940021c 100644 --- a/tests/compatibility/_consumer_workspace.py +++ b/tests/compatibility/_consumer_workspace.py @@ -15,6 +15,7 @@ from __future__ import annotations import os +import shlex import shutil import subprocess import sys @@ -43,13 +44,6 @@ def clone_consumer(name: str, parent_directory: Path) -> Path: ) module_file.write_text(module_contents, encoding="utf-8") - # Updating the consumer lockfile for the local override lets its production - # commands keep their normal --lockfile_mode=error setting. - subprocess.run( - ["bazel", "mod", "deps", "--lockfile_mode=update"], - cwd=workspace, - check=True, - ) return workspace @@ -58,7 +52,7 @@ def run_bazel( *arguments: str, extra_environment: dict[str, str] | None = None, ) -> None: - """Run one consumer-facing Bazel command, keeping failure output concise.""" + """Run Bazel and retain a readable status summary plus its complete output.""" environment = os.environ.copy() if extra_environment: environment.update(extra_environment) @@ -68,16 +62,37 @@ def run_bazel( command, cwd=workspace, env=environment, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False, - text=True, + encoding="utf-8", + errors="replace", ) + status = "FAILED" if result.returncode else "PASSED" + summary = ( + f"### {workspace.name}: bazel {arguments[0]} — {status}\n\n" + f"Command: `{shlex.join(command)}`\n\n" + f"Exit code: {result.returncode}\n" + ) + if result.returncode: + lines = result.stdout.splitlines() + # Compiler and repository errors can precede a long list of skipped + # tests. Show those diagnostics before the tail rather than losing them. + diagnostics = [line for line in lines if any(marker in line for marker in ("ERROR:", "FAIL:", "Error in "))] + excerpt = "\n".join(diagnostics[:10] + lines[-30:]) + summary += f"\n```text\n{excerpt}\n```\n" + + outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + if outputs_directory: + retained = Path(outputs_directory) / workspace.name + retained.mkdir(parents=True, exist_ok=True) + log_name = f"{arguments[0]}.log" + (retained / log_name).write_text(result.stdout, encoding="utf-8") + summary += f"\nFull output: `{log_name}` in the uploaded test artifacts.\n" + with (retained / "compatibility_summary.md").open("a", encoding="utf-8") as summary_file: + summary_file.write(summary + "\n") if result.returncode: - output = (result.stdout + result.stderr).splitlines() - output_tail = "\n".join(output[-80:]) - raise AssertionError( - f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}" - ) + raise AssertionError(summary) def retain_coverage_results(workspace: Path, archive_directory: str) -> None: @@ -89,7 +104,7 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None: if not archive.is_dir() or not outputs_directory: return - retained_report = Path(outputs_directory) / "coverage-report" + retained_report = Path(outputs_directory) / workspace.name / "coverage-report" summary_markdown = workspace / "coverage_summary.md" try: retained_report.mkdir(parents=True, exist_ok=True) @@ -106,12 +121,12 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None: shutil.copytree(result, retained_report / name) elif result.is_file(): shutil.copy2(result, retained_report / name) - except Exception as error: + except Exception as error: # pylint: disable=broad-exception-caught print(f"Could not copy all coverage report files: {error}", file=sys.stderr) try: report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report)) - except Exception as error: + except Exception as error: # pylint: disable=broad-exception-caught print(f"Could not archive the coverage report: {error}", file=sys.stderr) return @@ -151,7 +166,7 @@ def verify_coverage_results(workspace: Path, archive_directory: str) -> None: outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR" - report_archive = Path(outputs_directory) / "coverage-report.zip" + report_archive = Path(outputs_directory) / workspace.name / "coverage-report.zip" assert report_archive.is_file(), "Coverage report was not retained for Bazel" coverage_percent = 100 * lines_hit / lines_found diff --git a/tests/compatibility/test_baselibs.py b/tests/compatibility/test_baselibs.py deleted file mode 100644 index e0e66c9..0000000 --- a/tests/compatibility/test_baselibs.py +++ /dev/null @@ -1,60 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -"""Black-box test for baselibs' production LLVM coverage workflow.""" - -from pathlib import Path - -from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results - - -def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: - """The checked-in baselibs workflow produces a non-empty coverage report.""" - workspace = clone_consumer("baselibs", tmp_path) - - run_bazel( - workspace, - "coverage", - "--lockfile_mode=error", - "--config=llvm_cov", - "--build_tests_only", - "--", - "//score/...", - # This upstream-main test currently fails under the coverage config - # because its compact-JSON expectations conflict with default pretty printing. - "-//score/json/internal/writer/vajson:vajson_serialize_test", - "-//score/language/safecpp/aborts_upon_exception/...", - "-//score/language/safecpp/safe_math/details:floating_point_environment_test", - "-//score/os/linux/utils/test:network_interface_test", - ) - # The generator assembles artifacts before returning a gate failure. - try: - run_bazel( - workspace, - "run", - "--lockfile_mode=error", - "@score_coverage//:generate_coverage_html", - "--", - "--yaml", - "tools/coverage/coverage_justifications.yaml", - "--summary-md", - "coverage_summary.md", - "--testlogs-subdir", - "score", - "--archive-dir", - "coverage_artifact", - extra_environment={"COVERAGE_THRESHOLD": "0"}, - ) - finally: - retain_coverage_results(workspace, "coverage_artifact") - - verify_coverage_results(workspace, "coverage_artifact") diff --git a/tests/compatibility/test_consumers.py b/tests/compatibility/test_consumers.py new file mode 100644 index 0000000..477fe4a --- /dev/null +++ b/tests/compatibility/test_consumers.py @@ -0,0 +1,95 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Run coverage tests against real consumer repositories.""" + +from dataclasses import dataclass +from pathlib import Path + +import pytest +from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results + + +@dataclass(frozen=True) +class Consumer: + """Arguments needed to generate coverage reports for a downstream repository.""" + + coverage_args: tuple[str, ...] = ("--config=llvm_cov", "--build_tests_only", "--", "//score/...") + justification_file: str | None = "tools/coverage/coverage_justifications.yaml" + + +CONSUMERS = { + "baselibs": Consumer( + coverage_args=( + "--config=llvm_cov", + "--build_tests_only", + "--", + "//score/...", + # This test expects compact JSON, but the coverage config uses pretty printing. + "-//score/json/internal/writer/vajson:vajson_serialize_test", + # Exclusions from baselibs' LLVM coverage workflow: these tests + # require GCC semantics or fail with static linking. + "-//score/language/safecpp/aborts_upon_exception/...", + "-//score/language/safecpp/safe_math/details:floating_point_environment_test", + "-//score/os/linux/utils/test:network_interface_test", + ), + ), + "inc_security_crypto": Consumer(), + "inc_someip_gateway": Consumer( + coverage_args=("--build_tests_only", "--", "//..."), + justification_file=None, + ), + "kyron": Consumer( + coverage_args=("--config=llvm_cov", "--build_tests_only", "--", "//src/..."), + ), + "lifecycle": Consumer( + justification_file="quality/coverage/coverage_justifications.yaml", + ), + "logging": Consumer(justification_file=None), + "persistency": Consumer(justification_file="quality/coverage/coverage_justifications.yaml"), + "time": Consumer(), +} + + +@pytest.mark.parametrize("consumer,settings", CONSUMERS.items(), ids=CONSUMERS) +def test_consumer_coverage_workflow_reports_measured_coverage( + tmp_path: Path, consumer: str, settings: Consumer +) -> None: + """Each consumer must produce and retain a report with measured coverage.""" + workspace = clone_consumer(consumer, tmp_path) + if consumer == "inc_someip_gateway": + # The gateway's test wrappers use the production workflow's host selection. + (workspace / ".bazel_config").write_text("host\n", encoding="utf-8") + + # The local override changes the module graph; allow lockfile updates even + # when the consumer enables strict lockfile checks in its bazelrc. + run_bazel(workspace, "coverage", "--lockfile_mode=update", *settings.coverage_args) + report_arguments = ["--summary-md", "coverage_summary.md", "--archive-dir", "coverage_artifact"] + if settings.justification_file: + report_arguments.extend(("--yaml", settings.justification_file)) + + # Keep any report artifacts even if report generation fails. + try: + run_bazel( + workspace, + "run", + "--lockfile_mode=update", + "@score_coverage//:generate_coverage_html", + "--", + *report_arguments, + # Verify measured coverage below without imposing a consumer-specific gate. + extra_environment={"COVERAGE_THRESHOLD": "0"}, + ) + finally: + retain_coverage_results(workspace, "coverage_artifact") + + verify_coverage_results(workspace, "coverage_artifact") diff --git a/tests/compatibility/test_lifecycle.py b/tests/compatibility/test_lifecycle.py deleted file mode 100644 index ccbb158..0000000 --- a/tests/compatibility/test_lifecycle.py +++ /dev/null @@ -1,50 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -"""Black-box test for lifecycle's production Linux coverage workflow.""" - -from pathlib import Path - -from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results - - -def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: - """The checked-in workflow meets its gate and produces a coverage report.""" - workspace = clone_consumer("lifecycle", tmp_path) - - run_bazel( - workspace, - "coverage", - "--config=llvm_cov", - "//score/...", - "--lockfile_mode=error", - "--build_tests_only", - ) - # The generator assembles artifacts before returning a gate failure. - try: - run_bazel( - workspace, - "run", - "@score_coverage//:generate_coverage_html", - "--", - "--yaml", - "quality/coverage/coverage_justifications.yaml", - "--summary-md", - "coverage_summary.md", - "--archive-dir", - "coverage_artifacts", - extra_environment={"COVERAGE_THRESHOLD": "66"}, - ) - finally: - retain_coverage_results(workspace, "coverage_artifacts") - - verify_coverage_results(workspace, "coverage_artifacts") diff --git a/tests/unit/tools/BUILD b/tests/unit/tools/BUILD index 0f3ef0e..8726739 100644 --- a/tests/unit/tools/BUILD +++ b/tests/unit/tools/BUILD @@ -21,3 +21,11 @@ py_test( main = "self_coverage_gate_test.py", deps = ["//tools:self_coverage_gate_lib"], ) + +# Exercise malformed subprocess output and retained diagnostics in isolation; +# the real consumer tests remain separate because they need network/toolchains. +py_test( + name = "consumer_workspace_test", + srcs = ["consumer_workspace_test.py"], + deps = ["//tests/compatibility:consumer_workspace"], +) diff --git a/tests/unit/tools/consumer_workspace_test.py b/tests/unit/tools/consumer_workspace_test.py new file mode 100644 index 0000000..4bb51bc --- /dev/null +++ b/tests/unit/tools/consumer_workspace_test.py @@ -0,0 +1,84 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Regression checks for downstream command output and failure diagnostics.""" + +import os +import shlex +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from _consumer_workspace import run_bazel + + +class ConsumerWorkspaceTest(unittest.TestCase): + """Use a real subprocess so decoding behavior is exercised as well.""" + + def run_command(self, root: Path, source: str) -> None: + """Replace only Bazel itself; keep subprocess capture and artifact writes real.""" + workspace = root / "logging" + workspace.mkdir() + launcher = root / "bazel" + # Bazel's interpreter path can exceed the kernel's shebang length limit. + launcher.write_text( + f'#!/bin/sh\nexec {shlex.quote(sys.executable)} -c {shlex.quote(source)} "$@"\n', encoding="utf-8" + ) + launcher.chmod(0o755) + with patch.dict( + os.environ, + { + "PATH": str(root) + os.pathsep + os.environ["PATH"], + "TEST_UNDECLARED_OUTPUTS_DIR": str(root / "outputs"), + }, + ): + run_bazel(workspace, "coverage", "--config=llvm_cov") + + def test_successful_build_with_non_utf8_output_retains_readable_log(self) -> None: + """A consumer's arbitrary output bytes must not turn a successful build red.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + self.run_command(root, 'import os\nos.write(1, b"build succeeded\\n")\nos.write(2, b"message: \\xff\\n")\n') + + log = (root / "outputs/logging/coverage.log").read_text(encoding="utf-8") + self.assertEqual(log, "build succeeded\nmessage: \ufffd\n") + summary = (root / "outputs/logging/compatibility_summary.md").read_text(encoding="utf-8") + self.assertIn("logging: bazel coverage — PASSED", summary) + self.assertIn("Exit code: 0", summary) + + def test_failure_preserves_early_error_and_full_log(self) -> None: + """A long skipped-test list must not hide the actual repository error.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + with self.assertRaises(AssertionError) as failure: + self.run_command( + root, + 'import sys\nprint("ERROR: dependency checksum mismatch")\n' + 'print("skipped test\\n" * 100, end="")\nsys.exit(7)\n', + ) + + message = str(failure.exception) + self.assertIn("logging: bazel coverage — FAILED", message) + self.assertIn("bazel coverage --config=llvm_cov", message) + self.assertIn("Exit code: 7", message) + self.assertIn("ERROR: dependency checksum mismatch", message) + self.assertIn("coverage.log", message) + log = (root / "outputs/logging/coverage.log").read_text(encoding="utf-8") + self.assertEqual(log.count("skipped test"), 100) + summary = (root / "outputs/logging/compatibility_summary.md").read_text(encoding="utf-8") + self.assertIn(message, summary) + + +if __name__ == "__main__": + unittest.main()