From 90b3d2ec8e7bfd9a5f3284fdd3d05116f4355f4d Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Mon, 26 Oct 2026 14:13:31 +0100 Subject: [PATCH 1/3] test: extend downstream coverage compatibility checks Check six more consumer repositories with one parametrized test target. Keep reports separate for each consumer and require measured coverage without enforcing consumer-specific coverage thresholds. Signed-off-by: Alexander Lanin --- .github/workflows/compatibility_tests.yml | 10 +-- README.md | 3 + tests/compatibility/BUILD | 21 +---- tests/compatibility/_consumer_workspace.py | 11 +-- tests/compatibility/test_baselibs.py | 60 -------------- tests/compatibility/test_consumers.py | 92 ++++++++++++++++++++++ tests/compatibility/test_lifecycle.py | 50 ------------ 7 files changed, 106 insertions(+), 141 deletions(-) delete mode 100644 tests/compatibility/test_baselibs.py create mode 100644 tests/compatibility/test_consumers.py delete mode 100644 tests/compatibility/test_lifecycle.py diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index 8778d04..b9096fd 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - consumer: [baselibs, lifecycle] + consumer: [baselibs, inc_security_crypto, inc_someip_gateway, kyron, lifecycle, logging, persistency, time] steps: - name: Checkout coverage_tool uses: actions/checkout@v7.0.1 @@ -50,12 +50,12 @@ jobs: disk-cache-key: compatibility-${{ matrix.consumer }} - name: Run the consumer's coverage workflow - run: bazel test --lockfile_mode=error //tests/compatibility:${{ matrix.consumer }} + run: bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=${{ matrix.consumer }} - name: Publish coverage summary if: always() env: - REPORT_ARCHIVE: bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.outputs/coverage-report.zip + REPORT_ARCHIVE: bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/coverage-report.zip run: | if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY" @@ -67,7 +67,7 @@ jobs: with: name: compatibility-${{ matrix.consumer }}-results path: | - bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.xml - bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.outputs/coverage-report.zip + bazel-testlogs/tests/compatibility/consumers/test.xml + bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/coverage-report.zip if-no-files-found: ignore retention-days: 3 diff --git a/README.md b/README.md index 3833450..7347313 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,6 @@ from the main repository's, so nested Bazel commands do not wait on the outer test's lock. Run the end-to-end commands sequentially because they share the consumer's coverage report; the pytest CLI case deletes an existing report to exercise the missing-report diagnostic. + +Compatibility tests run coverage tests against real consumer repositories. Select one +with `bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=kyron`. diff --git a/tests/compatibility/BUILD b/tests/compatibility/BUILD index e17d07e..9dfb47a 100644 --- a/tests/compatibility/BUILD +++ b/tests/compatibility/BUILD @@ -17,26 +17,13 @@ load("@score_tools//score_pytest:pytest.bzl", "score_pytest") # minutes on a cold CI runner. The tests need network access to clone current # consumer main branches and run Bazel in those nested workspaces. # -# Keep each consumer scenario in its own target so CI and JUnit reports show -# exactly which real downstream workflow failed. +# The parametrized test runs all consumers by default. CI selects one with -k +# so each job reports its own result; report archives use consumer subdirectories. score_pytest( - name = "baselibs", + name = "consumers", size = "large", timeout = "eternal", - srcs = ["test_baselibs.py"], - data = ["_consumer_workspace.py"], - imports = ["."], - tags = [ - "integration", - "local", - ], -) - -score_pytest( - name = "lifecycle", - size = "large", - timeout = "eternal", - srcs = ["test_lifecycle.py"], + srcs = ["test_consumers.py"], data = ["_consumer_workspace.py"], imports = ["."], tags = [ diff --git a/tests/compatibility/_consumer_workspace.py b/tests/compatibility/_consumer_workspace.py index 5752890..a1b8349 100644 --- a/tests/compatibility/_consumer_workspace.py +++ b/tests/compatibility/_consumer_workspace.py @@ -43,13 +43,6 @@ def clone_consumer(name: str, parent_directory: Path) -> Path: ) module_file.write_text(module_contents, encoding="utf-8") - # Updating the consumer lockfile for the local override lets its production - # commands keep their normal --lockfile_mode=error setting. - subprocess.run( - ["bazel", "mod", "deps", "--lockfile_mode=update"], - cwd=workspace, - check=True, - ) return workspace @@ -89,7 +82,7 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None: if not archive.is_dir() or not outputs_directory: return - retained_report = Path(outputs_directory) / "coverage-report" + retained_report = Path(outputs_directory) / workspace.name / "coverage-report" summary_markdown = workspace / "coverage_summary.md" try: retained_report.mkdir(parents=True, exist_ok=True) @@ -151,7 +144,7 @@ def verify_coverage_results(workspace: Path, archive_directory: str) -> None: outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR" - report_archive = Path(outputs_directory) / "coverage-report.zip" + report_archive = Path(outputs_directory) / workspace.name / "coverage-report.zip" assert report_archive.is_file(), "Coverage report was not retained for Bazel" coverage_percent = 100 * lines_hit / lines_found diff --git a/tests/compatibility/test_baselibs.py b/tests/compatibility/test_baselibs.py deleted file mode 100644 index e0e66c9..0000000 --- a/tests/compatibility/test_baselibs.py +++ /dev/null @@ -1,60 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -"""Black-box test for baselibs' production LLVM coverage workflow.""" - -from pathlib import Path - -from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results - - -def test_baselibs_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: - """The checked-in baselibs workflow produces a non-empty coverage report.""" - workspace = clone_consumer("baselibs", tmp_path) - - run_bazel( - workspace, - "coverage", - "--lockfile_mode=error", - "--config=llvm_cov", - "--build_tests_only", - "--", - "//score/...", - # This upstream-main test currently fails under the coverage config - # because its compact-JSON expectations conflict with default pretty printing. - "-//score/json/internal/writer/vajson:vajson_serialize_test", - "-//score/language/safecpp/aborts_upon_exception/...", - "-//score/language/safecpp/safe_math/details:floating_point_environment_test", - "-//score/os/linux/utils/test:network_interface_test", - ) - # The generator assembles artifacts before returning a gate failure. - try: - run_bazel( - workspace, - "run", - "--lockfile_mode=error", - "@score_coverage//:generate_coverage_html", - "--", - "--yaml", - "tools/coverage/coverage_justifications.yaml", - "--summary-md", - "coverage_summary.md", - "--testlogs-subdir", - "score", - "--archive-dir", - "coverage_artifact", - extra_environment={"COVERAGE_THRESHOLD": "0"}, - ) - finally: - retain_coverage_results(workspace, "coverage_artifact") - - verify_coverage_results(workspace, "coverage_artifact") diff --git a/tests/compatibility/test_consumers.py b/tests/compatibility/test_consumers.py new file mode 100644 index 0000000..0992780 --- /dev/null +++ b/tests/compatibility/test_consumers.py @@ -0,0 +1,92 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Run coverage tests against real consumer repositories.""" + +from dataclasses import dataclass +from pathlib import Path + +import pytest +from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results + + +@dataclass(frozen=True) +class Consumer: + """Arguments needed to generate coverage reports for a downstream repository.""" + + coverage_args: tuple[str, ...] = ("--config=llvm_cov", "--build_tests_only", "--", "//score/...") + justification_file: str | None = "tools/coverage/coverage_justifications.yaml" + + +CONSUMERS = { + "baselibs": Consumer( + coverage_args=( + "--config=llvm_cov", + "--build_tests_only", + "--", + "//score/...", + # This test expects compact JSON, but the coverage config uses pretty printing. + "-//score/json/internal/writer/vajson:vajson_serialize_test", + # Exclusions from baselibs' LLVM coverage workflow: these tests + # require GCC semantics or fail with static linking. + "-//score/language/safecpp/aborts_upon_exception/...", + "-//score/language/safecpp/safe_math/details:floating_point_environment_test", + "-//score/os/linux/utils/test:network_interface_test", + ), + ), + "inc_security_crypto": Consumer(), + "inc_someip_gateway": Consumer( + coverage_args=("--build_tests_only", "--", "//..."), + justification_file=None, + ), + "kyron": Consumer( + coverage_args=("--config=llvm_cov", "--build_tests_only", "--", "//src/..."), + ), + "lifecycle": Consumer( + justification_file="quality/coverage/coverage_justifications.yaml", + ), + "logging": Consumer(justification_file=None), + "persistency": Consumer(justification_file="quality/coverage/coverage_justifications.yaml"), + "time": Consumer(), +} + + +@pytest.mark.parametrize("consumer,settings", CONSUMERS.items(), ids=CONSUMERS) +def test_consumer_coverage_workflow_reports_measured_coverage( + tmp_path: Path, consumer: str, settings: Consumer +) -> None: + """Each consumer must produce and retain a report with measured coverage.""" + workspace = clone_consumer(consumer, tmp_path) + if consumer == "inc_someip_gateway": + # The gateway's test wrappers use the production workflow's host selection. + (workspace / ".bazel_config").write_text("host\n", encoding="utf-8") + + run_bazel(workspace, "coverage", *settings.coverage_args) + report_arguments = ["--summary-md", "coverage_summary.md", "--archive-dir", "coverage_artifact"] + if settings.justification_file: + report_arguments.extend(("--yaml", settings.justification_file)) + + # Keep any report artifacts even if report generation fails. + try: + run_bazel( + workspace, + "run", + "@score_coverage//:generate_coverage_html", + "--", + *report_arguments, + # Verify measured coverage below without imposing a consumer-specific gate. + extra_environment={"COVERAGE_THRESHOLD": "0"}, + ) + finally: + retain_coverage_results(workspace, "coverage_artifact") + + verify_coverage_results(workspace, "coverage_artifact") diff --git a/tests/compatibility/test_lifecycle.py b/tests/compatibility/test_lifecycle.py deleted file mode 100644 index ccbb158..0000000 --- a/tests/compatibility/test_lifecycle.py +++ /dev/null @@ -1,50 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -"""Black-box test for lifecycle's production Linux coverage workflow.""" - -from pathlib import Path - -from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results - - -def test_lifecycle_coverage_workflow_reports_measured_coverage(tmp_path: Path) -> None: - """The checked-in workflow meets its gate and produces a coverage report.""" - workspace = clone_consumer("lifecycle", tmp_path) - - run_bazel( - workspace, - "coverage", - "--config=llvm_cov", - "//score/...", - "--lockfile_mode=error", - "--build_tests_only", - ) - # The generator assembles artifacts before returning a gate failure. - try: - run_bazel( - workspace, - "run", - "@score_coverage//:generate_coverage_html", - "--", - "--yaml", - "quality/coverage/coverage_justifications.yaml", - "--summary-md", - "coverage_summary.md", - "--archive-dir", - "coverage_artifacts", - extra_environment={"COVERAGE_THRESHOLD": "66"}, - ) - finally: - retain_coverage_results(workspace, "coverage_artifacts") - - verify_coverage_results(workspace, "coverage_artifacts") From 2d8907e08e1b7ef31bc0faea7ef7114ac8535783 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Mon, 26 Oct 2026 14:34:35 +0100 Subject: [PATCH 2/3] test: allow consumer lockfile updates explicitly Consumer bazelrc files can enforce strict lockfile checks. Override that setting for both nested Bazel commands so the local coverage-tool override can update the temporary workspace lockfile. Signed-off-by: Alexander Lanin --- tests/compatibility/test_consumers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/compatibility/test_consumers.py b/tests/compatibility/test_consumers.py index 0992780..477fe4a 100644 --- a/tests/compatibility/test_consumers.py +++ b/tests/compatibility/test_consumers.py @@ -70,7 +70,9 @@ def test_consumer_coverage_workflow_reports_measured_coverage( # The gateway's test wrappers use the production workflow's host selection. (workspace / ".bazel_config").write_text("host\n", encoding="utf-8") - run_bazel(workspace, "coverage", *settings.coverage_args) + # The local override changes the module graph; allow lockfile updates even + # when the consumer enables strict lockfile checks in its bazelrc. + run_bazel(workspace, "coverage", "--lockfile_mode=update", *settings.coverage_args) report_arguments = ["--summary-md", "coverage_summary.md", "--archive-dir", "coverage_artifact"] if settings.justification_file: report_arguments.extend(("--yaml", settings.justification_file)) @@ -80,6 +82,7 @@ def test_consumer_coverage_workflow_reports_measured_coverage( run_bazel( workspace, "run", + "--lockfile_mode=update", "@score_coverage//:generate_coverage_html", "--", *report_arguments, From 1960b239037127549f2c15f10d49a50c1c258947 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Mon, 26 Oct 2026 14:51:26 +0100 Subject: [PATCH 3/3] test: improve consumer command diagnostics Decode consumer output with replacement for invalid UTF-8 so successful builds do not fail while their output is read. Retain complete Bazel logs and publish concise command summaries with early errors and the log tail. Signed-off-by: Alexander Lanin --- .github/workflows/compatibility_tests.yml | 13 +++- tests/compatibility/BUILD | 14 +++- tests/compatibility/_consumer_workspace.py | 42 ++++++++--- tests/unit/tools/BUILD | 8 ++ tests/unit/tools/consumer_workspace_test.py | 84 +++++++++++++++++++++ 5 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 tests/unit/tools/consumer_workspace_test.py diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index b9096fd..651d99f 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -49,14 +49,18 @@ jobs: with: disk-cache-key: compatibility-${{ matrix.consumer }} - - name: Run the consumer's coverage workflow + - name: Run consumer coverage tests run: bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=${{ matrix.consumer }} - - name: Publish coverage summary + - name: Publish test and coverage summaries if: always() env: - REPORT_ARCHIVE: bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/coverage-report.zip + RESULTS_DIR: bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }} run: | + if [ -f "$RESULTS_DIR/compatibility_summary.md" ]; then + cat "$RESULTS_DIR/compatibility_summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + REPORT_ARCHIVE="$RESULTS_DIR/coverage-report.zip" if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY" fi @@ -68,6 +72,7 @@ jobs: name: compatibility-${{ matrix.consumer }}-results path: | bazel-testlogs/tests/compatibility/consumers/test.xml - bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/coverage-report.zip + bazel-testlogs/tests/compatibility/consumers/test.log + bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/ if-no-files-found: ignore retention-days: 3 diff --git a/tests/compatibility/BUILD b/tests/compatibility/BUILD index 9dfb47a..b190dce 100644 --- a/tests/compatibility/BUILD +++ b/tests/compatibility/BUILD @@ -11,8 +11,18 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +load("@rules_python//python:defs.bzl", "py_library") load("@score_tools//score_pytest:pytest.bzl", "score_pytest") +# Unit tests run the command helper with a fake Bazel executable to cover +# malformed output and failure reporting without downloading consumer toolchains. +py_library( + name = "consumer_workspace", + srcs = ["_consumer_workspace.py"], + imports = ["."], + visibility = ["//tests/unit/tools:__pkg__"], +) + # Each test runs the consumer's full coverage pipeline, which can take many # minutes on a cold CI runner. The tests need network access to clone current # consumer main branches and run Bazel in those nested workspaces. @@ -24,10 +34,10 @@ score_pytest( size = "large", timeout = "eternal", srcs = ["test_consumers.py"], - data = ["_consumer_workspace.py"], - imports = ["."], + args = ["--tb=short"], tags = [ "integration", "local", ], + deps = [":consumer_workspace"], ) diff --git a/tests/compatibility/_consumer_workspace.py b/tests/compatibility/_consumer_workspace.py index a1b8349..940021c 100644 --- a/tests/compatibility/_consumer_workspace.py +++ b/tests/compatibility/_consumer_workspace.py @@ -15,6 +15,7 @@ from __future__ import annotations import os +import shlex import shutil import subprocess import sys @@ -51,7 +52,7 @@ def run_bazel( *arguments: str, extra_environment: dict[str, str] | None = None, ) -> None: - """Run one consumer-facing Bazel command, keeping failure output concise.""" + """Run Bazel and retain a readable status summary plus its complete output.""" environment = os.environ.copy() if extra_environment: environment.update(extra_environment) @@ -61,16 +62,37 @@ def run_bazel( command, cwd=workspace, env=environment, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False, - text=True, + encoding="utf-8", + errors="replace", ) + status = "FAILED" if result.returncode else "PASSED" + summary = ( + f"### {workspace.name}: bazel {arguments[0]} — {status}\n\n" + f"Command: `{shlex.join(command)}`\n\n" + f"Exit code: {result.returncode}\n" + ) + if result.returncode: + lines = result.stdout.splitlines() + # Compiler and repository errors can precede a long list of skipped + # tests. Show those diagnostics before the tail rather than losing them. + diagnostics = [line for line in lines if any(marker in line for marker in ("ERROR:", "FAIL:", "Error in "))] + excerpt = "\n".join(diagnostics[:10] + lines[-30:]) + summary += f"\n```text\n{excerpt}\n```\n" + + outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + if outputs_directory: + retained = Path(outputs_directory) / workspace.name + retained.mkdir(parents=True, exist_ok=True) + log_name = f"{arguments[0]}.log" + (retained / log_name).write_text(result.stdout, encoding="utf-8") + summary += f"\nFull output: `{log_name}` in the uploaded test artifacts.\n" + with (retained / "compatibility_summary.md").open("a", encoding="utf-8") as summary_file: + summary_file.write(summary + "\n") if result.returncode: - output = (result.stdout + result.stderr).splitlines() - output_tail = "\n".join(output[-80:]) - raise AssertionError( - f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}" - ) + raise AssertionError(summary) def retain_coverage_results(workspace: Path, archive_directory: str) -> None: @@ -99,12 +121,12 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None: shutil.copytree(result, retained_report / name) elif result.is_file(): shutil.copy2(result, retained_report / name) - except Exception as error: + except Exception as error: # pylint: disable=broad-exception-caught print(f"Could not copy all coverage report files: {error}", file=sys.stderr) try: report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report)) - except Exception as error: + except Exception as error: # pylint: disable=broad-exception-caught print(f"Could not archive the coverage report: {error}", file=sys.stderr) return diff --git a/tests/unit/tools/BUILD b/tests/unit/tools/BUILD index 0f3ef0e..8726739 100644 --- a/tests/unit/tools/BUILD +++ b/tests/unit/tools/BUILD @@ -21,3 +21,11 @@ py_test( main = "self_coverage_gate_test.py", deps = ["//tools:self_coverage_gate_lib"], ) + +# Exercise malformed subprocess output and retained diagnostics in isolation; +# the real consumer tests remain separate because they need network/toolchains. +py_test( + name = "consumer_workspace_test", + srcs = ["consumer_workspace_test.py"], + deps = ["//tests/compatibility:consumer_workspace"], +) diff --git a/tests/unit/tools/consumer_workspace_test.py b/tests/unit/tools/consumer_workspace_test.py new file mode 100644 index 0000000..4bb51bc --- /dev/null +++ b/tests/unit/tools/consumer_workspace_test.py @@ -0,0 +1,84 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Regression checks for downstream command output and failure diagnostics.""" + +import os +import shlex +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from _consumer_workspace import run_bazel + + +class ConsumerWorkspaceTest(unittest.TestCase): + """Use a real subprocess so decoding behavior is exercised as well.""" + + def run_command(self, root: Path, source: str) -> None: + """Replace only Bazel itself; keep subprocess capture and artifact writes real.""" + workspace = root / "logging" + workspace.mkdir() + launcher = root / "bazel" + # Bazel's interpreter path can exceed the kernel's shebang length limit. + launcher.write_text( + f'#!/bin/sh\nexec {shlex.quote(sys.executable)} -c {shlex.quote(source)} "$@"\n', encoding="utf-8" + ) + launcher.chmod(0o755) + with patch.dict( + os.environ, + { + "PATH": str(root) + os.pathsep + os.environ["PATH"], + "TEST_UNDECLARED_OUTPUTS_DIR": str(root / "outputs"), + }, + ): + run_bazel(workspace, "coverage", "--config=llvm_cov") + + def test_successful_build_with_non_utf8_output_retains_readable_log(self) -> None: + """A consumer's arbitrary output bytes must not turn a successful build red.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + self.run_command(root, 'import os\nos.write(1, b"build succeeded\\n")\nos.write(2, b"message: \\xff\\n")\n') + + log = (root / "outputs/logging/coverage.log").read_text(encoding="utf-8") + self.assertEqual(log, "build succeeded\nmessage: \ufffd\n") + summary = (root / "outputs/logging/compatibility_summary.md").read_text(encoding="utf-8") + self.assertIn("logging: bazel coverage — PASSED", summary) + self.assertIn("Exit code: 0", summary) + + def test_failure_preserves_early_error_and_full_log(self) -> None: + """A long skipped-test list must not hide the actual repository error.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + with self.assertRaises(AssertionError) as failure: + self.run_command( + root, + 'import sys\nprint("ERROR: dependency checksum mismatch")\n' + 'print("skipped test\\n" * 100, end="")\nsys.exit(7)\n', + ) + + message = str(failure.exception) + self.assertIn("logging: bazel coverage — FAILED", message) + self.assertIn("bazel coverage --config=llvm_cov", message) + self.assertIn("Exit code: 7", message) + self.assertIn("ERROR: dependency checksum mismatch", message) + self.assertIn("coverage.log", message) + log = (root / "outputs/logging/coverage.log").read_text(encoding="utf-8") + self.assertEqual(log.count("skipped test"), 100) + summary = (root / "outputs/logging/compatibility_summary.md").read_text(encoding="utf-8") + self.assertIn(message, summary) + + +if __name__ == "__main__": + unittest.main()