test: extend downstream coverage compatibility checks

.github/workflows/compatibility_tests.yml

@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        consumer: [baselibs, lifecycle]
+        consumer: [baselibs, inc_security_crypto, inc_someip_gateway, kyron, lifecycle, logging, persistency, time]
     steps:
       - name: Checkout coverage_tool
         uses: actions/checkout@v7.0.1
@@ -49,14 +49,18 @@ jobs:
         with:
           disk-cache-key: compatibility-${{ matrix.consumer }}
 
-      - name: Run the consumer's coverage workflow
-        run: bazel test --lockfile_mode=error //tests/compatibility:${{ matrix.consumer }}
+      - name: Run consumer coverage tests
+        run: bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=${{ matrix.consumer }}
 
-      - name: Publish coverage summary
+      - name: Publish test and coverage summaries
         if: always()
         env:
-          REPORT_ARCHIVE: bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.outputs/coverage-report.zip
+          RESULTS_DIR: bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}
         run: |
+          if [ -f "$RESULTS_DIR/compatibility_summary.md" ]; then
+            cat "$RESULTS_DIR/compatibility_summary.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+          REPORT_ARCHIVE="$RESULTS_DIR/coverage-report.zip"
           if [ -f "$REPORT_ARCHIVE" ] && unzip -Z1 "$REPORT_ARCHIVE" | grep -x coverage_summary.md > /dev/null; then
             unzip -p "$REPORT_ARCHIVE" coverage_summary.md >> "$GITHUB_STEP_SUMMARY"
           fi
@@ -67,7 +71,8 @@ jobs:
         with:
           name: compatibility-${{ matrix.consumer }}-results
           path: |
-            bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.xml
-            bazel-testlogs/tests/compatibility/${{ matrix.consumer }}/test.outputs/coverage-report.zip
+            bazel-testlogs/tests/compatibility/consumers/test.xml
+            bazel-testlogs/tests/compatibility/consumers/test.log
+            bazel-testlogs/tests/compatibility/consumers/test.outputs/${{ matrix.consumer }}/
           if-no-files-found: ignore
           retention-days: 3

README.md

@@ -82,3 +82,6 @@ from the main repository's, so nested Bazel commands do not wait on the outer
 test's lock. Run the end-to-end commands sequentially because they share the
 consumer's coverage report; the pytest CLI case deletes an existing report to
 exercise the missing-report diagnostic.
+
+Compatibility tests run coverage tests against real consumer repositories. Select one
+with `bazel test //tests/compatibility:consumers --test_arg=-k --test_arg=kyron`.

tests/compatibility/BUILD

@@ -11,36 +11,33 @@
 # SPDX-License-Identifier: Apache-2.0
 # *******************************************************************************
 
+load("@rules_python//python:defs.bzl", "py_library")
 load("@score_tools//score_pytest:pytest.bzl", "score_pytest")
 
+# Unit tests run the command helper with a fake Bazel executable to cover
+# malformed output and failure reporting without downloading consumer toolchains.
+py_library(
+    name = "consumer_workspace",
+    srcs = ["_consumer_workspace.py"],
+    imports = ["."],
+    visibility = ["//tests/unit/tools:__pkg__"],
+)
+
 # Each test runs the consumer's full coverage pipeline, which can take many
 # minutes on a cold CI runner. The tests need network access to clone current
 # consumer main branches and run Bazel in those nested workspaces.
 #
-# Keep each consumer scenario in its own target so CI and JUnit reports show
-# exactly which real downstream workflow failed.
-score_pytest(
-    name = "baselibs",
-    size = "large",
-    timeout = "eternal",
-    srcs = ["test_baselibs.py"],
-    data = ["_consumer_workspace.py"],
-    imports = ["."],
-    tags = [
-        "integration",
-        "local",
-    ],
-)
-
+# The parametrized test runs all consumers by default. CI selects one with -k
+# so each job reports its own result; report archives use consumer subdirectories.
 score_pytest(
-    name = "lifecycle",
+    name = "consumers",
     size = "large",
     timeout = "eternal",
-    srcs = ["test_lifecycle.py"],
-    data = ["_consumer_workspace.py"],
-    imports = ["."],
+    srcs = ["test_consumers.py"],
+    args = ["--tb=short"],
     tags = [
         "integration",
         "local",
     ],
+    deps = [":consumer_workspace"],
 )

tests/compatibility/_consumer_workspace.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import os
+import shlex
 import shutil
 import subprocess
 import sys
@@ -43,13 +44,6 @@ def clone_consumer(name: str, parent_directory: Path) -> Path:
     )
     module_file.write_text(module_contents, encoding="utf-8")
 
-    # Updating the consumer lockfile for the local override lets its production
-    # commands keep their normal --lockfile_mode=error setting.
-    subprocess.run(
-        ["bazel", "mod", "deps", "--lockfile_mode=update"],
-        cwd=workspace,
-        check=True,
-    )
     return workspace
 
 
@@ -58,7 +52,7 @@ def run_bazel(
     *arguments: str,
     extra_environment: dict[str, str] | None = None,
 ) -> None:
-    """Run one consumer-facing Bazel command, keeping failure output concise."""
+    """Run Bazel and retain a readable status summary plus its complete output."""
     environment = os.environ.copy()
     if extra_environment:
         environment.update(extra_environment)
@@ -68,16 +62,37 @@ def run_bazel(
         command,
         cwd=workspace,
         env=environment,
-        capture_output=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
         check=False,
-        text=True,
+        encoding="utf-8",
+        errors="replace",
     )
+    status = "FAILED" if result.returncode else "PASSED"
+    summary = (
+        f"### {workspace.name}: bazel {arguments[0]} — {status}\n\n"
+        f"Command: `{shlex.join(command)}`\n\n"
+        f"Exit code: {result.returncode}\n"
+    )
+    if result.returncode:
+        lines = result.stdout.splitlines()
+        # Compiler and repository errors can precede a long list of skipped
+        # tests. Show those diagnostics before the tail rather than losing them.
+        diagnostics = [line for line in lines if any(marker in line for marker in ("ERROR:", "FAIL:", "Error in "))]
+        excerpt = "\n".join(diagnostics[:10] + lines[-30:])
+        summary += f"\n```text\n{excerpt}\n```\n"
+
+    outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR")
+    if outputs_directory:
+        retained = Path(outputs_directory) / workspace.name
+        retained.mkdir(parents=True, exist_ok=True)
+        log_name = f"{arguments[0]}.log"
+        (retained / log_name).write_text(result.stdout, encoding="utf-8")
+        summary += f"\nFull output: `{log_name}` in the uploaded test artifacts.\n"
+        with (retained / "compatibility_summary.md").open("a", encoding="utf-8") as summary_file:
+            summary_file.write(summary + "\n")
     if result.returncode:
-        output = (result.stdout + result.stderr).splitlines()
-        output_tail = "\n".join(output[-80:])
-        raise AssertionError(
-            f"`{' '.join(command)}` exited with code {result.returncode}.\nLast Bazel output lines:\n{output_tail}"
-        )
+        raise AssertionError(summary)
 
 
 def retain_coverage_results(workspace: Path, archive_directory: str) -> None:
@@ -89,7 +104,7 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None:
     if not archive.is_dir() or not outputs_directory:
         return
 
-    retained_report = Path(outputs_directory) / "coverage-report"
+    retained_report = Path(outputs_directory) / workspace.name / "coverage-report"
     summary_markdown = workspace / "coverage_summary.md"
     try:
         retained_report.mkdir(parents=True, exist_ok=True)
@@ -106,12 +121,12 @@ def retain_coverage_results(workspace: Path, archive_directory: str) -> None:
                 shutil.copytree(result, retained_report / name)
             elif result.is_file():
                 shutil.copy2(result, retained_report / name)
-    except Exception as error:
+    except Exception as error:  # pylint: disable=broad-exception-caught
         print(f"Could not copy all coverage report files: {error}", file=sys.stderr)
 
     try:
         report_archive = Path(shutil.make_archive(str(retained_report), "zip", retained_report))
-    except Exception as error:
+    except Exception as error:  # pylint: disable=broad-exception-caught
         print(f"Could not archive the coverage report: {error}", file=sys.stderr)
         return
 
@@ -151,7 +166,7 @@ def verify_coverage_results(workspace: Path, archive_directory: str) -> None:
 
     outputs_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR")
     assert outputs_directory, "Bazel did not provide TEST_UNDECLARED_OUTPUTS_DIR"
-    report_archive = Path(outputs_directory) / "coverage-report.zip"
+    report_archive = Path(outputs_directory) / workspace.name / "coverage-report.zip"
     assert report_archive.is_file(), "Coverage report was not retained for Bazel"
 
     coverage_percent = 100 * lines_hit / lines_found

tests/compatibility/test_consumers.py

@@ -0,0 +1,95 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Run coverage tests against real consumer repositories."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+from _consumer_workspace import clone_consumer, retain_coverage_results, run_bazel, verify_coverage_results
+
+
+@dataclass(frozen=True)
+class Consumer:
+    """Arguments needed to generate coverage reports for a downstream repository."""
+
+    coverage_args: tuple[str, ...] = ("--config=llvm_cov", "--build_tests_only", "--", "//score/...")
+    justification_file: str | None = "tools/coverage/coverage_justifications.yaml"
+
+
+CONSUMERS = {
+    "baselibs": Consumer(
+        coverage_args=(
+            "--config=llvm_cov",
+            "--build_tests_only",
+            "--",
+            "//score/...",
+            # This test expects compact JSON, but the coverage config uses pretty printing.
+            "-//score/json/internal/writer/vajson:vajson_serialize_test",
+            # Exclusions from baselibs' LLVM coverage workflow: these tests
+            # require GCC semantics or fail with static linking.
+            "-//score/language/safecpp/aborts_upon_exception/...",
+            "-//score/language/safecpp/safe_math/details:floating_point_environment_test",
+            "-//score/os/linux/utils/test:network_interface_test",
+        ),
+    ),
+    "inc_security_crypto": Consumer(),
+    "inc_someip_gateway": Consumer(
+        coverage_args=("--build_tests_only", "--", "//..."),
+        justification_file=None,
+    ),
+    "kyron": Consumer(
+        coverage_args=("--config=llvm_cov", "--build_tests_only", "--", "//src/..."),
+    ),
+    "lifecycle": Consumer(
+        justification_file="quality/coverage/coverage_justifications.yaml",
+    ),
+    "logging": Consumer(justification_file=None),
+    "persistency": Consumer(justification_file="quality/coverage/coverage_justifications.yaml"),
+    "time": Consumer(),
+}
+
+
+@pytest.mark.parametrize("consumer,settings", CONSUMERS.items(), ids=CONSUMERS)
+def test_consumer_coverage_workflow_reports_measured_coverage(
+    tmp_path: Path, consumer: str, settings: Consumer
+) -> None:
+    """Each consumer must produce and retain a report with measured coverage."""
+    workspace = clone_consumer(consumer, tmp_path)
+    if consumer == "inc_someip_gateway":
+        # The gateway's test wrappers use the production workflow's host selection.
+        (workspace / ".bazel_config").write_text("host\n", encoding="utf-8")
+
+    # The local override changes the module graph; allow lockfile updates even
+    # when the consumer enables strict lockfile checks in its bazelrc.
+    run_bazel(workspace, "coverage", "--lockfile_mode=update", *settings.coverage_args)
+    report_arguments = ["--summary-md", "coverage_summary.md", "--archive-dir", "coverage_artifact"]
+    if settings.justification_file:
+        report_arguments.extend(("--yaml", settings.justification_file))
+
+    # Keep any report artifacts even if report generation fails.
+    try:
+        run_bazel(
+            workspace,
+            "run",
+            "--lockfile_mode=update",
+            "@score_coverage//:generate_coverage_html",
+            "--",
+            *report_arguments,
+            # Verify measured coverage below without imposing a consumer-specific gate.
+            extra_environment={"COVERAGE_THRESHOLD": "0"},
+        )
+    finally:
+        retain_coverage_results(workspace, "coverage_artifact")
+
+    verify_coverage_results(workspace, "coverage_artifact")