From 37829f9589e877e1040eaf81fc012de28806e824 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 29 Oct 2026 14:54:28 +0100 Subject: [PATCH 1/2] Integrate coverage_tool reports into documentation --- .github/workflows/test_and_docs.yml | 4 +- .gitignore | 1 + .../score_modules_tooling.MODULE.bazel | 2 +- docs/integration_process/step_7_reporting.rst | 31 +++++ known_good.json | 2 +- scripts/module_tests.py | 112 ++++++++++++++++-- scripts/tests/test_module_tests.py | 106 ++++++++++++++++- 7 files changed, 239 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_and_docs.yml b/.github/workflows/test_and_docs.yml index 604dd58cdc7..725d3c82ddc 100644 --- a/.github/workflows/test_and_docs.yml +++ b/.github/workflows/test_and_docs.yml @@ -130,7 +130,9 @@ jobs: merge-multiple: true - name: Write unit test and coverage summaries # Module unit tests and their coverage come only from module_tests.yml, which runs each - # module's own tests against ref_int's resolved dependency versions. + # module's own tests against ref_int's resolved dependency versions. The report command + # also combines their rebased LCOV into coverage-report/coverage_report.dat, the input + # read by docs-as-code when //:docs_shim runs in this workspace below. run: python ./scripts/module_tests.py report _module_reports - name: Collect test reports for the documentation # The source code linker reads tests-report/ instead of bazel-testlogs/ when it exists, so diff --git a/.gitignore b/.gitignore index 42bd4efa127..103b491b857 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ _build ubproject.toml /docs/_collections /tests-report +/coverage-report # Workspace files /score_*/ diff --git a/bazel_common/score_modules_tooling.MODULE.bazel b/bazel_common/score_modules_tooling.MODULE.bazel index c0f401f18a7..31f5a6bf76c 100644 --- a/bazel_common/score_modules_tooling.MODULE.bazel +++ b/bazel_common/score_modules_tooling.MODULE.bazel @@ -69,7 +69,7 @@ git_override( bazel_dep(name = "score_docs_as_code") git_override( module_name = "score_docs_as_code", - commit = "f1b30375e49646d95ca52565c41f5b926b0aa884", + commit = "946fd135045841db63e18ffd532374963e84a220", remote = "https://github.com/eclipse-score/docs-as-code.git", ) diff --git a/docs/integration_process/step_7_reporting.rst b/docs/integration_process/step_7_reporting.rst index 8d9c00c3936..827846efe34 100644 --- a/docs/integration_process/step_7_reporting.rst +++ b/docs/integration_process/step_7_reporting.rst @@ -118,6 +118,37 @@ measured. Build a single module's report with: bazel run //rust_coverage:rust_coverage_score_my_module +Component coverage in the documentation +--------------------------------------- + +``scripts/module_tests.py`` uses ``--config=llvm_cov`` and exports reports through +``@score_coverage//:generate_coverage_html`` for the integrated Baselibs, +Persistency and Logging releases. Other modules run their unit tests +without coverage until a release using ``coverage_tool`` is integrated; the +activation points are commented in ``COVERAGE_MODULES`` in the script. + +The export uses ``COVERAGE_THRESHOLD=0`` to collect evidence without adding a +percentage gate. It contains plain LCOV at ``coverage/coverage_report.dat`` and +the detailed HTML report. Source paths are translated to the reference +integration's repository identities before upload; for example, +``score/foo.cpp`` in Baselibs becomes +``external/score_baselibs+/score/foo.cpp``. + +The ``report`` command combines these files into +``coverage-report/coverage_report.dat``, which docs-as-code reads when the +published documentation is built. Component coverage uses only the +``primary_need_id`` and ``code_targets`` already exported by the pinned modules. +Missing ownership or measurements remain unavailable; the integration does not +add component metadata through patches. + +To include coverage in a local documentation build: + +.. code-block:: bash + + python3 scripts/module_tests.py run score_persistency + python3 scripts/module_tests.py report _module_tests/reports + bazel run //:docs_shim + Verification & status reports ----------------------------- diff --git a/known_good.json b/known_good.json index 9f9be872721..b832d89f965 100644 --- a/known_good.json +++ b/known_good.json @@ -307,7 +307,7 @@ }, "score_docs_as_code": { "repo": "https://github.com/eclipse-score/docs-as-code.git", - "hash": "f1b30375e49646d95ca52565c41f5b926b0aa884" + "hash": "946fd135045841db63e18ffd532374963e84a220" }, "score_process_description": { "repo": "https://github.com/eclipse-score/process_description.git", diff --git a/scripts/module_tests.py b/scripts/module_tests.py index 46820c215ea..eed59c4f71f 100644 --- a/scripts/module_tests.py +++ b/scripts/module_tests.py @@ -41,6 +41,17 @@ REF_INT = Path(__file__).resolve().parents[1] UNIFIED_RC = REF_INT / "bazel_common" / "unified.bazelrc" UNIFIED_TOOLCHAINS = REF_INT / "bazel_common" / "unified_toolchains.MODULE.bazel" +# Only enable the coverage_tool flow for releases already integrated in known_good.json. +COVERAGE_MODULES = { + "score_baselibs", + "score_persistency", + "score_logging", + # "score_kyron", # Enable once ref_int integrates a release using coverage_tool. + # "score_communication", # Enable once ref_int integrates a release using coverage_tool. + # "score_lifecycle", # Enable once ref_int integrates a release using coverage_tool. + # "score_time", # Enable once ref_int integrates a release using coverage_tool. + # "score_config_management", # Enable once ref_int integrates a release using coverage_tool. +} # Bazel's per-target result lines, e.g. "//score/foo:bar_test FAILED in 2.1s". STATUS_LINE = re.compile(r"^(//\S+|@\S+)\s+(FAILED TO BUILD|FAILED|TIMEOUT|NO STATUS|INCOMPLETE|FLAKY)\b", re.M) @@ -108,18 +119,30 @@ def override_rc(paths: dict[str, Path], extra_test_config: list[str] = ()) -> st return rc + "".join(f"build --{flag}\n" for flag in extra_test_config) -def coverage_args(test_args: list[str], langs: list[str], excludes: list[str] = ()) -> list[str]: +def coverage_args( + test_args: list[str], langs: list[str], excludes: list[str] = (), *, enabled: bool = True +) -> list[str]: """The module's own test args with ref_int's unified config added before any ``--``. + Modules awaiting coverage_tool integration run their unit tests without coverage. ``excludes`` (the module's ``exclude_test_targets``) become negative patterns after ``--``. """ - ours = ["--config=unified", "--combined_report=lcov"] + ours = ["--config=unified"] + if enabled: + ours.append("--config=llvm_cov") if "rust" in langs: ours.append("--config=unified-rust") split = test_args.index("--") if "--" in test_args else len(test_args) targets = [*test_args[split + 1 :], *(f"-{target}" for target in excludes)] separator = ["--"] if targets else [] - return ["coverage", "--lockfile_mode=update", *test_args[:split], *ours, *separator, *targets] + return [ + "coverage" if enabled else "test", + "--lockfile_mode=update", + *test_args[:split], + *ours, + *separator, + *targets, + ] def _status(case: ET.Element) -> str: @@ -173,11 +196,64 @@ def _append_unified_toolchain(workspace: Path) -> None: f.write(f'\ninclude("//:{UNIFIED_TOOLCHAINS.name}")\n') -def _collect_coverage(workspace: Path, out: Path, env: dict) -> None: - (out / "coverage").mkdir() - lcov = Path(bazel(["info", "output_path"], workspace, env).stdout.strip()) / "_coverage" / "_coverage_report.dat" - if lcov.is_file(): - shutil.copy(lcov, out / "coverage" / "lcov.dat") +def normalize_lcov(text: str, module: str) -> str: + """Rebase coverage_tool's canonical paths into ref_int's external module namespace.""" + lines = [] + for line in text.splitlines(): + if line.startswith("SF:"): + path = line[3:] + if path.startswith("/"): + raise ValueError(f"{module}: coverage source must be repository-relative: {path}") + if path.startswith("../"): + path = "external/" + path[3:] + if not path.startswith("external/"): + path = f"external/{module}+/{path}" + parts = path.split("/") + if len(parts) < 3 or any(part in ("", ".", "..", "_virtual_includes") for part in parts): + raise ValueError(f"{module}: coverage source is not canonical: {path}") + if parts[1] == "_main": + parts[1] = f"{module}+" + line = "SF:" + "/".join(parts) + lines.append(line) + return "\n".join(lines) + "\n" if lines else "" + + +def combine_reports(reports: Path, modules: list[str], output: Path) -> None: + """Combine available module reports, leaving absent measurements unavailable.""" + chunks = [] + for module in sorted(modules): + report = reports / module / "coverage" / "coverage_report.dat" + if report.is_file(): + chunks.append(report.read_text(encoding="utf-8")) + else: + print(f"::warning::{module}: no documentation coverage report available") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("".join(chunk.rstrip("\n") + "\n" for chunk in chunks), encoding="utf-8") + + +def _collect_coverage(workspace: Path, out: Path, env: dict) -> int: + # Export through coverage_tool's public CLI. It owns the backend's report packaging; + # ref_int only translates the exported source identities into its workspace context. + # Threshold zero collects evidence without adding a new percentage gate to module tests. + result = bazel( + [ + "run", + "--lockfile_mode=update", + "@score_coverage//:generate_coverage_html", + "--", + "--testlogs-subdir=", + "--archive-dir", + str(out / "coverage"), + ], + workspace, + {**env, "COVERAGE_THRESHOLD": "0"}, + ) + print(result.stdout) + if result.returncode: + return result.returncode + lcov = out / "coverage" / "coverage_report.dat" + lcov.write_text(normalize_lcov(lcov.read_text(encoding="utf-8"), out.name), encoding="utf-8") + return 0 def _write_summary( @@ -241,7 +317,10 @@ def run(args: argparse.Namespace) -> int: unpinned = sorted(shared - set(pinned)) shutil.copy(rc, out / "overrides.bazelrc") - command = coverage_args(module.metadata.test_args, module.metadata.langs, module.metadata.exclude_test_targets) + coverage_enabled = module.name in COVERAGE_MODULES + command = coverage_args( + module.metadata.test_args, module.metadata.langs, module.metadata.exclude_test_targets, enabled=coverage_enabled + ) exit_code, log = streamed(["bazel", *command], workspace, env) results = list((workspace / "bazel-testlogs").glob("**/test.xml")) @@ -252,7 +331,9 @@ def run(args: argparse.Namespace) -> int: # Bazel exits 0 when every test is skipped (e.g. all incompatible with the platform). no_tests = not results exit_code = exit_code or int(no_tests) - _collect_coverage(workspace, out, env) + if coverage_enabled: + coverage_exit = _collect_coverage(workspace, out, env) + exit_code = exit_code or coverage_exit (out / "result.json").write_text(json.dumps({"exit_code": exit_code}) + "\n") _write_summary(module.name, out, exit_code, log, no_tests=no_tests, pinned=pinned, unpinned=unpinned) return exit_code @@ -270,10 +351,10 @@ def report(args: argparse.Namespace) -> int: result_file = reports / "result.json" result = json.loads(result_file.read_text()) if result_file.is_file() else {"exit_code": 1} unit_tests[module.name] = {**count_testcases(reports / "tests"), "exit_code": result["exit_code"]} - lcov = reports / "coverage" / "lcov.dat" - if "cpp" in module.metadata.langs: + lcov = reports / "coverage" / "coverage_report.dat" + if module.name in COVERAGE_MODULES: numbers = lcov_percentages(lcov) if lcov.is_file() else {} - coverage[f"{module.name}_cpp"] = {**numbers, "exit_code": 0 if numbers.get("lines") else 1} + coverage[module.name] = {**numbers, "exit_code": 0 if numbers.get("lines") else 1} args.out_dir.mkdir(parents=True, exist_ok=True) unit_tests_md = generate_markdown_report( @@ -289,6 +370,11 @@ def report(args: argparse.Namespace) -> int: output_path=args.out_dir / "coverage_summary.md", ) append_to_step_summary(unit_tests_md, coverage_md) + combine_reports( + args.reports_dir, + sorted(COVERAGE_MODULES & known.modules["target_sw"].keys()), + REF_INT / "coverage-report" / "coverage_report.dat", + ) return 0 diff --git a/scripts/tests/test_module_tests.py b/scripts/tests/test_module_tests.py index 138c447fb5a..bd90eec63d1 100644 --- a/scripts/tests/test_module_tests.py +++ b/scripts/tests/test_module_tests.py @@ -11,14 +11,20 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* from pathlib import Path +from types import SimpleNamespace +import pytest from module_tests import ( STATUS_LINE, + _collect_coverage, + combine_reports, count_testcases, coverage_args, graph_modules, lcov_percentages, + normalize_lcov, override_rc, + report, ) @@ -51,7 +57,7 @@ def test_coverage_args_put_unified_config_before_target_separator(): "--lockfile_mode=update", "--config=x86_64-linux", "--config=unified", - "--combined_report=lcov", + "--config=llvm_cov", "--", "//score/...", "-//score/test/...", @@ -60,7 +66,7 @@ def test_coverage_args_put_unified_config_before_target_separator(): def test_coverage_args_register_ferrocene_only_for_rust(): args = coverage_args(["//src/..."], ["rust"]) - assert args[-2:] == ["--combined_report=lcov", "--config=unified-rust"] + assert args[-2:] == ["--config=llvm_cov", "--config=unified-rust"] assert "--config=unified-rust" not in coverage_args(["//score/..."], ["cpp"]) @@ -69,7 +75,7 @@ def test_coverage_args_exclude_targets_after_separator(): assert args[-3:] == ["--config=unified-rust", "--", "-//score/rust:miri_tests"] # An existing separator is reused, the excludes go after the module's own targets. args = coverage_args(["--", "//score/...", "-//score/test/..."], ["cpp"], ["//score/a:b"]) - assert args[-5:] == ["--combined_report=lcov", "--", "//score/...", "-//score/test/...", "-//score/a:b"] + assert args[-5:] == ["--config=llvm_cov", "--", "//score/...", "-//score/test/...", "-//score/a:b"] def test_status_line_reads_bazel_failures(): @@ -102,3 +108,97 @@ def test_lcov_percentages_sums_all_records(tmp_path): lcov = tmp_path / "lcov.dat" lcov.write_text("SF:a.cpp\nLF:10\nLH:5\nFNF:2\nFNH:2\nend_of_record\nSF:b.cpp\nLF:10\nLH:10\nend_of_record\n") assert lcov_percentages(lcov) == {"lines": "75.0%", "functions": "100.0%", "branches": ""} + + +def test_equal_relative_paths_in_different_modules_remain_distinct(tmp_path): + for module, hits in [("first", 1), ("second", 0)]: + report = tmp_path / module / "coverage" / "coverage_report.dat" + report.parent.mkdir(parents=True) + report.write_text(normalize_lcov(f"SF:src/lib.rs\nDA:1,{hits}\nend_of_record\n", module)) + output = tmp_path / "coverage-report" / "coverage_report.dat" + combine_reports(tmp_path, ["second", "first", "missing"], output) + assert output.read_text() == ( + "SF:external/first+/src/lib.rs\nDA:1,1\nend_of_record\nSF:external/second+/src/lib.rs\nDA:1,0\nend_of_record\n" + ) + + +@pytest.mark.parametrize("prefix", ["external/dep+/", "../dep+/"]) +def test_external_dependency_is_not_reassigned_to_tested_module(prefix): + assert normalize_lcov(f"SF:{prefix}src/lib.h\nDA:2,1\n", "owner") == ("SF:external/dep+/src/lib.h\nDA:2,1\n") + + +@pytest.mark.parametrize( + "path", ["/unknown/lib.cpp", "src/../../lib.cpp", "../dep+/../lib.cpp", "_virtual_includes/a.h"] +) +def test_unresolvable_sources_fail_instead_of_silently_matching_wrong_files(path): + with pytest.raises(ValueError, match="coverage source"): + normalize_lcov(f"SF:{path}\n", "owner") + + +def test_coverage_tool_export_is_rebased_for_documentation(tmp_path, monkeypatch): + """Packaging belongs to coverage_tool; ref_int consumes its exported plain LCOV.""" + workspace = tmp_path / "workspace" + out = tmp_path / "reports" / "score_persistency" + out.mkdir(parents=True) + + def export(args, cwd, env): + assert cwd == workspace + assert env == {"EXISTING": "value", "COVERAGE_THRESHOLD": "0"} + assert args == [ + "run", + "--lockfile_mode=update", + "@score_coverage//:generate_coverage_html", + "--", + "--testlogs-subdir=", + "--archive-dir", + str(out / "coverage"), + ] + report = out / "coverage" / "coverage_report.dat" + report.parent.mkdir() + report.write_text("SF:score/kvs/kvs.cpp\nDA:1,3\nBRDA:1,0,0,-\nend_of_record\n") + return SimpleNamespace(stdout="exported", returncode=0) + + monkeypatch.setattr("module_tests.bazel", export) + assert _collect_coverage(workspace, out, {"EXISTING": "value"}) == 0 + assert (out / "coverage" / "coverage_report.dat").read_text() == ( + "SF:external/score_persistency+/score/kvs/kvs.cpp\nDA:1,3\nBRDA:1,0,0,-\nend_of_record\n" + ) + + +def test_coverage_export_failure_is_propagated(tmp_path, monkeypatch): + monkeypatch.setattr("module_tests.bazel", lambda *_args: SimpleNamespace(stdout="export failed", returncode=2)) + assert _collect_coverage(tmp_path, tmp_path / "score_persistency", {}) == 2 + + +def test_modules_awaiting_coverage_tool_still_run_tests(): + args = coverage_args(["--config=x86_64-linux", "--", "//score/..."], ["cpp"], enabled=False) + assert args[0] == "test" + assert "--config=unified" in args + assert "--config=llvm_cov" not in args + assert args[-2:] == ["--", "//score/..."] + + +def test_documentation_ignores_coverage_from_modules_awaiting_coverage_tool(tmp_path, monkeypatch): + """Old artifacts must not bring an unsupported module back into the documentation report.""" + modules = {name: SimpleNamespace(name=name) for name in ("score_persistency", "score_communication")} + monkeypatch.setattr("module_tests.REF_INT", tmp_path) + monkeypatch.setattr("module_tests.load_known_good", lambda _path: SimpleNamespace(modules={"target_sw": modules})) + for name in modules: + lcov = tmp_path / "reports" / name / "coverage" / "coverage_report.dat" + lcov.parent.mkdir(parents=True) + lcov.write_text(f"SF:external/{name}+/source.cpp\nDA:1,1\nLF:1\nLH:1\nend_of_record\n") + assert report(SimpleNamespace(reports_dir=tmp_path / "reports", out_dir=tmp_path / "summaries")) == 0 + combined = (tmp_path / "coverage-report" / "coverage_report.dat").read_text() + assert "score_persistency+" in combined + assert "score_communication+" not in combined + summary = (tmp_path / "summaries" / "coverage_summary.md").read_text() + assert "score_persistency" in summary + assert "score_communication" not in summary + + +def test_missing_reports_remove_stale_coverage_instead_of_reusing_it(tmp_path): + output = tmp_path / "coverage-report" / "coverage_report.dat" + output.parent.mkdir() + output.write_text("SF:external/old+/a.cpp\nDA:1,1\n") + combine_reports(tmp_path, ["missing"], output) + assert output.read_text() == "" From f34d346efa762eb269542a549d46d0d202cde577 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Thu, 29 Oct 2026 21:29:29 +0100 Subject: [PATCH 2/2] Fix Baselibs LLVM coverage test selection --- scripts/module_tests.py | 30 ++++++++++++++++++++++++----- scripts/tests/test_module_tests.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/scripts/module_tests.py b/scripts/module_tests.py index eed59c4f71f..97e2e57641f 100644 --- a/scripts/module_tests.py +++ b/scripts/module_tests.py @@ -120,7 +120,7 @@ def override_rc(paths: dict[str, Path], extra_test_config: list[str] = ()) -> st def coverage_args( - test_args: list[str], langs: list[str], excludes: list[str] = (), *, enabled: bool = True + test_args: list[str], langs: list[str], excludes: list[str] = (), *, enabled: bool = True, module: str = "" ) -> list[str]: """The module's own test args with ref_int's unified config added before any ``--``. @@ -129,16 +129,32 @@ def coverage_args( """ ours = ["--config=unified"] if enabled: - ours.append("--config=llvm_cov") + ours.extend(["--build_tests_only", "--config=llvm_cov"]) if "rust" in langs: ours.append("--config=unified-rust") split = test_args.index("--") if "--" in test_args else len(test_args) - targets = [*test_args[split + 1 :], *(f"-{target}" for target in excludes)] + flags = test_args[:split] + targets = list(test_args[split + 1 :]) + if enabled and module == "score_baselibs": + # Let llvm_cov select Clang without the module's GCC configuration. + flags = [arg for arg in flags if arg != "--config=score-linux-x86_64"] + targets.extend( + [ + # This library interposes __cxa_allocate_exception; ld.lld rejects the + # duplicate definition from the statically linked C++ runtime. + "-//score/language/safecpp/aborts_upon_exception/...", + # These tests require GCC FP semantics or fail gMock's leak check + # with static linking. Keep the exclusions specific to coverage. + "-//score/language/safecpp/safe_math/details:floating_point_environment_test", + "-//score/os/linux/utils/test:network_interface_test", + ] + ) + targets.extend(f"-{target}" for target in excludes) separator = ["--"] if targets else [] return [ "coverage" if enabled else "test", "--lockfile_mode=update", - *test_args[:split], + *flags, *ours, *separator, *targets, @@ -319,7 +335,11 @@ def run(args: argparse.Namespace) -> int: coverage_enabled = module.name in COVERAGE_MODULES command = coverage_args( - module.metadata.test_args, module.metadata.langs, module.metadata.exclude_test_targets, enabled=coverage_enabled + module.metadata.test_args, + module.metadata.langs, + module.metadata.exclude_test_targets, + enabled=coverage_enabled, + module=module.name, ) exit_code, log = streamed(["bazel", *command], workspace, env) diff --git a/scripts/tests/test_module_tests.py b/scripts/tests/test_module_tests.py index bd90eec63d1..b51b2a8cfb8 100644 --- a/scripts/tests/test_module_tests.py +++ b/scripts/tests/test_module_tests.py @@ -57,6 +57,7 @@ def test_coverage_args_put_unified_config_before_target_separator(): "--lockfile_mode=update", "--config=x86_64-linux", "--config=unified", + "--build_tests_only", "--config=llvm_cov", "--", "//score/...", @@ -78,6 +79,36 @@ def test_coverage_args_exclude_targets_after_separator(): assert args[-5:] == ["--config=llvm_cov", "--", "//score/...", "-//score/test/...", "-//score/a:b"] +def test_baselibs_coverage_preserves_targets_and_only_adds_llvm_exclusions(): + """LLVM exceptions must not disable these tests in a regular GCC test run.""" + test_args = ["--config=score-linux-x86_64", "--", "//..."] + normal = coverage_args(test_args, ["cpp"], enabled=False, module="score_baselibs") + assert normal == [ + "test", + "--lockfile_mode=update", + "--config=score-linux-x86_64", + "--config=unified", + "--", + "//...", + ] + + coverage = coverage_args(test_args, ["cpp"], module="score_baselibs") + assert "--config=score-linux-x86_64" not in coverage + assert "--build_tests_only" in coverage + assert coverage[coverage.index("--") + 1 :] == [ + "//...", + "-//score/language/safecpp/aborts_upon_exception/...", + "-//score/language/safecpp/safe_math/details:floating_point_environment_test", + "-//score/os/linux/utils/test:network_interface_test", + ] + # ref_int selects nlohmann JSON, so the compatibility test's vajson-specific + # pretty-printing exclusion is unnecessary here. + assert not any("vajson" in target for target in coverage) + assert "-//score/os/linux/utils/test:network_interface_test" not in coverage_args( + ["--", "//score/..."], ["cpp"], module="score_logging" + ) + + def test_status_line_reads_bazel_failures(): log = ( "//a:ok_test PASSED in 1.0s\n"