diff --git a/.github/scripts/check-file-coverage-report.sh b/.github/scripts/check-file-coverage-report.sh new file mode 100755 index 0000000..288d73f --- /dev/null +++ b/.github/scripts/check-file-coverage-report.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +minimum="${1:-90}" +report="${2:-coverage.json}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +workspace="${3:-$script_dir/../..}" +workspace_root="$(cd "$workspace" && pwd -P)/" +# Gate handwritten production files under crates//src/. Integration +# tests, colocated test.rs modules, generated macro code in vendor/, and other +# worktrees are not production files. The browser tests still run and exercise +# the production files counted here. +source_root="${workspace_root}crates/" + +covered_files="$(jq --arg source_root "$source_root" ' + [ + .data[].files[] + | select(.filename | startswith($source_root)) + | select(.filename | contains("/src/")) + | select(.filename | endswith("/test.rs") | not) + | select(.filename | endswith("/tests.rs") | not) + | select(.summary.lines.count > 0) + ] + | length +' "$report")" + +if [[ "$covered_files" -eq 0 ]]; then + echo "coverage report contains no production files with executable lines under crates/" >&2 + exit 1 +fi + +summary="$(jq -r --arg workspace_root "$workspace_root" --arg source_root "$source_root" ' + .data[].files[] + | select(.filename | startswith($source_root)) + | select(.filename | contains("/src/")) + | select(.filename | endswith("/test.rs") | not) + | select(.filename | endswith("/tests.rs") | not) + | select(.summary.lines.count > 0) + | [ + (.filename | ltrimstr($workspace_root)), + (.summary.lines.percent | tostring), + (.summary.lines.covered | tostring), + (.summary.lines.count | tostring) + ] + | @tsv +' "$report")" + +printf 'File\tLine coverage\tCovered lines\tCoverable lines\n' +while IFS=$'\t' read -r file percent covered count; do + printf '%s\t%.2f%%\t%s\t%s\n' "$file" "$percent" "$covered" "$count" +done <<< "$summary" + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + printf '### Per-file line coverage\n\n' + printf '| File | Coverage | Lines |\n' + printf '| --- | ---: | ---: |\n' + while IFS=$'\t' read -r file percent covered count; do + printf '| %s | %.2f%% | %s/%s |\n' "$file" "$percent" "$covered" "$count" + done <<< "$summary" + } >> "$GITHUB_STEP_SUMMARY" +fi + +failures="$(jq -r \ + --arg workspace_root "$workspace_root" \ + --arg source_root "$source_root" \ + --argjson minimum "$minimum" ' + .data[].files[] + | select(.filename | startswith($source_root)) + | select(.filename | contains("/src/")) + | select(.filename | endswith("/test.rs") | not) + | select(.filename | endswith("/tests.rs") | not) + | select(.summary.lines.count > 0) + | select(.summary.lines.percent < $minimum) + | "\(.filename | ltrimstr($workspace_root)): \(.summary.lines.percent)%" + ' "$report")" + +if [[ -n "$failures" ]]; then + printf '\nFiles below %s%% line coverage:\n%s\n' "$minimum" "$failures" >&2 + exit 1 +fi diff --git a/.github/scripts/check-file-coverage.sh b/.github/scripts/check-file-coverage.sh index df41e23..024c193 100755 --- a/.github/scripts/check-file-coverage.sh +++ b/.github/scripts/check-file-coverage.sh @@ -3,12 +3,25 @@ set -euo pipefail minimum="${1:-90}" report="${2:-coverage.json}" -workspace_root="$(pwd -P)/" -# Every crate lives under `crates//src/`, so one prefix covers the -# whole workspace. Vendored submodules and `worktrees/` sit outside it and are -# excluded by the same test. -source_root="${workspace_root}crates/" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +workspace_root="$(cd "$script_dir/../.." && pwd -P)" +if [[ "$(pwd -P)" != "$workspace_root" ]]; then + echo "run the coverage gate from the repository root: $workspace_root" >&2 + exit 1 +fi +if [[ "${TINYBROWSER_LIVE_TESTS:-}" != 1 ]]; then + echo "coverage gate requires TINYBROWSER_LIVE_TESTS=1 and a usable Chrome" >&2 + exit 1 +fi +if [[ -n "${TINYBROWSER_CHROME:-}" && ! -x "$TINYBROWSER_CHROME" ]]; then + echo "configured TINYBROWSER_CHROME is not executable" >&2 + exit 1 +fi + +# The Chrome suites serve their pages on loopback and fail if Chrome is absent. +# Requiring explicit opt-in prevents an apparently green coverage run whose +# browser tests silently skipped on a developer machine without Chrome. cargo llvm-cov \ --locked \ --workspace \ @@ -17,61 +30,4 @@ cargo llvm-cov \ --json \ --output-path "$report" -covered_files="$(jq --arg source_root "$source_root" ' - [ - .data[].files[] - | select(.filename | startswith($source_root)) - | select(.summary.lines.count > 0) - ] - | length -' "$report")" - -if [[ "$covered_files" -eq 0 ]]; then - echo "coverage report contains no files with executable lines under crates/" >&2 - exit 1 -fi - -summary="$(jq -r --arg workspace_root "$workspace_root" --arg source_root "$source_root" ' - .data[].files[] - | select(.filename | startswith($source_root)) - | select(.summary.lines.count > 0) - | [ - (.filename | ltrimstr($workspace_root)), - (.summary.lines.percent | tostring), - (.summary.lines.covered | tostring), - (.summary.lines.count | tostring) - ] - | @tsv -' "$report")" - -printf 'File\tLine coverage\tCovered lines\tCoverable lines\n' -while IFS=$'\t' read -r file percent covered count; do - printf '%s\t%.2f%%\t%s\t%s\n' "$file" "$percent" "$covered" "$count" -done <<< "$summary" - -if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - { - printf '### Per-file line coverage\n\n' - printf '| File | Coverage | Lines |\n' - printf '| --- | ---: | ---: |\n' - while IFS=$'\t' read -r file percent covered count; do - printf '| %s | %.2f%% | %s/%s |\n' "$file" "$percent" "$covered" "$count" - done <<< "$summary" - } >> "$GITHUB_STEP_SUMMARY" -fi - -failures="$(jq -r \ - --arg workspace_root "$workspace_root" \ - --arg source_root "$source_root" \ - --argjson minimum "$minimum" ' - .data[].files[] - | select(.filename | startswith($source_root)) - | select(.summary.lines.count > 0) - | select(.summary.lines.percent < $minimum) - | "\(.filename | ltrimstr($workspace_root)): \(.summary.lines.percent)%" - ' "$report")" - -if [[ -n "$failures" ]]; then - printf '\nFiles below %s%% line coverage:\n%s\n' "$minimum" "$failures" >&2 - exit 1 -fi +"$script_dir/check-file-coverage-report.sh" "$minimum" "$report" "$workspace_root" diff --git a/.github/scripts/test-check-file-coverage.sh b/.github/scripts/test-check-file-coverage.sh new file mode 100755 index 0000000..e2c0e98 --- /dev/null +++ b/.github/scripts/test-check-file-coverage.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +repo_root="$(cd "$script_dir/../.." && pwd -P)" +scratch="$(mktemp -d)" +scratch="$(cd "$scratch" && pwd -P)" +trap 'rm -rf "$scratch"' EXIT + +python3 - "$scratch" <<'PY' > "$scratch/good.json" +import json +import sys + +root = sys.argv[1] + +def file(path, covered, count): + return { + "filename": f"{root}/{path}", + "summary": {"lines": {"covered": covered, "count": count, + "percent": covered / count * 100}}, + } + +print(json.dumps({"data": [{"files": [ + file("crates/example/src/ops.rs", 9, 10), + file("crates/example/src/test.rs", 0, 10), + file("crates/example/tests/integration.rs", 0, 10), + file("vendor/tinybus/crates/tinybus/src/lib.rs", 0, 10), +]}]})) +PY + +python3 - "$scratch" <<'PY' > "$scratch/bad.json" +import json +import sys + +root = sys.argv[1] +print(json.dumps({"data": [{"files": [{ + "filename": f"{root}/crates/example/src/ops.rs", + "summary": {"lines": {"covered": 8, "count": 10, "percent": 80}}, +}]}]})) +PY + +python3 - "$scratch" <<'PY' > "$scratch/test-only.json" +import json +import sys + +root = sys.argv[1] +print(json.dumps({"data": [{"files": [{ + "filename": f"{root}/crates/example/src/test.rs", + "summary": {"lines": {"covered": 10, "count": 10, "percent": 100}}, +}]}]})) +PY + +( + cd "$scratch" + "$script_dir/check-file-coverage-report.sh" 90 good.json "$scratch" > good.out + grep -q 'crates/example/src/ops.rs' good.out + if grep -Eq 'test.rs|integration.rs|vendor/' good.out; then + echo "test-only or vendored file was counted" >&2 + exit 1 + fi + if "$script_dir/check-file-coverage-report.sh" 90 bad.json "$scratch" > bad.out 2>&1; then + echo "production file below 90% passed" >&2 + exit 1 + fi + grep -q 'crates/example/src/ops.rs: 80%' bad.out + if "$script_dir/check-file-coverage-report.sh" 90 test-only.json "$scratch" > empty.out 2>&1; then + echo "report with no production files passed" >&2 + exit 1 + fi + grep -q 'no production files' empty.out +) + +mkdir "$scratch/fakebin" +cat > "$scratch/fakebin/cargo" <<'FAKE_CARGO' +#!/usr/bin/env bash +set -euo pipefail +[[ "${TINYBROWSER_LIVE_TESTS:-}" == 1 ]] || { + echo "coverage did not opt in to live browser tests" >&2 + exit 1 +} +[[ "$1" == llvm-cov ]] +shift +for flag in --locked --workspace --all-targets --all-features --json; do + [[ " $* " == *" $flag "* ]] || exit 1 +done +while [[ "$1" != --output-path ]]; do shift; done +cp "$COVERAGE_FIXTURE" "$2" +FAKE_CARGO +chmod +x "$scratch/fakebin/cargo" +python3 - "$scratch/good.json" "$scratch" "$repo_root" <<'PY' > "$scratch/wrapper-good.json" +import json +import sys + +report = json.load(open(sys.argv[1])) +for group in report["data"]: + for file in group["files"]: + file["filename"] = file["filename"].replace(sys.argv[2] + "/", sys.argv[3] + "/", 1) +print(json.dumps(report)) +PY +( + cd "$repo_root" + TINYBROWSER_LIVE_TESTS=1 PATH="$scratch/fakebin:$PATH" \ + COVERAGE_FIXTURE="$scratch/wrapper-good.json" \ + "$script_dir/check-file-coverage.sh" 90 "$scratch/wrapped.json" > "$scratch/wrapped.out" + grep -q 'crates/example/src/ops.rs' "$scratch/wrapped.out" + if ( + unset TINYBROWSER_LIVE_TESTS + PATH="$scratch/fakebin:$PATH" COVERAGE_FIXTURE="$scratch/wrapper-good.json" \ + "$script_dir/check-file-coverage.sh" 90 "$scratch/not-opted-in.json" > "$scratch/not-opted-in.out" 2>&1 + ); then + echo "coverage gate accepted missing live-test opt-in" >&2 + exit 1 + fi + grep -q 'requires TINYBROWSER_LIVE_TESTS=1' "$scratch/not-opted-in.out" + if TINYBROWSER_CHROME="$scratch/missing-chrome" \ + TINYBROWSER_LIVE_TESTS=1 PATH="$scratch/fakebin:$PATH" \ + COVERAGE_FIXTURE="$scratch/wrapper-good.json" \ + "$script_dir/check-file-coverage.sh" 90 "$scratch/missing.json" > "$scratch/missing.out" 2>&1; then + echo "missing configured Chrome passed" >&2 + exit 1 + fi + grep -q 'not executable' "$scratch/missing.out" +) +if ( + cd "$scratch" + TINYBROWSER_LIVE_TESTS=1 PATH="$scratch/fakebin:$PATH" \ + COVERAGE_FIXTURE="$scratch/wrapper-good.json" \ + "$script_dir/check-file-coverage.sh" 90 "$scratch/wrong-directory.json" > "$scratch/wrong-directory.out" 2>&1 +); then + echo "coverage gate accepted a different working directory" >&2 + exit 1 +fi +grep -q 'run the coverage gate from the repository root' "$scratch/wrong-directory.out" + +echo "coverage script tests passed" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e26de19..4abde47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,8 +26,6 @@ jobs: # protocol conversation would be both unexercised and uncovered, which is # precisely the half that cannot be checked any other way. # - # GitHub's Ubuntu runners ship Google Chrome at a path the module already - # looks in, so there is nothing to install. TINYBROWSER_LIVE_TESTS: "1" # Runners restrict unprivileged user namespaces, so Chrome has no usable # sandbox here. Accepting that in a disposable CI VM is a different @@ -52,30 +50,17 @@ jobs: - uses: Swatinem/rust-cache@v2 - # Named explicitly rather than left to the module's own discovery order. - # The runner has several browsers on it, including a snap-packaged - # `/usr/bin/chromium` that starts, says nothing, and is timed out — and a - # build should not go red because the search order changed under it. This - # also prints which browser was chosen, so a future failure says so in the - # log instead of costing a round trip to find out. - - name: Select a browser for the live tests + - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2.2.0 + id: chrome + + - name: Select Chrome for the live tests + env: + CHROME_PATH: ${{ steps.chrome.outputs.chrome-path }} run: | set -euo pipefail - for candidate in \ - /usr/bin/google-chrome \ - /usr/bin/google-chrome-stable \ - /opt/google/chrome/chrome \ - /usr/bin/chromium-browser \ - /usr/bin/chromium - do - [ -x "$candidate" ] || continue - echo "selected $candidate" - "$candidate" --version || true - echo "TINYBROWSER_CHROME=$candidate" >> "$GITHUB_ENV" - exit 0 - done - echo "no browser on this runner; the live tests cannot run" >&2 - exit 1 + test -x "$CHROME_PATH" + "$CHROME_PATH" --version + echo "TINYBROWSER_CHROME=$CHROME_PATH" >> "$GITHUB_ENV" - name: Check formatting run: cargo fmt --all -- --check @@ -121,7 +106,10 @@ jobs: exit 1 fi - - name: Require 90% line coverage in every source file + - name: Verify the coverage gate + run: .github/scripts/test-check-file-coverage.sh + + - name: Require 90% line coverage in every production source file run: .github/scripts/check-file-coverage.sh 90 coverage.json - name: Upload coverage report diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f20cff8..3c39827 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,10 @@ jobs: name: Prepare release if: ${{ github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest + env: + # The release coverage gate opts into Chrome tests on a disposable runner. + TINYBROWSER_LIVE_TESTS: "1" + TINYBROWSER_CHROME_ARGS: --no-sandbox outputs: crate_name: ${{ steps.version.outputs.crate_name }} next_version: ${{ steps.version.outputs.next_version }} @@ -51,6 +55,18 @@ jobs: - uses: Swatinem/rust-cache@v2 + - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2.2.0 + id: chrome + + - name: Select Chrome for release coverage + env: + CHROME_PATH: ${{ steps.chrome.outputs.chrome-path }} + run: | + set -euo pipefail + test -x "$CHROME_PATH" + "$CHROME_PATH" --version + echo "TINYBROWSER_CHROME=$CHROME_PATH" >> "$GITHUB_ENV" + - name: Check formatting run: cargo fmt --all -- --check @@ -63,7 +79,10 @@ jobs: - name: Test run: cargo test --all-features - - name: Require 90% line coverage in every source file + - name: Verify the coverage gate + run: .github/scripts/test-check-file-coverage.sh + + - name: Require 90% line coverage in every production source file run: .github/scripts/check-file-coverage.sh 90 target/coverage.json - name: Build documentation diff --git a/AGENTS.md b/AGENTS.md index 0924feb..e065745 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -264,9 +264,11 @@ gitlink here. Their licences and roles are recorded in `THIRD-PARTY.md`. quietly does nothing reports green for a build in which nothing was checked. - The unit suites must not need a browser. Anything that would is a sign the judgement belongs in a pure function that can be tested without one. -- Maintain at least 90% line coverage in every source file. Add or update tests - with every behavior change, and note any deliberately untested edge case in - the pull request description. +- Maintain at least 90% line coverage in every production source file. The + coverage gate opts into the hermetic live Chrome tests; `test.rs` modules, + integration-test sources, generated vendor code, and worktrees do not count + as production files. Add or update tests with every behavior change, and note + any deliberately untested edge case in the pull request description. Write the test first when fixing a bug: a failing test that reproduces the report, then the fix that turns it green. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1424f9d..8a93fce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,11 +18,13 @@ cargo build --all-targets --all-features cargo test --all-features ``` -CI also requires at least 90% line coverage in every source file. After -installing `cargo-llvm-cov`, run the same gate locally: +CI also requires at least 90% line coverage in every production source file. +The gate runs the loopback-only live Chrome tests and fails if Chrome is +unavailable; install Chrome and `cargo-llvm-cov` before running it locally: ```sh -.github/scripts/check-file-coverage.sh 90 target/coverage.json +TINYBROWSER_LIVE_TESTS=1 .github/scripts/check-file-coverage.sh 90 target/coverage.json +.github/scripts/test-check-file-coverage.sh ``` The bundled example should also run: diff --git a/docs/specs/tinybus-module-release.md b/docs/specs/tinybus-module-release.md index d16a687..2419666 100644 --- a/docs/specs/tinybus-module-release.md +++ b/docs/specs/tinybus-module-release.md @@ -26,8 +26,9 @@ distributable without also shipping the TinyBus host runtime. ## Verification -CI exercises the bus interface through TinyBus's in-memory transport, enforces -90% line coverage in every source file, and builds the `cdylib`. The release +CI exercises the bus interface through TinyBus's in-memory transport and +the loopback-only live Chrome suite, enforces 90% line coverage in every +production source file, and builds the `cdylib`. The release workflow builds each native module from the tagged source and records its exact digest in the adjacent allowlist. After publishing, it downloads the Ubuntu x86_64 archive through TinyBus's GitHub release API and calls `Greet` over an