Run live Chrome coverage in release preparation

.github/scripts/check-file-coverage-report.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+minimum="${1:-90}"
+report="${2:-coverage.json}"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+workspace="${3:-$script_dir/../..}"
+workspace_root="$(cd "$workspace" && pwd -P)/"
+# Gate handwritten production files under crates/<package>/src/. Integration
+# tests, colocated test.rs modules, generated macro code in vendor/, and other
+# worktrees are not production files. The browser tests still run and exercise
+# the production files counted here.
+source_root="${workspace_root}crates/"
+
+covered_files="$(jq --arg source_root "$source_root" '
+  [
+    .data[].files[]
+    | select(.filename | startswith($source_root))
+    | select(.filename | contains("/src/"))
+    | select(.filename | endswith("/test.rs") | not)
+    | select(.filename | endswith("/tests.rs") | not)
+    | select(.summary.lines.count > 0)
+  ]
+  | length
+' "$report")"
+
+if [[ "$covered_files" -eq 0 ]]; then
+  echo "coverage report contains no production files with executable lines under crates/" >&2
+  exit 1
+fi
+
+summary="$(jq -r --arg workspace_root "$workspace_root" --arg source_root "$source_root" '
+  .data[].files[]
+  | select(.filename | startswith($source_root))
+  | select(.filename | contains("/src/"))
+  | select(.filename | endswith("/test.rs") | not)
+  | select(.filename | endswith("/tests.rs") | not)
+  | select(.summary.lines.count > 0)
+  | [
+      (.filename | ltrimstr($workspace_root)),
+      (.summary.lines.percent | tostring),
+      (.summary.lines.covered | tostring),
+      (.summary.lines.count | tostring)
+    ]
+  | @tsv
+' "$report")"
+
+printf 'File\tLine coverage\tCovered lines\tCoverable lines\n'
+while IFS=$'\t' read -r file percent covered count; do
+  printf '%s\t%.2f%%\t%s\t%s\n' "$file" "$percent" "$covered" "$count"
+done <<< "$summary"
+
+if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+  {
+    printf '### Per-file line coverage\n\n'
+    printf '| File | Coverage | Lines |\n'
+    printf '| --- | ---: | ---: |\n'
+    while IFS=$'\t' read -r file percent covered count; do
+      printf '| %s | %.2f%% | %s/%s |\n' "$file" "$percent" "$covered" "$count"
+    done <<< "$summary"
+  } >> "$GITHUB_STEP_SUMMARY"
+fi
+
+failures="$(jq -r \
+  --arg workspace_root "$workspace_root" \
+  --arg source_root "$source_root" \
+  --argjson minimum "$minimum" '
+    .data[].files[]
+    | select(.filename | startswith($source_root))
+    | select(.filename | contains("/src/"))
+    | select(.filename | endswith("/test.rs") | not)
+    | select(.filename | endswith("/tests.rs") | not)
+    | select(.summary.lines.count > 0)
+    | select(.summary.lines.percent < $minimum)
+    | "\(.filename | ltrimstr($workspace_root)): \(.summary.lines.percent)%"
+  ' "$report")"
+
+if [[ -n "$failures" ]]; then
+  printf '\nFiles below %s%% line coverage:\n%s\n' "$minimum" "$failures" >&2
+  exit 1
+fi

.github/scripts/check-file-coverage.sh

@@ -3,12 +3,25 @@ set -euo pipefail
 
 minimum="${1:-90}"
 report="${2:-coverage.json}"
-workspace_root="$(pwd -P)/"
-# Every crate lives under `crates/<package>/src/`, so one prefix covers the
-# whole workspace. Vendored submodules and `worktrees/` sit outside it and are
-# excluded by the same test.
-source_root="${workspace_root}crates/"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+workspace_root="$(cd "$script_dir/../.." && pwd -P)"
+if [[ "$(pwd -P)" != "$workspace_root" ]]; then
+  echo "run the coverage gate from the repository root: $workspace_root" >&2
+  exit 1
+fi
 
+if [[ "${TINYBROWSER_LIVE_TESTS:-}" != 1 ]]; then
+  echo "coverage gate requires TINYBROWSER_LIVE_TESTS=1 and a usable Chrome" >&2
+  exit 1
+fi
+if [[ -n "${TINYBROWSER_CHROME:-}" && ! -x "$TINYBROWSER_CHROME" ]]; then
+  echo "configured TINYBROWSER_CHROME is not executable" >&2
+  exit 1
+fi
+
+# The Chrome suites serve their pages on loopback and fail if Chrome is absent.
+# Requiring explicit opt-in prevents an apparently green coverage run whose
+# browser tests silently skipped on a developer machine without Chrome.
 cargo llvm-cov \
   --locked \
   --workspace \
@@ -17,61 +30,4 @@ cargo llvm-cov \
   --json \
   --output-path "$report"
 
-covered_files="$(jq --arg source_root "$source_root" '
-  [
-    .data[].files[]
-    | select(.filename | startswith($source_root))
-    | select(.summary.lines.count > 0)
-  ]
-  | length
-' "$report")"
-
-if [[ "$covered_files" -eq 0 ]]; then
-  echo "coverage report contains no files with executable lines under crates/" >&2
-  exit 1
-fi
-
-summary="$(jq -r --arg workspace_root "$workspace_root" --arg source_root "$source_root" '
-  .data[].files[]
-  | select(.filename | startswith($source_root))
-  | select(.summary.lines.count > 0)
-  | [
-      (.filename | ltrimstr($workspace_root)),
-      (.summary.lines.percent | tostring),
-      (.summary.lines.covered | tostring),
-      (.summary.lines.count | tostring)
-    ]
-  | @tsv
-' "$report")"
-
-printf 'File\tLine coverage\tCovered lines\tCoverable lines\n'
-while IFS=$'\t' read -r file percent covered count; do
-  printf '%s\t%.2f%%\t%s\t%s\n' "$file" "$percent" "$covered" "$count"
-done <<< "$summary"
-
-if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
-  {
-    printf '### Per-file line coverage\n\n'
-    printf '| File | Coverage | Lines |\n'
-    printf '| --- | ---: | ---: |\n'
-    while IFS=$'\t' read -r file percent covered count; do
-      printf '| %s | %.2f%% | %s/%s |\n' "$file" "$percent" "$covered" "$count"
-    done <<< "$summary"
-  } >> "$GITHUB_STEP_SUMMARY"
-fi
-
-failures="$(jq -r \
-  --arg workspace_root "$workspace_root" \
-  --arg source_root "$source_root" \
-  --argjson minimum "$minimum" '
-    .data[].files[]
-    | select(.filename | startswith($source_root))
-    | select(.summary.lines.count > 0)
-    | select(.summary.lines.percent < $minimum)
-    | "\(.filename | ltrimstr($workspace_root)): \(.summary.lines.percent)%"
-  ' "$report")"
-
-if [[ -n "$failures" ]]; then
-  printf '\nFiles below %s%% line coverage:\n%s\n' "$minimum" "$failures" >&2
-  exit 1
-fi
+"$script_dir/check-file-coverage-report.sh" "$minimum" "$report" "$workspace_root"

.github/scripts/test-check-file-coverage.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+repo_root="$(cd "$script_dir/../.." && pwd -P)"
+scratch="$(mktemp -d)"
+scratch="$(cd "$scratch" && pwd -P)"
+trap 'rm -rf "$scratch"' EXIT
+
+python3 - "$scratch" <<'PY' > "$scratch/good.json"
+import json
+import sys
+
+root = sys.argv[1]
+
+def file(path, covered, count):
+    return {
+        "filename": f"{root}/{path}",
+        "summary": {"lines": {"covered": covered, "count": count,
+                              "percent": covered / count * 100}},
+    }
+
+print(json.dumps({"data": [{"files": [
+    file("crates/example/src/ops.rs", 9, 10),
+    file("crates/example/src/test.rs", 0, 10),
+    file("crates/example/tests/integration.rs", 0, 10),
+    file("vendor/tinybus/crates/tinybus/src/lib.rs", 0, 10),
+]}]}))
+PY
+
+python3 - "$scratch" <<'PY' > "$scratch/bad.json"
+import json
+import sys
+
+root = sys.argv[1]
+print(json.dumps({"data": [{"files": [{
+    "filename": f"{root}/crates/example/src/ops.rs",
+    "summary": {"lines": {"covered": 8, "count": 10, "percent": 80}},
+}]}]}))
+PY
+
+python3 - "$scratch" <<'PY' > "$scratch/test-only.json"
+import json
+import sys
+
+root = sys.argv[1]
+print(json.dumps({"data": [{"files": [{
+    "filename": f"{root}/crates/example/src/test.rs",
+    "summary": {"lines": {"covered": 10, "count": 10, "percent": 100}},
+}]}]}))
+PY
+
+(
+  cd "$scratch"
+  "$script_dir/check-file-coverage-report.sh" 90 good.json "$scratch" > good.out
+  grep -q 'crates/example/src/ops.rs' good.out
+  if grep -Eq 'test.rs|integration.rs|vendor/' good.out; then
+    echo "test-only or vendored file was counted" >&2
+    exit 1
+  fi
+  if "$script_dir/check-file-coverage-report.sh" 90 bad.json "$scratch" > bad.out 2>&1; then
+    echo "production file below 90% passed" >&2
+    exit 1
+  fi
+  grep -q 'crates/example/src/ops.rs: 80%' bad.out
+  if "$script_dir/check-file-coverage-report.sh" 90 test-only.json "$scratch" > empty.out 2>&1; then
+    echo "report with no production files passed" >&2
+    exit 1
+  fi
+  grep -q 'no production files' empty.out
+)
+
+mkdir "$scratch/fakebin"
+cat > "$scratch/fakebin/cargo" <<'FAKE_CARGO'
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "${TINYBROWSER_LIVE_TESTS:-}" == 1 ]] || {
+  echo "coverage did not opt in to live browser tests" >&2
+  exit 1
+}
+[[ "$1" == llvm-cov ]]
+shift
+for flag in --locked --workspace --all-targets --all-features --json; do
+  [[ " $* " == *" $flag "* ]] || exit 1
+done
+while [[ "$1" != --output-path ]]; do shift; done
+cp "$COVERAGE_FIXTURE" "$2"
+FAKE_CARGO
+chmod +x "$scratch/fakebin/cargo"
+python3 - "$scratch/good.json" "$scratch" "$repo_root" <<'PY' > "$scratch/wrapper-good.json"
+import json
+import sys
+
+report = json.load(open(sys.argv[1]))
+for group in report["data"]:
+    for file in group["files"]:
+        file["filename"] = file["filename"].replace(sys.argv[2] + "/", sys.argv[3] + "/", 1)
+print(json.dumps(report))
+PY
+(
+  cd "$repo_root"
+  TINYBROWSER_LIVE_TESTS=1 PATH="$scratch/fakebin:$PATH" \
+    COVERAGE_FIXTURE="$scratch/wrapper-good.json" \
+    "$script_dir/check-file-coverage.sh" 90 "$scratch/wrapped.json" > "$scratch/wrapped.out"
+  grep -q 'crates/example/src/ops.rs' "$scratch/wrapped.out"
+  if (
+      unset TINYBROWSER_LIVE_TESTS
+      PATH="$scratch/fakebin:$PATH" COVERAGE_FIXTURE="$scratch/wrapper-good.json" \
+        "$script_dir/check-file-coverage.sh" 90 "$scratch/not-opted-in.json" > "$scratch/not-opted-in.out" 2>&1
+  ); then
+    echo "coverage gate accepted missing live-test opt-in" >&2
+    exit 1
+  fi
+  grep -q 'requires TINYBROWSER_LIVE_TESTS=1' "$scratch/not-opted-in.out"
+  if TINYBROWSER_CHROME="$scratch/missing-chrome" \
+      TINYBROWSER_LIVE_TESTS=1 PATH="$scratch/fakebin:$PATH" \
+      COVERAGE_FIXTURE="$scratch/wrapper-good.json" \
+      "$script_dir/check-file-coverage.sh" 90 "$scratch/missing.json" > "$scratch/missing.out" 2>&1; then
+    echo "missing configured Chrome passed" >&2
+    exit 1
+  fi
+  grep -q 'not executable' "$scratch/missing.out"
+)
+if (
+  cd "$scratch"
+  TINYBROWSER_LIVE_TESTS=1 PATH="$scratch/fakebin:$PATH" \
+    COVERAGE_FIXTURE="$scratch/wrapper-good.json" \
+    "$script_dir/check-file-coverage.sh" 90 "$scratch/wrong-directory.json" > "$scratch/wrong-directory.out" 2>&1
+); then
+  echo "coverage gate accepted a different working directory" >&2
+  exit 1
+fi
+grep -q 'run the coverage gate from the repository root' "$scratch/wrong-directory.out"
+
+echo "coverage script tests passed"

.github/workflows/ci.yml

@@ -26,8 +26,6 @@ jobs:
       # protocol conversation would be both unexercised and uncovered, which is
       # precisely the half that cannot be checked any other way.
       #
-      # GitHub's Ubuntu runners ship Google Chrome at a path the module already
-      # looks in, so there is nothing to install.
       TINYBROWSER_LIVE_TESTS: "1"
       # Runners restrict unprivileged user namespaces, so Chrome has no usable
       # sandbox here. Accepting that in a disposable CI VM is a different
@@ -52,30 +50,17 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
-      # Named explicitly rather than left to the module's own discovery order.
-      # The runner has several browsers on it, including a snap-packaged
-      # `/usr/bin/chromium` that starts, says nothing, and is timed out — and a
-      # build should not go red because the search order changed under it. This
-      # also prints which browser was chosen, so a future failure says so in the
-      # log instead of costing a round trip to find out.
-      - name: Select a browser for the live tests
+      - uses: browser-actions/setup-chrome@48ad923757ca74d66703209fe939badbdf80f2f4 # v2.2.0
+        id: chrome
+
+      - name: Select Chrome for the live tests
+        env:
+          CHROME_PATH: ${{ steps.chrome.outputs.chrome-path }}
         run: |
           set -euo pipefail
-          for candidate in \
-            /usr/bin/google-chrome \
-            /usr/bin/google-chrome-stable \
-            /opt/google/chrome/chrome \
-            /usr/bin/chromium-browser \
-            /usr/bin/chromium
-          do
-            [ -x "$candidate" ] || continue
-            echo "selected $candidate"
-            "$candidate" --version || true
-            echo "TINYBROWSER_CHROME=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          done
-          echo "no browser on this runner; the live tests cannot run" >&2
-          exit 1
+          test -x "$CHROME_PATH"
+          "$CHROME_PATH" --version
+          echo "TINYBROWSER_CHROME=$CHROME_PATH" >> "$GITHUB_ENV"
 
       - name: Check formatting
         run: cargo fmt --all -- --check
@@ -121,7 +106,10 @@ jobs:
             exit 1
           fi
 
-      - name: Require 90% line coverage in every source file
+      - name: Verify the coverage gate
+        run: .github/scripts/test-check-file-coverage.sh
+
+      - name: Require 90% line coverage in every production source file
         run: .github/scripts/check-file-coverage.sh 90 coverage.json
 
       - name: Upload coverage report