diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69d872a..dd43a5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,3 +28,88 @@ jobs: - run: bun run lint - run: bun run typecheck - run: bun run test + + # The engineering quality gates on the lines a pull request changes (issue #20, .quality/gate.py, + # .quality/README.md). Only the first two steps (checkout and the render of the description) get the token, + # and both run before any code from the pull request (bun install is the first); no later step or post step + # gets it. + quality: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + runs-on: ubuntu-latest + # Mutation stops itself inside a 30-minute budget and still writes its report; setup stays well under the rest. + timeout-minutes: 45 + steps: + - name: Check out the pull request + # Not actions/checkout: it keeps github.token for its post step, which runs at the end of the job, after + # code from the pull request. This fetches GitHub's merge commit with full history (the gate diffs against + # its first parent and checks the base out), passes the token to git through the environment only for the + # fetch, and stores no credential. + env: + GH_TOKEN: ${{ github.token }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + MERGE_REF: ${{ github.ref }} + run: | + set -o pipefail + find . -mindepth 1 -maxdepth 1 -exec rm -rf {} + + git init --quiet . + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraheader GIT_CONFIG_VALUE_0="AUTHORIZATION: basic $auth" \ + git fetch --quiet --no-tags "$REPO_URL" "+$MERGE_REF:refs/remotes/pull/merge" + git checkout --quiet --detach refs/remotes/pull/merge + - name: Render the pull request description + # The gate reads "Equivalent mutant:" waivers from GitHub's own rendering of the description (POST + # /markdown, gfm), so a waiver counts only where a reviewer sees it on the PR page. Fetched now rather than + # taken from the event, so re-running this job after editing the description sees the edit. python3 -I + # keeps the checkout off sys.path. The HTML's sha256 is a step output, which no later step can change. + id: description + env: + GH_TOKEN: ${{ github.token }} + PR_URL: ${{ github.event.pull_request.url }} + API_URL: ${{ github.api_url }} + run: | + set -o pipefail + curl --proto '=https' --tlsv1.2 --retry 5 --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" --header "Accept: application/vnd.github+json" "$PR_URL" \ + | python3 -I -c 'import json, sys; json.dump({"text": json.load(sys.stdin)["body"] or "", "mode": "gfm"}, sys.stdout)' \ + | curl --proto '=https' --tlsv1.2 --retry 5 --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" --header "Accept: application/vnd.github+json" \ + --header "Content-Type: application/json" --data-binary @- "$API_URL/markdown" \ + > "$RUNNER_TEMP/pr-body.html" + echo "sha256=$(sha256sum "$RUNNER_TEMP/pr-body.html" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: No sudo for the gate + # The gate runs code from the pull request and every mutant of it, and none of that may run as root. The + # hosted runner's passwordless sudo rule is dropped, and the job fails if sudo still works. + run: | + if command -v sudo >/dev/null 2>&1; then + sudo -n rm -f /etc/sudoers.d/runner 2>/dev/null || true + if sudo -n true 2>/dev/null; then echo "sudo still works"; exit 1; fi + fi + # Both actions default `token` to github.token; with an empty token they send no auth. + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.2 + no-cache: true + token: "" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + token: "" + - run: bun install --frozen-lockfile + - name: Install quality tools + # Pinned commits and versions in .quality/requirements.txt, in a venv that is thrown away with the job. + run: | + python -m venv "$RUNNER_TEMP/quality-venv" + "$RUNNER_TEMP/quality-venv/bin/pip" install --quiet -r .quality/requirements.txt + - name: Gate self-tests + run: | + export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" + python -m unittest discover -s .quality -p 'test_*.py' + - name: Quality gate + env: + DESCRIPTION_SHA256: ${{ steps.description.outputs.sha256 }} + run: | + export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" + python .quality/gate.py --description-html "$RUNNER_TEMP/pr-body.html" --description-sha256 "$DESCRIPTION_SHA256" diff --git a/.github/workflows/mutation-baseline.yml b/.github/workflows/mutation-baseline.yml new file mode 100644 index 0000000..6a96cbb --- /dev/null +++ b/.github/workflows/mutation-baseline.yml @@ -0,0 +1,89 @@ +name: mutation-baseline + +# The full mutation score of src/ for .quality/baseline.json. Never part of an ordinary pull request: +# a PR is judged on its changed lines by the quality job in ci.yml. It runs by hand, or on a pull request that changes +# the baseline script, and always measures the BASE branch's src/ code (the PR supplies the tests and the gate). +# Result: the RESULT line in the log and the step summary; copy the figures and this run's URL into baseline.json. + +on: + workflow_dispatch: + pull_request: + paths: + - ".quality/full_baseline.py" + - ".github/workflows/mutation-baseline.yml" + +permissions: + contents: read + +# A running baseline is never cancelled: a path filter on pull_request matches the whole PR's diff, so every push to +# such a PR starts this workflow again. The new run waits, and exits at once unless that push changed these files. +concurrency: + group: mutation-baseline-${{ github.ref }} + cancel-in-progress: false + +jobs: + baseline: + runs-on: ubuntu-latest + # The script stops itself at 330 minutes and still prints its totals; GitHub's own limit is 360. + timeout-minutes: 350 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Did this push change the baseline? + id: wanted + env: + ACTION: ${{ github.event.action }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.pull_request.head.sha }} + run: | + set -eu + run=true + if [ "$ACTION" = synchronize ] && changed=$(git diff --name-only "$BEFORE" "$AFTER" 2>/dev/null); then + printf '%s\n' "$changed" | grep -qxE '\.quality/full_baseline\.py|\.github/workflows/mutation-baseline\.yml' || run=false + fi + echo "run=$run" | tee -a "$GITHUB_OUTPUT" + - name: Measure the base branch's code + if: github.event_name == 'pull_request' && steps.wanted.outputs.run == 'true' + run: | + set -eu + # The source under measurement is the base's; the tests and the gate are the pull request's. The base is the + # merge commit's first parent, the tip the merge was built on (the event's base sha can be older than that). + BASE_SHA=$(git rev-parse HEAD^1) + rm -rf src + git checkout "$BASE_SHA" -- src + echo "src differs from $BASE_SHA in:"; git diff --stat "$BASE_SHA" -- src + - name: No sudo for the measured code + # The hosted runner's passwordless sudo rule is dropped, and the job fails if sudo still works. + if: steps.wanted.outputs.run == 'true' + run: | + if command -v sudo >/dev/null 2>&1; then + sudo -n rm -f /etc/sudoers.d/runner 2>/dev/null || true + if sudo -n true 2>/dev/null; then echo "sudo still works"; exit 1; fi + fi + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + if: steps.wanted.outputs.run == 'true' + with: + bun-version: 1.4.2 + no-cache: true + token: "" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + if: steps.wanted.outputs.run == 'true' + with: + python-version: "3.12" + token: "" + - if: steps.wanted.outputs.run == 'true' + run: bun install --frozen-lockfile + - name: Install quality tools + if: steps.wanted.outputs.run == 'true' + run: | + python -m venv "$RUNNER_TEMP/quality-venv" + "$RUNNER_TEMP/quality-venv/bin/pip" install --quiet -r .quality/requirements.txt + - name: Mutate every site + if: steps.wanted.outputs.run == 'true' + env: + BASELINE_BUDGET_SECONDS: "19800" + run: | + export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" + python .quality/full_baseline.py diff --git a/.gitignore b/.gitignore index c4f0007..9e8e2ff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ coverage/ .claude/settings.local.json *.log target/ +__pycache__/ diff --git a/.quality/README.md b/.quality/README.md new file mode 100644 index 0000000..ce60df7 --- /dev/null +++ b/.quality/README.md @@ -0,0 +1,106 @@ +# Quality gate + +The `quality` job in `.github/workflows/ci.yml` runs on every pull request and calls `gate.py`. It follows the +engineering quality standard as a ratchet: only code the PR adds or changes is judged, so debt already on main never +blocks a PR. Everything lives in this folder, that job and `mutation-baseline.yml`, so one revert removes it. + +| Check | Fails when | Tool | +|---|---|---| +| Coverage | an executable line the PR adds or changes has no test hit, or a changed file is missing from the report | Vitest, v8 coverage (lcov) | +| CRAP | a function the PR adds or changes scores 10 or more, unless it was already 10 or more on base and scores no higher now | crapper | +| Duplication | the PR adds a structural duplicate pair at 0.8 similarity or above | dryer | +| Mutation | a mutant on a changed line survives, mutator fails on a file, or the budget runs out | mutator | + +The thresholds (CRAP 10, similarity 0.8, a mutation budget of 5 to 30 minutes) are constants in `gate.py`, and a +self-test pins them. Changing one is an owner decision recorded in the engineering standard. + +The gate is a port of the one in the sibling Bun repository (buildboard), adapted to Vitest: one scope, Vitest's +coverage, and a mutant command that copes with Vite resolving symbolic links (see Mutation). + +## Scope + +TypeScript under `src/`, minus tests and `.d.ts`. The test helpers under `test/`, the fake app server included, are +not judged. + +A pure-deletion hunk judges no line. Every diff runs with `--no-color --no-ext-diff --no-textconv`, so git settings +cannot change what the gate reads, and a file whose hunks do not add up to git's own count stops the gate. + +## Coverage + +The gate runs `bun run coverage` itself (with the lcov report pinned), after deleting any old report, so a stale +report never stands in for it. `vitest.config.ts` lists every file under `src/`, so a file no test loads is in the +report at 0 hits. + +The CLI entry points, `src/cli/bin.ts` and `src/conformance/cli.ts`, only run as child processes in the tests, which +v8 coverage does not see. Each changed line in them is reported on its own and can be waived (see Waivers). Keep them +thin: the logic belongs in `main.ts` and `runner.ts`, which the tests import. + +The repo total (lines and functions, `src/` only) is compared with `baseline.json` and reported. A drop alone does +not fail the PR, because deleting covered code can cause it. A PR that raises a number updates `baseline.json`. + +The gate and every mutant run with `TETHER_KEYCHAIN=off` and without `TETHER_REAL_SERVICE`, so nothing it runs +touches the OS keychain or installs a real launchd agent. + +## CRAP ratchet + +A changed function that already scores 10 or more on base passes when its CRAP is no higher than on base; the report +lists it as "still over the limit" with both scores. A new, moved or renamed function, and one that was under 10 on +base, must stay under 10. Functions match by file, name and order among same-named functions in the file; when the +count of a name in a file changes, every function of that name counts as new. A base function with no coverage +record is not scored (0% would be the highest base possible), so it is judged as new. + +The base score is measured, not stored, and only when a touched function is over 10: the gate checks out base in a +throwaway worktree that borrows this checkout's `node_modules`, and runs the same coverage there. + +## Mutation + +Mutation runs only after every other check passed, on the changed lines of the changed files. The standard's +example command, `mutator --changed --test-command "bun run test"`, does not work here for two reasons, so the gate +calls mutator per file with `--lines` and its own test command, `mutant-tests.sh`: + +- mutator runs each mutant in a worker under `target/mutation-workers` made of symbolic links to the tree and a copy + of the mutated file. Vite resolves every module to its real path, so a linked test would import the original file + and every mutant would survive. `mutant-tests.sh` first replaces the links with real copies (all but + `node_modules`). +- The whole suite takes about 45 seconds. The script runs `vitest related` for the file first (the tests that import + it, directly or not), on one Vitest worker, and the whole suite only for a mutant that survived: the entry points + are reached only through child processes, which `related` cannot see. + +A mutant is killed only when Vitest reports a failed test or a test file that failed to load. A run that writes no +report (a broken config, a crash) is a harness error and never a kill: the mutant shows as a survivor, and the gate +fails where a reviewer can see it. + +The budget is sized to the change: the time already spent, plus for each file the rounds its sites need on its +workers (plus mutator's own unmutated run), times the measured time of the whole suite. It is never below 5 minutes +and never above 30. When it runs out, the check fails cleanly and still reports what finished and what was not +checked; the job timeout (45 minutes) sits above the largest budget. + +The quality job removes the hosted runner's sudo rule before any code from the pull request runs, so no mutant can +run anything as root. + +## Waivers + +A surviving mutant or an untested changed line passes only with this line in the PR description, as a paragraph of +its own: + + Equivalent mutant: src/file.ts:42 why no test can tell the difference + +One line waives every mutant on that line. The gate reads GitHub's own rendering of the description, fetched and +hashed before any code from the pull request runs, so a waiver counts only where a reviewer sees it. After editing +the description, re-run the quality job: it reads the description as it is when the job starts. + +## Baseline + +`baseline.json` holds the default branch's numbers. Line coverage moves by about 0.06 points between runs of the +same code (some paths depend on timing), which is one more reason a drop alone never fails a PR: coverage, functions measured, functions over CRAP 10, duplicate +pairs, and the full mutation score from `full_baseline.py`, which mutates every site under `src/` with the gate's own +machinery. `.github/workflows/mutation-baseline.yml` runs it on a GitHub runner, by hand or on a pull request that +changes the script or that workflow, always on the base branch's `src/`. Copy its `RESULT` line and the run's URL +into `baseline.json`. + +## Running it locally + + python3.12 -m venv /tmp/qv && /tmp/qv/bin/pip install -r .quality/requirements.txt + PATH=/tmp/qv/bin:$PATH python3.12 -m unittest discover -s .quality -p 'test_*.py' + PATH=/tmp/qv/bin:$PATH python3.12 .quality/gate.py --base origin/main + PATH=/tmp/qv/bin:$PATH python3.12 .quality/full_baseline.py diff --git a/.quality/baseline.json b/.quality/baseline.json new file mode 100644 index 0000000..1585a31 --- /dev/null +++ b/.quality/baseline.json @@ -0,0 +1,17 @@ +{ + "measuredOn": "main at 41d3e6c, 2026-11-18", + "coverage": { + "lines": 97.11, + "functions": 98.96 + }, + "functionsMeasured": 242, + "functionsOverCrap10": 0, + "duplicatePairs": 0, + "mutation": { + "score": 81.3, + "killed": 514, + "survived": 118, + "sites": 636, + "run": "https://github.com/farzai/tether/actions/runs/44246737986" + } +} diff --git a/.quality/full_baseline.py b/.quality/full_baseline.py new file mode 100644 index 0000000..d0103f9 --- /dev/null +++ b/.quality/full_baseline.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Full mutation baseline: every mutation site under src/, not only the lines a PR changes. + +Run by hand from the repository root (see .quality/README.md). It runs the suite with coverage, then uses the gate's own machinery +(worker plan, memory limit, per-mutant isolation, the same test command) with every file's whole range, prints one +line per file as it finishes, and ends with the totals that belong in .quality/baseline.json. Uncovered sites are not +run (mutator skips them), so the score is killed / (killed + survived). Files the budget did not reach are listed, +and the score covers only the files that finished. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.dont_write_bytecode = True +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import gate # noqa: E402 + + +def source_files(root: Path) -> list[str]: + found = [] + for scope in gate.SCOPE: + for path in (root / scope).rglob("*.ts"): + name = path.relative_to(root).as_posix() + if gate.in_scope(name): + found.append(name) + return sorted(found) + + +class Totals: + """Running totals of a full run, one file at a time.""" + + def __init__(self, started: float) -> None: + self.started = started + self.killed = 0 + self.survived = 0 + self.unchecked: list[str] = [] + self.failed: list[str] = [] + + def record(self, path: str, outcome) -> None: + status, k, survivors, detail = outcome + self.killed += k + self.survived += len(survivors) + if status == "failed": + self.failed.append(path) + elif status == "unchecked": + self.unchecked.append(path) + elapsed = time.monotonic() - self.started + print(f"{status:9} {path} killed={k} survived={len(survivors)} elapsed={elapsed:.0f}s", flush=True) + if status == "failed": + print(str(detail)[-600:], flush=True) + + def result(self, files: int, sites: int) -> dict: + done = self.killed + self.survived + return { + "files": files, + "sites": sites, + "killed": self.killed, + "survived": self.survived, + "score": round(100 * self.killed / done, 1) if done else None, + "filesNotChecked": sorted(self.unchecked), + "filesFailed": sorted(self.failed), + "seconds": round(time.monotonic() - self.started), + } + + +def publish(result: dict) -> None: + print("RESULT " + json.dumps(result), flush=True) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write("## Full mutation baseline\n\n```json\n" + json.dumps(result, indent=2) + "\n```\n") + + +def main() -> int: + root = Path.cwd() + budget = float(os.environ.get("BASELINE_BUDGET_SECONDS", "19800")) # 330 minutes + started = time.monotonic() + suite = gate.run_coverage(root) + files = source_files(root) + ranges = {f: [(1, len((root / f).read_text(encoding="utf-8").splitlines()))] for f in files} + cores = gate.available_cores() + limit = gate.worker_limit(cores, gate.memory_limit()) + with ThreadPoolExecutor(max_workers=max(1, min(len(files), cores))) as pool: + sites = dict(zip(files, pool.map(lambda f: gate.count_sites(f, gate.wanted_lines(ranges[f]), root), files))) + known = sum(n for n in sites.values() if isinstance(n, int)) + todo = sorted(gate.plan_workers(sites, limit), key=lambda f: -(sites[f] if isinstance(sites[f], int) else 1)) + print(f"suite {suite:.0f}s, {len(files)} files, {known} sites, {len(todo)} to mutate, limit {limit} ({cores} cores)", flush=True) + + totals = Totals(started) + # One file at a time with every slot: a share proportional to the sites leaves a big file on one worker for hours. + for path in todo: + totals.record(path, gate.mutate_file(path, ranges[path], root, limit, started + budget)) + gate.shutil.rmtree(root / "target" / "mutation-workers", ignore_errors=True) + result = totals.result(len(files), known) + publish(result) + return 1 if result["filesNotChecked"] or result["filesFailed"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.quality/gate.py b/.quality/gate.py new file mode 100644 index 0000000..e264dab --- /dev/null +++ b/.quality/gate.py @@ -0,0 +1,884 @@ +#!/usr/bin/env python3 +"""Pull request quality gate (engineering quality standard, issue #20). + +Ratchet, not big bang: only code the pull request adds or changes is judged, so debt that is already on main +never blocks a PR. Four checks, all of them in this one file so a revert removes the gate: + + coverage every executable line the PR adds or changes must be hit by a test (Vitest's v8 lcov); the repo total is + compared with .quality/baseline.json and reported, but a drop alone never fails a PR, so deleting + covered code cannot block a cleanup + crap a function the PR adds or changes must score under 10 (crapper), unless it was already 10 or more on + base and scores no higher now (the base is measured in a throwaway worktree with the same test run) + duplicate the PR must not add a structural duplicate pair at 0.8 or above (dryer) + mutation no surviving mutant on the lines the PR changes (mutator), inside a budget sized to the change +Both coverage and mutation can be waived for one line (never line 0, a whole file missing from the coverage report) +with a line in the PR description, "Equivalent mutant: : " (the reason is 10 or more characters). +One such line waives every mutant on that line. It counts only as a top-level paragraph of its own in GitHub's +rendering of the description, which an earlier workflow step writes to a file (--description-html) with its sha256. + +Scope for every check is the TypeScript under src/ (tests and .d.ts excluded). The test helpers under test/, +the fake app server included, are not judged. + +Run from the repository root with bun, and with crapper, dryer and mutator on the PATH (.quality/requirements.txt). +The gate runs the coverage itself, so a stale report can never stand in for it. + +Exit 0 when every check passes, 1 when any fails. The report goes to stdout and to GITHUB_STEP_SUMMARY. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from html.parser import HTMLParser +from pathlib import Path + +# Never leave __pycache__ in the tree. +sys.dont_write_bytecode = True + +# The standard's thresholds. Changing one is an owner decision, never a way to get a PR through. +CRAP_LIMIT = 10 +DUPLICATE_THRESHOLD = 0.8 +MUTATION_BUDGET_MIN = 300.0 +MUTATION_BUDGET_MAX = 1800.0 + +SCOPE = ("src",) +BASELINE = Path(".quality/baseline.json") +MUTANT_TESTS = ".quality/mutant-tests.sh" +LCOV = Path("coverage/lcov.info") +# The package script, so the gate measures what `bun run coverage` measures (vitest.config.ts lists every file under +# src/, so a file no test loads is still in the report, at 0 hits). The flags pin the report the gate reads. +COVERAGE_COMMAND = ["bun", "run", "coverage", "--coverage.reporter=lcov", "--coverage.reportsDirectory=coverage"] +# No test the gate runs may install a real launchd agent (test/service-real.test.ts) or reach the OS keychain. +DROPPED_VARIABLES = ("TETHER_REAL_SERVICE",) +PINNED_VARIABLES = {"TETHER_KEYCHAIN": "off"} +# Every diff the gate reads: git settings (colour, an external diff, a textconv driver) cannot change what it sees. +DIFF = ["--no-color", "--no-ext-diff", "--no-textconv", "--no-renames"] +# Line 0 never matches: it stands for a whole file missing from the coverage report, and no waiver covers that. +EQUIVALENT = re.compile(r"Equivalent mutant:\s*(\S+?):([1-9]\d*)\s+(\S.{9,})") +# A bidirectional control could make the reason read differently from what it says, so none may appear anywhere in +# the paragraph, on any of its lines. +BIDI = re.compile(r"[؜‎‏‪-‮⁦-⁩]") + +Ranges = dict[str, list[tuple[int, int]]] + + +def run(args: list[str], cwd: Path | None = None, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", env=env) + if check and result.returncode != 0: + sys.exit(f"command failed ({result.returncode}): {' '.join(args)}\n{result.stdout[-3000:]}\n{result.stderr[-3000:]}") + return result + + +def test_env() -> dict[str, str]: + kept = {key: value for key, value in os.environ.items() if key not in DROPPED_VARIABLES} + return {**kept, **PINNED_VARIABLES} + + +def resolve_base(explicit: str | None) -> str: + """The commit the PR is measured against. + + A pull_request checkout is GitHub's merge commit, whose first parent is the current base tip, so HEAD^1 is + exact even when the base moved after the PR was opened. Otherwise use --base. + """ + if run(["git", "rev-parse", "--verify", "-q", "HEAD^2"], check=False).returncode == 0: + return run(["git", "rev-parse", "HEAD^1"]).stdout.strip() + if not explicit: + sys.exit("HEAD is not a merge commit; pass --base ") + return explicit + + +def is_test_path(path: str) -> bool: + parts = path.split("/") + return "test" in parts[:-1] or parts[-1].endswith((".test.ts", ".spec.ts")) + + +def in_scope(path: str) -> bool: + """TypeScript under src/: no tests, no declaration files.""" + if not path.startswith(tuple(f"{root}/" for root in SCOPE)): + return False + return path.endswith(".ts") and not path.endswith(".d.ts") and not is_test_path(path) + + +def added_counts(base: str) -> dict[str, int]: + """File -> lines it adds, by name as git stores it. -z keeps a name with a space, a tab or a non-ASCII + character whole: a patch's +++ line ends such a name with a tab or quotes it.""" + listed = run(["git", "diff", *DIFF, "--numstat", "-z", "--diff-filter=AMT", base, "HEAD", "--", *SCOPE]).stdout + counts: dict[str, int] = {} + for record in filter(None, listed.split("\0")): + added, _deleted, path = record.split("\t", 2) + if not in_scope(path): + continue + if added == "-": # git counts no lines in a file it reads as binary: it must never pass unjudged + sys.exit(f"git reads {path!r} as a binary file, so its changed lines cannot be judged") + if int(added) > 0: # a pure deletion adds no line to judge + counts[path] = int(added) + return counts + + +HUNK = re.compile(r"^@@ -\S+ \+(\d+)(?:,(\d+))? @@", re.MULTILINE) + + +def file_diff(base: str, path: str) -> str: + """The -U0 patch of one file from base to HEAD; the name is a literal path, never a pattern.""" + return run(["git", "--literal-pathspecs", "diff", *DIFF, "-U0", base, "HEAD", "--", path]).stdout + + +def changed_ranges(base: str) -> Ranges: + """Added or modified lines (new-file numbering) per file. A hunk that adds nothing (+N,0) is skipped. Each file's + hunks must add up to the lines git counts for it; when they do not, the gate stops, because a file it cannot + read must never pass as unchanged.""" + found: Ranges = {} + for path, added in added_counts(base).items(): + hunks = [(int(start), 1 if count == "" else int(count)) for start, count in HUNK.findall(file_diff(base, path))] + ranges = [(start, start + count - 1) for start, count in hunks if count > 0] + if sum(end - start + 1 for start, end in ranges) != added: + sys.exit(f"could not read the changed lines of {path!r}: git counts {added} added, its hunks do not") + found[path] = ranges + return found + + +def touches(ranges: list[tuple[int, int]], first: int, last: int) -> bool: + return any(start <= last and end >= first for start, end in ranges) + + +def run_coverage(root: Path) -> float: + """Run the suite with coverage into coverage/lcov.info, after deleting any old report. Returns the seconds it took, + which is also the gate's measure of one mutant's worst case (one mutant runs at most the whole suite).""" + shutil.rmtree(root / "coverage", ignore_errors=True) + started = time.monotonic() + result = run(COVERAGE_COMMAND, cwd=root, check=False, env=test_env()) + if result.returncode != 0: + sys.exit(f"the tests failed under coverage ({result.returncode}):\n{result.stdout[-3000:]}\n{result.stderr[-3000:]}") + if not (root / LCOV).is_file(): + sys.exit(f"the coverage run wrote no {LCOV}") + return time.monotonic() - started + + +def parse_lcov(text: str) -> dict[str, dict[int, int]]: + """File -> {line: hits} for the executable lines in an lcov report.""" + files: dict[str, dict[int, int]] = {} + current: dict[int, int] | None = None + for line in text.splitlines(): + if line.startswith("SF:"): + current = files.setdefault(line[3:].strip(), {}) + elif line.startswith("DA:") and current is not None: + number, _, rest = line[3:].partition(",") + current[int(number)] = int(rest.split(",")[0]) + return files + + +def lcov_totals(text: str) -> dict[str, float]: + """Line and function coverage of the in-scope files, in percent (two decimals), from the LF/LH/FNF/FNH records. + """ + sums = {"LF": 0, "LH": 0, "FNF": 0, "FNH": 0} + counted = False + for line in text.splitlines(): + if line.startswith("SF:"): + counted = in_scope(line[3:].strip()) + elif counted and line[:3].rstrip(":") in sums: + key, _, value = line.partition(":") + sums[key] += int(value) + + def pct(hit: int, found: int) -> float: + return round(100 * hit / found, 2) if found else 100.0 + + return {"lines": pct(sums["LH"], sums["LF"]), "functions": pct(sums["FNH"], sums["FNF"])} + + +def lines_of(report: dict[str, dict[int, int]], path: str) -> dict[int, int] | None: + for name, lines in report.items(): + if name == path or name.endswith("/" + path): + return lines + return None + + +def uncovered_changed(changed: Ranges, lcov_text: str) -> list[tuple[str, int]]: + """Changed executable lines that no test hits. vitest.config.ts lists every file under src/, so a file the tests + only run as a separate process (src/cli/bin.ts) is in the report at 0 hits, and each of its changed lines is + reported on its own. A file the report does not list at all is reported as line 0, which no waiver names: the + report cannot vouch for any of its lines.""" + report = parse_lcov(lcov_text) + found: list[tuple[str, int]] = [] + for path, ranges in sorted(changed.items()): + lines = lines_of(report, path) + if lines is None: + found.append((path, 0)) + continue + for number in sorted(n for n, hits in lines.items() if hits == 0 and touches(ranges, n, n)): + found.append((path, number)) + return found + + +def coverage_verdict( + uncovered: list[tuple[str, int]], allowed: dict[tuple[str, int], str], now: dict[str, float], baseline: dict +) -> tuple[bool, list[str]]: + """The PR fails on its own untested lines, never on the repo total.""" + lines = totals_report(now, baseline["coverage"]) + bad = [item for item in uncovered if item not in allowed] + for path, number in uncovered: + reason = allowed.get((path, number)) + where = f"{path}:{number}" if number else f"{path} (not in the coverage report)" + lines.append(f" untested changed line {where}" + (f" (explained: {reason})" if reason else "")) + if bad: + lines.append( + "Add a test that runs each line, or add 'Equivalent mutant: : ' to the PR description " + "when no test can reach it." + ) + else: + lines.insert(0, "Every executable line the PR adds or changes is hit by a test.") + return not bad, lines + + +def totals_report(now: dict[str, float], floor: dict[str, float]) -> list[str]: + """The repo totals against the baseline: reported, and a rise asks for the baseline to be updated.""" + lines = [ + f"Repo coverage: lines {now['lines']}% (baseline {floor['lines']}%), " + f"functions {now['functions']}% (baseline {floor['functions']}%)." + ] + keys = ("lines", "functions") + if any(now[key] > floor[key] for key in keys): + lines.append("Coverage improved: update coverage in .quality/baseline.json in this PR.") + if any(now[key] < floor[key] for key in keys): + lines.append("Below the baseline; that alone does not fail the PR (deleting covered code can cause it).") + return lines + + +def check_coverage(lcov_path: Path, changed: Ranges, baseline: dict, allowed: dict[tuple[str, int], str]) -> tuple[bool, list[str]]: + text = lcov_path.read_text(encoding="utf-8") + return coverage_verdict(uncovered_changed(changed, text), allowed, lcov_totals(text), baseline) + + +Scored = tuple[tuple[str, str, int], int, float, int, float] +"""One function's score: (key, start line, CRAP, complexity, coverage percent). The key is (path, name, nth function of +that name in the file), so a function that moved to another file or was renamed has no key on base: it is new.""" + + +def scored_functions(root: Path, paths: list[str], need_coverage: bool = False) -> list[Scored]: + """Every function in the given files with its CRAP, from the coverage under root. + + A function with no coverage record scores as 0% covered, unless `need_coverage`: then it is left out, because + 0% is the highest score it could have, which would let any head score pass against it.""" + from crapper.coverage import load_bundle + from crapper.crap import make_entry + from crapper.languages import functions_in_file + + bundle = load_bundle(root) + bundle.bind_sources(sorted(paths)) + found: list[Scored] = [] + for path in sorted(paths): + source = (root / path).read_text(encoding="utf-8", errors="replace") + seen: dict[str, int] = {} + for function in functions_in_file("typescript", source, path, str(root)): + nth = seen.get(function.name, 0) + seen[function.name] = nth + 1 + percent = bundle.percent_for(function) + if percent is None and need_coverage: + continue + entry = make_entry(function, 0.0 if percent is None else percent) + if entry.crap is not None: + found.append(((path, function.name, nth), function.start_line, entry.crap, entry.complexity, entry.coverage)) + return found + + +def crap_verdict( + head: list[Scored], touched: set[tuple[str, str, int]], base: dict[tuple[str, str, int], float] | None +) -> tuple[bool, list[str]]: + """A function the PR touches passes under the limit. One that is over the limit passes only when it was already + over on base and scores no higher now (raw scores; one decimal is only how they print); a new, moved or renamed + one has to be under, and so is every function of a name whose count in its file changed (order cannot tell which + one is the old one).""" + + def counts(keys) -> dict[tuple[str, str], int]: + found: dict[tuple[str, str], int] = {} + for path, name, _nth in keys: + found[(path, name)] = found.get((path, name), 0) + 1 + return found + + on_head = counts(key for key, *_ in head) + on_base = counts(base or {}) + bad: list[str] = [] + still: list[str] = [] + judged = [item for item in head if item[0] in touched] + for key, line, crap, complexity, coverage in judged: + if crap < CRAP_LIMIT: + continue + label = f"{key[0]}:{line} {key[1]} CRAP {crap:.1f} (complexity {complexity}, coverage {coverage:.0f}%)" + before = (base or {}).get(key) if on_head.get(key[:2]) == on_base.get(key[:2]) else None + failure = crap_failure(crap, before) + if failure: + bad.append(f"{label}, {failure}") + else: + still.append(f"{label}, base {before:.1f}") + checked = len(judged) + lines = [f"{checked} added or changed function(s) scored, limit is under {CRAP_LIMIT}."] + lines += [f" over the limit: {item}" for item in bad] + lines += [f" still over the limit, no worse than base: {item}" for item in still] + return not bad, lines + + +def crap_failure(crap: float, before: float | None) -> str | None: + """Why a function at or over the limit fails, or None when it may stay: it was already over on base and is no + higher now.""" + if before is None: + return "new function (no score on base under this file and name)" + if before < CRAP_LIMIT: + return f"was {before:.1f} on base (under the limit)" + if crap > before: + return f"was {before:.1f} on base (higher)" + return None + + +def base_worktree(root: Path, base: str) -> tuple[Path, Path]: + """A throwaway detached worktree of base. Returns (its parent, the tree).""" + parent = Path(tempfile.mkdtemp(prefix="quality-base-")) + tree = parent / "tree" + run(["git", "worktree", "add", "--detach", str(tree), base], cwd=root) + return parent, tree + + +def drop_worktree(root: Path, parent: Path) -> None: + run(["git", "worktree", "remove", "--force", str(parent / "tree")], cwd=root, check=False) + shutil.rmtree(parent, ignore_errors=True) + + +def base_scores(base: str, root: Path, paths: list[str]) -> dict[tuple[str, str, int], float]: + """CRAP on base for the functions in the given files: the base tree, with the same coverage run as head (the base + borrows this checkout's node_modules).""" + parent, tree = base_worktree(root, base) + try: + existing = [path for path in paths if (tree / path).is_file()] + if not existing: + return {} + (tree / "node_modules").symlink_to((root / "node_modules").resolve(), target_is_directory=True) + run_coverage(tree) + return { + key: crap for key, _line, crap, _complexity, _coverage in scored_functions(tree, existing, need_coverage=True) + } + finally: + drop_worktree(root, parent) + + +def touched_functions(root: Path, changed: Ranges) -> set[tuple[str, str, int]]: + """Functions with a changed line in them.""" + from crapper.languages import functions_in_file + + found: set[tuple[str, str, int]] = set() + for path, ranges in changed.items(): + source = (root / path).read_text(encoding="utf-8", errors="replace") + seen: dict[str, int] = {} + for function in functions_in_file("typescript", source, path, str(root)): + nth = seen.get(function.name, 0) + seen[function.name] = nth + 1 + if touches(ranges, function.start_line, function.end_line): + found.add((path, function.name, nth)) + return found + + +def check_crap(changed: Ranges, root: Path, base: str) -> tuple[bool, list[str]]: + head = scored_functions(root, sorted(changed)) + touched = touched_functions(root, changed) + over = sorted({key[0] for key, _l, crap, _c, _v in head if key in touched and crap >= CRAP_LIMIT}) + return crap_verdict(head, touched, base_scores(base, root, over) if over else None) + + +Pair = tuple[float, tuple[str, int, int], tuple[str, int, int]] +# The start of each block in dryer's report; a line, so a path or note that holds the word cannot add one. +DUPLICATE = re.compile(r"^DUPLICATE ", re.MULTILINE) + + +def parse_dryer(out: str) -> list[Pair]: + pairs: list[Pair] = [] + # The text before the first marker is not a block. + for block in DUPLICATE.split(out)[1:]: + score = re.match(r"score=([\d.]+)", block) + places = re.findall(r"^\s+(\S+):(\d+)-(\d+)$", block, re.MULTILINE) + if score and len(places) == 2: + left, right = ((p, int(a), int(b)) for p, a, b in places) + pairs.append((float(score.group(1)), left, right)) + # A block the gate cannot read could hide a new pair, so it stops the gate rather than being skipped. + blocks = len(DUPLICATE.findall(out)) + if blocks != len(pairs): + sys.exit(f"dryer printed {blocks} DUPLICATE blocks but {len(pairs)} pairs could be read:\n{out[-3000:]}") + return pairs + + +def dryer_pairs(root: Path) -> list[Pair]: + """Pairs at the threshold or above under src/ (dryer skips test folders and files on its own).""" + targets = [name for name in SCOPE if (root / name).is_dir()] + if not targets: + return [] + out = run(["dryer", "--threshold", str(DUPLICATE_THRESHOLD), *targets], cwd=root).stdout + shutil.rmtree(root / ".metrics", ignore_errors=True) + return parse_dryer(out) + + +def duplicate_verdict(changed: Ranges, before: list[Pair], after: list[Pair]) -> tuple[bool, list[str]]: + """A pair is new when it touches a changed line and its two files hold more pairs than they did on base.""" + + def key(pair: Pair) -> tuple[str, str]: + return tuple(sorted((pair[1][0], pair[2][0]))) # type: ignore[return-value] + + def count(pairs: list[Pair], wanted: tuple[str, str]) -> int: + return sum(1 for p in pairs if key(p) == wanted) + + bad = [] + for pair in after: + score, left, right = pair + touched = any(f in changed and touches(changed[f], a, b) for f, a, b in (left, right)) + if touched and count(after, key(pair)) > count(before, key(pair)): + bad.append(f"{left[0]}:{left[1]}-{left[2]} and {right[0]}:{right[1]}-{right[2]} (similarity {score:.2f})") + head = f"{len(after)} duplicate pair(s) at {DUPLICATE_THRESHOLD} or above on this PR, {len(before)} on base." + return not bad, [head] + [f" new duplicate: {item}" for item in bad] + + +def check_duplicates(changed: Ranges, base: str, root: Path) -> tuple[bool, list[str]]: + parent, tree = base_worktree(root, base) + try: + before = dryer_pairs(tree) + finally: + drop_worktree(root, parent) + return duplicate_verdict(changed, before, dryer_pairs(root)) + + +# Elements that never hold content, so they never wrap a waiver. +VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}) + + +class TopLevelParagraphs(HTMLParser): + """The text of each top-level

in GitHub's rendering that holds text and nothing else. Anything inside another + element (

,
, a list, a quote) or next to markup in its paragraph is left out, and an element that is
+    never closed covers everything after it."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.open: list[str] = []
+        self.text: list[str] | None = None
+        self.found: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.text = [] if tag == "p" and not self.open and not attrs else None
+        if tag not in VOID:
+            self.open.append(tag)
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.text = None
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag not in self.open:
+            return
+        while self.open.pop() != tag:
+            pass
+        # Text is collected only from a top-level 

with no tag since, so this end tag closes that very paragraph; + # nothing after it belongs to it. + if self.text is not None: + self.found.append("".join(self.text)) + self.text = None + + def handle_data(self, data: str) -> None: + if self.text is not None: + self.text.append(data) + + +def survivors_allowed(html: str) -> dict[tuple[str, int], str]: + """The waivers in GitHub's rendering of the description: each must be the whole text of a top-level paragraph, so + a reviewer always sees it on the PR page.""" + parser = TopLevelParagraphs() + parser.feed(html) + parser.close() + found = (EQUIVALENT.fullmatch(text.strip()) for text in parser.found if not BIDI.search(text)) + return {(m.group(1), int(m.group(2))): m.group(3).strip() for m in found if m} + + +def read_waivers(path: str | None, sha256: str | None) -> tuple[bool, list[str], dict[tuple[str, int], str]]: + """The waivers that count, from GitHub's rendering of the description, written to `path` by the workflow step that + holds the token, before any code from the pull request runs, together with its sha256 (a step output, which no + later step can change). Without a file nothing is waived; a file that cannot be read, or comes without its hash or + does not match it, fails, because then the gate cannot tell which waivers a reviewer sees.""" + if path is None: + return True, ["No rendered description was given: nothing is waived."], {} + try: + data = Path(path).read_bytes() + except OSError as error: + return False, [f"The rendered description cannot be read, so no waiver can be: {error}"], {} + if not sha256 or hashlib.sha256(data).hexdigest() != sha256.strip().lower(): + return False, ["The rendered description does not match the sha256 it was rendered with, so no waiver can be read."], {} + allowed = survivors_allowed(data.decode("utf-8", errors="replace")) + lines = [f"{len(allowed)} waiver(s) count, each the whole text of a top-level paragraph on GitHub."] + lines.extend(f" {file}:{line}" for file, line in sorted(allowed)) + return True, lines, allowed + + +SAFE_PATH = re.compile(r"^[A-Za-z0-9_./@+-]+$") + + +def safe_path(path: str) -> bool: + return bool(SAFE_PATH.match(path)) and ".." not in path.split("/") + + +def _kill_pass(tag: str) -> int: + """One scan of /proc: kill every process that carries the tag. Returns how many it killed.""" + needle = f"QUALITY_GATE_TAG={tag}".encode() + proc_dir = Path("/proc") + if not proc_dir.is_dir(): + return 0 + others = (e for e in proc_dir.iterdir() if e.name.isdigit() and int(e.name) != os.getpid()) + return sum(1 for entry in others if _kill_if_tagged(entry, needle)) + + +def _kill_if_tagged(entry: Path, needle: bytes) -> bool: + """Kill the process behind one /proc entry when its environment holds the tag. A process that is gone, or not + ours to read, is skipped.""" + try: + if needle not in (entry / "environ").read_bytes().split(b"\0"): + return False + os.kill(int(entry.name), signal.SIGKILL) + return True + except OSError: + return False + + +def kill_tagged(tag: str, passes: int = 20) -> None: + """Kill every process that carries QUALITY_GATE_TAG= in its environment (Linux, via /proc). + + mutator starts each test run in its own session, so killing mutator's process group leaves them running; they + inherit the tag from the environment, which is how they are found. A process can fork between the scan and the + kill, so scan again until a pass finds none. + """ + for _ in range(passes): + if _kill_pass(tag) == 0: + return + time.sleep(0.05) + + +def run_limited(args: list[str], cwd: Path, env: dict, seconds: float) -> tuple[int | None, str, str]: + """Run args; on timeout kill the process group and everything tagged by this run, and return code None.""" + tag = uuid.uuid4().hex + proc = subprocess.Popen( + args, cwd=cwd, env={**env, "QUALITY_GATE_TAG": tag}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True, + ) + try: + out, err = proc.communicate(timeout=max(seconds, 0.1)) + return proc.returncode, out, err + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + kill_tagged(tag) + proc.communicate() + return None, "", "" + + +def mutator_args(path: str, wanted: list[int], *extra: str) -> list[str]: + return ["mutator", path, "--lines", ",".join(map(str, wanted)), "--use-existing-coverage", *extra] + + +def count_sites(path: str, wanted: list[int], root: Path) -> int | str: + """Mutation sites on the wanted lines (a scan runs no tests), or "unknown" when the scan does not finish.""" + code, out, _ = run_limited(mutator_args(path, wanted, "--scan"), root, dict(os.environ), 30) + if code is None: + return "unknown" + return len([line for line in out.splitlines() if re.match(r"\s+\S+:\d+\s", line)]) + + +CGROUP = Path("/sys/fs/cgroup") +BYTES_PER_MUTANT = 1 << 30 # one Vitest run of the whole suite stays well under 1 GB +MEMORY_RESERVE = 2 << 30 # for the gate, the mutator processes and the system + + +def available_cores(cgroup: Path = CGROUP) -> int: + """Cores this process may use: the CPU affinity, capped by a cgroup v2 CPU quota when there is one. + + os.cpu_count() reports the host, which on a container runner can be far more than the job is given. + """ + n = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1) + try: + quota, period = (cgroup / "cpu.max").read_text().split()[:2] + if quota != "max": + n = min(n, max(1, int(int(quota) / int(period)))) + except (OSError, ValueError): + pass + return max(1, n) + + +def memory_limit(cgroup: Path = CGROUP, meminfo: Path = Path("/proc/meminfo")) -> int | None: + """Bytes of memory this job may use: the cgroup limit (v2, then v1) or, without one, the machine's total.""" + for name in ("memory.max", "memory/memory.limit_in_bytes"): + try: + text = (cgroup / name).read_text().strip() + if text != "max" and int(text) < (1 << 60): + return int(text) + except (OSError, ValueError): + continue + try: + for line in meminfo.read_text().splitlines(): + if line.startswith("MemTotal:"): + return int(line.split()[1]) * 1024 + except (OSError, ValueError): + pass + return None + + +def worker_limit(cores: int, memory: int | None) -> int: + """Mutants allowed at once: the cores, or what the memory holds after a 2 GB reserve at 1 GB per mutant.""" + if memory is None: + return max(1, cores) + return max(1, min(cores, (memory - MEMORY_RESERVE) // BYTES_PER_MUTANT)) + + +def plan_workers(sites: dict[str, int | str], limit: int) -> dict[str, int]: + """Mutants to run at once per file; the total never goes over `limit`. + + A file with many sites gets many workers, so a big file does not finish last on a few workers while other cores + sit idle. With more files than `limit` every file gets one worker and the files queue for a free slot (the caller + runs at most `limit` mutator processes at a time). + """ + # A file whose scan did not finish ("unknown") still gets mutated, so it takes one slot of the plan. + sites = {path: (count if isinstance(count, int) else 1) for path, count in sites.items()} + sites = {path: count for path, count in sites.items() if count > 0} + if not sites: + return {} + if len(sites) >= limit: + return {path: 1 for path in sites} + total = sum(sites.values()) + extra = limit - len(sites) + exact = {path: extra * count / total for path, count in sites.items()} + plan = {path: 1 + int(exact[path]) for path in sites} + leftover = limit - sum(plan.values()) + for path in sorted(sites, key=lambda p: (exact[p] - int(exact[p]), sites[p]), reverse=True)[:leftover]: + plan[path] += 1 + return plan + + +def mutation_budget(spent: float, sites: dict[str, int | str], plan: dict[str, int], limit: int, mutant_seconds: float) -> float: + """The whole gate's time budget, sized to the change (engineering standard, "Mutation budget"): the time already + spent, plus for each file the rounds its sites need on its workers, plus mutator's own unmutated run of the file, + times the measured time of one mutant. Files beyond `limit` queue, so the per-file rounds are spread over the + slots. Never below 5 minutes, never above 30.""" + rounds = 0 + for path, workers in plan.items(): + count = sites[path] if isinstance(sites[path], int) else 1 + rounds += math.ceil(count / workers) + 1 # type: ignore[operator] + files_at_once = max(1, min(len(plan), limit)) + wanted = spent + math.ceil(rounds / files_at_once) * mutant_seconds + return max(MUTATION_BUDGET_MIN, min(MUTATION_BUDGET_MAX, wanted)) + + +def wanted_lines(ranges: list[tuple[int, int]]) -> list[int]: + return sorted({n for start, end in ranges for n in range(start, end + 1)}) + + +def mutate_file(path: str, ranges: list[tuple[int, int]], root: Path, workers: int, deadline: float): + """Mutate one file's changed lines. Returns (status, killed, survivors, detail); status is ok, unchecked or failed.""" + wanted = wanted_lines(ranges) + remaining = deadline - time.monotonic() + if remaining <= 0: + return "unchecked", 0, [], count_sites(path, wanted, root) + # The file reaches the test command through the environment, never through the command string. + env = {**test_env(), "MUTANT_FILE": path} + code, out, err = run_limited( + mutator_args(path, wanted, "--mutate-all", "--max-workers", str(workers), "--test-command", f"./{MUTANT_TESTS}"), + root, env, remaining, + ) + if code is None: + return "unchecked", 0, [], count_sites(path, wanted, root) + if code not in (0, 3): + return "failed", 0, [], f"mutator failed on {path} (exit {code}):\n{out[-1500:]}\n{err[-1500:]}" + killed = 0 + survivors: list[tuple[str, int, str]] = [] + for line in out.splitlines(): + m = re.match(r"(KILLED|SURVIVED)\s+(\S+?):(\d+)\s+(.*)", line) + if not m: + continue + if m.group(1) == "KILLED": + killed += 1 + else: + survivors.append((m.group(2), int(m.group(3)), m.group(4))) + return "ok", killed, survivors, "" + + +def tally(outcomes: dict[str, tuple]) -> tuple[int, list[tuple[str, int, str]], list[tuple[str, int | str]], list[str]]: + """Killed, survivors, files not checked (with their sites) and failure details, over every file's outcome.""" + survived: list[tuple[str, int, str]] = [] + unchecked: list[tuple[str, int | str]] = [] + failed: list[str] = [] + killed = 0 + for path in sorted(outcomes): + status, k, survivors, detail = outcomes[path] + if status == "failed": + failed.append(detail) + elif status == "unchecked": + unchecked.append((path, detail)) + killed += k + survived.extend(survivors) + return killed, survived, unchecked, failed + + +def unchecked_report(unchecked: list[tuple[str, int | str]], budget: float) -> list[str]: + if not unchecked: + return [] + return [ + f"The {budget:.0f}s budget ran out before every file was checked. Not checked:", + *(f" {path} ({sites} mutation site(s))" for path, sites in unchecked), + "The runner may have been busy: re-run this job once before doing anything else. If it runs out of time " + "again, the PR is too large for the gate to finish and has to be split, or its tests made faster.", + ] + + +def mutation_report( + outcomes: dict[str, tuple], allowed: dict[tuple[str, int], str], budget: float, elapsed: float, how: str +) -> tuple[bool, list[str]]: + """The verdict and report from each file's outcome. A file mutator failed on fails the check, and so does a file + the budget ran out on; the files that finished are still reported.""" + killed, survived, unchecked, failed = tally(outcomes) + unexplained = [s for s in survived if (s[0], s[1]) not in allowed] + lines = [ + f"{killed} mutant(s) killed, {len(survived)} survived on the changed lines, {elapsed:.0f}s of a " + f"{budget:.0f}s budget ({how})." + ] + for path, line, op in survived: + reason = allowed.get((path, line)) + tag = f"explained as equivalent: {reason}" if reason else "NOT explained" + lines.append(f" survivor {path}:{line} {op} ({tag})") + if unexplained: + lines.append( + "Kill each survivor with a test, or add 'Equivalent mutant: : ' " + "to the PR description (one such line waives every mutant on that line)." + ) + lines.extend(failed) + lines.extend(unchecked_report(unchecked, budget)) + return not unexplained and not unchecked and not failed, lines + + +def check_mutation( + changed: Ranges, + root: Path, + allowed: dict[tuple[str, int], str], + started: float, + mutant_seconds: float, + workers: int | None = None, +) -> tuple[bool, list[str]]: + """Mutate the changed lines inside a budget sized to the change, as many mutants at once as the machine allows. + + Each mutant runs in its own copy of the tree (mutator's worker directories), so parallel mutants never share + files. When the budget runs out the check fails cleanly: the report names what was not checked, instead of the + runner killing the job and the report with it. + """ + for path in changed: + if not safe_path(path): + return False, [f"refusing to run mutator on an unusual file name: {path!r}"] + cores = available_cores() + limit = workers or worker_limit(cores, memory_limit()) + shutil.rmtree(root / ".metrics", ignore_errors=True) + paths = sorted(changed) + # Scans run no tests, so counting every file's sites first is cheap. + with ThreadPoolExecutor(max_workers=max(1, min(len(paths), cores))) as pool: + sites = dict(zip(paths, pool.map(lambda p: count_sites(p, wanted_lines(changed[p]), root), paths))) + plan = plan_workers(sites, limit) + budget = mutation_budget(time.monotonic() - started, sites, plan, limit, mutant_seconds) + deadline = started + budget + # A file with no mutation site needs no run; a scan that did not finish is planned as one site. + todo = [p for p in paths if p in plan] + with ThreadPoolExecutor(max_workers=max(1, min(len(todo), limit))) as pool: + outcomes = dict(zip(todo, pool.map(lambda p: mutate_file(p, changed[p], root, plan[p], deadline), todo))) + shutil.rmtree(root / "target" / "mutation-workers", ignore_errors=True) + shutil.rmtree(root / ".metrics", ignore_errors=True) + per_file = ", ".join(f"{Path(p).name} x{n} ({sites[p]} sites)" for p, n in sorted(plan.items())) or "no sites" + how = f"limit {limit} mutants at once from {cores} core(s), one mutant measured at {mutant_seconds:.0f}s: {per_file}" + return mutation_report(outcomes, allowed, budget, time.monotonic() - started, how) + + +def main() -> int: + started = time.monotonic() + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base", help="base commit when HEAD is not a merge commit") + parser.add_argument("--workers", type=int, default=None, help="mutants at once; default from cores and memory") + parser.add_argument("--skip-coverage-run", action="store_true", help="read coverage/lcov.info as it is (self-tests)") + parser.add_argument("--skip-mutation", action="store_true", help="local runs only") + parser.add_argument("--description-html", help="GitHub's rendering of the PR description, from an earlier step") + parser.add_argument("--description-sha256", help="the sha256 of that file, as the step that wrote it gave it") + opts = parser.parse_args() + + root = Path.cwd() + base = resolve_base(opts.base) + results, before_mutation = gate_results(opts, root, base, started) + report = [f"## Quality gate (base {base[:8]})", "", f"Every check before mutation took {before_mutation:.0f}s.", ""] + for name, passed, lines in results: + report.append(f"### {name}: {'pass' if passed else 'FAIL'}") + report.extend(lines) + report.append("") + text = "\n".join(report) + print(text) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write(text + "\n") + return 0 if all(passed for _, passed, _ in results) else 1 + + +Result = tuple[str, bool, list[str]] + + +def guarded(check, *args) -> tuple[bool, list[str]]: + """Run one check. A check that breaks (a tool that fails, a report it cannot read, a bug) fails with the error's + type and message, and the other checks and the summary still run.""" + try: + return check(*args) + except (Exception, SystemExit) as error: # noqa: BLE001 - every failure must reach the report + return False, [f"The check broke: {type(error).__name__}: {str(error)[-3000:]}"] + + +def gate_results(opts: argparse.Namespace, root: Path, base: str, started: float) -> tuple[list[Result], float]: + """Every check in order, and the seconds spent before mutation.""" + baseline = json.loads(BASELINE.read_text(encoding="utf-8")) + changed = changed_ranges(base) + ok, lines, allowed = read_waivers(opts.description_html, opts.description_sha256) + results: list[Result] = [("Waivers", ok, lines)] + + timing: list[float] = [] + ok, lines = guarded(coverage_step, opts, root, changed, baseline, allowed, timing) + results.append(("Coverage", ok, lines)) + suite_seconds = timing[0] if timing else 0.0 + nothing = "No lines were added or changed in TypeScript under src/." + results.append(("CRAP", *guarded(check_crap, changed, root, base)) if changed else ("CRAP", True, [nothing])) + results.append( + ("Duplication", *guarded(check_duplicates, changed, base, root)) if changed else ("Duplication", True, [nothing]) + ) + before_mutation = time.monotonic() - started + results.append(mutation_result(opts, changed, root, allowed, started, suite_seconds, results, nothing)) + return results, before_mutation + + +def mutation_result(opts, changed: Ranges, root: Path, allowed, started: float, suite_seconds: float, earlier, nothing: str) -> Result: + """Mutation runs last, and only when there is something to mutate and every earlier check passed.""" + if not changed: + return ("Mutation", True, [nothing]) + if not all(passed for _, passed, _ in earlier): + return ("Mutation", False, ["Not run: an earlier check failed. Fix it first; mutation runs after it passes."]) + if opts.skip_mutation: + return ("Mutation", True, ["Skipped (--skip-mutation)."]) + return ("Mutation", *guarded(check_mutation, changed, root, allowed, started, suite_seconds, opts.workers)) + + +def coverage_step(opts, root: Path, changed: Ranges, baseline: dict, allowed, timing: list[float]) -> tuple[bool, list[str]]: + """Run the suite with coverage (unless told to read the report as it is), time it into `timing`, and judge it.""" + if opts.skip_coverage_run: + return check_coverage(root / LCOV, changed, baseline, allowed) + timing.append(run_coverage(root)) + print(f"tests with coverage: {timing[0]:.0f}s", flush=True) + ok, lines = check_coverage(root / LCOV, changed, baseline, allowed) + return ok, [*lines, f"The suite with coverage took {timing[0]:.0f}s."] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.quality/mutant-tests.sh b/.quality/mutant-tests.sh new file mode 100755 index 0000000..8b08359 --- /dev/null +++ b/.quality/mutant-tests.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# Test command for mutator: the Vitest tests that can see one mutated source file under src/. +# +# mutator runs each mutant in a worker directory under target/mutation-workers: symbolic links to the real +# tree and a private copy of the mutated file. Vite resolves every module to its real path, so in that layout +# a test would import the original, unmutated file and every mutant would survive. This script first replaces +# the links with real copies: every top-level entry except node_modules (and folders no test reads), and, +# inside the folders mutator expanded on the way to the mutated file, every linked child. The mutated file is +# already a real file and is left alone. In the repository root (mutator's own unmutated run) nothing is a +# link and nothing moves. +# +# The file comes in MUTANT_FILE (set by gate.py), so no file name is ever part of a command string. +# +# A mutant is killed only when Vitest reports a failed test or a test file that failed to load. A run that +# writes no report (a broken config, a crash) is a harness error, never a kill: it passes here, so the mutant +# shows as a survivor and the gate fails where a reviewer can see it. +set -eu + +file=${MUTANT_FILE:?MUTANT_FILE is not set} +# No mutant may install a real launchd agent or touch the OS keychain or the user's own ~/.tether. +unset TETHER_REAL_SERVICE +export TETHER_KEYCHAIN=off +export TETHER_HOME="${TMPDIR:-/tmp}/tether-mutant-home-$$" +report=$(mktemp) +trap 'rm -rf "$TETHER_HOME" "$report"' EXIT + +materialize() { + for entry in "$1"/* "$1"/.[!.]*; do + [ -L "$entry" ] || continue + case "$entry" in + ./node_modules | ./.git | ./target | ./coverage | ./.metrics) continue ;; + esac + target=$(readlink "$entry") + rm -f "$entry" + cp -R "$target" "$entry" + done +} + +materialize . +dir=$(dirname "$file") +while [ "$dir" != "." ] && [ "$dir" != "/" ]; do + materialize "./$dir" + dir=$(dirname "$dir") +done + +vitest=./node_modules/.bin/vitest + +# Runs Vitest with the given arguments. Exits the script with 1 when a test failed or a test file failed to +# load; returns otherwise, a harness error included. +tier() { + rm -f "$report" + "$vitest" "$@" --bail=1 --coverage.enabled=false --reporter=json --outputFile="$report" \ + >/dev/null 2>&1 || true + [ -s "$report" ] || return 0 + failed=$(python3 -I -c 'import json, sys; d = json.load(open(sys.argv[1])); print(d["numFailedTests"] + d["numFailedTestSuites"])' \ + "$report" 2>/dev/null) || return 0 + if [ "$failed" -gt 0 ]; then + exit 1 + fi +} + +# Tier 1: the tests that import the file, directly or through other modules (Vitest's module graph). Most +# mutants die here in seconds. +# One Vitest worker: mutator already runs several mutants at once. +tier related --run --passWithNoTests --maxWorkers=1 "$file" +# Tier 2: the whole suite, for what reaches the file without importing it: the CLI entry points +# (src/cli/bin.ts, src/conformance/cli.ts) only run as child processes. Only a mutant that survived tier 1 +# pays for this, and it keeps Vitest's own workers: the suite waits on child processes, and on one worker it +# takes minutes, which mutator's unmutated run of every file would pay too. +tier run diff --git a/.quality/rendered-descriptions.json b/.quality/rendered-descriptions.json new file mode 100644 index 0000000..a6f6f39 --- /dev/null +++ b/.quality/rendered-descriptions.json @@ -0,0 +1,98 @@ +{ + "plain": { + "body": "Summary.\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Summary.

\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "backtick_fence": { + "body": "Example:\n\n```text\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n```\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Example:

\n
Equivalent mutant: src/a.ts:4 the two branches return the same value\n
\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "tilde_fence": { + "body": "~~~~\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n```\nstill code\n~~~~\n", + "html": "
Equivalent mutant: src/a.ts:3 the two branches return the same value\n```\nstill code\n
" + }, + "unclosed_fence": { + "body": " ```\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "
Equivalent mutant: src/a.ts:3 the two branches return the same value\n
" + }, + "comment": { + "body": "\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "unclosed_comment": { + "body": "text Equivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "Equivalent mutant: src/a.ts:3 the two branches return the same value" + }, + "link_reference_title": { + "body": "[x]: /u \"\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n\"\n", + "html": "" + }, + "multi_line_alt": { + "body": "\"\nEquivalent\n", + "html": "

\"\nEquivalent

" + }, + "attribute_across_blank_lines": { + "body": "
\n", + "html": "" + }, + "hidden_elements": { + "body": "\n\n\n\n\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "<script>\n\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n\n</script>\n<STYLE>\n\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n\n</STYLE>\n\n

Equivalent mutant: src/a.ts:4 the two branches return the same value

\n\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "continues_paragraph": { + "body": "Some text\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Some text
\nEquivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "indented": { + "body": "Text.\n\n Equivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Text.

\n
Equivalent mutant: src/a.ts:3 the two branches return the same value\n
" + }, + "adjacent": { + "body": "Equivalent mutant: src/a.ts:3 the two branches return the same value\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n\nEquivalent mutant: src/a.ts:5 the two branches return the same value\n", + "html": "

Equivalent mutant: src/a.ts:3 the two branches return the same value
\nEquivalent mutant: src/a.ts:4 the two branches return the same value

\n

Equivalent mutant: src/a.ts:5 the two branches return the same value

" + }, + "crlf": { + "body": "```\r\ncode\r\n```\r\n\r\nEquivalent mutant: src/a.ts:3 the two branches return the same value\r\n", + "html": "
code\n
\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "backtick_in_info_string": { + "body": "```a`b\n\n", + "html": "

```a`b

\n" + }, + "fence_inside_html_block": { + "body": "
\n```\n
\nant: src/a.ts:3 {REASON}

" + self.assertEqual(set(gate.survivors_allowed(html)), {("src/a.ts", 3)}) + + def test_a_stray_end_tag_after_a_waiver_paragraph_adds_nothing_to_it(self) -> None: + # Once its paragraph closes, nothing that follows (a stray end tag, then text) is collected for it. + waiver = f"Equivalent mutant: src/a.ts:3 {REASON}" + parser = gate.TopLevelParagraphs() + parser.feed(f"

{waiver}

tail") + self.assertEqual(parser.found, [waiver]) + self.assertIsNone(parser.text) + parser.feed(f"

Equivalent mutant: src/a.ts:4 {REASON}

") + self.assertEqual(parser.found, [waiver, f"Equivalent mutant: src/a.ts:4 {REASON}"]) + + def test_a_bidi_control_on_a_later_line_of_the_paragraph_waives_nothing(self) -> None: + self.assertEqual(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:3\nthe two \u202ebranches return the same

"), {}) + self.assertEqual(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:3\n{REASON}

"), {("src/a.ts", 3): REASON}) + + def test_a_waiver_for_line_0_does_not_parse(self) -> None: + for line in ("0", "00"): + with self.subTest(line=line): + self.assertEqual(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:{line} {REASON}

"), {}) + self.assertEqual(set(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:10 {REASON}

")), {("src/a.ts", 10)}) + + def test_closing_an_element_closes_what_it_left_open(self) -> None: + waiver = f"Equivalent mutant: src/a.ts:3 {REASON}" + self.assertEqual(set(gate.survivors_allowed(f"

{waiver}

")), {("src/a.ts", 3)}) + + +class ReadWaivers(unittest.TestCase): + """The waivers come from a file an earlier workflow step rendered, checked against the sha256 that step gave.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.path = Path(self._tmp.name) / "pr-body.html" + + def tearDown(self) -> None: + self._tmp.cleanup() + + def rendered(self, html: str) -> tuple[str, str]: + self.path.write_bytes(html.encode()) + return str(self.path), hashlib.sha256(html.encode()).hexdigest() + + def test_waivers_are_read_from_the_rendered_file_that_matches_its_hash(self) -> None: + path, digest = self.rendered(RENDERED["plain"]["html"]) + ok, lines, allowed = gate.read_waivers(path, digest) + self.assertTrue(ok) + self.assertEqual(allowed, {("src/a.ts", 3): REASON}) + self.assertEqual(lines, ["1 waiver(s) count, each the whole text of a top-level paragraph on GitHub.", " src/a.ts:3"]) + + def test_the_hash_is_compared_without_regard_to_case_or_surrounding_space(self) -> None: + path, digest = self.rendered(RENDERED["plain"]["html"]) + ok, _lines, allowed = gate.read_waivers(path, f" {digest.upper()}\n") + self.assertTrue(ok) + self.assertEqual(set(allowed), {("src/a.ts", 3)}) + + def test_without_a_rendered_file_nothing_is_waived(self) -> None: + ok, lines, allowed = gate.read_waivers(None, None) + self.assertEqual((ok, allowed), (True, {})) + self.assertEqual(lines, ["No rendered description was given: nothing is waived."]) + + def test_a_file_that_cannot_be_read_fails(self) -> None: + ok, lines, allowed = gate.read_waivers(str(self.path), "0" * 64) + self.assertEqual((ok, allowed), (False, {})) + self.assertIn("cannot be read", lines[0]) + + def test_a_file_that_does_not_match_its_hash_fails(self) -> None: + path, digest = self.rendered(RENDERED["plain"]["html"]) + self.path.write_bytes(RENDERED["plain"]["html"].encode() + b"

x

") + for sha in (digest, None, ""): + with self.subTest(sha=sha): + ok, lines, allowed = gate.read_waivers(path, sha) + self.assertEqual((ok, allowed), (False, {})) + self.assertIn("does not match the sha256", lines[0]) + + def test_the_gate_no_longer_talks_to_the_network(self) -> None: + source = Path(gate.__file__).read_text(encoding="utf-8") + self.assertNotIn("urllib", source) + self.assertNotIn("GITHUB_TOKEN", source) + + +WORKFLOW = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "ci.yml" + + +def job_text(name: str) -> str: + """One job of ci.yml without its comment lines, so only what runs is checked.""" + lines = WORKFLOW.read_text(encoding="utf-8").splitlines(keepends=True) + text = "".join(line for line in lines if not line.lstrip().startswith("#")) + start = text.index(f"\n {name}:\n") + 1 + rest = text[start + 1 :] + following = [rest.index(f"\n {m}") for m in ("check:", "quality:") if f"\n {m}" in rest] + return text[start : start + 1 + min(following)] if following else text[start:] + + +def workflow_steps() -> list[str]: + """Each step of the quality job as its block of text, in order.""" + text = job_text("quality") + body = text[text.index(" steps:\n") :] + return [block for block in body.split("\n - ")[1:]] + + +class QualityWorkflow(unittest.TestCase): + """The token reaches only the steps that run before any code from the pull request, and no action's post step.""" + + CREDENTIAL = ("github.token", "secrets.", "GITHUB_TOKEN") + + def step(self, needle: str) -> tuple[int, str]: + found = [(n, block) for n, block in enumerate(workflow_steps()) if needle in block] + self.assertEqual(len(found), 1, needle) + return found[0] + + def test_the_job_runs_on_every_pull_request_on_one_hosted_linux_runner(self) -> None: + text = WORKFLOW.read_text(encoding="utf-8") + on = text[text.index("\non:") : text.index("\njobs:")] + self.assertNotIn("paths", on) # a path filter would let a PR skip the gate + job = job_text("quality") + self.assertIn("runs-on: ubuntu-latest", job) + self.assertIn("if: github.event_name == 'pull_request'", job) + + def test_only_the_checkout_and_render_steps_get_the_token(self) -> None: + checkout, _ = self.step("name: Check out the pull request") + render, _ = self.step("name: Render the pull request description") + install, _ = self.step("run: bun install --frozen-lockfile") + self.assertEqual((checkout, render), (0, 1)) + self.assertLess(render, install) + holders = [n for n, block in enumerate(workflow_steps()) if any(word in block for word in self.CREDENTIAL)] + self.assertEqual(holders, [checkout, render]) + job = job_text("quality") + self.assertEqual(sum(job.count(word) for word in self.CREDENTIAL), 2) # not at job level either + + def test_no_action_takes_the_token_by_default(self) -> None: + self.assertNotIn("actions/checkout", job_text("quality")) + for action in ("oven-sh/setup-bun@", "actions/setup-python@"): + with self.subTest(action=action): + _, block = self.step(action) + self.assertIn('token: ""', block) + uses = [block for block in workflow_steps() if block.startswith("uses:")] + self.assertEqual(len(uses), 2) + + def test_steps_with_the_token_fail_on_any_broken_command_and_inline_no_expression(self) -> None: + for name in ("name: Check out the pull request", "name: Render the pull request description"): + with self.subTest(step=name): + _, block = self.step(name) + script = block[block.index("run: |") :] + self.assertIn("set -o pipefail", script) + self.assertNotIn("${{", script) + _, render = self.step("name: Render the pull request description") + self.assertIn("/markdown", render) + self.assertEqual(render.count("--fail "), 2) + + def test_the_token_can_read_the_pull_request_and_nothing_more(self) -> None: + job = job_text("quality") + block = job[job.index(" permissions:\n") :].split("\n runs-on")[0] + self.assertEqual(block.splitlines()[1:], [" contents: read", " pull-requests: read"]) + text = WORKFLOW.read_text(encoding="utf-8") + top = text[text.index("\npermissions:") : text.index("\njobs:")] + self.assertNotIn("write", top) + + def test_the_gate_reads_the_rendered_file_and_its_hash_and_outlives_its_budget(self) -> None: + _, block = self.step("name: Quality gate") + self.assertIn('--description-html "$RUNNER_TEMP/pr-body.html"', block) + self.assertIn("--description-sha256", block) + self.assertIn("steps.description.outputs.sha256", block) + minutes = int(job_text("quality").split("timeout-minutes:")[1].split()[0]) + # The job must outlive the largest mutation budget, or the runner kills it and the report with it. + self.assertGreater(minutes * 60, gate.MUTATION_BUDGET_MAX + 600) + + +LCOV = textwrap.dedent( + """\ + TN: + SF:src/a.ts + FNF:4 + FNH:3 + DA:1,3 + DA:2,0 + DA:5,0 + DA:6,1 + LF:4 + LH:2 + end_of_record + SF:/abs/checkout/src/b.ts + FNF:1 + FNH:1 + DA:1,0 + LF:6 + LH:5 + end_of_record + SF:test/helpers.ts + FNF:10 + FNH:0 + LF:100 + LH:0 + end_of_record + """ +) + + +class UncoveredChangedLines(unittest.TestCase): + def test_an_executable_changed_line_with_no_hits_is_reported(self) -> None: + found = gate.uncovered_changed({"src/a.ts": [(1, 2), (6, 6)]}, LCOV) + self.assertEqual(found, [("src/a.ts", 2)]) + + def test_a_changed_line_that_is_not_executable_is_ignored(self) -> None: + self.assertEqual(gate.uncovered_changed({"src/a.ts": [(3, 4)]}, LCOV), []) + + def test_absolute_paths_in_the_report_match_by_suffix(self) -> None: + self.assertEqual(gate.uncovered_changed({"src/b.ts": [(1, 1)]}, LCOV), [("src/b.ts", 1)]) + + def test_a_file_missing_from_the_report_is_reported_whole(self) -> None: + self.assertEqual(gate.uncovered_changed({"src/c.ts": [(4, 5)]}, LCOV), [("src/c.ts", 0)]) + + +class LcovTotals(unittest.TestCase): + def test_the_totals_count_only_in_scope_files(self) -> None: + # src/a.ts 2 of 4 lines and 3 of 4 functions; the absolute b.ts path is not in scope by name, and + # the test helper never counts. + self.assertEqual(gate.lcov_totals(LCOV), {"lines": 50.0, "functions": 75.0}) + + def test_an_empty_report_is_full(self) -> None: + self.assertEqual(gate.lcov_totals(""), {"lines": 100.0, "functions": 100.0}) + + +class CoverageVerdict(unittest.TestCase): + BASELINE = {"coverage": {"lines": 98.92, "functions": 95.07}} + + def verdict(self, uncovered, allowed=None, now=None): + return gate.coverage_verdict(uncovered, allowed or {}, now or {"lines": 90.0, "functions": 90.0}, self.BASELINE) + + def test_a_lower_repo_total_alone_does_not_fail_but_is_reported(self) -> None: + ok, lines = self.verdict([]) + self.assertTrue(ok) + self.assertIn("lines 90.0% (baseline 98.92%)", lines[1]) + self.assertTrue(any("Below the baseline" in line for line in lines)) + + def test_a_higher_repo_total_asks_for_the_baseline_to_be_updated(self) -> None: + _, lines = self.verdict([], now={"lines": 99.0, "functions": 95.07}) + self.assertTrue(any("update coverage in .quality/baseline.json" in line for line in lines)) + + def test_an_uncovered_changed_line_fails(self) -> None: + ok, _ = self.verdict([("src/a.ts", 2)]) + self.assertFalse(ok) + + def test_an_override_with_a_reason_waives_the_line(self) -> None: + ok, _ = self.verdict([("src/a.ts", 2)], {("src/a.ts", 2): "only reachable as another user"}) + self.assertTrue(ok) + + def test_no_waiver_covers_a_file_missing_from_the_coverage_report(self) -> None: + allowed = gate.survivors_allowed(f"

Equivalent mutant: src/c.ts:0 {REASON}

") + ok, lines = self.verdict(gate.uncovered_changed({"src/c.ts": [(4, 5)]}, LCOV), allowed) + self.assertFalse(ok) + self.assertTrue(any("src/c.ts (not in the coverage report" in line for line in lines)) + + +DRYER = ( + "Scanning src\n" + "DUPLICATE score=0.91 kind=function\n src/a.ts:1-9\n src/b.ts:3-11\n" + "DUPLICATE score=0.85 kind=function\n src/c.ts:2-8\n src/d.ts:4-10\n" +) + + +class DryerOutput(unittest.TestCase): + def test_the_output_is_parsed_into_pairs(self) -> None: + self.assertEqual( + gate.parse_dryer(DRYER), + [(0.91, ("src/a.ts", 1, 9), ("src/b.ts", 3, 11)), + (0.85, ("src/c.ts", 2, 8), ("src/d.ts", 4, 10))], + ) + + def test_no_duplicates_is_no_pairs(self) -> None: + self.assertEqual(gate.parse_dryer("No duplicate candidates found.\n"), []) + + def test_a_block_the_gate_cannot_read_stops_the_gate(self) -> None: + for block in ( + "DUPLICATE kind=function\n src/e.ts:1-2\n src/f.ts:1-2\n", + "DUPLICATE score=0.9\n src/e.ts:1-2\n", + ): + with self.subTest(block=block), self.assertRaises(SystemExit) as stop: + gate.parse_dryer(DRYER + block) + self.assertIn("3 DUPLICATE blocks but 2 pairs", str(stop.exception.code)) + + def test_only_a_marker_at_the_start_of_a_line_starts_a_block(self) -> None: + preamble = "score=0.5 in a header\n src/e.ts:1-2\n src/f.ts:1-2\n" + self.assertEqual(len(gate.parse_dryer(preamble + DRYER)), 2) + + +class DuplicateVerdict(unittest.TestCase): + PAIR = (0.9, ("src/a.ts", 10, 20), ("src/b.ts", 30, 40)) + + def test_a_new_pair_on_a_changed_line_fails(self) -> None: + ok, lines = gate.duplicate_verdict({"src/a.ts": [(12, 12)]}, [], [self.PAIR]) + self.assertFalse(ok) + self.assertIn(" new duplicate: src/a.ts:10-20 and src/b.ts:30-40 (similarity 0.90)", lines) + + def test_a_pair_already_on_base_passes(self) -> None: + ok, _ = gate.duplicate_verdict({"src/a.ts": [(12, 12)]}, [self.PAIR], [self.PAIR]) + self.assertTrue(ok) + + def test_a_pair_the_pr_did_not_touch_passes(self) -> None: + ok, _ = gate.duplicate_verdict({"src/a.ts": [(1, 5)]}, [], [self.PAIR]) + self.assertTrue(ok) + + +class DryerRun(InRepo): + @unittest.skipUnless(shutil.which("dryer"), "needs dryer") + def test_a_copied_function_is_found_by_the_real_dryer_and_tests_are_skipped(self) -> None: + body = "export function f{n}(a: number[]): number {{\n let t = 0\n for (const x of a) {{\n if (x > 2) {{\n t += x * 3\n }} else {{\n t -= x\n }}\n }}\n return t\n}}\n" + self.write("src/c.ts", body.format(n=1)) + self.write("src/d.ts", body.format(n=2)) + self.write("test/e.ts", body.format(n=3)) + pairs = gate.dryer_pairs(self.repo) + self.assertEqual({tuple(sorted((p[1][0], p[2][0]))) for p in pairs}, {("src/c.ts", "src/d.ts")}) + self.assertFalse((self.repo / ".metrics").exists()) + + +SLOW = 'case "$*" in *--scan*) echo " src/x.ts:1 a -> b";; *) sleep 30;; esac\n' + + +class MutationBudget(InRepo): + def stub_mutator(self, body: str) -> Path: + bin_dir = self.repo / "bin" + bin_dir.mkdir(exist_ok=True) + script = bin_dir / "mutator" + script.write_text("#!/bin/sh\n" + body) + script.chmod(script.stat().st_mode | stat.S_IEXEC) + return bin_dir + + def check(self, body: str, changed, seconds_left: float, workers: int = 1): + """check_mutation with the whole budget (30 minutes, the most there is) spent but `seconds_left`.""" + bin_dir = self.stub_mutator(body) + started = time.monotonic() - gate.MUTATION_BUDGET_MAX + seconds_left + with mock.patch.dict(os.environ, {"PATH": f"{bin_dir}:{os.environ['PATH']}"}): + return gate.check_mutation(changed, self.repo, {}, started, 1000.0, workers) + + def test_running_out_of_budget_fails_cleanly_with_what_was_not_checked(self) -> None: + ok, lines = self.check(SLOW, {"src/a.ts": [(1, 3)], "src/b.ts": [(1, 2)]}, 1.0) + text = "\n".join(lines) + self.assertFalse(ok) + self.assertIn("budget ran out", text) + self.assertIn("src/a.ts", text) + self.assertIn("src/b.ts", text) + self.assertLess(text.index("re-run"), text.index("split")) + + def test_the_slow_mutator_is_actually_stopped(self) -> None: + start = time.monotonic() + self.check(SLOW, {"src/a.ts": [(1, 3)]}, 1.0) + self.assertLess(time.monotonic() - start, 10) + + def test_a_finished_run_inside_the_budget_reports_survivors(self) -> None: + body = 'case "$*" in *--scan*) echo " src/a.ts:1 > -> >=";; *) echo "SURVIVED src/a.ts:1 > -> >="; exit 3;; esac\n' + ok, lines = self.check(body, {"src/a.ts": [(1, 1)]}, 60.0) + self.assertFalse(ok) + self.assertIn("survivor src/a.ts:1", "\n".join(lines)) + + def test_a_mutator_failure_on_one_file_still_reports_the_others(self) -> None: + body = ( + 'case "$*" in *--scan*) echo " $1:1 a -> b"; exit 0;; esac\n' + 'case "$1" in *a.ts) echo "KILLED $1:1 a -> b"; exit 0;; *) echo boom; exit 2;; esac\n' + ) + ok, lines = self.check(body, {"src/a.ts": [(1, 1)], "src/b.ts": [(1, 1)]}, 60.0, workers=2) + text = "\n".join(lines) + self.assertFalse(ok) + self.assertIn("1 mutant(s) killed", text) + self.assertIn("mutator failed on src/b.ts (exit 2)", text) + + def test_the_file_reaches_the_test_command_through_the_environment_only(self) -> None: + record = self.repo / "record.txt" + body = ( + 'case "$*" in *--scan*) echo " x:1 a -> b"; exit 0;; esac\n' + f'echo "$*" > "{record}"; echo "env=$MUTANT_FILE real=${{TETHER_REAL_SERVICE:-unset}} keychain=${{TETHER_KEYCHAIN:-unset}}" >> "{record}"\n' + ) + with mock.patch.dict(os.environ, {"TETHER_REAL_SERVICE": "1", "TETHER_KEYCHAIN": "on"}): + self.check(body, {"src/client/policy.ts": [(1, 2)]}, 60.0) + args, env_line = record.read_text().splitlines()[:2] + self.assertNotIn("policy.ts", args.split("--test-command", 1)[1]) + self.assertEqual(env_line, "env=src/client/policy.ts real=unset keychain=off") # no mutant may install a launchd agent or reach the keychain + + def test_a_file_name_is_not_run_through_a_shell(self) -> None: + self.assertFalse(gate.safe_path("src/a.ts; touch pwned")) + self.assertFalse(gate.safe_path("src/../../etc.ts")) + self.assertTrue(gate.safe_path("src/client/policy.ts")) + + def test_no_more_mutator_processes_than_the_limit_run_at_once(self) -> None: + running = self.repo / "running" + counts = self.repo / "counts.txt" + running.mkdir() + body = ( + 'case "$*" in *--scan*) echo " $1:1 a -> b"; exit 0;; esac\n' + f'touch "{running}/$$"; ls "{running}" | wc -l >> "{counts}"; sleep 1; rm "{running}/$$"\n' + ) + self.check(body, {f"src/f{i}.ts": [(1, 1)] for i in range(6)}, 120.0, workers=2) + peak = max(int(line) for line in counts.read_text().split()) + self.assertLessEqual(peak, 2) + self.assertEqual(len(counts.read_text().split()), 6) + + def test_the_kill_scans_again_until_no_tagged_process_is_left(self) -> None: + with mock.patch.object(gate, "_kill_pass", side_effect=[2, 1, 0]) as scan: + gate.kill_tagged("tag") + self.assertEqual(scan.call_count, 3) + + +class BudgetSizing(unittest.TestCase): + def test_a_small_change_gets_the_five_minute_floor(self) -> None: + self.assertEqual(gate.mutation_budget(60.0, {"a": 2}, {"a": 2}, 2, 30.0), 300.0) + + def test_the_budget_grows_with_the_rounds_the_sites_need(self) -> None: + # 20 sites on 2 workers: 10 rounds plus mutator's unmutated run, 11 x 40 s, after 100 s already spent. + self.assertEqual(gate.mutation_budget(100.0, {"a": 20}, {"a": 2}, 2, 40.0), 540.0) + + def test_files_beyond_the_slots_queue(self) -> None: + # Three files of 1 site on 1 worker each, 2 slots: 6 rounds over 2 slots is 3, 3 x 100 s. + self.assertEqual(gate.mutation_budget(0.0, {"a": 1, "b": 1, "c": 1}, {"a": 1, "b": 1, "c": 1}, 2, 100.0), 300.0) + self.assertEqual(gate.mutation_budget(0.0, {"a": 1, "b": 1, "c": 1}, {"a": 1, "b": 1, "c": 1}, 2, 150.0), 450.0) + + def test_the_budget_never_passes_thirty_minutes(self) -> None: + self.assertEqual(gate.mutation_budget(0.0, {"a": 500}, {"a": 2}, 2, 96.0), 1800.0) + + def test_a_scan_that_did_not_finish_counts_as_one_site(self) -> None: + self.assertEqual(gate.mutation_budget(0.0, {"a": "unknown"}, {"a": 1}, 1, 200.0), 400.0) + + +class Workers(unittest.TestCase): + def test_workers_follow_the_sites_of_each_file(self) -> None: + self.assertEqual(gate.plan_workers({"a": 100}, 4), {"a": 4}) + self.assertEqual(gate.plan_workers({"a": 0}, 4), {}) + self.assertEqual(gate.plan_workers({"a": 5, "b": 5, "c": 5}, 2), {"a": 1, "b": 1, "c": 1}) + + def test_the_total_never_goes_over_the_limit(self) -> None: + many = {"big": 100, **{f"f{i}": 1 for i in range(23)}} + self.assertEqual(sum(gate.plan_workers(many, 24).values()), 24) + plan = gate.plan_workers({"a": "unknown", "b": 100, "c": "unknown"}, 24) + self.assertEqual(sum(plan.values()), 24) + + def test_a_cgroup_cpu_quota_caps_the_cores(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "cpu.max").write_text("200000 100000\n") + self.assertEqual(gate.available_cores(Path(tmp)), min(2, gate.available_cores(Path(tmp) / "none"))) + + def test_memory_caps_the_workers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "memory.max").write_text(str(8 << 30) + "\n") + self.assertEqual(gate.worker_limit(24, gate.memory_limit(Path(tmp))), 6) + Path(tmp, "memory.max").write_text("max\n") + meminfo = Path(tmp, "meminfo") + meminfo.write_text("MemTotal: 16777216 kB\n") + self.assertEqual(gate.memory_limit(Path(tmp), meminfo), 16 << 30) + self.assertEqual(gate.worker_limit(4, None), 4) + self.assertEqual(gate.worker_limit(4, 1 << 30), 1) + + +class CrapVerdict(unittest.TestCase): + """The CRAP ratchet: a function already over the limit on base may stay, never get worse.""" + + def verdict(self, head, base, touched=None): + keys = {key for key, *_ in head} if touched is None else touched + return gate.crap_verdict(head, keys, base) + + @staticmethod + def fn(crap, name="big", path="src/a.ts", nth=0, line=10): + return ((path, name, nth), line, crap, 40, 90.0) + + def test_same_score_as_base_passes_and_says_it_is_still_over(self) -> None: + ok, lines = self.verdict([self.fn(67.3)], {("src/a.ts", "big", 0): 67.3}) + self.assertTrue(ok) + self.assertTrue(any("still over the limit" in line and "67.3" in line and "base 67.3" in line for line in lines)) + self.assertFalse(any("over the limit:" in line for line in lines)) + + def test_lower_score_than_base_passes(self) -> None: + ok, _ = self.verdict([self.fn(60.0)], {("src/a.ts", "big", 0): 67.3}) + self.assertTrue(ok) + + def test_higher_score_than_base_fails_and_prints_both(self) -> None: + ok, lines = self.verdict([self.fn(67.4)], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + self.assertTrue(any("over the limit:" in line and "67.4" in line and "was 67.3 on base (higher)" in line for line in lines)) + + def test_a_higher_raw_score_fails_even_when_it_prints_the_same(self) -> None: + ok, lines = self.verdict([self.fn(67.34)], {("src/a.ts", "big", 0): 67.26}) + self.assertFalse(ok) + self.assertTrue(any("67.3" in line and "higher" in line for line in lines)) + + def test_an_equal_raw_score_passes(self) -> None: + ok, _ = self.verdict([self.fn(67.26)], {("src/a.ts", "big", 0): 67.26}) + self.assertTrue(ok) + + def test_a_new_function_with_a_name_the_file_already_had_is_new(self) -> None: + # Head has two "big"; base had one. Matching by order would compare the new first one with the old one. + head = [self.fn(15.0, nth=0), self.fn(67.0, nth=1)] + ok, lines = self.verdict(head, {("src/a.ts", "big", 0): 67.0}) + self.assertFalse(ok) + self.assertTrue(any("new function" in line for line in lines)) + + def test_a_function_removed_from_a_same_named_pair_is_new_too(self) -> None: + ok, _ = self.verdict([self.fn(67.0)], {("src/a.ts", "big", 0): 67.0, ("src/a.ts", "big", 1): 80.0}) + self.assertFalse(ok) + + def test_the_same_count_of_a_name_still_matches_by_order(self) -> None: + head = [self.fn(20.0, nth=0), self.fn(25.0, nth=1)] + ok, _ = self.verdict(head, {("src/a.ts", "big", 0): 20.0, ("src/a.ts", "big", 1): 25.0}) + self.assertTrue(ok) + + def test_names_in_another_file_do_not_change_the_count(self) -> None: + head = [self.fn(67.0), self.fn(5.0, path="src/b.ts", nth=0), self.fn(6.0, path="src/b.ts", nth=1)] + ok, _ = self.verdict(head, {("src/a.ts", "big", 0): 67.0}) + self.assertTrue(ok) + + def test_under_the_limit_on_base_and_over_now_fails(self) -> None: + ok, lines = self.verdict([self.fn(10.0)], {("src/a.ts", "big", 0): 9.9}) + self.assertFalse(ok) + self.assertTrue(any("was 9.9 on base (under the limit)" in line for line in lines)) + + def test_a_new_function_over_the_limit_fails(self) -> None: + ok, lines = self.verdict([self.fn(12.0, name="fresh")], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + self.assertTrue(any("new function" in line for line in lines)) + + def test_a_renamed_function_is_new(self) -> None: + ok, _ = self.verdict([self.fn(67.3, name="renamed")], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + + def test_a_moved_function_is_new(self) -> None: + ok, _ = self.verdict([self.fn(67.3, path="src/b.ts")], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + + def test_a_second_function_of_the_same_name_is_matched_by_order(self) -> None: + head = [self.fn(20.0, nth=0), self.fn(30.0, nth=1)] + ok, _ = self.verdict(head, {("src/a.ts", "big", 0): 20.0, ("src/a.ts", "big", 1): 25.0}) + self.assertFalse(ok) + + def test_a_function_under_the_limit_passes_without_a_base(self) -> None: + ok, lines = self.verdict([self.fn(9.9, name="small")], None) + self.assertTrue(ok) + self.assertIn("1 added or changed function(s) scored", lines[0]) + + def test_functions_the_pr_did_not_touch_are_not_judged(self) -> None: + ok, lines = self.verdict([self.fn(99.0)], {}, touched=set()) + self.assertTrue(ok) + self.assertIn("0 added or changed function(s) scored", lines[0]) + + + +class CrapBase(InRepo): + """base_scores, touched_functions and crapper on an lcov report, against a throwaway repository.""" + + def test_touched_functions_names_only_the_functions_with_a_changed_line(self) -> None: + self.write("src/f.ts", "function one() {\n return 1;\n}\n\nfunction two() {\n return 2;\n}\n\nfunction two() {\n return 3;\n}\n") + self.assertEqual(gate.touched_functions(self.repo, {"src/f.ts": [(6, 6)]}), {("src/f.ts", "two", 0)}) + self.assertEqual( + gate.touched_functions(self.repo, {"src/f.ts": [(1, 1), (10, 10)]}), + {("src/f.ts", "one", 0), ("src/f.ts", "two", 1)}, + ) + + def test_crapper_scores_from_the_line_records(self) -> None: + source = ( + "export function pick(a: number, b: number): number {\n" + " if (a > b) {\n return a\n }\n if (a < 0) {\n return 0\n }\n return b\n}\n" + ) + self.write("src/p.ts", source) + # Only FNF/FNH counts and DA lines, so the score comes from the lines. Lines 3 and 6 never ran. + hits = {1: 4, 2: 4, 3: 0, 4: 4, 5: 4, 6: 0, 7: 4, 8: 4, 9: 4} + self.write( + "coverage/lcov.info", + "TN:\nSF:src/p.ts\nFNF:1\nFNH:1\n" + "".join(f"DA:{n},{h}\n" for n, h in hits.items()) + + "LF:9\nLH:7\nend_of_record\n", + ) + [(key, _line, crap, complexity, coverage)] = gate.scored_functions(self.repo, ["src/p.ts"]) + self.assertEqual(key, ("src/p.ts", "pick", 0)) + self.assertEqual(complexity, 3) + self.assertGreater(coverage, 50.0) # read from the DA lines, not 0% for a missing FN record + self.assertLess(coverage, 100.0) + self.assertLess(crap, 9 + 3) # 0% coverage would score complexity squared plus complexity + + def run_base_scores(self, scored, paths): + runs: list[Path] = [] + (self.repo / "node_modules").mkdir() + with mock.patch.object(gate, "run_coverage", side_effect=lambda root: runs.append(root) or 1.0), \ + mock.patch.object(gate, "scored_functions", scored): + return gate.base_scores(self.base, self.repo, paths), runs + + def test_base_scores_measures_the_files_that_exist_on_base_with_coverage_required(self) -> None: + seen = {} + + def scored(root, paths, need_coverage=False): + seen.update(need_coverage=need_coverage, modules=(root / "node_modules").is_symlink(), paths=paths) + return [(("src/a.ts", "f", 0), 1, 42.0, 7, 80.0)] + + result, runs = self.run_base_scores(scored, ["src/a.ts", "src/new.ts"]) + self.assertEqual(result, {("src/a.ts", "f", 0): 42.0}) + self.assertEqual(seen, {"need_coverage": True, "modules": True, "paths": ["src/a.ts"]}) + self.assertEqual(len(runs), 1) + self.assertNotEqual(runs[0], self.repo) # the base tree, not this checkout + + def test_base_scores_runs_nothing_when_no_file_is_on_base_and_removes_its_worktree(self) -> None: + result, runs = self.run_base_scores(lambda root, paths, need_coverage=False: [], ["src/new.ts"]) + self.assertEqual((result, runs), ({}, [])) + self.assertNotIn("quality-base", git(self.repo, "worktree", "list")) + + def test_a_base_function_without_a_coverage_record_is_not_scored(self) -> None: + class Bundle: + def bind_sources(self, paths): + pass + + def percent_for(self, function): + return None + + class Fn: + name = "big" + start_line = 1 + end_line = 3 + + entry = mock.Mock(crap=9999.0, complexity=70, coverage=0.0) + with ( + mock.patch("crapper.coverage.load_bundle", return_value=Bundle()), + mock.patch("crapper.languages.functions_in_file", return_value=[Fn()]), + mock.patch("crapper.crap.make_entry", return_value=entry), + ): + self.write("src/f.ts", "x\n") + self.assertEqual(gate.scored_functions(self.repo, ["src/f.ts"], need_coverage=True), []) + self.assertEqual(len(gate.scored_functions(self.repo, ["src/f.ts"])), 1) + + +REPO = Path(__file__).resolve().parent.parent +VITEST_CONFIG = 'import { defineConfig } from "vitest/config";\nexport default defineConfig({ test: { include: ["test/**/*.test.ts"] } });\n' + + +@unittest.skipUnless((REPO / "node_modules/.bin/vitest").exists(), "needs bun install") +class MutantTestsScript(unittest.TestCase): + """mutant-tests.sh with the real Vitest, on a throwaway project that borrows this checkout's node_modules.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) / "project" + root = self.root + (root / ".quality").mkdir(parents=True) + shutil.copy(REPO / ".quality/mutant-tests.sh", root / ".quality/mutant-tests.sh") + (root / "node_modules").symlink_to(REPO / "node_modules", target_is_directory=True) + (root / "vitest.config.ts").write_text(VITEST_CONFIG) + (root / "src/lib").mkdir(parents=True) + (root / "test").mkdir() + (root / "src/lib/a.ts").write_text("export function one(): number {\n return 1;\n}\n") + (root / "test/a.test.ts").write_text( + 'import { expect, test } from "vitest";\nimport { one } from "../src/lib/a.ts";\n' + 'test("one", () => { expect(one()).toBe(1); });\n' + 'test("isolated", () => {\n expect(process.env.TETHER_REAL_SERVICE).toBeUndefined();\n' + ' expect(process.env.TETHER_KEYCHAIN).toBe("off");\n expect(process.env.TETHER_HOME).toContain("tether-mutant-home");\n});\n' + ) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def run_script(self, cwd: Path, file: str = "src/lib/a.ts") -> subprocess.CompletedProcess[str]: + env = {**os.environ, "MUTANT_FILE": file, "TETHER_REAL_SERVICE": "1", "TETHER_KEYCHAIN": "on"} + return subprocess.run(["sh", ".quality/mutant-tests.sh"], cwd=cwd, env=env, capture_output=True, text=True) + + def worker(self, mutated: str) -> Path: + """mutator's layout: links to every top-level entry, src and src/lib expanded, a.ts a private mutated copy.""" + worker = Path(self._tmp.name) / "worker" + (worker / "src/lib").mkdir(parents=True) + for entry in self.root.iterdir(): + if entry.name != "src": + (worker / entry.name).symlink_to(entry) + (worker / "src/lib/a.ts").write_text(mutated) + return worker + + def test_the_tests_pass_on_the_unmutated_file_with_the_service_and_keychain_kept_away(self) -> None: + result = self.run_script(self.root) + self.assertEqual(result.returncode, 0, result.stdout[-1500:] + result.stderr[-1500:]) + + def test_a_mutant_the_tests_catch_fails_the_command(self) -> None: + (self.root / "src/lib/a.ts").write_text("export function one(): number {\n return 2;\n}\n") + self.assertNotEqual(self.run_script(self.root).returncode, 0) + + def test_in_a_worker_of_links_the_tests_see_the_mutated_copy(self) -> None: + # Vite resolves links to their real path, so a linked test would import the original file and pass. + worker = self.worker("export function one(): number {\n return 2;\n}\n") + self.assertNotEqual(self.run_script(worker).returncode, 0) + self.assertFalse((worker / "test").is_symlink()) + self.assertTrue((worker / "node_modules").is_symlink()) + + def test_in_a_worker_of_links_the_unmutated_copy_passes(self) -> None: + worker = self.worker((self.root / "src/lib/a.ts").read_text()) + result = self.run_script(worker) + self.assertEqual(result.returncode, 0, result.stdout[-1500:] + result.stderr[-1500:]) + + def entry_point(self, source: str) -> None: + """src/entry.ts, which no test imports: a test reads it the way the CLI tests start bin.ts, as a process.""" + (self.root / "src/entry.ts").write_text(source) + (self.root / "test/entry.test.ts").write_text( + 'import { readFileSync } from "node:fs";\nimport { expect, test } from "vitest";\n' + 'test("entry", () => { expect(readFileSync("src/entry.ts", "utf8")).toContain("exit(0)"); });\n' + ) + + def test_a_file_no_test_imports_passes_unmutated(self) -> None: + self.entry_point("process.exit(0);\n") + result = self.run_script(self.root, "src/entry.ts") + self.assertEqual(result.returncode, 0, result.stdout[-1500:] + result.stderr[-1500:]) + + def test_a_mutant_in_a_file_no_test_imports_is_killed_by_the_whole_suite(self) -> None: + self.entry_point("process.exit(1);\n") + self.assertNotEqual(self.run_script(self.root, "src/entry.ts").returncode, 0) + + def test_a_broken_harness_is_never_a_kill(self) -> None: + (self.root / "vitest.config.ts").write_text('throw new Error("broken config");\n') + (self.root / "src/lib/a.ts").write_text("export function one(): number {\n return 2;\n}\n") + self.assertEqual(self.run_script(self.root).returncode, 0) + + +class Baseline(unittest.TestCase): + def test_the_committed_baseline_holds_every_number_the_standard_asks_for(self) -> None: + baseline = json.loads((REPO / gate.BASELINE).read_text(encoding="utf-8")) + for key in ("lines", "functions"): + self.assertIsInstance(baseline["coverage"][key], float) + for key in ("functionsMeasured", "functionsOverCrap10", "duplicatePairs"): + self.assertIsInstance(baseline[key], int) + self.assertIsInstance(baseline["mutation"]["score"], float) + + +class CrapFailure(unittest.TestCase): + def test_each_reason_a_function_over_the_limit_fails_or_stays(self) -> None: + self.assertIn("new function", gate.crap_failure(12.0, None)) + self.assertIn("under the limit", gate.crap_failure(12.0, 9.9)) + self.assertIn("higher", gate.crap_failure(12.1, 12.0)) + self.assertIsNone(gate.crap_failure(12.0, 12.0)) + + +class CheckCrap(InRepo): + def test_the_base_is_measured_only_for_files_with_a_touched_function_over_the_limit(self) -> None: + head = [(("src/a.ts", "big", 0), 1, 20.0, 4, 0.0), (("src/b.ts", "ok", 0), 1, 2.0, 1, 100.0)] + touched = {("src/a.ts", "big", 0), ("src/b.ts", "ok", 0)} + with mock.patch.object(gate, "scored_functions", return_value=head), \ + mock.patch.object(gate, "touched_functions", return_value=touched), \ + mock.patch.object(gate, "base_scores", return_value={("src/a.ts", "big", 0): 20.0}) as base: + ok, lines = gate.check_crap({"src/a.ts": [(1, 1)], "src/b.ts": [(1, 1)]}, self.repo, self.base) + base.assert_called_once_with(self.base, self.repo, ["src/a.ts"]) + self.assertTrue(ok) + self.assertIn("still over the limit, no worse than base", "\n".join(lines)) + + def test_no_base_run_when_nothing_touched_is_over_the_limit(self) -> None: + head = [(("src/b.ts", "ok", 0), 1, 2.0, 1, 100.0)] + with mock.patch.object(gate, "scored_functions", return_value=head), \ + mock.patch.object(gate, "touched_functions", return_value={("src/b.ts", "ok", 0)}), \ + mock.patch.object(gate, "base_scores") as base: + ok, _ = gate.check_crap({"src/b.ts": [(1, 1)]}, self.repo, self.base) + base.assert_not_called() + self.assertTrue(ok) + + +class RunCoverage(InRepo): + def stub_bun(self, body: str) -> Path: + bin_dir = self.repo / "bin" + bin_dir.mkdir() + (bin_dir / "bun").write_text("#!/bin/sh\n" + body) + (bin_dir / "bun").chmod(0o755) + return bin_dir + + def run_with(self, body: str): + bin_dir = self.stub_bun(body) + with mock.patch.dict(os.environ, {"PATH": f"{bin_dir}:{os.environ['PATH']}", "TETHER_REAL_SERVICE": "1"}): + return gate.run_coverage(self.repo) + + def test_a_stale_report_is_deleted_and_the_run_writes_a_new_one_without_the_real_service_or_keychain(self) -> None: + self.write("coverage/stale.txt", "old\n") + body = 'mkdir -p coverage; echo "args=$* real=${TETHER_REAL_SERVICE:-unset} keychain=${TETHER_KEYCHAIN:-unset}" > coverage/lcov.info\n' + self.assertGreaterEqual(self.run_with(body), 0.0) + self.assertFalse((self.repo / "coverage/stale.txt").exists()) + text = (self.repo / "coverage/lcov.info").read_text() + self.assertIn("run coverage --coverage.reporter=lcov --coverage.reportsDirectory=coverage", text) + self.assertIn("real=unset keychain=off", text) + + def test_failing_tests_stop_the_gate(self) -> None: + with self.assertRaises(SystemExit) as stop: + self.run_with("echo boom; exit 1\n") + self.assertIn("the tests failed under coverage", str(stop.exception.code)) + + def test_a_run_that_writes_no_report_stops_the_gate(self) -> None: + with self.assertRaises(SystemExit) as stop: + self.run_with("exit 0\n") + self.assertIn("wrote no coverage/lcov.info", str(stop.exception.code)) + + +class KillTagged(unittest.TestCase): + def test_only_a_process_with_the_tag_is_killed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + entry = Path(tmp, "4242") + entry.mkdir() + (entry / "environ").write_bytes(b"A=1\0QUALITY_GATE_TAG=t\0") + with mock.patch.object(gate.os, "kill") as kill: + self.assertTrue(gate._kill_if_tagged(entry, b"QUALITY_GATE_TAG=t")) + self.assertFalse(gate._kill_if_tagged(entry, b"QUALITY_GATE_TAG=other")) + self.assertFalse(gate._kill_if_tagged(Path(tmp, "gone"), b"QUALITY_GATE_TAG=t")) + kill.assert_called_once_with(4242, gate.signal.SIGKILL) + + @unittest.skipUnless(Path("/proc").is_dir(), "needs /proc") + def test_a_test_run_in_its_own_session_is_killed_with_the_gate_run(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + pidfile = Path(tmp, "child.pid") + script = Path(tmp, "spawner.sh") + script.write_text( + "#!/bin/sh\n" + f'python3 -c "import os,sys,time; os.setsid(); open(sys.argv[1],\'w\').write(str(os.getpid())); time.sleep(60)" "{pidfile}" &\n' + "sleep 60\n" + ) + script.chmod(0o755) + code, _, _ = gate.run_limited([str(script)], Path(tmp), dict(os.environ), 2.0) + self.assertIsNone(code) + time.sleep(0.5) + self.assertFalse(Path(f"/proc/{pidfile.read_text()}").exists(), "the child in its own session survived the kill") + + +class FullBaseline(InRepo): + def setUp(self) -> None: + super().setUp() + import full_baseline + + self.fb = full_baseline + + def test_every_in_scope_source_file_is_listed(self) -> None: + self.write("src/client/x.ts", "export const x = 1\n") + self.write("test/a.test.ts", "x\n") + self.write("test/fake-app/server.ts", "export const s = 1\n") + self.write("src/types.d.ts", "declare const d: 1\n") + self.assertEqual(self.fb.source_files(self.repo), ["src/a.ts", "src/client/x.ts"]) + + def test_the_totals_score_only_what_finished_and_list_the_rest(self) -> None: + totals = self.fb.Totals(time.monotonic()) + with redirect_stdout(io.StringIO()): + totals.record("a.ts", ("ok", 3, [("a.ts", 1, "x")], "")) + totals.record("b.ts", ("unchecked", 0, [], 7)) + totals.record("c.ts", ("failed", 0, [], "boom")) + result = totals.result(3, 20) + self.assertEqual((result["killed"], result["survived"], result["score"]), (3, 1, 75.0)) + self.assertEqual((result["filesNotChecked"], result["filesFailed"]), (["b.ts"], ["c.ts"])) + self.assertIsNone(self.fb.Totals(time.monotonic()).result(0, 0)["score"]) + + def test_the_result_goes_to_the_log_and_the_step_summary(self) -> None: + summary = self.repo / "summary.md" + out = io.StringIO() + with mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}), redirect_stdout(out): + self.fb.publish({"score": 50.0}) + self.assertIn('RESULT {"score": 50.0}', out.getvalue()) + self.assertIn("## Full mutation baseline", summary.read_text()) + + def test_main_mutates_each_file_with_every_slot_and_fails_on_a_file_not_checked(self) -> None: + calls = [] + + def mutate(path, ranges, root, workers, deadline): + calls.append((path, workers)) + return ("ok", 1, [], "") if path.endswith("a.ts") else ("unchecked", 0, [], 2) + + self.write("src/b.ts", "export const b = 1\n") + with mock.patch.object(gate, "run_coverage", return_value=1.0), \ + mock.patch.object(gate, "count_sites", return_value=2), \ + mock.patch.object(gate, "worker_limit", return_value=3), \ + mock.patch.object(gate, "mutate_file", side_effect=mutate), \ + mock.patch.dict(os.environ, {}, clear=False), redirect_stdout(io.StringIO()) as out: + os.environ.pop("GITHUB_STEP_SUMMARY", None) + code = self.fb.main() + self.assertEqual(code, 1) + self.assertEqual(sorted(calls), [("src/a.ts", 3), ("src/b.ts", 3)]) + self.assertIn('"filesNotChecked": ["src/b.ts"]', out.getvalue()) + + +if __name__ == "__main__": + unittest.main()