From 353a3f44adcb664b33a015fff8500d4abdbe505e Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 12:29:23 +0700 Subject: [PATCH 1/7] test(quality): add the gate self-tests, the gate and the Vitest mutant command Port the changed-line quality gate from the sibling Bun repository and adapt it to Vitest: one scope (src/), the package coverage script with a pinned lcov report, and a mutant command that copies mutator's linked worker tree so Vite loads the mutated file. The workflow job and the baseline do not exist yet, so their self-tests fail (9 errors, 119 pass). --- .gitignore | 1 + .quality/full_baseline.py | 107 +++ .quality/gate.py | 884 ++++++++++++++++++++ .quality/mutant-tests.sh | 70 ++ .quality/rendered-descriptions.json | 98 +++ .quality/requirements.txt | 8 + .quality/test_gate.py | 1182 +++++++++++++++++++++++++++ 7 files changed, 2350 insertions(+) create mode 100644 .quality/full_baseline.py create mode 100644 .quality/gate.py create mode 100755 .quality/mutant-tests.sh create mode 100644 .quality/rendered-descriptions.json create mode 100644 .quality/requirements.txt create mode 100644 .quality/test_gate.py diff --git a/.gitignore b/.gitignore index c4f0007..9e8e2ff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ coverage/ .claude/settings.local.json *.log target/ +__pycache__/ diff --git a/.quality/full_baseline.py b/.quality/full_baseline.py new file mode 100644 index 0000000..d0103f9 --- /dev/null +++ b/.quality/full_baseline.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Full mutation baseline: every mutation site under src/, not only the lines a PR changes. + +Run by hand from the repository root (see .quality/README.md). It runs the suite with coverage, then uses the gate's own machinery +(worker plan, memory limit, per-mutant isolation, the same test command) with every file's whole range, prints one +line per file as it finishes, and ends with the totals that belong in .quality/baseline.json. Uncovered sites are not +run (mutator skips them), so the score is killed / (killed + survived). Files the budget did not reach are listed, +and the score covers only the files that finished. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.dont_write_bytecode = True +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import gate # noqa: E402 + + +def source_files(root: Path) -> list[str]: + found = [] + for scope in gate.SCOPE: + for path in (root / scope).rglob("*.ts"): + name = path.relative_to(root).as_posix() + if gate.in_scope(name): + found.append(name) + return sorted(found) + + +class Totals: + """Running totals of a full run, one file at a time.""" + + def __init__(self, started: float) -> None: + self.started = started + self.killed = 0 + self.survived = 0 + self.unchecked: list[str] = [] + self.failed: list[str] = [] + + def record(self, path: str, outcome) -> None: + status, k, survivors, detail = outcome + self.killed += k + self.survived += len(survivors) + if status == "failed": + self.failed.append(path) + elif status == "unchecked": + self.unchecked.append(path) + elapsed = time.monotonic() - self.started + print(f"{status:9} {path} killed={k} survived={len(survivors)} elapsed={elapsed:.0f}s", flush=True) + if status == "failed": + print(str(detail)[-600:], flush=True) + + def result(self, files: int, sites: int) -> dict: + done = self.killed + self.survived + return { + "files": files, + "sites": sites, + "killed": self.killed, + "survived": self.survived, + "score": round(100 * self.killed / done, 1) if done else None, + "filesNotChecked": sorted(self.unchecked), + "filesFailed": sorted(self.failed), + "seconds": round(time.monotonic() - self.started), + } + + +def publish(result: dict) -> None: + print("RESULT " + json.dumps(result), flush=True) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write("## Full mutation baseline\n\n```json\n" + json.dumps(result, indent=2) + "\n```\n") + + +def main() -> int: + root = Path.cwd() + budget = float(os.environ.get("BASELINE_BUDGET_SECONDS", "19800")) # 330 minutes + started = time.monotonic() + suite = gate.run_coverage(root) + files = source_files(root) + ranges = {f: [(1, len((root / f).read_text(encoding="utf-8").splitlines()))] for f in files} + cores = gate.available_cores() + limit = gate.worker_limit(cores, gate.memory_limit()) + with ThreadPoolExecutor(max_workers=max(1, min(len(files), cores))) as pool: + sites = dict(zip(files, pool.map(lambda f: gate.count_sites(f, gate.wanted_lines(ranges[f]), root), files))) + known = sum(n for n in sites.values() if isinstance(n, int)) + todo = sorted(gate.plan_workers(sites, limit), key=lambda f: -(sites[f] if isinstance(sites[f], int) else 1)) + print(f"suite {suite:.0f}s, {len(files)} files, {known} sites, {len(todo)} to mutate, limit {limit} ({cores} cores)", flush=True) + + totals = Totals(started) + # One file at a time with every slot: a share proportional to the sites leaves a big file on one worker for hours. + for path in todo: + totals.record(path, gate.mutate_file(path, ranges[path], root, limit, started + budget)) + gate.shutil.rmtree(root / "target" / "mutation-workers", ignore_errors=True) + result = totals.result(len(files), known) + publish(result) + return 1 if result["filesNotChecked"] or result["filesFailed"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.quality/gate.py b/.quality/gate.py new file mode 100644 index 0000000..e264dab --- /dev/null +++ b/.quality/gate.py @@ -0,0 +1,884 @@ +#!/usr/bin/env python3 +"""Pull request quality gate (engineering quality standard, issue #20). + +Ratchet, not big bang: only code the pull request adds or changes is judged, so debt that is already on main +never blocks a PR. Four checks, all of them in this one file so a revert removes the gate: + + coverage every executable line the PR adds or changes must be hit by a test (Vitest's v8 lcov); the repo total is + compared with .quality/baseline.json and reported, but a drop alone never fails a PR, so deleting + covered code cannot block a cleanup + crap a function the PR adds or changes must score under 10 (crapper), unless it was already 10 or more on + base and scores no higher now (the base is measured in a throwaway worktree with the same test run) + duplicate the PR must not add a structural duplicate pair at 0.8 or above (dryer) + mutation no surviving mutant on the lines the PR changes (mutator), inside a budget sized to the change +Both coverage and mutation can be waived for one line (never line 0, a whole file missing from the coverage report) +with a line in the PR description, "Equivalent mutant: : " (the reason is 10 or more characters). +One such line waives every mutant on that line. It counts only as a top-level paragraph of its own in GitHub's +rendering of the description, which an earlier workflow step writes to a file (--description-html) with its sha256. + +Scope for every check is the TypeScript under src/ (tests and .d.ts excluded). The test helpers under test/, +the fake app server included, are not judged. + +Run from the repository root with bun, and with crapper, dryer and mutator on the PATH (.quality/requirements.txt). +The gate runs the coverage itself, so a stale report can never stand in for it. + +Exit 0 when every check passes, 1 when any fails. The report goes to stdout and to GITHUB_STEP_SUMMARY. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from html.parser import HTMLParser +from pathlib import Path + +# Never leave __pycache__ in the tree. +sys.dont_write_bytecode = True + +# The standard's thresholds. Changing one is an owner decision, never a way to get a PR through. +CRAP_LIMIT = 10 +DUPLICATE_THRESHOLD = 0.8 +MUTATION_BUDGET_MIN = 300.0 +MUTATION_BUDGET_MAX = 1800.0 + +SCOPE = ("src",) +BASELINE = Path(".quality/baseline.json") +MUTANT_TESTS = ".quality/mutant-tests.sh" +LCOV = Path("coverage/lcov.info") +# The package script, so the gate measures what `bun run coverage` measures (vitest.config.ts lists every file under +# src/, so a file no test loads is still in the report, at 0 hits). The flags pin the report the gate reads. +COVERAGE_COMMAND = ["bun", "run", "coverage", "--coverage.reporter=lcov", "--coverage.reportsDirectory=coverage"] +# No test the gate runs may install a real launchd agent (test/service-real.test.ts) or reach the OS keychain. +DROPPED_VARIABLES = ("TETHER_REAL_SERVICE",) +PINNED_VARIABLES = {"TETHER_KEYCHAIN": "off"} +# Every diff the gate reads: git settings (colour, an external diff, a textconv driver) cannot change what it sees. +DIFF = ["--no-color", "--no-ext-diff", "--no-textconv", "--no-renames"] +# Line 0 never matches: it stands for a whole file missing from the coverage report, and no waiver covers that. +EQUIVALENT = re.compile(r"Equivalent mutant:\s*(\S+?):([1-9]\d*)\s+(\S.{9,})") +# A bidirectional control could make the reason read differently from what it says, so none may appear anywhere in +# the paragraph, on any of its lines. +BIDI = re.compile(r"[؜‎‏‪-‮⁦-⁩]") + +Ranges = dict[str, list[tuple[int, int]]] + + +def run(args: list[str], cwd: Path | None = None, check: bool = True, env: dict | None = None) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", env=env) + if check and result.returncode != 0: + sys.exit(f"command failed ({result.returncode}): {' '.join(args)}\n{result.stdout[-3000:]}\n{result.stderr[-3000:]}") + return result + + +def test_env() -> dict[str, str]: + kept = {key: value for key, value in os.environ.items() if key not in DROPPED_VARIABLES} + return {**kept, **PINNED_VARIABLES} + + +def resolve_base(explicit: str | None) -> str: + """The commit the PR is measured against. + + A pull_request checkout is GitHub's merge commit, whose first parent is the current base tip, so HEAD^1 is + exact even when the base moved after the PR was opened. Otherwise use --base. + """ + if run(["git", "rev-parse", "--verify", "-q", "HEAD^2"], check=False).returncode == 0: + return run(["git", "rev-parse", "HEAD^1"]).stdout.strip() + if not explicit: + sys.exit("HEAD is not a merge commit; pass --base ") + return explicit + + +def is_test_path(path: str) -> bool: + parts = path.split("/") + return "test" in parts[:-1] or parts[-1].endswith((".test.ts", ".spec.ts")) + + +def in_scope(path: str) -> bool: + """TypeScript under src/: no tests, no declaration files.""" + if not path.startswith(tuple(f"{root}/" for root in SCOPE)): + return False + return path.endswith(".ts") and not path.endswith(".d.ts") and not is_test_path(path) + + +def added_counts(base: str) -> dict[str, int]: + """File -> lines it adds, by name as git stores it. -z keeps a name with a space, a tab or a non-ASCII + character whole: a patch's +++ line ends such a name with a tab or quotes it.""" + listed = run(["git", "diff", *DIFF, "--numstat", "-z", "--diff-filter=AMT", base, "HEAD", "--", *SCOPE]).stdout + counts: dict[str, int] = {} + for record in filter(None, listed.split("\0")): + added, _deleted, path = record.split("\t", 2) + if not in_scope(path): + continue + if added == "-": # git counts no lines in a file it reads as binary: it must never pass unjudged + sys.exit(f"git reads {path!r} as a binary file, so its changed lines cannot be judged") + if int(added) > 0: # a pure deletion adds no line to judge + counts[path] = int(added) + return counts + + +HUNK = re.compile(r"^@@ -\S+ \+(\d+)(?:,(\d+))? @@", re.MULTILINE) + + +def file_diff(base: str, path: str) -> str: + """The -U0 patch of one file from base to HEAD; the name is a literal path, never a pattern.""" + return run(["git", "--literal-pathspecs", "diff", *DIFF, "-U0", base, "HEAD", "--", path]).stdout + + +def changed_ranges(base: str) -> Ranges: + """Added or modified lines (new-file numbering) per file. A hunk that adds nothing (+N,0) is skipped. Each file's + hunks must add up to the lines git counts for it; when they do not, the gate stops, because a file it cannot + read must never pass as unchanged.""" + found: Ranges = {} + for path, added in added_counts(base).items(): + hunks = [(int(start), 1 if count == "" else int(count)) for start, count in HUNK.findall(file_diff(base, path))] + ranges = [(start, start + count - 1) for start, count in hunks if count > 0] + if sum(end - start + 1 for start, end in ranges) != added: + sys.exit(f"could not read the changed lines of {path!r}: git counts {added} added, its hunks do not") + found[path] = ranges + return found + + +def touches(ranges: list[tuple[int, int]], first: int, last: int) -> bool: + return any(start <= last and end >= first for start, end in ranges) + + +def run_coverage(root: Path) -> float: + """Run the suite with coverage into coverage/lcov.info, after deleting any old report. Returns the seconds it took, + which is also the gate's measure of one mutant's worst case (one mutant runs at most the whole suite).""" + shutil.rmtree(root / "coverage", ignore_errors=True) + started = time.monotonic() + result = run(COVERAGE_COMMAND, cwd=root, check=False, env=test_env()) + if result.returncode != 0: + sys.exit(f"the tests failed under coverage ({result.returncode}):\n{result.stdout[-3000:]}\n{result.stderr[-3000:]}") + if not (root / LCOV).is_file(): + sys.exit(f"the coverage run wrote no {LCOV}") + return time.monotonic() - started + + +def parse_lcov(text: str) -> dict[str, dict[int, int]]: + """File -> {line: hits} for the executable lines in an lcov report.""" + files: dict[str, dict[int, int]] = {} + current: dict[int, int] | None = None + for line in text.splitlines(): + if line.startswith("SF:"): + current = files.setdefault(line[3:].strip(), {}) + elif line.startswith("DA:") and current is not None: + number, _, rest = line[3:].partition(",") + current[int(number)] = int(rest.split(",")[0]) + return files + + +def lcov_totals(text: str) -> dict[str, float]: + """Line and function coverage of the in-scope files, in percent (two decimals), from the LF/LH/FNF/FNH records. + """ + sums = {"LF": 0, "LH": 0, "FNF": 0, "FNH": 0} + counted = False + for line in text.splitlines(): + if line.startswith("SF:"): + counted = in_scope(line[3:].strip()) + elif counted and line[:3].rstrip(":") in sums: + key, _, value = line.partition(":") + sums[key] += int(value) + + def pct(hit: int, found: int) -> float: + return round(100 * hit / found, 2) if found else 100.0 + + return {"lines": pct(sums["LH"], sums["LF"]), "functions": pct(sums["FNH"], sums["FNF"])} + + +def lines_of(report: dict[str, dict[int, int]], path: str) -> dict[int, int] | None: + for name, lines in report.items(): + if name == path or name.endswith("/" + path): + return lines + return None + + +def uncovered_changed(changed: Ranges, lcov_text: str) -> list[tuple[str, int]]: + """Changed executable lines that no test hits. vitest.config.ts lists every file under src/, so a file the tests + only run as a separate process (src/cli/bin.ts) is in the report at 0 hits, and each of its changed lines is + reported on its own. A file the report does not list at all is reported as line 0, which no waiver names: the + report cannot vouch for any of its lines.""" + report = parse_lcov(lcov_text) + found: list[tuple[str, int]] = [] + for path, ranges in sorted(changed.items()): + lines = lines_of(report, path) + if lines is None: + found.append((path, 0)) + continue + for number in sorted(n for n, hits in lines.items() if hits == 0 and touches(ranges, n, n)): + found.append((path, number)) + return found + + +def coverage_verdict( + uncovered: list[tuple[str, int]], allowed: dict[tuple[str, int], str], now: dict[str, float], baseline: dict +) -> tuple[bool, list[str]]: + """The PR fails on its own untested lines, never on the repo total.""" + lines = totals_report(now, baseline["coverage"]) + bad = [item for item in uncovered if item not in allowed] + for path, number in uncovered: + reason = allowed.get((path, number)) + where = f"{path}:{number}" if number else f"{path} (not in the coverage report)" + lines.append(f" untested changed line {where}" + (f" (explained: {reason})" if reason else "")) + if bad: + lines.append( + "Add a test that runs each line, or add 'Equivalent mutant: : ' to the PR description " + "when no test can reach it." + ) + else: + lines.insert(0, "Every executable line the PR adds or changes is hit by a test.") + return not bad, lines + + +def totals_report(now: dict[str, float], floor: dict[str, float]) -> list[str]: + """The repo totals against the baseline: reported, and a rise asks for the baseline to be updated.""" + lines = [ + f"Repo coverage: lines {now['lines']}% (baseline {floor['lines']}%), " + f"functions {now['functions']}% (baseline {floor['functions']}%)." + ] + keys = ("lines", "functions") + if any(now[key] > floor[key] for key in keys): + lines.append("Coverage improved: update coverage in .quality/baseline.json in this PR.") + if any(now[key] < floor[key] for key in keys): + lines.append("Below the baseline; that alone does not fail the PR (deleting covered code can cause it).") + return lines + + +def check_coverage(lcov_path: Path, changed: Ranges, baseline: dict, allowed: dict[tuple[str, int], str]) -> tuple[bool, list[str]]: + text = lcov_path.read_text(encoding="utf-8") + return coverage_verdict(uncovered_changed(changed, text), allowed, lcov_totals(text), baseline) + + +Scored = tuple[tuple[str, str, int], int, float, int, float] +"""One function's score: (key, start line, CRAP, complexity, coverage percent). The key is (path, name, nth function of +that name in the file), so a function that moved to another file or was renamed has no key on base: it is new.""" + + +def scored_functions(root: Path, paths: list[str], need_coverage: bool = False) -> list[Scored]: + """Every function in the given files with its CRAP, from the coverage under root. + + A function with no coverage record scores as 0% covered, unless `need_coverage`: then it is left out, because + 0% is the highest score it could have, which would let any head score pass against it.""" + from crapper.coverage import load_bundle + from crapper.crap import make_entry + from crapper.languages import functions_in_file + + bundle = load_bundle(root) + bundle.bind_sources(sorted(paths)) + found: list[Scored] = [] + for path in sorted(paths): + source = (root / path).read_text(encoding="utf-8", errors="replace") + seen: dict[str, int] = {} + for function in functions_in_file("typescript", source, path, str(root)): + nth = seen.get(function.name, 0) + seen[function.name] = nth + 1 + percent = bundle.percent_for(function) + if percent is None and need_coverage: + continue + entry = make_entry(function, 0.0 if percent is None else percent) + if entry.crap is not None: + found.append(((path, function.name, nth), function.start_line, entry.crap, entry.complexity, entry.coverage)) + return found + + +def crap_verdict( + head: list[Scored], touched: set[tuple[str, str, int]], base: dict[tuple[str, str, int], float] | None +) -> tuple[bool, list[str]]: + """A function the PR touches passes under the limit. One that is over the limit passes only when it was already + over on base and scores no higher now (raw scores; one decimal is only how they print); a new, moved or renamed + one has to be under, and so is every function of a name whose count in its file changed (order cannot tell which + one is the old one).""" + + def counts(keys) -> dict[tuple[str, str], int]: + found: dict[tuple[str, str], int] = {} + for path, name, _nth in keys: + found[(path, name)] = found.get((path, name), 0) + 1 + return found + + on_head = counts(key for key, *_ in head) + on_base = counts(base or {}) + bad: list[str] = [] + still: list[str] = [] + judged = [item for item in head if item[0] in touched] + for key, line, crap, complexity, coverage in judged: + if crap < CRAP_LIMIT: + continue + label = f"{key[0]}:{line} {key[1]} CRAP {crap:.1f} (complexity {complexity}, coverage {coverage:.0f}%)" + before = (base or {}).get(key) if on_head.get(key[:2]) == on_base.get(key[:2]) else None + failure = crap_failure(crap, before) + if failure: + bad.append(f"{label}, {failure}") + else: + still.append(f"{label}, base {before:.1f}") + checked = len(judged) + lines = [f"{checked} added or changed function(s) scored, limit is under {CRAP_LIMIT}."] + lines += [f" over the limit: {item}" for item in bad] + lines += [f" still over the limit, no worse than base: {item}" for item in still] + return not bad, lines + + +def crap_failure(crap: float, before: float | None) -> str | None: + """Why a function at or over the limit fails, or None when it may stay: it was already over on base and is no + higher now.""" + if before is None: + return "new function (no score on base under this file and name)" + if before < CRAP_LIMIT: + return f"was {before:.1f} on base (under the limit)" + if crap > before: + return f"was {before:.1f} on base (higher)" + return None + + +def base_worktree(root: Path, base: str) -> tuple[Path, Path]: + """A throwaway detached worktree of base. Returns (its parent, the tree).""" + parent = Path(tempfile.mkdtemp(prefix="quality-base-")) + tree = parent / "tree" + run(["git", "worktree", "add", "--detach", str(tree), base], cwd=root) + return parent, tree + + +def drop_worktree(root: Path, parent: Path) -> None: + run(["git", "worktree", "remove", "--force", str(parent / "tree")], cwd=root, check=False) + shutil.rmtree(parent, ignore_errors=True) + + +def base_scores(base: str, root: Path, paths: list[str]) -> dict[tuple[str, str, int], float]: + """CRAP on base for the functions in the given files: the base tree, with the same coverage run as head (the base + borrows this checkout's node_modules).""" + parent, tree = base_worktree(root, base) + try: + existing = [path for path in paths if (tree / path).is_file()] + if not existing: + return {} + (tree / "node_modules").symlink_to((root / "node_modules").resolve(), target_is_directory=True) + run_coverage(tree) + return { + key: crap for key, _line, crap, _complexity, _coverage in scored_functions(tree, existing, need_coverage=True) + } + finally: + drop_worktree(root, parent) + + +def touched_functions(root: Path, changed: Ranges) -> set[tuple[str, str, int]]: + """Functions with a changed line in them.""" + from crapper.languages import functions_in_file + + found: set[tuple[str, str, int]] = set() + for path, ranges in changed.items(): + source = (root / path).read_text(encoding="utf-8", errors="replace") + seen: dict[str, int] = {} + for function in functions_in_file("typescript", source, path, str(root)): + nth = seen.get(function.name, 0) + seen[function.name] = nth + 1 + if touches(ranges, function.start_line, function.end_line): + found.add((path, function.name, nth)) + return found + + +def check_crap(changed: Ranges, root: Path, base: str) -> tuple[bool, list[str]]: + head = scored_functions(root, sorted(changed)) + touched = touched_functions(root, changed) + over = sorted({key[0] for key, _l, crap, _c, _v in head if key in touched and crap >= CRAP_LIMIT}) + return crap_verdict(head, touched, base_scores(base, root, over) if over else None) + + +Pair = tuple[float, tuple[str, int, int], tuple[str, int, int]] +# The start of each block in dryer's report; a line, so a path or note that holds the word cannot add one. +DUPLICATE = re.compile(r"^DUPLICATE ", re.MULTILINE) + + +def parse_dryer(out: str) -> list[Pair]: + pairs: list[Pair] = [] + # The text before the first marker is not a block. + for block in DUPLICATE.split(out)[1:]: + score = re.match(r"score=([\d.]+)", block) + places = re.findall(r"^\s+(\S+):(\d+)-(\d+)$", block, re.MULTILINE) + if score and len(places) == 2: + left, right = ((p, int(a), int(b)) for p, a, b in places) + pairs.append((float(score.group(1)), left, right)) + # A block the gate cannot read could hide a new pair, so it stops the gate rather than being skipped. + blocks = len(DUPLICATE.findall(out)) + if blocks != len(pairs): + sys.exit(f"dryer printed {blocks} DUPLICATE blocks but {len(pairs)} pairs could be read:\n{out[-3000:]}") + return pairs + + +def dryer_pairs(root: Path) -> list[Pair]: + """Pairs at the threshold or above under src/ (dryer skips test folders and files on its own).""" + targets = [name for name in SCOPE if (root / name).is_dir()] + if not targets: + return [] + out = run(["dryer", "--threshold", str(DUPLICATE_THRESHOLD), *targets], cwd=root).stdout + shutil.rmtree(root / ".metrics", ignore_errors=True) + return parse_dryer(out) + + +def duplicate_verdict(changed: Ranges, before: list[Pair], after: list[Pair]) -> tuple[bool, list[str]]: + """A pair is new when it touches a changed line and its two files hold more pairs than they did on base.""" + + def key(pair: Pair) -> tuple[str, str]: + return tuple(sorted((pair[1][0], pair[2][0]))) # type: ignore[return-value] + + def count(pairs: list[Pair], wanted: tuple[str, str]) -> int: + return sum(1 for p in pairs if key(p) == wanted) + + bad = [] + for pair in after: + score, left, right = pair + touched = any(f in changed and touches(changed[f], a, b) for f, a, b in (left, right)) + if touched and count(after, key(pair)) > count(before, key(pair)): + bad.append(f"{left[0]}:{left[1]}-{left[2]} and {right[0]}:{right[1]}-{right[2]} (similarity {score:.2f})") + head = f"{len(after)} duplicate pair(s) at {DUPLICATE_THRESHOLD} or above on this PR, {len(before)} on base." + return not bad, [head] + [f" new duplicate: {item}" for item in bad] + + +def check_duplicates(changed: Ranges, base: str, root: Path) -> tuple[bool, list[str]]: + parent, tree = base_worktree(root, base) + try: + before = dryer_pairs(tree) + finally: + drop_worktree(root, parent) + return duplicate_verdict(changed, before, dryer_pairs(root)) + + +# Elements that never hold content, so they never wrap a waiver. +VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}) + + +class TopLevelParagraphs(HTMLParser): + """The text of each top-level

in GitHub's rendering that holds text and nothing else. Anything inside another + element (

,
, a list, a quote) or next to markup in its paragraph is left out, and an element that is
+    never closed covers everything after it."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.open: list[str] = []
+        self.text: list[str] | None = None
+        self.found: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.text = [] if tag == "p" and not self.open and not attrs else None
+        if tag not in VOID:
+            self.open.append(tag)
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.text = None
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag not in self.open:
+            return
+        while self.open.pop() != tag:
+            pass
+        # Text is collected only from a top-level 

with no tag since, so this end tag closes that very paragraph; + # nothing after it belongs to it. + if self.text is not None: + self.found.append("".join(self.text)) + self.text = None + + def handle_data(self, data: str) -> None: + if self.text is not None: + self.text.append(data) + + +def survivors_allowed(html: str) -> dict[tuple[str, int], str]: + """The waivers in GitHub's rendering of the description: each must be the whole text of a top-level paragraph, so + a reviewer always sees it on the PR page.""" + parser = TopLevelParagraphs() + parser.feed(html) + parser.close() + found = (EQUIVALENT.fullmatch(text.strip()) for text in parser.found if not BIDI.search(text)) + return {(m.group(1), int(m.group(2))): m.group(3).strip() for m in found if m} + + +def read_waivers(path: str | None, sha256: str | None) -> tuple[bool, list[str], dict[tuple[str, int], str]]: + """The waivers that count, from GitHub's rendering of the description, written to `path` by the workflow step that + holds the token, before any code from the pull request runs, together with its sha256 (a step output, which no + later step can change). Without a file nothing is waived; a file that cannot be read, or comes without its hash or + does not match it, fails, because then the gate cannot tell which waivers a reviewer sees.""" + if path is None: + return True, ["No rendered description was given: nothing is waived."], {} + try: + data = Path(path).read_bytes() + except OSError as error: + return False, [f"The rendered description cannot be read, so no waiver can be: {error}"], {} + if not sha256 or hashlib.sha256(data).hexdigest() != sha256.strip().lower(): + return False, ["The rendered description does not match the sha256 it was rendered with, so no waiver can be read."], {} + allowed = survivors_allowed(data.decode("utf-8", errors="replace")) + lines = [f"{len(allowed)} waiver(s) count, each the whole text of a top-level paragraph on GitHub."] + lines.extend(f" {file}:{line}" for file, line in sorted(allowed)) + return True, lines, allowed + + +SAFE_PATH = re.compile(r"^[A-Za-z0-9_./@+-]+$") + + +def safe_path(path: str) -> bool: + return bool(SAFE_PATH.match(path)) and ".." not in path.split("/") + + +def _kill_pass(tag: str) -> int: + """One scan of /proc: kill every process that carries the tag. Returns how many it killed.""" + needle = f"QUALITY_GATE_TAG={tag}".encode() + proc_dir = Path("/proc") + if not proc_dir.is_dir(): + return 0 + others = (e for e in proc_dir.iterdir() if e.name.isdigit() and int(e.name) != os.getpid()) + return sum(1 for entry in others if _kill_if_tagged(entry, needle)) + + +def _kill_if_tagged(entry: Path, needle: bytes) -> bool: + """Kill the process behind one /proc entry when its environment holds the tag. A process that is gone, or not + ours to read, is skipped.""" + try: + if needle not in (entry / "environ").read_bytes().split(b"\0"): + return False + os.kill(int(entry.name), signal.SIGKILL) + return True + except OSError: + return False + + +def kill_tagged(tag: str, passes: int = 20) -> None: + """Kill every process that carries QUALITY_GATE_TAG= in its environment (Linux, via /proc). + + mutator starts each test run in its own session, so killing mutator's process group leaves them running; they + inherit the tag from the environment, which is how they are found. A process can fork between the scan and the + kill, so scan again until a pass finds none. + """ + for _ in range(passes): + if _kill_pass(tag) == 0: + return + time.sleep(0.05) + + +def run_limited(args: list[str], cwd: Path, env: dict, seconds: float) -> tuple[int | None, str, str]: + """Run args; on timeout kill the process group and everything tagged by this run, and return code None.""" + tag = uuid.uuid4().hex + proc = subprocess.Popen( + args, cwd=cwd, env={**env, "QUALITY_GATE_TAG": tag}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True, + ) + try: + out, err = proc.communicate(timeout=max(seconds, 0.1)) + return proc.returncode, out, err + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + kill_tagged(tag) + proc.communicate() + return None, "", "" + + +def mutator_args(path: str, wanted: list[int], *extra: str) -> list[str]: + return ["mutator", path, "--lines", ",".join(map(str, wanted)), "--use-existing-coverage", *extra] + + +def count_sites(path: str, wanted: list[int], root: Path) -> int | str: + """Mutation sites on the wanted lines (a scan runs no tests), or "unknown" when the scan does not finish.""" + code, out, _ = run_limited(mutator_args(path, wanted, "--scan"), root, dict(os.environ), 30) + if code is None: + return "unknown" + return len([line for line in out.splitlines() if re.match(r"\s+\S+:\d+\s", line)]) + + +CGROUP = Path("/sys/fs/cgroup") +BYTES_PER_MUTANT = 1 << 30 # one Vitest run of the whole suite stays well under 1 GB +MEMORY_RESERVE = 2 << 30 # for the gate, the mutator processes and the system + + +def available_cores(cgroup: Path = CGROUP) -> int: + """Cores this process may use: the CPU affinity, capped by a cgroup v2 CPU quota when there is one. + + os.cpu_count() reports the host, which on a container runner can be far more than the job is given. + """ + n = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1) + try: + quota, period = (cgroup / "cpu.max").read_text().split()[:2] + if quota != "max": + n = min(n, max(1, int(int(quota) / int(period)))) + except (OSError, ValueError): + pass + return max(1, n) + + +def memory_limit(cgroup: Path = CGROUP, meminfo: Path = Path("/proc/meminfo")) -> int | None: + """Bytes of memory this job may use: the cgroup limit (v2, then v1) or, without one, the machine's total.""" + for name in ("memory.max", "memory/memory.limit_in_bytes"): + try: + text = (cgroup / name).read_text().strip() + if text != "max" and int(text) < (1 << 60): + return int(text) + except (OSError, ValueError): + continue + try: + for line in meminfo.read_text().splitlines(): + if line.startswith("MemTotal:"): + return int(line.split()[1]) * 1024 + except (OSError, ValueError): + pass + return None + + +def worker_limit(cores: int, memory: int | None) -> int: + """Mutants allowed at once: the cores, or what the memory holds after a 2 GB reserve at 1 GB per mutant.""" + if memory is None: + return max(1, cores) + return max(1, min(cores, (memory - MEMORY_RESERVE) // BYTES_PER_MUTANT)) + + +def plan_workers(sites: dict[str, int | str], limit: int) -> dict[str, int]: + """Mutants to run at once per file; the total never goes over `limit`. + + A file with many sites gets many workers, so a big file does not finish last on a few workers while other cores + sit idle. With more files than `limit` every file gets one worker and the files queue for a free slot (the caller + runs at most `limit` mutator processes at a time). + """ + # A file whose scan did not finish ("unknown") still gets mutated, so it takes one slot of the plan. + sites = {path: (count if isinstance(count, int) else 1) for path, count in sites.items()} + sites = {path: count for path, count in sites.items() if count > 0} + if not sites: + return {} + if len(sites) >= limit: + return {path: 1 for path in sites} + total = sum(sites.values()) + extra = limit - len(sites) + exact = {path: extra * count / total for path, count in sites.items()} + plan = {path: 1 + int(exact[path]) for path in sites} + leftover = limit - sum(plan.values()) + for path in sorted(sites, key=lambda p: (exact[p] - int(exact[p]), sites[p]), reverse=True)[:leftover]: + plan[path] += 1 + return plan + + +def mutation_budget(spent: float, sites: dict[str, int | str], plan: dict[str, int], limit: int, mutant_seconds: float) -> float: + """The whole gate's time budget, sized to the change (engineering standard, "Mutation budget"): the time already + spent, plus for each file the rounds its sites need on its workers, plus mutator's own unmutated run of the file, + times the measured time of one mutant. Files beyond `limit` queue, so the per-file rounds are spread over the + slots. Never below 5 minutes, never above 30.""" + rounds = 0 + for path, workers in plan.items(): + count = sites[path] if isinstance(sites[path], int) else 1 + rounds += math.ceil(count / workers) + 1 # type: ignore[operator] + files_at_once = max(1, min(len(plan), limit)) + wanted = spent + math.ceil(rounds / files_at_once) * mutant_seconds + return max(MUTATION_BUDGET_MIN, min(MUTATION_BUDGET_MAX, wanted)) + + +def wanted_lines(ranges: list[tuple[int, int]]) -> list[int]: + return sorted({n for start, end in ranges for n in range(start, end + 1)}) + + +def mutate_file(path: str, ranges: list[tuple[int, int]], root: Path, workers: int, deadline: float): + """Mutate one file's changed lines. Returns (status, killed, survivors, detail); status is ok, unchecked or failed.""" + wanted = wanted_lines(ranges) + remaining = deadline - time.monotonic() + if remaining <= 0: + return "unchecked", 0, [], count_sites(path, wanted, root) + # The file reaches the test command through the environment, never through the command string. + env = {**test_env(), "MUTANT_FILE": path} + code, out, err = run_limited( + mutator_args(path, wanted, "--mutate-all", "--max-workers", str(workers), "--test-command", f"./{MUTANT_TESTS}"), + root, env, remaining, + ) + if code is None: + return "unchecked", 0, [], count_sites(path, wanted, root) + if code not in (0, 3): + return "failed", 0, [], f"mutator failed on {path} (exit {code}):\n{out[-1500:]}\n{err[-1500:]}" + killed = 0 + survivors: list[tuple[str, int, str]] = [] + for line in out.splitlines(): + m = re.match(r"(KILLED|SURVIVED)\s+(\S+?):(\d+)\s+(.*)", line) + if not m: + continue + if m.group(1) == "KILLED": + killed += 1 + else: + survivors.append((m.group(2), int(m.group(3)), m.group(4))) + return "ok", killed, survivors, "" + + +def tally(outcomes: dict[str, tuple]) -> tuple[int, list[tuple[str, int, str]], list[tuple[str, int | str]], list[str]]: + """Killed, survivors, files not checked (with their sites) and failure details, over every file's outcome.""" + survived: list[tuple[str, int, str]] = [] + unchecked: list[tuple[str, int | str]] = [] + failed: list[str] = [] + killed = 0 + for path in sorted(outcomes): + status, k, survivors, detail = outcomes[path] + if status == "failed": + failed.append(detail) + elif status == "unchecked": + unchecked.append((path, detail)) + killed += k + survived.extend(survivors) + return killed, survived, unchecked, failed + + +def unchecked_report(unchecked: list[tuple[str, int | str]], budget: float) -> list[str]: + if not unchecked: + return [] + return [ + f"The {budget:.0f}s budget ran out before every file was checked. Not checked:", + *(f" {path} ({sites} mutation site(s))" for path, sites in unchecked), + "The runner may have been busy: re-run this job once before doing anything else. If it runs out of time " + "again, the PR is too large for the gate to finish and has to be split, or its tests made faster.", + ] + + +def mutation_report( + outcomes: dict[str, tuple], allowed: dict[tuple[str, int], str], budget: float, elapsed: float, how: str +) -> tuple[bool, list[str]]: + """The verdict and report from each file's outcome. A file mutator failed on fails the check, and so does a file + the budget ran out on; the files that finished are still reported.""" + killed, survived, unchecked, failed = tally(outcomes) + unexplained = [s for s in survived if (s[0], s[1]) not in allowed] + lines = [ + f"{killed} mutant(s) killed, {len(survived)} survived on the changed lines, {elapsed:.0f}s of a " + f"{budget:.0f}s budget ({how})." + ] + for path, line, op in survived: + reason = allowed.get((path, line)) + tag = f"explained as equivalent: {reason}" if reason else "NOT explained" + lines.append(f" survivor {path}:{line} {op} ({tag})") + if unexplained: + lines.append( + "Kill each survivor with a test, or add 'Equivalent mutant: : ' " + "to the PR description (one such line waives every mutant on that line)." + ) + lines.extend(failed) + lines.extend(unchecked_report(unchecked, budget)) + return not unexplained and not unchecked and not failed, lines + + +def check_mutation( + changed: Ranges, + root: Path, + allowed: dict[tuple[str, int], str], + started: float, + mutant_seconds: float, + workers: int | None = None, +) -> tuple[bool, list[str]]: + """Mutate the changed lines inside a budget sized to the change, as many mutants at once as the machine allows. + + Each mutant runs in its own copy of the tree (mutator's worker directories), so parallel mutants never share + files. When the budget runs out the check fails cleanly: the report names what was not checked, instead of the + runner killing the job and the report with it. + """ + for path in changed: + if not safe_path(path): + return False, [f"refusing to run mutator on an unusual file name: {path!r}"] + cores = available_cores() + limit = workers or worker_limit(cores, memory_limit()) + shutil.rmtree(root / ".metrics", ignore_errors=True) + paths = sorted(changed) + # Scans run no tests, so counting every file's sites first is cheap. + with ThreadPoolExecutor(max_workers=max(1, min(len(paths), cores))) as pool: + sites = dict(zip(paths, pool.map(lambda p: count_sites(p, wanted_lines(changed[p]), root), paths))) + plan = plan_workers(sites, limit) + budget = mutation_budget(time.monotonic() - started, sites, plan, limit, mutant_seconds) + deadline = started + budget + # A file with no mutation site needs no run; a scan that did not finish is planned as one site. + todo = [p for p in paths if p in plan] + with ThreadPoolExecutor(max_workers=max(1, min(len(todo), limit))) as pool: + outcomes = dict(zip(todo, pool.map(lambda p: mutate_file(p, changed[p], root, plan[p], deadline), todo))) + shutil.rmtree(root / "target" / "mutation-workers", ignore_errors=True) + shutil.rmtree(root / ".metrics", ignore_errors=True) + per_file = ", ".join(f"{Path(p).name} x{n} ({sites[p]} sites)" for p, n in sorted(plan.items())) or "no sites" + how = f"limit {limit} mutants at once from {cores} core(s), one mutant measured at {mutant_seconds:.0f}s: {per_file}" + return mutation_report(outcomes, allowed, budget, time.monotonic() - started, how) + + +def main() -> int: + started = time.monotonic() + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--base", help="base commit when HEAD is not a merge commit") + parser.add_argument("--workers", type=int, default=None, help="mutants at once; default from cores and memory") + parser.add_argument("--skip-coverage-run", action="store_true", help="read coverage/lcov.info as it is (self-tests)") + parser.add_argument("--skip-mutation", action="store_true", help="local runs only") + parser.add_argument("--description-html", help="GitHub's rendering of the PR description, from an earlier step") + parser.add_argument("--description-sha256", help="the sha256 of that file, as the step that wrote it gave it") + opts = parser.parse_args() + + root = Path.cwd() + base = resolve_base(opts.base) + results, before_mutation = gate_results(opts, root, base, started) + report = [f"## Quality gate (base {base[:8]})", "", f"Every check before mutation took {before_mutation:.0f}s.", ""] + for name, passed, lines in results: + report.append(f"### {name}: {'pass' if passed else 'FAIL'}") + report.extend(lines) + report.append("") + text = "\n".join(report) + print(text) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write(text + "\n") + return 0 if all(passed for _, passed, _ in results) else 1 + + +Result = tuple[str, bool, list[str]] + + +def guarded(check, *args) -> tuple[bool, list[str]]: + """Run one check. A check that breaks (a tool that fails, a report it cannot read, a bug) fails with the error's + type and message, and the other checks and the summary still run.""" + try: + return check(*args) + except (Exception, SystemExit) as error: # noqa: BLE001 - every failure must reach the report + return False, [f"The check broke: {type(error).__name__}: {str(error)[-3000:]}"] + + +def gate_results(opts: argparse.Namespace, root: Path, base: str, started: float) -> tuple[list[Result], float]: + """Every check in order, and the seconds spent before mutation.""" + baseline = json.loads(BASELINE.read_text(encoding="utf-8")) + changed = changed_ranges(base) + ok, lines, allowed = read_waivers(opts.description_html, opts.description_sha256) + results: list[Result] = [("Waivers", ok, lines)] + + timing: list[float] = [] + ok, lines = guarded(coverage_step, opts, root, changed, baseline, allowed, timing) + results.append(("Coverage", ok, lines)) + suite_seconds = timing[0] if timing else 0.0 + nothing = "No lines were added or changed in TypeScript under src/." + results.append(("CRAP", *guarded(check_crap, changed, root, base)) if changed else ("CRAP", True, [nothing])) + results.append( + ("Duplication", *guarded(check_duplicates, changed, base, root)) if changed else ("Duplication", True, [nothing]) + ) + before_mutation = time.monotonic() - started + results.append(mutation_result(opts, changed, root, allowed, started, suite_seconds, results, nothing)) + return results, before_mutation + + +def mutation_result(opts, changed: Ranges, root: Path, allowed, started: float, suite_seconds: float, earlier, nothing: str) -> Result: + """Mutation runs last, and only when there is something to mutate and every earlier check passed.""" + if not changed: + return ("Mutation", True, [nothing]) + if not all(passed for _, passed, _ in earlier): + return ("Mutation", False, ["Not run: an earlier check failed. Fix it first; mutation runs after it passes."]) + if opts.skip_mutation: + return ("Mutation", True, ["Skipped (--skip-mutation)."]) + return ("Mutation", *guarded(check_mutation, changed, root, allowed, started, suite_seconds, opts.workers)) + + +def coverage_step(opts, root: Path, changed: Ranges, baseline: dict, allowed, timing: list[float]) -> tuple[bool, list[str]]: + """Run the suite with coverage (unless told to read the report as it is), time it into `timing`, and judge it.""" + if opts.skip_coverage_run: + return check_coverage(root / LCOV, changed, baseline, allowed) + timing.append(run_coverage(root)) + print(f"tests with coverage: {timing[0]:.0f}s", flush=True) + ok, lines = check_coverage(root / LCOV, changed, baseline, allowed) + return ok, [*lines, f"The suite with coverage took {timing[0]:.0f}s."] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.quality/mutant-tests.sh b/.quality/mutant-tests.sh new file mode 100755 index 0000000..8b08359 --- /dev/null +++ b/.quality/mutant-tests.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# Test command for mutator: the Vitest tests that can see one mutated source file under src/. +# +# mutator runs each mutant in a worker directory under target/mutation-workers: symbolic links to the real +# tree and a private copy of the mutated file. Vite resolves every module to its real path, so in that layout +# a test would import the original, unmutated file and every mutant would survive. This script first replaces +# the links with real copies: every top-level entry except node_modules (and folders no test reads), and, +# inside the folders mutator expanded on the way to the mutated file, every linked child. The mutated file is +# already a real file and is left alone. In the repository root (mutator's own unmutated run) nothing is a +# link and nothing moves. +# +# The file comes in MUTANT_FILE (set by gate.py), so no file name is ever part of a command string. +# +# A mutant is killed only when Vitest reports a failed test or a test file that failed to load. A run that +# writes no report (a broken config, a crash) is a harness error, never a kill: it passes here, so the mutant +# shows as a survivor and the gate fails where a reviewer can see it. +set -eu + +file=${MUTANT_FILE:?MUTANT_FILE is not set} +# No mutant may install a real launchd agent or touch the OS keychain or the user's own ~/.tether. +unset TETHER_REAL_SERVICE +export TETHER_KEYCHAIN=off +export TETHER_HOME="${TMPDIR:-/tmp}/tether-mutant-home-$$" +report=$(mktemp) +trap 'rm -rf "$TETHER_HOME" "$report"' EXIT + +materialize() { + for entry in "$1"/* "$1"/.[!.]*; do + [ -L "$entry" ] || continue + case "$entry" in + ./node_modules | ./.git | ./target | ./coverage | ./.metrics) continue ;; + esac + target=$(readlink "$entry") + rm -f "$entry" + cp -R "$target" "$entry" + done +} + +materialize . +dir=$(dirname "$file") +while [ "$dir" != "." ] && [ "$dir" != "/" ]; do + materialize "./$dir" + dir=$(dirname "$dir") +done + +vitest=./node_modules/.bin/vitest + +# Runs Vitest with the given arguments. Exits the script with 1 when a test failed or a test file failed to +# load; returns otherwise, a harness error included. +tier() { + rm -f "$report" + "$vitest" "$@" --bail=1 --coverage.enabled=false --reporter=json --outputFile="$report" \ + >/dev/null 2>&1 || true + [ -s "$report" ] || return 0 + failed=$(python3 -I -c 'import json, sys; d = json.load(open(sys.argv[1])); print(d["numFailedTests"] + d["numFailedTestSuites"])' \ + "$report" 2>/dev/null) || return 0 + if [ "$failed" -gt 0 ]; then + exit 1 + fi +} + +# Tier 1: the tests that import the file, directly or through other modules (Vitest's module graph). Most +# mutants die here in seconds. +# One Vitest worker: mutator already runs several mutants at once. +tier related --run --passWithNoTests --maxWorkers=1 "$file" +# Tier 2: the whole suite, for what reaches the file without importing it: the CLI entry points +# (src/cli/bin.ts, src/conformance/cli.ts) only run as child processes. Only a mutant that survived tier 1 +# pays for this, and it keeps Vitest's own workers: the suite waits on child processes, and on one worker it +# takes minutes, which mutator's unmutated run of every file would pay too. +tier run diff --git a/.quality/rendered-descriptions.json b/.quality/rendered-descriptions.json new file mode 100644 index 0000000..a6f6f39 --- /dev/null +++ b/.quality/rendered-descriptions.json @@ -0,0 +1,98 @@ +{ + "plain": { + "body": "Summary.\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Summary.

\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "backtick_fence": { + "body": "Example:\n\n```text\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n```\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Example:

\n
Equivalent mutant: src/a.ts:4 the two branches return the same value\n
\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "tilde_fence": { + "body": "~~~~\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n```\nstill code\n~~~~\n", + "html": "
Equivalent mutant: src/a.ts:3 the two branches return the same value\n```\nstill code\n
" + }, + "unclosed_fence": { + "body": " ```\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "
Equivalent mutant: src/a.ts:3 the two branches return the same value\n
" + }, + "comment": { + "body": "\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "unclosed_comment": { + "body": "text Equivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "Equivalent mutant: src/a.ts:3 the two branches return the same value" + }, + "link_reference_title": { + "body": "[x]: /u \"\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n\"\n", + "html": "" + }, + "multi_line_alt": { + "body": "\"\nEquivalent\n", + "html": "

\"\nEquivalent

" + }, + "attribute_across_blank_lines": { + "body": "
\n", + "html": "" + }, + "hidden_elements": { + "body": "\n\n\n\n\n\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "<script>\n\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n\n</script>\n<STYLE>\n\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n\n</STYLE>\n\n

Equivalent mutant: src/a.ts:4 the two branches return the same value

\n\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "continues_paragraph": { + "body": "Some text\nEquivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Some text
\nEquivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "indented": { + "body": "Text.\n\n Equivalent mutant: src/a.ts:3 the two branches return the same value\n", + "html": "

Text.

\n
Equivalent mutant: src/a.ts:3 the two branches return the same value\n
" + }, + "adjacent": { + "body": "Equivalent mutant: src/a.ts:3 the two branches return the same value\nEquivalent mutant: src/a.ts:4 the two branches return the same value\n\nEquivalent mutant: src/a.ts:5 the two branches return the same value\n", + "html": "

Equivalent mutant: src/a.ts:3 the two branches return the same value
\nEquivalent mutant: src/a.ts:4 the two branches return the same value

\n

Equivalent mutant: src/a.ts:5 the two branches return the same value

" + }, + "crlf": { + "body": "```\r\ncode\r\n```\r\n\r\nEquivalent mutant: src/a.ts:3 the two branches return the same value\r\n", + "html": "
code\n
\n

Equivalent mutant: src/a.ts:3 the two branches return the same value

" + }, + "backtick_in_info_string": { + "body": "```a`b\n\n", + "html": "

```a`b

\n" + }, + "fence_inside_html_block": { + "body": "
\n```\n
\nant: src/a.ts:3 {REASON}

" + self.assertEqual(set(gate.survivors_allowed(html)), {("src/a.ts", 3)}) + + def test_a_stray_end_tag_after_a_waiver_paragraph_adds_nothing_to_it(self) -> None: + # Once its paragraph closes, nothing that follows (a stray end tag, then text) is collected for it. + waiver = f"Equivalent mutant: src/a.ts:3 {REASON}" + parser = gate.TopLevelParagraphs() + parser.feed(f"

{waiver}

tail") + self.assertEqual(parser.found, [waiver]) + self.assertIsNone(parser.text) + parser.feed(f"

Equivalent mutant: src/a.ts:4 {REASON}

") + self.assertEqual(parser.found, [waiver, f"Equivalent mutant: src/a.ts:4 {REASON}"]) + + def test_a_bidi_control_on_a_later_line_of_the_paragraph_waives_nothing(self) -> None: + self.assertEqual(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:3\nthe two \u202ebranches return the same

"), {}) + self.assertEqual(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:3\n{REASON}

"), {("src/a.ts", 3): REASON}) + + def test_a_waiver_for_line_0_does_not_parse(self) -> None: + for line in ("0", "00"): + with self.subTest(line=line): + self.assertEqual(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:{line} {REASON}

"), {}) + self.assertEqual(set(gate.survivors_allowed(f"

Equivalent mutant: src/a.ts:10 {REASON}

")), {("src/a.ts", 10)}) + + def test_closing_an_element_closes_what_it_left_open(self) -> None: + waiver = f"Equivalent mutant: src/a.ts:3 {REASON}" + self.assertEqual(set(gate.survivors_allowed(f"

{waiver}

")), {("src/a.ts", 3)}) + + +class ReadWaivers(unittest.TestCase): + """The waivers come from a file an earlier workflow step rendered, checked against the sha256 that step gave.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.path = Path(self._tmp.name) / "pr-body.html" + + def tearDown(self) -> None: + self._tmp.cleanup() + + def rendered(self, html: str) -> tuple[str, str]: + self.path.write_bytes(html.encode()) + return str(self.path), hashlib.sha256(html.encode()).hexdigest() + + def test_waivers_are_read_from_the_rendered_file_that_matches_its_hash(self) -> None: + path, digest = self.rendered(RENDERED["plain"]["html"]) + ok, lines, allowed = gate.read_waivers(path, digest) + self.assertTrue(ok) + self.assertEqual(allowed, {("src/a.ts", 3): REASON}) + self.assertEqual(lines, ["1 waiver(s) count, each the whole text of a top-level paragraph on GitHub.", " src/a.ts:3"]) + + def test_the_hash_is_compared_without_regard_to_case_or_surrounding_space(self) -> None: + path, digest = self.rendered(RENDERED["plain"]["html"]) + ok, _lines, allowed = gate.read_waivers(path, f" {digest.upper()}\n") + self.assertTrue(ok) + self.assertEqual(set(allowed), {("src/a.ts", 3)}) + + def test_without_a_rendered_file_nothing_is_waived(self) -> None: + ok, lines, allowed = gate.read_waivers(None, None) + self.assertEqual((ok, allowed), (True, {})) + self.assertEqual(lines, ["No rendered description was given: nothing is waived."]) + + def test_a_file_that_cannot_be_read_fails(self) -> None: + ok, lines, allowed = gate.read_waivers(str(self.path), "0" * 64) + self.assertEqual((ok, allowed), (False, {})) + self.assertIn("cannot be read", lines[0]) + + def test_a_file_that_does_not_match_its_hash_fails(self) -> None: + path, digest = self.rendered(RENDERED["plain"]["html"]) + self.path.write_bytes(RENDERED["plain"]["html"].encode() + b"

x

") + for sha in (digest, None, ""): + with self.subTest(sha=sha): + ok, lines, allowed = gate.read_waivers(path, sha) + self.assertEqual((ok, allowed), (False, {})) + self.assertIn("does not match the sha256", lines[0]) + + def test_the_gate_no_longer_talks_to_the_network(self) -> None: + source = Path(gate.__file__).read_text(encoding="utf-8") + self.assertNotIn("urllib", source) + self.assertNotIn("GITHUB_TOKEN", source) + + +WORKFLOW = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "ci.yml" + + +def job_text(name: str) -> str: + """One job of ci.yml without its comment lines, so only what runs is checked.""" + lines = WORKFLOW.read_text(encoding="utf-8").splitlines(keepends=True) + text = "".join(line for line in lines if not line.lstrip().startswith("#")) + start = text.index(f"\n {name}:\n") + 1 + rest = text[start + 1 :] + following = [rest.index(f"\n {m}") for m in ("check:", "quality:") if f"\n {m}" in rest] + return text[start : start + 1 + min(following)] if following else text[start:] + + +def workflow_steps() -> list[str]: + """Each step of the quality job as its block of text, in order.""" + text = job_text("quality") + body = text[text.index(" steps:\n") :] + return [block for block in body.split("\n - ")[1:]] + + +class QualityWorkflow(unittest.TestCase): + """The token reaches only the steps that run before any code from the pull request, and no action's post step.""" + + CREDENTIAL = ("github.token", "secrets.", "GITHUB_TOKEN") + + def step(self, needle: str) -> tuple[int, str]: + found = [(n, block) for n, block in enumerate(workflow_steps()) if needle in block] + self.assertEqual(len(found), 1, needle) + return found[0] + + def test_the_job_runs_on_every_pull_request_on_one_hosted_linux_runner(self) -> None: + text = WORKFLOW.read_text(encoding="utf-8") + on = text[text.index("\non:") : text.index("\njobs:")] + self.assertNotIn("paths", on) # a path filter would let a PR skip the gate + job = job_text("quality") + self.assertIn("runs-on: ubuntu-latest", job) + self.assertIn("if: github.event_name == 'pull_request'", job) + + def test_only_the_checkout_and_render_steps_get_the_token(self) -> None: + checkout, _ = self.step("name: Check out the pull request") + render, _ = self.step("name: Render the pull request description") + install, _ = self.step("run: bun install --frozen-lockfile") + self.assertEqual((checkout, render), (0, 1)) + self.assertLess(render, install) + holders = [n for n, block in enumerate(workflow_steps()) if any(word in block for word in self.CREDENTIAL)] + self.assertEqual(holders, [checkout, render]) + job = job_text("quality") + self.assertEqual(sum(job.count(word) for word in self.CREDENTIAL), 2) # not at job level either + + def test_no_action_takes_the_token_by_default(self) -> None: + self.assertNotIn("actions/checkout", job_text("quality")) + for action in ("oven-sh/setup-bun@", "actions/setup-python@"): + with self.subTest(action=action): + _, block = self.step(action) + self.assertIn("token: ''", block) + uses = [block for block in workflow_steps() if block.startswith("uses:")] + self.assertEqual(len(uses), 2) + + def test_steps_with_the_token_fail_on_any_broken_command_and_inline_no_expression(self) -> None: + for name in ("name: Check out the pull request", "name: Render the pull request description"): + with self.subTest(step=name): + _, block = self.step(name) + script = block[block.index("run: |") :] + self.assertIn("set -o pipefail", script) + self.assertNotIn("${{", script) + _, render = self.step("name: Render the pull request description") + self.assertIn("/markdown", render) + self.assertEqual(render.count("--fail "), 2) + + def test_the_token_can_read_the_pull_request_and_nothing_more(self) -> None: + job = job_text("quality") + block = job[job.index(" permissions:\n") :].split("\n runs-on")[0] + self.assertEqual(block.splitlines()[1:], [" contents: read", " pull-requests: read"]) + text = WORKFLOW.read_text(encoding="utf-8") + top = text[text.index("\npermissions:") : text.index("\njobs:")] + self.assertNotIn("write", top) + + def test_the_gate_reads_the_rendered_file_and_its_hash_and_outlives_its_budget(self) -> None: + _, block = self.step("name: Quality gate") + self.assertIn('--description-html "$RUNNER_TEMP/pr-body.html"', block) + self.assertIn("--description-sha256", block) + self.assertIn("steps.description.outputs.sha256", block) + minutes = int(job_text("quality").split("timeout-minutes:")[1].split()[0]) + # The job must outlive the largest mutation budget, or the runner kills it and the report with it. + self.assertGreater(minutes * 60, gate.MUTATION_BUDGET_MAX + 600) + + +LCOV = textwrap.dedent( + """\ + TN: + SF:src/a.ts + FNF:4 + FNH:3 + DA:1,3 + DA:2,0 + DA:5,0 + DA:6,1 + LF:4 + LH:2 + end_of_record + SF:/abs/checkout/src/b.ts + FNF:1 + FNH:1 + DA:1,0 + LF:6 + LH:5 + end_of_record + SF:test/helpers.ts + FNF:10 + FNH:0 + LF:100 + LH:0 + end_of_record + """ +) + + +class UncoveredChangedLines(unittest.TestCase): + def test_an_executable_changed_line_with_no_hits_is_reported(self) -> None: + found = gate.uncovered_changed({"src/a.ts": [(1, 2), (6, 6)]}, LCOV) + self.assertEqual(found, [("src/a.ts", 2)]) + + def test_a_changed_line_that_is_not_executable_is_ignored(self) -> None: + self.assertEqual(gate.uncovered_changed({"src/a.ts": [(3, 4)]}, LCOV), []) + + def test_absolute_paths_in_the_report_match_by_suffix(self) -> None: + self.assertEqual(gate.uncovered_changed({"src/b.ts": [(1, 1)]}, LCOV), [("src/b.ts", 1)]) + + def test_a_file_missing_from_the_report_is_reported_whole(self) -> None: + self.assertEqual(gate.uncovered_changed({"src/c.ts": [(4, 5)]}, LCOV), [("src/c.ts", 0)]) + + +class LcovTotals(unittest.TestCase): + def test_the_totals_count_only_in_scope_files(self) -> None: + # src/a.ts 2 of 4 lines and 3 of 4 functions; the absolute b.ts path is not in scope by name, and + # the test helper never counts. + self.assertEqual(gate.lcov_totals(LCOV), {"lines": 50.0, "functions": 75.0}) + + def test_an_empty_report_is_full(self) -> None: + self.assertEqual(gate.lcov_totals(""), {"lines": 100.0, "functions": 100.0}) + + +class CoverageVerdict(unittest.TestCase): + BASELINE = {"coverage": {"lines": 98.92, "functions": 95.07}} + + def verdict(self, uncovered, allowed=None, now=None): + return gate.coverage_verdict(uncovered, allowed or {}, now or {"lines": 90.0, "functions": 90.0}, self.BASELINE) + + def test_a_lower_repo_total_alone_does_not_fail_but_is_reported(self) -> None: + ok, lines = self.verdict([]) + self.assertTrue(ok) + self.assertIn("lines 90.0% (baseline 98.92%)", lines[1]) + self.assertTrue(any("Below the baseline" in line for line in lines)) + + def test_a_higher_repo_total_asks_for_the_baseline_to_be_updated(self) -> None: + _, lines = self.verdict([], now={"lines": 99.0, "functions": 95.07}) + self.assertTrue(any("update coverage in .quality/baseline.json" in line for line in lines)) + + def test_an_uncovered_changed_line_fails(self) -> None: + ok, _ = self.verdict([("src/a.ts", 2)]) + self.assertFalse(ok) + + def test_an_override_with_a_reason_waives_the_line(self) -> None: + ok, _ = self.verdict([("src/a.ts", 2)], {("src/a.ts", 2): "only reachable as another user"}) + self.assertTrue(ok) + + def test_no_waiver_covers_a_file_missing_from_the_coverage_report(self) -> None: + allowed = gate.survivors_allowed(f"

Equivalent mutant: src/c.ts:0 {REASON}

") + ok, lines = self.verdict(gate.uncovered_changed({"src/c.ts": [(4, 5)]}, LCOV), allowed) + self.assertFalse(ok) + self.assertTrue(any("src/c.ts (not in the coverage report" in line for line in lines)) + + +DRYER = ( + "Scanning src\n" + "DUPLICATE score=0.91 kind=function\n src/a.ts:1-9\n src/b.ts:3-11\n" + "DUPLICATE score=0.85 kind=function\n src/c.ts:2-8\n src/d.ts:4-10\n" +) + + +class DryerOutput(unittest.TestCase): + def test_the_output_is_parsed_into_pairs(self) -> None: + self.assertEqual( + gate.parse_dryer(DRYER), + [(0.91, ("src/a.ts", 1, 9), ("src/b.ts", 3, 11)), + (0.85, ("src/c.ts", 2, 8), ("src/d.ts", 4, 10))], + ) + + def test_no_duplicates_is_no_pairs(self) -> None: + self.assertEqual(gate.parse_dryer("No duplicate candidates found.\n"), []) + + def test_a_block_the_gate_cannot_read_stops_the_gate(self) -> None: + for block in ( + "DUPLICATE kind=function\n src/e.ts:1-2\n src/f.ts:1-2\n", + "DUPLICATE score=0.9\n src/e.ts:1-2\n", + ): + with self.subTest(block=block), self.assertRaises(SystemExit) as stop: + gate.parse_dryer(DRYER + block) + self.assertIn("3 DUPLICATE blocks but 2 pairs", str(stop.exception.code)) + + def test_only_a_marker_at_the_start_of_a_line_starts_a_block(self) -> None: + preamble = "score=0.5 in a header\n src/e.ts:1-2\n src/f.ts:1-2\n" + self.assertEqual(len(gate.parse_dryer(preamble + DRYER)), 2) + + +class DuplicateVerdict(unittest.TestCase): + PAIR = (0.9, ("src/a.ts", 10, 20), ("src/b.ts", 30, 40)) + + def test_a_new_pair_on_a_changed_line_fails(self) -> None: + ok, lines = gate.duplicate_verdict({"src/a.ts": [(12, 12)]}, [], [self.PAIR]) + self.assertFalse(ok) + self.assertIn(" new duplicate: src/a.ts:10-20 and src/b.ts:30-40 (similarity 0.90)", lines) + + def test_a_pair_already_on_base_passes(self) -> None: + ok, _ = gate.duplicate_verdict({"src/a.ts": [(12, 12)]}, [self.PAIR], [self.PAIR]) + self.assertTrue(ok) + + def test_a_pair_the_pr_did_not_touch_passes(self) -> None: + ok, _ = gate.duplicate_verdict({"src/a.ts": [(1, 5)]}, [], [self.PAIR]) + self.assertTrue(ok) + + +class DryerRun(InRepo): + @unittest.skipUnless(shutil.which("dryer"), "needs dryer") + def test_a_copied_function_is_found_by_the_real_dryer_and_tests_are_skipped(self) -> None: + body = "export function f{n}(a: number[]): number {{\n let t = 0\n for (const x of a) {{\n if (x > 2) {{\n t += x * 3\n }} else {{\n t -= x\n }}\n }}\n return t\n}}\n" + self.write("src/c.ts", body.format(n=1)) + self.write("src/d.ts", body.format(n=2)) + self.write("test/e.ts", body.format(n=3)) + pairs = gate.dryer_pairs(self.repo) + self.assertEqual({tuple(sorted((p[1][0], p[2][0]))) for p in pairs}, {("src/c.ts", "src/d.ts")}) + self.assertFalse((self.repo / ".metrics").exists()) + + +SLOW = 'case "$*" in *--scan*) echo " src/x.ts:1 a -> b";; *) sleep 30;; esac\n' + + +class MutationBudget(InRepo): + def stub_mutator(self, body: str) -> Path: + bin_dir = self.repo / "bin" + bin_dir.mkdir(exist_ok=True) + script = bin_dir / "mutator" + script.write_text("#!/bin/sh\n" + body) + script.chmod(script.stat().st_mode | stat.S_IEXEC) + return bin_dir + + def check(self, body: str, changed, seconds_left: float, workers: int = 1): + """check_mutation with the whole budget (30 minutes, the most there is) spent but `seconds_left`.""" + bin_dir = self.stub_mutator(body) + started = time.monotonic() - gate.MUTATION_BUDGET_MAX + seconds_left + with mock.patch.dict(os.environ, {"PATH": f"{bin_dir}:{os.environ['PATH']}"}): + return gate.check_mutation(changed, self.repo, {}, started, 1000.0, workers) + + def test_running_out_of_budget_fails_cleanly_with_what_was_not_checked(self) -> None: + ok, lines = self.check(SLOW, {"src/a.ts": [(1, 3)], "src/b.ts": [(1, 2)]}, 1.0) + text = "\n".join(lines) + self.assertFalse(ok) + self.assertIn("budget ran out", text) + self.assertIn("src/a.ts", text) + self.assertIn("src/b.ts", text) + self.assertLess(text.index("re-run"), text.index("split")) + + def test_the_slow_mutator_is_actually_stopped(self) -> None: + start = time.monotonic() + self.check(SLOW, {"src/a.ts": [(1, 3)]}, 1.0) + self.assertLess(time.monotonic() - start, 10) + + def test_a_finished_run_inside_the_budget_reports_survivors(self) -> None: + body = 'case "$*" in *--scan*) echo " src/a.ts:1 > -> >=";; *) echo "SURVIVED src/a.ts:1 > -> >="; exit 3;; esac\n' + ok, lines = self.check(body, {"src/a.ts": [(1, 1)]}, 60.0) + self.assertFalse(ok) + self.assertIn("survivor src/a.ts:1", "\n".join(lines)) + + def test_a_mutator_failure_on_one_file_still_reports_the_others(self) -> None: + body = ( + 'case "$*" in *--scan*) echo " $1:1 a -> b"; exit 0;; esac\n' + 'case "$1" in *a.ts) echo "KILLED $1:1 a -> b"; exit 0;; *) echo boom; exit 2;; esac\n' + ) + ok, lines = self.check(body, {"src/a.ts": [(1, 1)], "src/b.ts": [(1, 1)]}, 60.0, workers=2) + text = "\n".join(lines) + self.assertFalse(ok) + self.assertIn("1 mutant(s) killed", text) + self.assertIn("mutator failed on src/b.ts (exit 2)", text) + + def test_the_file_reaches_the_test_command_through_the_environment_only(self) -> None: + record = self.repo / "record.txt" + body = ( + 'case "$*" in *--scan*) echo " x:1 a -> b"; exit 0;; esac\n' + f'echo "$*" > "{record}"; echo "env=$MUTANT_FILE real=${{TETHER_REAL_SERVICE:-unset}} keychain=${{TETHER_KEYCHAIN:-unset}}" >> "{record}"\n' + ) + with mock.patch.dict(os.environ, {"TETHER_REAL_SERVICE": "1", "TETHER_KEYCHAIN": "on"}): + self.check(body, {"src/client/policy.ts": [(1, 2)]}, 60.0) + args, env_line = record.read_text().splitlines()[:2] + self.assertNotIn("policy.ts", args.split("--test-command", 1)[1]) + self.assertEqual(env_line, "env=src/client/policy.ts real=unset keychain=off") # no mutant may install a launchd agent or reach the keychain + + def test_a_file_name_is_not_run_through_a_shell(self) -> None: + self.assertFalse(gate.safe_path("src/a.ts; touch pwned")) + self.assertFalse(gate.safe_path("src/../../etc.ts")) + self.assertTrue(gate.safe_path("src/client/policy.ts")) + + def test_no_more_mutator_processes_than_the_limit_run_at_once(self) -> None: + running = self.repo / "running" + counts = self.repo / "counts.txt" + running.mkdir() + body = ( + 'case "$*" in *--scan*) echo " $1:1 a -> b"; exit 0;; esac\n' + f'touch "{running}/$$"; ls "{running}" | wc -l >> "{counts}"; sleep 1; rm "{running}/$$"\n' + ) + self.check(body, {f"src/f{i}.ts": [(1, 1)] for i in range(6)}, 120.0, workers=2) + peak = max(int(line) for line in counts.read_text().split()) + self.assertLessEqual(peak, 2) + self.assertEqual(len(counts.read_text().split()), 6) + + def test_the_kill_scans_again_until_no_tagged_process_is_left(self) -> None: + with mock.patch.object(gate, "_kill_pass", side_effect=[2, 1, 0]) as scan: + gate.kill_tagged("tag") + self.assertEqual(scan.call_count, 3) + + +class BudgetSizing(unittest.TestCase): + def test_a_small_change_gets_the_five_minute_floor(self) -> None: + self.assertEqual(gate.mutation_budget(60.0, {"a": 2}, {"a": 2}, 2, 30.0), 300.0) + + def test_the_budget_grows_with_the_rounds_the_sites_need(self) -> None: + # 20 sites on 2 workers: 10 rounds plus mutator's unmutated run, 11 x 40 s, after 100 s already spent. + self.assertEqual(gate.mutation_budget(100.0, {"a": 20}, {"a": 2}, 2, 40.0), 540.0) + + def test_files_beyond_the_slots_queue(self) -> None: + # Three files of 1 site on 1 worker each, 2 slots: 6 rounds over 2 slots is 3, 3 x 100 s. + self.assertEqual(gate.mutation_budget(0.0, {"a": 1, "b": 1, "c": 1}, {"a": 1, "b": 1, "c": 1}, 2, 100.0), 300.0) + self.assertEqual(gate.mutation_budget(0.0, {"a": 1, "b": 1, "c": 1}, {"a": 1, "b": 1, "c": 1}, 2, 150.0), 450.0) + + def test_the_budget_never_passes_thirty_minutes(self) -> None: + self.assertEqual(gate.mutation_budget(0.0, {"a": 500}, {"a": 2}, 2, 96.0), 1800.0) + + def test_a_scan_that_did_not_finish_counts_as_one_site(self) -> None: + self.assertEqual(gate.mutation_budget(0.0, {"a": "unknown"}, {"a": 1}, 1, 200.0), 400.0) + + +class Workers(unittest.TestCase): + def test_workers_follow_the_sites_of_each_file(self) -> None: + self.assertEqual(gate.plan_workers({"a": 100}, 4), {"a": 4}) + self.assertEqual(gate.plan_workers({"a": 0}, 4), {}) + self.assertEqual(gate.plan_workers({"a": 5, "b": 5, "c": 5}, 2), {"a": 1, "b": 1, "c": 1}) + + def test_the_total_never_goes_over_the_limit(self) -> None: + many = {"big": 100, **{f"f{i}": 1 for i in range(23)}} + self.assertEqual(sum(gate.plan_workers(many, 24).values()), 24) + plan = gate.plan_workers({"a": "unknown", "b": 100, "c": "unknown"}, 24) + self.assertEqual(sum(plan.values()), 24) + + def test_a_cgroup_cpu_quota_caps_the_cores(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "cpu.max").write_text("200000 100000\n") + self.assertEqual(gate.available_cores(Path(tmp)), min(2, gate.available_cores(Path(tmp) / "none"))) + + def test_memory_caps_the_workers(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "memory.max").write_text(str(8 << 30) + "\n") + self.assertEqual(gate.worker_limit(24, gate.memory_limit(Path(tmp))), 6) + Path(tmp, "memory.max").write_text("max\n") + meminfo = Path(tmp, "meminfo") + meminfo.write_text("MemTotal: 16777216 kB\n") + self.assertEqual(gate.memory_limit(Path(tmp), meminfo), 16 << 30) + self.assertEqual(gate.worker_limit(4, None), 4) + self.assertEqual(gate.worker_limit(4, 1 << 30), 1) + + +class CrapVerdict(unittest.TestCase): + """The CRAP ratchet: a function already over the limit on base may stay, never get worse.""" + + def verdict(self, head, base, touched=None): + keys = {key for key, *_ in head} if touched is None else touched + return gate.crap_verdict(head, keys, base) + + @staticmethod + def fn(crap, name="big", path="src/a.ts", nth=0, line=10): + return ((path, name, nth), line, crap, 40, 90.0) + + def test_same_score_as_base_passes_and_says_it_is_still_over(self) -> None: + ok, lines = self.verdict([self.fn(67.3)], {("src/a.ts", "big", 0): 67.3}) + self.assertTrue(ok) + self.assertTrue(any("still over the limit" in line and "67.3" in line and "base 67.3" in line for line in lines)) + self.assertFalse(any("over the limit:" in line for line in lines)) + + def test_lower_score_than_base_passes(self) -> None: + ok, _ = self.verdict([self.fn(60.0)], {("src/a.ts", "big", 0): 67.3}) + self.assertTrue(ok) + + def test_higher_score_than_base_fails_and_prints_both(self) -> None: + ok, lines = self.verdict([self.fn(67.4)], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + self.assertTrue(any("over the limit:" in line and "67.4" in line and "was 67.3 on base (higher)" in line for line in lines)) + + def test_a_higher_raw_score_fails_even_when_it_prints_the_same(self) -> None: + ok, lines = self.verdict([self.fn(67.34)], {("src/a.ts", "big", 0): 67.26}) + self.assertFalse(ok) + self.assertTrue(any("67.3" in line and "higher" in line for line in lines)) + + def test_an_equal_raw_score_passes(self) -> None: + ok, _ = self.verdict([self.fn(67.26)], {("src/a.ts", "big", 0): 67.26}) + self.assertTrue(ok) + + def test_a_new_function_with_a_name_the_file_already_had_is_new(self) -> None: + # Head has two "big"; base had one. Matching by order would compare the new first one with the old one. + head = [self.fn(15.0, nth=0), self.fn(67.0, nth=1)] + ok, lines = self.verdict(head, {("src/a.ts", "big", 0): 67.0}) + self.assertFalse(ok) + self.assertTrue(any("new function" in line for line in lines)) + + def test_a_function_removed_from_a_same_named_pair_is_new_too(self) -> None: + ok, _ = self.verdict([self.fn(67.0)], {("src/a.ts", "big", 0): 67.0, ("src/a.ts", "big", 1): 80.0}) + self.assertFalse(ok) + + def test_the_same_count_of_a_name_still_matches_by_order(self) -> None: + head = [self.fn(20.0, nth=0), self.fn(25.0, nth=1)] + ok, _ = self.verdict(head, {("src/a.ts", "big", 0): 20.0, ("src/a.ts", "big", 1): 25.0}) + self.assertTrue(ok) + + def test_names_in_another_file_do_not_change_the_count(self) -> None: + head = [self.fn(67.0), self.fn(5.0, path="src/b.ts", nth=0), self.fn(6.0, path="src/b.ts", nth=1)] + ok, _ = self.verdict(head, {("src/a.ts", "big", 0): 67.0}) + self.assertTrue(ok) + + def test_under_the_limit_on_base_and_over_now_fails(self) -> None: + ok, lines = self.verdict([self.fn(10.0)], {("src/a.ts", "big", 0): 9.9}) + self.assertFalse(ok) + self.assertTrue(any("was 9.9 on base (under the limit)" in line for line in lines)) + + def test_a_new_function_over_the_limit_fails(self) -> None: + ok, lines = self.verdict([self.fn(12.0, name="fresh")], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + self.assertTrue(any("new function" in line for line in lines)) + + def test_a_renamed_function_is_new(self) -> None: + ok, _ = self.verdict([self.fn(67.3, name="renamed")], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + + def test_a_moved_function_is_new(self) -> None: + ok, _ = self.verdict([self.fn(67.3, path="src/b.ts")], {("src/a.ts", "big", 0): 67.3}) + self.assertFalse(ok) + + def test_a_second_function_of_the_same_name_is_matched_by_order(self) -> None: + head = [self.fn(20.0, nth=0), self.fn(30.0, nth=1)] + ok, _ = self.verdict(head, {("src/a.ts", "big", 0): 20.0, ("src/a.ts", "big", 1): 25.0}) + self.assertFalse(ok) + + def test_a_function_under_the_limit_passes_without_a_base(self) -> None: + ok, lines = self.verdict([self.fn(9.9, name="small")], None) + self.assertTrue(ok) + self.assertIn("1 added or changed function(s) scored", lines[0]) + + def test_functions_the_pr_did_not_touch_are_not_judged(self) -> None: + ok, lines = self.verdict([self.fn(99.0)], {}, touched=set()) + self.assertTrue(ok) + self.assertIn("0 added or changed function(s) scored", lines[0]) + + + +class CrapBase(InRepo): + """base_scores, touched_functions and crapper on an lcov report, against a throwaway repository.""" + + def test_touched_functions_names_only_the_functions_with_a_changed_line(self) -> None: + self.write("src/f.ts", "function one() {\n return 1;\n}\n\nfunction two() {\n return 2;\n}\n\nfunction two() {\n return 3;\n}\n") + self.assertEqual(gate.touched_functions(self.repo, {"src/f.ts": [(6, 6)]}), {("src/f.ts", "two", 0)}) + self.assertEqual( + gate.touched_functions(self.repo, {"src/f.ts": [(1, 1), (10, 10)]}), + {("src/f.ts", "one", 0), ("src/f.ts", "two", 1)}, + ) + + def test_crapper_scores_from_the_line_records(self) -> None: + source = ( + "export function pick(a: number, b: number): number {\n" + " if (a > b) {\n return a\n }\n if (a < 0) {\n return 0\n }\n return b\n}\n" + ) + self.write("src/p.ts", source) + # Only FNF/FNH counts and DA lines, so the score comes from the lines. Lines 3 and 6 never ran. + hits = {1: 4, 2: 4, 3: 0, 4: 4, 5: 4, 6: 0, 7: 4, 8: 4, 9: 4} + self.write( + "coverage/lcov.info", + "TN:\nSF:src/p.ts\nFNF:1\nFNH:1\n" + "".join(f"DA:{n},{h}\n" for n, h in hits.items()) + + "LF:9\nLH:7\nend_of_record\n", + ) + [(key, _line, crap, complexity, coverage)] = gate.scored_functions(self.repo, ["src/p.ts"]) + self.assertEqual(key, ("src/p.ts", "pick", 0)) + self.assertEqual(complexity, 3) + self.assertGreater(coverage, 50.0) # read from the DA lines, not 0% for a missing FN record + self.assertLess(coverage, 100.0) + self.assertLess(crap, 9 + 3) # 0% coverage would score complexity squared plus complexity + + def run_base_scores(self, scored, paths): + runs: list[Path] = [] + (self.repo / "node_modules").mkdir() + with mock.patch.object(gate, "run_coverage", side_effect=lambda root: runs.append(root) or 1.0), \ + mock.patch.object(gate, "scored_functions", scored): + return gate.base_scores(self.base, self.repo, paths), runs + + def test_base_scores_measures_the_files_that_exist_on_base_with_coverage_required(self) -> None: + seen = {} + + def scored(root, paths, need_coverage=False): + seen.update(need_coverage=need_coverage, modules=(root / "node_modules").is_symlink(), paths=paths) + return [(("src/a.ts", "f", 0), 1, 42.0, 7, 80.0)] + + result, runs = self.run_base_scores(scored, ["src/a.ts", "src/new.ts"]) + self.assertEqual(result, {("src/a.ts", "f", 0): 42.0}) + self.assertEqual(seen, {"need_coverage": True, "modules": True, "paths": ["src/a.ts"]}) + self.assertEqual(len(runs), 1) + self.assertNotEqual(runs[0], self.repo) # the base tree, not this checkout + + def test_base_scores_runs_nothing_when_no_file_is_on_base_and_removes_its_worktree(self) -> None: + result, runs = self.run_base_scores(lambda root, paths, need_coverage=False: [], ["src/new.ts"]) + self.assertEqual((result, runs), ({}, [])) + self.assertNotIn("quality-base", git(self.repo, "worktree", "list")) + + def test_a_base_function_without_a_coverage_record_is_not_scored(self) -> None: + class Bundle: + def bind_sources(self, paths): + pass + + def percent_for(self, function): + return None + + class Fn: + name = "big" + start_line = 1 + end_line = 3 + + entry = mock.Mock(crap=9999.0, complexity=70, coverage=0.0) + with ( + mock.patch("crapper.coverage.load_bundle", return_value=Bundle()), + mock.patch("crapper.languages.functions_in_file", return_value=[Fn()]), + mock.patch("crapper.crap.make_entry", return_value=entry), + ): + self.write("src/f.ts", "x\n") + self.assertEqual(gate.scored_functions(self.repo, ["src/f.ts"], need_coverage=True), []) + self.assertEqual(len(gate.scored_functions(self.repo, ["src/f.ts"])), 1) + + +REPO = Path(__file__).resolve().parent.parent +VITEST_CONFIG = 'import { defineConfig } from "vitest/config";\nexport default defineConfig({ test: { include: ["test/**/*.test.ts"] } });\n' + + +@unittest.skipUnless((REPO / "node_modules/.bin/vitest").exists(), "needs bun install") +class MutantTestsScript(unittest.TestCase): + """mutant-tests.sh with the real Vitest, on a throwaway project that borrows this checkout's node_modules.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) / "project" + root = self.root + (root / ".quality").mkdir(parents=True) + shutil.copy(REPO / ".quality/mutant-tests.sh", root / ".quality/mutant-tests.sh") + (root / "node_modules").symlink_to(REPO / "node_modules", target_is_directory=True) + (root / "vitest.config.ts").write_text(VITEST_CONFIG) + (root / "src/lib").mkdir(parents=True) + (root / "test").mkdir() + (root / "src/lib/a.ts").write_text("export function one(): number {\n return 1;\n}\n") + (root / "test/a.test.ts").write_text( + 'import { expect, test } from "vitest";\nimport { one } from "../src/lib/a.ts";\n' + 'test("one", () => { expect(one()).toBe(1); });\n' + 'test("isolated", () => {\n expect(process.env.TETHER_REAL_SERVICE).toBeUndefined();\n' + ' expect(process.env.TETHER_KEYCHAIN).toBe("off");\n expect(process.env.TETHER_HOME).toContain("tether-mutant-home");\n});\n' + ) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def run_script(self, cwd: Path, file: str = "src/lib/a.ts") -> subprocess.CompletedProcess[str]: + env = {**os.environ, "MUTANT_FILE": file, "TETHER_REAL_SERVICE": "1", "TETHER_KEYCHAIN": "on"} + return subprocess.run(["sh", ".quality/mutant-tests.sh"], cwd=cwd, env=env, capture_output=True, text=True) + + def worker(self, mutated: str) -> Path: + """mutator's layout: links to every top-level entry, src and src/lib expanded, a.ts a private mutated copy.""" + worker = Path(self._tmp.name) / "worker" + (worker / "src/lib").mkdir(parents=True) + for entry in self.root.iterdir(): + if entry.name != "src": + (worker / entry.name).symlink_to(entry) + (worker / "src/lib/a.ts").write_text(mutated) + return worker + + def test_the_tests_pass_on_the_unmutated_file_with_the_service_and_keychain_kept_away(self) -> None: + result = self.run_script(self.root) + self.assertEqual(result.returncode, 0, result.stdout[-1500:] + result.stderr[-1500:]) + + def test_a_mutant_the_tests_catch_fails_the_command(self) -> None: + (self.root / "src/lib/a.ts").write_text("export function one(): number {\n return 2;\n}\n") + self.assertNotEqual(self.run_script(self.root).returncode, 0) + + def test_in_a_worker_of_links_the_tests_see_the_mutated_copy(self) -> None: + # Vite resolves links to their real path, so a linked test would import the original file and pass. + worker = self.worker("export function one(): number {\n return 2;\n}\n") + self.assertNotEqual(self.run_script(worker).returncode, 0) + self.assertFalse((worker / "test").is_symlink()) + self.assertTrue((worker / "node_modules").is_symlink()) + + def test_in_a_worker_of_links_the_unmutated_copy_passes(self) -> None: + worker = self.worker((self.root / "src/lib/a.ts").read_text()) + result = self.run_script(worker) + self.assertEqual(result.returncode, 0, result.stdout[-1500:] + result.stderr[-1500:]) + + def entry_point(self, source: str) -> None: + """src/entry.ts, which no test imports: a test reads it the way the CLI tests start bin.ts, as a process.""" + (self.root / "src/entry.ts").write_text(source) + (self.root / "test/entry.test.ts").write_text( + 'import { readFileSync } from "node:fs";\nimport { expect, test } from "vitest";\n' + 'test("entry", () => { expect(readFileSync("src/entry.ts", "utf8")).toContain("exit(0)"); });\n' + ) + + def test_a_file_no_test_imports_passes_unmutated(self) -> None: + self.entry_point("process.exit(0);\n") + result = self.run_script(self.root, "src/entry.ts") + self.assertEqual(result.returncode, 0, result.stdout[-1500:] + result.stderr[-1500:]) + + def test_a_mutant_in_a_file_no_test_imports_is_killed_by_the_whole_suite(self) -> None: + self.entry_point("process.exit(1);\n") + self.assertNotEqual(self.run_script(self.root, "src/entry.ts").returncode, 0) + + def test_a_broken_harness_is_never_a_kill(self) -> None: + (self.root / "vitest.config.ts").write_text('throw new Error("broken config");\n') + (self.root / "src/lib/a.ts").write_text("export function one(): number {\n return 2;\n}\n") + self.assertEqual(self.run_script(self.root).returncode, 0) + + +class Baseline(unittest.TestCase): + def test_the_committed_baseline_holds_every_number_the_standard_asks_for(self) -> None: + baseline = json.loads((REPO / gate.BASELINE).read_text(encoding="utf-8")) + for key in ("lines", "functions"): + self.assertIsInstance(baseline["coverage"][key], float) + for key in ("functionsMeasured", "functionsOverCrap10", "duplicatePairs"): + self.assertIsInstance(baseline[key], int) + self.assertIsInstance(baseline["mutation"]["score"], float) + + +class CrapFailure(unittest.TestCase): + def test_each_reason_a_function_over_the_limit_fails_or_stays(self) -> None: + self.assertIn("new function", gate.crap_failure(12.0, None)) + self.assertIn("under the limit", gate.crap_failure(12.0, 9.9)) + self.assertIn("higher", gate.crap_failure(12.1, 12.0)) + self.assertIsNone(gate.crap_failure(12.0, 12.0)) + + +class CheckCrap(InRepo): + def test_the_base_is_measured_only_for_files_with_a_touched_function_over_the_limit(self) -> None: + head = [(("src/a.ts", "big", 0), 1, 20.0, 4, 0.0), (("src/b.ts", "ok", 0), 1, 2.0, 1, 100.0)] + touched = {("src/a.ts", "big", 0), ("src/b.ts", "ok", 0)} + with mock.patch.object(gate, "scored_functions", return_value=head), \ + mock.patch.object(gate, "touched_functions", return_value=touched), \ + mock.patch.object(gate, "base_scores", return_value={("src/a.ts", "big", 0): 20.0}) as base: + ok, lines = gate.check_crap({"src/a.ts": [(1, 1)], "src/b.ts": [(1, 1)]}, self.repo, self.base) + base.assert_called_once_with(self.base, self.repo, ["src/a.ts"]) + self.assertTrue(ok) + self.assertIn("still over the limit, no worse than base", "\n".join(lines)) + + def test_no_base_run_when_nothing_touched_is_over_the_limit(self) -> None: + head = [(("src/b.ts", "ok", 0), 1, 2.0, 1, 100.0)] + with mock.patch.object(gate, "scored_functions", return_value=head), \ + mock.patch.object(gate, "touched_functions", return_value={("src/b.ts", "ok", 0)}), \ + mock.patch.object(gate, "base_scores") as base: + ok, _ = gate.check_crap({"src/b.ts": [(1, 1)]}, self.repo, self.base) + base.assert_not_called() + self.assertTrue(ok) + + +class RunCoverage(InRepo): + def stub_bun(self, body: str) -> Path: + bin_dir = self.repo / "bin" + bin_dir.mkdir() + (bin_dir / "bun").write_text("#!/bin/sh\n" + body) + (bin_dir / "bun").chmod(0o755) + return bin_dir + + def run_with(self, body: str): + bin_dir = self.stub_bun(body) + with mock.patch.dict(os.environ, {"PATH": f"{bin_dir}:{os.environ['PATH']}", "TETHER_REAL_SERVICE": "1"}): + return gate.run_coverage(self.repo) + + def test_a_stale_report_is_deleted_and_the_run_writes_a_new_one_without_the_real_service_or_keychain(self) -> None: + self.write("coverage/stale.txt", "old\n") + body = 'mkdir -p coverage; echo "args=$* real=${TETHER_REAL_SERVICE:-unset} keychain=${TETHER_KEYCHAIN:-unset}" > coverage/lcov.info\n' + self.assertGreaterEqual(self.run_with(body), 0.0) + self.assertFalse((self.repo / "coverage/stale.txt").exists()) + text = (self.repo / "coverage/lcov.info").read_text() + self.assertIn("run coverage --coverage.reporter=lcov --coverage.reportsDirectory=coverage", text) + self.assertIn("real=unset keychain=off", text) + + def test_failing_tests_stop_the_gate(self) -> None: + with self.assertRaises(SystemExit) as stop: + self.run_with("echo boom; exit 1\n") + self.assertIn("the tests failed under coverage", str(stop.exception.code)) + + def test_a_run_that_writes_no_report_stops_the_gate(self) -> None: + with self.assertRaises(SystemExit) as stop: + self.run_with("exit 0\n") + self.assertIn("wrote no coverage/lcov.info", str(stop.exception.code)) + + +class KillTagged(unittest.TestCase): + def test_only_a_process_with_the_tag_is_killed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + entry = Path(tmp, "4242") + entry.mkdir() + (entry / "environ").write_bytes(b"A=1\0QUALITY_GATE_TAG=t\0") + with mock.patch.object(gate.os, "kill") as kill: + self.assertTrue(gate._kill_if_tagged(entry, b"QUALITY_GATE_TAG=t")) + self.assertFalse(gate._kill_if_tagged(entry, b"QUALITY_GATE_TAG=other")) + self.assertFalse(gate._kill_if_tagged(Path(tmp, "gone"), b"QUALITY_GATE_TAG=t")) + kill.assert_called_once_with(4242, gate.signal.SIGKILL) + + @unittest.skipUnless(Path("/proc").is_dir(), "needs /proc") + def test_a_test_run_in_its_own_session_is_killed_with_the_gate_run(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + pidfile = Path(tmp, "child.pid") + script = Path(tmp, "spawner.sh") + script.write_text( + "#!/bin/sh\n" + f'python3 -c "import os,sys,time; os.setsid(); open(sys.argv[1],\'w\').write(str(os.getpid())); time.sleep(60)" "{pidfile}" &\n' + "sleep 60\n" + ) + script.chmod(0o755) + code, _, _ = gate.run_limited([str(script)], Path(tmp), dict(os.environ), 2.0) + self.assertIsNone(code) + time.sleep(0.5) + self.assertFalse(Path(f"/proc/{pidfile.read_text()}").exists(), "the child in its own session survived the kill") + + +class FullBaseline(InRepo): + def setUp(self) -> None: + super().setUp() + import full_baseline + + self.fb = full_baseline + + def test_every_in_scope_source_file_is_listed(self) -> None: + self.write("src/client/x.ts", "export const x = 1\n") + self.write("test/a.test.ts", "x\n") + self.write("test/fake-app/server.ts", "export const s = 1\n") + self.write("src/types.d.ts", "declare const d: 1\n") + self.assertEqual(self.fb.source_files(self.repo), ["src/a.ts", "src/client/x.ts"]) + + def test_the_totals_score_only_what_finished_and_list_the_rest(self) -> None: + totals = self.fb.Totals(time.monotonic()) + with redirect_stdout(io.StringIO()): + totals.record("a.ts", ("ok", 3, [("a.ts", 1, "x")], "")) + totals.record("b.ts", ("unchecked", 0, [], 7)) + totals.record("c.ts", ("failed", 0, [], "boom")) + result = totals.result(3, 20) + self.assertEqual((result["killed"], result["survived"], result["score"]), (3, 1, 75.0)) + self.assertEqual((result["filesNotChecked"], result["filesFailed"]), (["b.ts"], ["c.ts"])) + self.assertIsNone(self.fb.Totals(time.monotonic()).result(0, 0)["score"]) + + def test_the_result_goes_to_the_log_and_the_step_summary(self) -> None: + summary = self.repo / "summary.md" + out = io.StringIO() + with mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}), redirect_stdout(out): + self.fb.publish({"score": 50.0}) + self.assertIn('RESULT {"score": 50.0}', out.getvalue()) + self.assertIn("## Full mutation baseline", summary.read_text()) + + def test_main_mutates_each_file_with_every_slot_and_fails_on_a_file_not_checked(self) -> None: + calls = [] + + def mutate(path, ranges, root, workers, deadline): + calls.append((path, workers)) + return ("ok", 1, [], "") if path.endswith("a.ts") else ("unchecked", 0, [], 2) + + self.write("src/b.ts", "export const b = 1\n") + with mock.patch.object(gate, "run_coverage", return_value=1.0), \ + mock.patch.object(gate, "count_sites", return_value=2), \ + mock.patch.object(gate, "worker_limit", return_value=3), \ + mock.patch.object(gate, "mutate_file", side_effect=mutate), \ + mock.patch.dict(os.environ, {}, clear=False), redirect_stdout(io.StringIO()) as out: + os.environ.pop("GITHUB_STEP_SUMMARY", None) + code = self.fb.main() + self.assertEqual(code, 1) + self.assertEqual(sorted(calls), [("src/a.ts", 3), ("src/b.ts", 3)]) + self.assertIn('"filesNotChecked": ["src/b.ts"]', out.getvalue()) + + +if __name__ == "__main__": + unittest.main() From 51b939774d1f7e55117c24242207a0ed7fecfa9f Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 12:42:30 +0700 Subject: [PATCH 2/7] ci: run the quality gate on every pull request One quality job on ubuntu-latest, pull requests only. The token reaches only the checkout and the render of the description, both before any code from the pull request runs; sudo is dropped before the gate. The README says what each check judges and why mutation uses its own test command. Workflow self-tests pass; only the baseline test still fails. --- .github/workflows/ci.yml | 85 ++++++++++++++++++++++++++++++++ .quality/README.md | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 .quality/README.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69d872a..f91fb1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,3 +28,88 @@ jobs: - run: bun run lint - run: bun run typecheck - run: bun run test + + # The engineering quality gates on the lines a pull request changes (issue #20, .quality/gate.py, + # .quality/README.md). Only the first two steps (checkout and the render of the description) get the token, + # and both run before any code from the pull request (bun install is the first); no later step or post step + # gets it. + quality: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + runs-on: ubuntu-latest + # Mutation stops itself inside a 30-minute budget and still writes its report; setup stays well under the rest. + timeout-minutes: 45 + steps: + - name: Check out the pull request + # Not actions/checkout: it keeps github.token for its post step, which runs at the end of the job, after + # code from the pull request. This fetches GitHub's merge commit with full history (the gate diffs against + # its first parent and checks the base out), passes the token to git through the environment only for the + # fetch, and stores no credential. + env: + GH_TOKEN: ${{ github.token }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + MERGE_REF: ${{ github.ref }} + run: | + set -o pipefail + find . -mindepth 1 -maxdepth 1 -exec rm -rf {} + + git init --quiet . + auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraheader GIT_CONFIG_VALUE_0="AUTHORIZATION: basic $auth" \ + git fetch --quiet --no-tags "$REPO_URL" "+$MERGE_REF:refs/remotes/pull/merge" + git checkout --quiet --detach refs/remotes/pull/merge + - name: Render the pull request description + # The gate reads "Equivalent mutant:" waivers from GitHub's own rendering of the description (POST + # /markdown, gfm), so a waiver counts only where a reviewer sees it on the PR page. Fetched now rather than + # taken from the event, so re-running this job after editing the description sees the edit. python3 -I + # keeps the checkout off sys.path. The HTML's sha256 is a step output, which no later step can change. + id: description + env: + GH_TOKEN: ${{ github.token }} + PR_URL: ${{ github.event.pull_request.url }} + API_URL: ${{ github.api_url }} + run: | + set -o pipefail + curl --proto '=https' --tlsv1.2 --retry 5 --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" --header "Accept: application/vnd.github+json" "$PR_URL" \ + | python3 -I -c 'import json, sys; json.dump({"text": json.load(sys.stdin)["body"] or "", "mode": "gfm"}, sys.stdout)' \ + | curl --proto '=https' --tlsv1.2 --retry 5 --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" --header "Accept: application/vnd.github+json" \ + --header "Content-Type: application/json" --data-binary @- "$API_URL/markdown" \ + > "$RUNNER_TEMP/pr-body.html" + echo "sha256=$(sha256sum "$RUNNER_TEMP/pr-body.html" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: No sudo for the gate + # The gate runs code from the pull request and every mutant of it, and none of that may run as root. The + # hosted runner's passwordless sudo rule is dropped, and the job fails if sudo still works. + run: | + if command -v sudo >/dev/null 2>&1; then + sudo -n rm -f /etc/sudoers.d/runner 2>/dev/null || true + if sudo -n true 2>/dev/null; then echo "sudo still works"; exit 1; fi + fi + # Both actions default `token` to github.token; with an empty token they send no auth. + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.2 + no-cache: true + token: '' + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + token: '' + - run: bun install --frozen-lockfile + - name: Install quality tools + # Pinned commits and versions in .quality/requirements.txt, in a venv that is thrown away with the job. + run: | + python -m venv "$RUNNER_TEMP/quality-venv" + "$RUNNER_TEMP/quality-venv/bin/pip" install --quiet -r .quality/requirements.txt + - name: Gate self-tests + run: | + export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" + python -m unittest discover -s .quality -p 'test_*.py' + - name: Quality gate + env: + DESCRIPTION_SHA256: ${{ steps.description.outputs.sha256 }} + run: | + export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" + python .quality/gate.py --description-html "$RUNNER_TEMP/pr-body.html" --description-sha256 "$DESCRIPTION_SHA256" diff --git a/.quality/README.md b/.quality/README.md new file mode 100644 index 0000000..a182d94 --- /dev/null +++ b/.quality/README.md @@ -0,0 +1,104 @@ +# Quality gate + +The `quality` job in `.github/workflows/ci.yml` runs on every pull request and calls `gate.py`. It follows the +engineering quality standard as a ratchet: only code the PR adds or changes is judged, so debt already on main never +blocks a PR. Everything lives in this folder and that one job, so one revert removes it. + +| Check | Fails when | Tool | +|---|---|---| +| Coverage | an executable line the PR adds or changes has no test hit, or a changed file is missing from the report | Vitest, v8 coverage (lcov) | +| CRAP | a function the PR adds or changes scores 10 or more, unless it was already 10 or more on base and scores no higher now | crapper | +| Duplication | the PR adds a structural duplicate pair at 0.8 similarity or above | dryer | +| Mutation | a mutant on a changed line survives, mutator fails on a file, or the budget runs out | mutator | + +The thresholds (CRAP 10, similarity 0.8, a mutation budget of 5 to 30 minutes) are constants in `gate.py`, and a +self-test pins them. Changing one is an owner decision recorded in the engineering standard. + +The gate is a port of the one in the sibling Bun repository (buildboard), adapted to Vitest: one scope, Vitest's +coverage, and a mutant command that copes with Vite resolving symbolic links (see Mutation). + +## Scope + +TypeScript under `src/`, minus tests and `.d.ts`. The test helpers under `test/`, the fake app server included, are +not judged. + +A pure-deletion hunk judges no line. Every diff runs with `--no-color --no-ext-diff --no-textconv`, so git settings +cannot change what the gate reads, and a file whose hunks do not add up to git's own count stops the gate. + +## Coverage + +The gate runs `bun run coverage` itself (with the lcov report pinned), after deleting any old report, so a stale +report never stands in for it. `vitest.config.ts` lists every file under `src/`, so a file no test loads is in the +report at 0 hits. + +The CLI entry points, `src/cli/bin.ts` and `src/conformance/cli.ts`, only run as child processes in the tests, which +v8 coverage does not see. Each changed line in them is reported on its own and can be waived (see Waivers). Keep them +thin: the logic belongs in `main.ts` and `runner.ts`, which the tests import. + +The repo total (lines and functions, `src/` only) is compared with `baseline.json` and reported. A drop alone does +not fail the PR, because deleting covered code can cause it. A PR that raises a number updates `baseline.json`. + +The gate and every mutant run with `TETHER_KEYCHAIN=off` and without `TETHER_REAL_SERVICE`, so nothing it runs +touches the OS keychain or installs a real launchd agent. + +## CRAP ratchet + +A changed function that already scores 10 or more on base passes when its CRAP is no higher than on base; the report +lists it as "still over the limit" with both scores. A new, moved or renamed function, and one that was under 10 on +base, must stay under 10. Functions match by file, name and order among same-named functions in the file; when the +count of a name in a file changes, every function of that name counts as new. A base function with no coverage +record is not scored (0% would be the highest base possible), so it is judged as new. + +The base score is measured, not stored, and only when a touched function is over 10: the gate checks out base in a +throwaway worktree that borrows this checkout's `node_modules`, and runs the same coverage there. + +## Mutation + +Mutation runs only after every other check passed, on the changed lines of the changed files. The standard's +example command, `mutator --changed --test-command "bun run test"`, does not work here for two reasons, so the gate +calls mutator per file with `--lines` and its own test command, `mutant-tests.sh`: + +- mutator runs each mutant in a worker under `target/mutation-workers` made of symbolic links to the tree and a copy + of the mutated file. Vite resolves every module to its real path, so a linked test would import the original file + and every mutant would survive. `mutant-tests.sh` first replaces the links with real copies (all but + `node_modules`). +- The whole suite takes about 45 seconds. The script runs `vitest related` for the file first (the tests that import + it, directly or not), on one Vitest worker, and the whole suite only for a mutant that survived: the entry points + are reached only through child processes, which `related` cannot see. + +A mutant is killed only when Vitest reports a failed test or a test file that failed to load. A run that writes no +report (a broken config, a crash) is a harness error and never a kill: the mutant shows as a survivor, and the gate +fails where a reviewer can see it. + +The budget is sized to the change: the time already spent, plus for each file the rounds its sites need on its +workers (plus mutator's own unmutated run), times the measured time of the whole suite. It is never below 5 minutes +and never above 30. When it runs out, the check fails cleanly and still reports what finished and what was not +checked; the job timeout (45 minutes) sits above the largest budget. + +The quality job removes the hosted runner's sudo rule before any code from the pull request runs, so no mutant can +run anything as root. + +## Waivers + +A surviving mutant or an untested changed line passes only with this line in the PR description, as a paragraph of +its own: + + Equivalent mutant: src/file.ts:42 why no test can tell the difference + +One line waives every mutant on that line. The gate reads GitHub's own rendering of the description, fetched and +hashed before any code from the pull request runs, so a waiver counts only where a reviewer sees it. After editing +the description, re-run the quality job: it reads the description as it is when the job starts. + +## Baseline + +`baseline.json` holds the default branch's numbers: coverage, functions measured, functions over CRAP 10, duplicate +pairs, and the full mutation score from `full_baseline.py`, which mutates every site under `src/` with the gate's own +machinery. Run it by hand on the default branch (it takes a long time) and copy its `RESULT` line into +`baseline.json`. + +## Running it locally + + python3.12 -m venv /tmp/qv && /tmp/qv/bin/pip install -r .quality/requirements.txt + PATH=/tmp/qv/bin:$PATH python3.12 -m unittest discover -s .quality -p 'test_*.py' + PATH=/tmp/qv/bin:$PATH python3.12 .quality/gate.py --base origin/main + PATH=/tmp/qv/bin:$PATH python3.12 .quality/full_baseline.py From ac72fa45dde19446361e0654213fa0f89c2dbca7 Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 12:47:50 +0700 Subject: [PATCH 3/7] ci(quality): record the static baseline and measure the mutation score Coverage, functions over CRAP 10 and duplicate pairs measured on main at 41d3e6c (242 functions, none over 10, no pair at 0.8). The full mutation run (636 sites) runs on a hosted runner through mutation-baseline.yml; its score fills in baseline.json once that run finishes. --- .github/workflows/mutation-baseline.yml | 89 +++++++++++++++++++++++++ .quality/README.md | 7 +- .quality/baseline.json | 11 +++ 3 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/mutation-baseline.yml create mode 100644 .quality/baseline.json diff --git a/.github/workflows/mutation-baseline.yml b/.github/workflows/mutation-baseline.yml new file mode 100644 index 0000000..5649432 --- /dev/null +++ b/.github/workflows/mutation-baseline.yml @@ -0,0 +1,89 @@ +name: mutation-baseline + +# The full mutation score of src/ for .quality/baseline.json. Never part of an ordinary pull request: +# a PR is judged on its changed lines by the quality job in ci.yml. It runs by hand, or on a pull request that changes +# the baseline script, and always measures the BASE branch's src/ code (the PR supplies the tests and the gate). +# Result: the RESULT line in the log and the step summary; copy the figures and this run's URL into baseline.json. + +on: + workflow_dispatch: + pull_request: + paths: + - ".quality/full_baseline.py" + - ".github/workflows/mutation-baseline.yml" + +permissions: + contents: read + +# A running baseline is never cancelled: a path filter on pull_request matches the whole PR's diff, so every push to +# such a PR starts this workflow again. The new run waits, and exits at once unless that push changed these files. +concurrency: + group: mutation-baseline-${{ github.ref }} + cancel-in-progress: false + +jobs: + baseline: + runs-on: ubuntu-latest + # The script stops itself at 330 minutes and still prints its totals; GitHub's own limit is 360. + timeout-minutes: 350 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Did this push change the baseline? + id: wanted + env: + ACTION: ${{ github.event.action }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.pull_request.head.sha }} + run: | + set -eu + run=true + if [ "$ACTION" = synchronize ] && changed=$(git diff --name-only "$BEFORE" "$AFTER" 2>/dev/null); then + printf '%s\n' "$changed" | grep -qxE '\.quality/full_baseline\.py|\.github/workflows/mutation-baseline\.yml' || run=false + fi + echo "run=$run" | tee -a "$GITHUB_OUTPUT" + - name: Measure the base branch's code + if: github.event_name == 'pull_request' && steps.wanted.outputs.run == 'true' + run: | + set -eu + # The source under measurement is the base's; the tests and the gate are the pull request's. The base is the + # merge commit's first parent, the tip the merge was built on (the event's base sha can be older than that). + BASE_SHA=$(git rev-parse HEAD^1) + rm -rf src + git checkout "$BASE_SHA" -- src + echo "src differs from $BASE_SHA in:"; git diff --stat "$BASE_SHA" -- src + - name: No sudo for the measured code + # The hosted runner's passwordless sudo rule is dropped, and the job fails if sudo still works. + if: steps.wanted.outputs.run == 'true' + run: | + if command -v sudo >/dev/null 2>&1; then + sudo -n rm -f /etc/sudoers.d/runner 2>/dev/null || true + if sudo -n true 2>/dev/null; then echo "sudo still works"; exit 1; fi + fi + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + if: steps.wanted.outputs.run == 'true' + with: + bun-version: 1.4.2 + no-cache: true + token: '' + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + if: steps.wanted.outputs.run == 'true' + with: + python-version: '3.12' + token: '' + - if: steps.wanted.outputs.run == 'true' + run: bun install --frozen-lockfile + - name: Install quality tools + if: steps.wanted.outputs.run == 'true' + run: | + python -m venv "$RUNNER_TEMP/quality-venv" + "$RUNNER_TEMP/quality-venv/bin/pip" install --quiet -r .quality/requirements.txt + - name: Mutate every site + if: steps.wanted.outputs.run == 'true' + env: + BASELINE_BUDGET_SECONDS: '19800' + run: | + export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" + python .quality/full_baseline.py diff --git a/.quality/README.md b/.quality/README.md index a182d94..e37c6c8 100644 --- a/.quality/README.md +++ b/.quality/README.md @@ -2,7 +2,7 @@ The `quality` job in `.github/workflows/ci.yml` runs on every pull request and calls `gate.py`. It follows the engineering quality standard as a ratchet: only code the PR adds or changes is judged, so debt already on main never -blocks a PR. Everything lives in this folder and that one job, so one revert removes it. +blocks a PR. Everything lives in this folder, that job and `mutation-baseline.yml`, so one revert removes it. | Check | Fails when | Tool | |---|---|---| @@ -93,8 +93,9 @@ the description, re-run the quality job: it reads the description as it is when `baseline.json` holds the default branch's numbers: coverage, functions measured, functions over CRAP 10, duplicate pairs, and the full mutation score from `full_baseline.py`, which mutates every site under `src/` with the gate's own -machinery. Run it by hand on the default branch (it takes a long time) and copy its `RESULT` line into -`baseline.json`. +machinery. `.github/workflows/mutation-baseline.yml` runs it on a GitHub runner, by hand or on a pull request that +changes the script or that workflow, always on the base branch's `src/`. Copy its `RESULT` line and the run's URL +into `baseline.json`. ## Running it locally diff --git a/.quality/baseline.json b/.quality/baseline.json new file mode 100644 index 0000000..f1f12a7 --- /dev/null +++ b/.quality/baseline.json @@ -0,0 +1,11 @@ +{ + "measuredOn": "main at 41d3e6c, 2026-11-18", + "coverage": { + "lines": 97.11, + "functions": 98.96 + }, + "functionsMeasured": 242, + "functionsOverCrap10": 0, + "duplicatePairs": 0, + "mutation": null +} From e94491f276b8bda8a7fda5dd5e8b6d5cc844c23f Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 12:59:28 +0700 Subject: [PATCH 4/7] style(ci): format the workflows the way biome does Biome formats YAML too, and lint failed on the single-quoted strings in the new jobs. The workflow self-test now looks for the double-quoted empty token. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/mutation-baseline.yml | 8 ++++---- .quality/test_gate.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f91fb1e..dd43a5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,11 +92,11 @@ jobs: with: bun-version: 1.4.2 no-cache: true - token: '' + token: "" - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: '3.12' - token: '' + python-version: "3.12" + token: "" - run: bun install --frozen-lockfile - name: Install quality tools # Pinned commits and versions in .quality/requirements.txt, in a venv that is thrown away with the job. diff --git a/.github/workflows/mutation-baseline.yml b/.github/workflows/mutation-baseline.yml index 5649432..6a96cbb 100644 --- a/.github/workflows/mutation-baseline.yml +++ b/.github/workflows/mutation-baseline.yml @@ -67,12 +67,12 @@ jobs: with: bun-version: 1.4.2 no-cache: true - token: '' + token: "" - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 if: steps.wanted.outputs.run == 'true' with: - python-version: '3.12' - token: '' + python-version: "3.12" + token: "" - if: steps.wanted.outputs.run == 'true' run: bun install --frozen-lockfile - name: Install quality tools @@ -83,7 +83,7 @@ jobs: - name: Mutate every site if: steps.wanted.outputs.run == 'true' env: - BASELINE_BUDGET_SECONDS: '19800' + BASELINE_BUDGET_SECONDS: "19800" run: | export PATH="$RUNNER_TEMP/quality-venv/bin:$PATH" python .quality/full_baseline.py diff --git a/.quality/test_gate.py b/.quality/test_gate.py index 69ab7ca..ef7f56a 100644 --- a/.quality/test_gate.py +++ b/.quality/test_gate.py @@ -451,7 +451,7 @@ def test_no_action_takes_the_token_by_default(self) -> None: for action in ("oven-sh/setup-bun@", "actions/setup-python@"): with self.subTest(action=action): _, block = self.step(action) - self.assertIn("token: ''", block) + self.assertIn('token: ""', block) uses = [block for block in workflow_steps() if block.startswith("uses:")] self.assertEqual(len(uses), 2) From e992ee0bf7bc2f0c465d796f01face2495349cf9 Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 14:04:47 +0700 Subject: [PATCH 5/7] ci(quality): record the full mutation score of main 514 of 632 covered mutants killed (81.3%) over all 636 sites of src/, every file checked, measured by mutation-baseline.yml on the hosted runner against main at 41d3e6c. --- .quality/baseline.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.quality/baseline.json b/.quality/baseline.json index f1f12a7..1585a31 100644 --- a/.quality/baseline.json +++ b/.quality/baseline.json @@ -7,5 +7,11 @@ "functionsMeasured": 242, "functionsOverCrap10": 0, "duplicatePairs": 0, - "mutation": null + "mutation": { + "score": 81.3, + "killed": 514, + "survived": 118, + "sites": 636, + "run": "https://github.com/farzai/tether/actions/runs/44246737986" + } } From 63fed41489cd2d825832c21c7b8df462fe1a29f9 Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 14:07:09 +0700 Subject: [PATCH 6/7] ci(quality): take the baseline coverage from the quality job's runner The gate compares coverage on ubuntu-latest, where main's src measures 97.17% of lines (macOS locally: 97.11%, platform-specific paths). The gate's first CI run asked for this update. --- .quality/README.md | 3 ++- .quality/baseline.json | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.quality/README.md b/.quality/README.md index e37c6c8..70aee05 100644 --- a/.quality/README.md +++ b/.quality/README.md @@ -91,7 +91,8 @@ the description, re-run the quality job: it reads the description as it is when ## Baseline -`baseline.json` holds the default branch's numbers: coverage, functions measured, functions over CRAP 10, duplicate +`baseline.json` holds the default branch's numbers, measured where the gate compares them (the quality job's +Linux runner; macOS measures slightly lower line coverage, since some code paths are platform-specific): coverage, functions measured, functions over CRAP 10, duplicate pairs, and the full mutation score from `full_baseline.py`, which mutates every site under `src/` with the gate's own machinery. `.github/workflows/mutation-baseline.yml` runs it on a GitHub runner, by hand or on a pull request that changes the script or that workflow, always on the base branch's `src/`. Copy its `RESULT` line and the run's URL diff --git a/.quality/baseline.json b/.quality/baseline.json index 1585a31..791e96e 100644 --- a/.quality/baseline.json +++ b/.quality/baseline.json @@ -1,7 +1,7 @@ { - "measuredOn": "main at 41d3e6c, 2026-11-18", + "measuredOn": "main at 41d3e6c, 2026-11-18, on the quality job's runner (ubuntu-latest)", "coverage": { - "lines": 97.11, + "lines": 97.17, "functions": 98.96 }, "functionsMeasured": 242, From 73beec509631613164e92f8957ebdf09d5aba363 Mon Sep 17 00:00:00 2001 From: parsilver Date: Wed, 18 Nov 2026 14:09:13 +0700 Subject: [PATCH 7/7] ci(quality): keep the baseline line coverage at 97.11% A second CI run of the same code measured 97.11%, not 97.17%: line coverage moves by about 0.06 points between runs because some paths depend on timing, not on the platform as the previous commit assumed. --- .quality/README.md | 4 ++-- .quality/baseline.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.quality/README.md b/.quality/README.md index 70aee05..ce60df7 100644 --- a/.quality/README.md +++ b/.quality/README.md @@ -91,8 +91,8 @@ the description, re-run the quality job: it reads the description as it is when ## Baseline -`baseline.json` holds the default branch's numbers, measured where the gate compares them (the quality job's -Linux runner; macOS measures slightly lower line coverage, since some code paths are platform-specific): coverage, functions measured, functions over CRAP 10, duplicate +`baseline.json` holds the default branch's numbers. Line coverage moves by about 0.06 points between runs of the +same code (some paths depend on timing), which is one more reason a drop alone never fails a PR: coverage, functions measured, functions over CRAP 10, duplicate pairs, and the full mutation score from `full_baseline.py`, which mutates every site under `src/` with the gate's own machinery. `.github/workflows/mutation-baseline.yml` runs it on a GitHub runner, by hand or on a pull request that changes the script or that workflow, always on the base branch's `src/`. Copy its `RESULT` line and the run's URL diff --git a/.quality/baseline.json b/.quality/baseline.json index 791e96e..1585a31 100644 --- a/.quality/baseline.json +++ b/.quality/baseline.json @@ -1,7 +1,7 @@ { - "measuredOn": "main at 41d3e6c, 2026-11-18, on the quality job's runner (ubuntu-latest)", + "measuredOn": "main at 41d3e6c, 2026-11-18", "coverage": { - "lines": 97.17, + "lines": 97.11, "functions": 98.96 }, "functionsMeasured": 242,