From d4fd82da1e3cd40561dff7a4936549b06d2d2bb6 Mon Sep 17 00:00:00 2001 From: Andreas Trawoeger Date: Mon, 26 Oct 2026 08:56:58 +0100 Subject: [PATCH 1/2] fix(gate): the mutation harness writes its mutants through its own --emit mode The validator's review of opencharly/docs#191 raised one real gap in the mutation harness this PR adds: the `--emit` mode is a documented utility that the workflow never exercises. The fix is not to delete it -- it is to make it the WRITER. Both harnesses in this repo asserted the mutation anchor in TWO places: once in `emit()` and once again in `main()`, which then wrote the mutant itself. So the same rule was carried twice in one file, and the copy in `main()` was the only one the workflow ever ran -- the `--emit` path, the one a diagnostic caller reaches, had no coverage and could drift away from the copy that gates. `main()` no longer asserts the anchor and no longer writes the mutant. It shells out to this harness's own `--emit` mode to write each mutant, so: * the mode is covered by construction -- the harness runs the very path a diagnostic caller runs; * the anchor assertion lives in exactly one place (`emit`), so the two copies cannot drift; * the mutant the harness tests IS the mutant `--emit` produces, so the diagnostic path and the gating path cannot diverge. This is the same fix in both harnesses: the step-deadline harness this repo already had, and the advisory harness this PR adds. Fixing only the new one would have left the repo with two harnesses of the same mechanism in two different shapes -- the duplication R2 forbids -- so both move together. The gate's OUTCOME is unchanged: all five mutants in each harness still turn the self-test red, and each still turns it red on the case that covers the mechanism it removes. Refs opencharly/docs#163 Assisted-by: Claude Code minimax/minimax-m3.1-flash:cloud (analysed on a live system) --- .../test_check_dependency_advisories.py | 43 +++++++++++++------ .github/scripts/test_check_step_timeouts.py | 42 ++++++++++++------ 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/.github/scripts/test_check_dependency_advisories.py b/.github/scripts/test_check_dependency_advisories.py index 5ca166ad..6de6e1e6 100644 --- a/.github/scripts/test_check_dependency_advisories.py +++ b/.github/scripts/test_check_dependency_advisories.py @@ -33,6 +33,14 @@ (`.github/scripts/`), because the checker derives its repository root from its own path. +THE `--emit` MODE IS THE WRITER, NOT A SECOND IMPLEMENTATION. Every mutant +this harness tests is written THROUGH `--emit` -- the same mode a diagnostic +caller uses -- so the mode is covered by construction rather than by a +parallel copy of the mutation, and the anchor assertion lives in exactly one +place (`emit`). A harness that mutated the source itself while shipping an +unexercised `--emit` beside it would carry the same rule twice and prove +neither copy. + usage: python3 .github/scripts/test_check_dependency_advisories.py python3 .github/scripts/test_check_dependency_advisories.py --emit exit: 0 = every mutant turned the self-test red; 1 = a mutant still passed. @@ -96,9 +104,10 @@ def emit(name: str, destination: str) -> int: """Write ONE mutant to a path, so another harness can use it as a control. - The anchor assertion is the same one the harness makes: an anchor that has - moved must FAIL LOUDLY here too, because a mutant that is not the mutation - it claims to be is a control that proves nothing. + This is the ONLY place the anchor is asserted and the ONLY place the + mutation is applied. An anchor that has moved must FAIL LOUDLY here, + because a mutant that is not the mutation it claims to be is a control that + proves nothing. """ text = SOURCE.read_text(encoding="utf-8") for candidate, anchor, replacement, what, _case in MUTANTS: @@ -107,7 +116,8 @@ def emit(name: str, destination: str) -> int: count = text.count(anchor) if count != 1: print( - f"FAIL: the anchor for {what} occurs {count} times, expected 1", + f"the anchor for {what} occurs {count} times, expected 1 — " + f"the mutation cannot be applied, so it would prove nothing", file=sys.stderr, ) return 1 @@ -128,23 +138,28 @@ def main() -> int: return 2 return emit(arguments[1], arguments[2]) - text = SOURCE.read_text(encoding="utf-8") failures: list[str] = [] passed = 0 - for name, anchor, replacement, what, case in MUTANTS: - count = text.count(anchor) - if count != 1: - failures.append( - f"{name}: the anchor for {what} occurs {count} times, expected 1 — " - f"the mutation cannot be applied, so it would prove nothing" - ) - continue + for name, _anchor, _replacement, what, case in MUTANTS: with tempfile.TemporaryDirectory() as temporary: scripts = Path(temporary) / ".github" / "scripts" scripts.mkdir(parents=True) mutant = scripts / f"{name}.py" - mutant.write_text(text.replace(anchor, replacement), encoding="utf-8") + # The mutant is written THROUGH this harness's own `--emit` mode -- + # the same path a diagnostic caller uses -- so the mode is covered + # by construction, and the anchor assertion happens exactly once. + written = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--emit", name, str(mutant)], + capture_output=True, + text=True, + ) + if written.returncode != 0: + failures.append( + f"{name}: the mutation could not be applied — " + f"{(written.stderr.strip() or written.stdout.strip())}" + ) + continue completed = subprocess.run( [sys.executable, str(mutant), "--self-test"], capture_output=True, diff --git a/.github/scripts/test_check_step_timeouts.py b/.github/scripts/test_check_step_timeouts.py index 819782e4..ebaf640f 100644 --- a/.github/scripts/test_check_step_timeouts.py +++ b/.github/scripts/test_check_step_timeouts.py @@ -35,6 +35,13 @@ root and would turn the self-test red for the WRONG reason, which would let a broken mutation pass unnoticed. +THE `--emit` MODE IS THE WRITER, NOT A SECOND IMPLEMENTATION. Every mutant +this harness tests is written THROUGH `--emit` — the same mode a diagnostic +caller uses — so the mode is covered by construction rather than by a parallel +copy of the mutation, and the anchor assertion lives in exactly one place +(`emit`). A harness that mutated the source itself while shipping an unexercised +`--emit` beside it would carry the same rule twice and prove neither copy. + usage: python3 .github/scripts/test_check_step_timeouts.py python3 .github/scripts/test_check_step_timeouts.py --emit exit: 0 = every mutant turned the self-test red; 1 = a mutant still passed. @@ -92,9 +99,10 @@ def emit(name: str, destination: str) -> int: """Write ONE mutant to a path, so another harness can use it as a control. - The anchor assertion is the same one the harness makes: an anchor that has - moved must FAIL LOUDLY here too, because a mutant that is not the mutation - it claims to be is a control that proves nothing. + This is the ONLY place the anchor is asserted and the ONLY place the + mutation is applied. An anchor that has moved must FAIL LOUDLY here, + because a mutant that is not the mutation it claims to be is a control that + proves nothing. """ text = SOURCE.read_text(encoding="utf-8") for candidate, anchor, replacement, what, _case in MUTANTS: @@ -103,7 +111,8 @@ def emit(name: str, destination: str) -> int: count = text.count(anchor) if count != 1: print( - f"FAIL: the anchor for {what} occurs {count} times, expected 1", + f"the anchor for {what} occurs {count} times, expected 1 — " + f"the mutation cannot be applied, so it would prove nothing", file=sys.stderr, ) return 1 @@ -124,23 +133,28 @@ def main() -> int: return 2 return emit(arguments[1], arguments[2]) - text = SOURCE.read_text(encoding="utf-8") failures: list[str] = [] passed = 0 - for name, anchor, replacement, what, case in MUTANTS: - count = text.count(anchor) - if count != 1: - failures.append( - f"{name}: the anchor for {what} occurs {count} times, expected 1 — " - f"the mutation cannot be applied, so it would prove nothing" - ) - continue + for name, _anchor, _replacement, what, case in MUTANTS: with tempfile.TemporaryDirectory() as temporary: scripts = Path(temporary) / ".github" / "scripts" scripts.mkdir(parents=True) mutant = scripts / f"{name}.py" - mutant.write_text(text.replace(anchor, replacement), encoding="utf-8") + # The mutant is written THROUGH this harness's own `--emit` mode -- + # the same path a diagnostic caller uses -- so the mode is covered + # by construction, and the anchor assertion happens exactly once. + written = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--emit", name, str(mutant)], + capture_output=True, + text=True, + ) + if written.returncode != 0: + failures.append( + f"{name}: the mutation could not be applied — " + f"{(written.stderr.strip() or written.stdout.strip())}" + ) + continue completed = subprocess.run( [sys.executable, str(mutant), "--self-test"], capture_output=True, From 1d136d9d7699904c4d55e7b184e23a255e74e66f Mon Sep 17 00:00:00 2001 From: Andreas Trawoeger Date: Mon, 26 Oct 2026 09:11:13 +0100 Subject: [PATCH 2/2] fix(gate): the two harness failure messages read alike The validator's review of opencharly/docs#192 raised one real inconsistency and one suggestion. The inconsistency is fixed here; the suggestion is rebutted on the PR thread with evidence. THE INCONSISTENCY. A harness has two ways to fail before it mutates anything: the named mutant does not exist, or the mutant's anchor has moved. The first printed "FAIL: no mutant named 'x'"; the second printed "the anchor for ... occurs N times, expected 1" with no marker at all. The exit code is the real signal -- nothing in CI greps for the prefix -- but two failure paths in one tool should read alike, and a reader scanning a CI log should not have to know which of the two messages carries a marker. The fix is the prefix, in BOTH harnesses -- the advisory one this PR rewrites and the step-deadline one it rewrites beside it. Fixing only the advisory one would have left the two siblings of the same mechanism reading differently, which is the duplication R2 forbids and the cutover rule (HC) names: every sibling with the same mechanism moves together. The gate's OUTCOME is unchanged: five mutants per harness, every one still turning the self-test red on the case that covers the mechanism it removes. Nothing asserts on these messages -- they are read by a human. THE SUGGESTION, REBUTTED. The validator notes that the harness's subprocess calls carry no timeout, and calls it harmless. It is, and adding one would be a REGRESSION in the failure mode: * the step already carries a deadline, and this repo's OWN gate asserts that every step does (5 min for the step-deadline harness, 10 min for the advisory one), so the set of child processes is bounded; * the advisory harness's children make LIVE network calls to the npm advisory database, and npm's own default fetch timeout is 300000 ms -- so any per-call timeout tighter than that would turn "the registry is slow" into a RED harness; * a red harness says "the self-test can no longer fail", which is exactly the false signal this harness exists to avoid. A timeout would trade a visible timeout for a misleading verdict. The full reasoning, and the evidence for it, is on the PR thread. Refs opencharly/docs#163 Refs opencharly/docs#192 Assisted-by: Claude Code minimax/minimax-m3.1-flash:cloud (analysed on a live system) --- .github/scripts/test_check_dependency_advisories.py | 2 +- .github/scripts/test_check_step_timeouts.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_check_dependency_advisories.py b/.github/scripts/test_check_dependency_advisories.py index 6de6e1e6..ad80b456 100644 --- a/.github/scripts/test_check_dependency_advisories.py +++ b/.github/scripts/test_check_dependency_advisories.py @@ -116,7 +116,7 @@ def emit(name: str, destination: str) -> int: count = text.count(anchor) if count != 1: print( - f"the anchor for {what} occurs {count} times, expected 1 — " + f"FAIL: the anchor for {what} occurs {count} times, expected 1 — " f"the mutation cannot be applied, so it would prove nothing", file=sys.stderr, ) diff --git a/.github/scripts/test_check_step_timeouts.py b/.github/scripts/test_check_step_timeouts.py index ebaf640f..fa444022 100644 --- a/.github/scripts/test_check_step_timeouts.py +++ b/.github/scripts/test_check_step_timeouts.py @@ -111,7 +111,7 @@ def emit(name: str, destination: str) -> int: count = text.count(anchor) if count != 1: print( - f"the anchor for {what} occurs {count} times, expected 1 — " + f"FAIL: the anchor for {what} occurs {count} times, expected 1 — " f"the mutation cannot be applied, so it would prove nothing", file=sys.stderr, )