diff --git a/.github/scripts/check_step_timeouts.py b/.github/scripts/check_step_timeouts.py new file mode 100644 index 00000000..36b0f8aa --- /dev/null +++ b/.github/scripts/check_step_timeouts.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +"""Require every workflow step and every job in this repo to carry a deadline. + +WHY. A GitHub Actions step with no `timeout-minutes` has no bound of its own: +if the command it runs hangs -- a clone that never finishes, a generator waiting +on a lock, a network call with no response -- the step runs until the runner's +own limit (six hours) and the job never reports WHICH step died. A step bound +turns that stall into a time-bounded failure that NAMES the step it died in; a +job bound turns the whole job's stall into a failure with a total deadline. + +This repo's deploy workflow had no bound on any of its sixteen steps and no +total deadline on its job (opencharly/docs#162), so the rule is enforced here, +on the workflows this repo OWNS. + +One file in `.github/workflows/` is not this repo's to fix: `tag-on-merge.yml` +is a DISTRIBUTED COPY of opencharly/.github's canonical template, installed +into every repo by that repo's bootstrap and bound back to the template by its +reconcile script. A deadline added to the copy here would be erased by that +reconcile pass, and the deadline belongs to the template. The copy is +therefore REPORTED by this checker, never silently skipped -- and a path named +explicitly on the command line is always asserted. + +NO YAML PARSER, ON PURPOSE. The docs site carries no YAML dependency +(package.json: astro, starlight, sharp, wrangler), and a check that needs a +package the repo does not carry is a check that does not run. The scanner below +is an indentation reader for the shape GitHub Actions defines, and the +self-test proves it against fixtures -- including the trap a grep falls into: a +`timeout-minutes:` that appears inside a `run:` block is TEXT, not a key, and +must not satisfy the rule. +""" + +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parent.parent.parent +WORKFLOW_DIR = ROOT / ".github" / "workflows" + +# A mapping key: its own indentation, its name, and whatever follows the colon. +KEY_RE = re.compile(r"^(?P[ \t]*)(?P[A-Za-z0-9_.\-]+):(?P[ ].*|)$") +# A block-scalar header: `|`, `>`, with the usual chomping/indent indicators. +BLOCK_RE = re.compile(r"^[|>][-+]?\d*$") +# A timeout value: an integer, optionally followed by a comment. +TIMEOUT_RE = re.compile(r"^(?P\d+)\s*(#.*)?$") +# GitHub's own ceiling for both job and step timeouts. +MAX_TIMEOUT = 360 +# The keys that label a step when it has no `name:`. +LABEL_KEYS = ("name", "uses", "run") + +# Workflow files this repo does NOT own, and why each is not asserted here. +# +# `.github/workflows/tag-on-merge.yml` is a DISTRIBUTED COPY of opencharly/ +# .github's canonical template (`.github/workflows/tag-on-merge-dispatcher.yml`): +# that repo's `scripts/bootstrap-repo-main.sh` installs the template into every +# repo, and its `scripts/reconcile-tag-on-merge-callers.sh` binds the copies +# back to the template (opencharly/.github#192). A deadline added to the COPY +# here would be erased by that reconcile pass, and the deadline belongs to the +# TEMPLATE, which is what the org-wide sweep owns (opencharly/.github#256). +# The copy is REPORTED, never silently skipped: a skipped surface that says +# nothing is a gap nobody can see. +ORG_DISTRIBUTED = { + ".github/workflows/tag-on-merge.yml": ( + "a distributed copy of opencharly/.github's tag-on-merge-dispatcher.yml; " + "its deadline belongs in that template (opencharly/.github#256), and the " + "copy-to-template binding is opencharly/.github#192" + ), +} + + +def _commented_out(value: str) -> str: + """The value with a trailing `# comment` removed. + + Only ever applied to a value that is a bare scalar (a number, or a block + header), where a `#` cannot be inside a quoted string. + """ + return value.split("#", 1)[0].strip() + + +def _mark_block_scalars(lines: list[str]) -> list[bool]: + """Mark every line that is CONTENT of a block scalar, not YAML structure. + + A `run: |` step body is a shell script, not a mapping: a line inside it + that looks like `timeout-minutes: 10` is text the shell would see, and a + step whose only "timeout" is inside such a body is UNBOUND. The content of + a block scalar is every following line indented deeper than its key. + """ + skip = [False] * len(lines) + index = 0 + while index < len(lines): + line = lines[index] + stripped = line.strip() + if not stripped or stripped.startswith("#"): + index += 1 + continue + match = KEY_RE.match(line) + if not match: + index += 1 + continue + key_indent = len(match.group("indent")) + if not BLOCK_RE.match(_commented_out(match.group("rest"))): + index += 1 + continue + # A block scalar: swallow the deeper-indented body that follows. + cursor = index + 1 + while cursor < len(lines): + body = lines[cursor] + if not body.strip(): + skip[cursor] = True + cursor += 1 + continue + if len(body) - len(body.lstrip()) > key_indent: + skip[cursor] = True + cursor += 1 + continue + break + index = cursor + return skip + + +def _structure(lines: list[str]) -> tuple[list[tuple[int, int, str, str]], list[tuple[int, int, str]]]: + """The mapping keys and the list items of a workflow, with their columns. + + Returns (keys, items) where a key is (line index, column, name, value) and + an item is (line index, column, text) -- a `- ` list entry. + """ + skip = _mark_block_scalars(lines) + keys: list[tuple[int, int, str, str]] = [] + items: list[tuple[int, int, str]] = [] + for index, line in enumerate(lines): + if skip[index]: + continue + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + match = KEY_RE.match(line) + if match: + keys.append( + ( + index, + len(match.group("indent")), + match.group("key"), + _commented_out(match.group("rest")), + ) + ) + continue + if stripped.startswith("- "): + dash = len(line) - len(line.lstrip()) + text = stripped[2:].strip() + items.append((index, dash, text)) + return keys, items + + +def _label_of(text: str) -> str: + """The human label of a list item: the key it opens with.""" + match = KEY_RE.match(text) + if not match: + return text[:60] + name = match.group("key") + if name in LABEL_KEYS: + value = _commented_out(match.group("rest")) + if name == "run": + value = value.splitlines()[0] if value else "" + return f"{name}: {value}" if value else name + return name + + +def _timeout_at(keys, start: int, end: int, column: int) -> tuple[bool, str]: + """Whether a `timeout-minutes:` key sits at `column` between two lines.""" + for index, indent, name, value in keys: + if index < start or index >= end: + continue + if name != "timeout-minutes" or indent != column: + continue + match = TIMEOUT_RE.match(value) + if not match: + return False, f"timeout-minutes is not a number: {value!r}" + minutes = int(match.group("value")) + if minutes < 1 or minutes > MAX_TIMEOUT: + return False, f"timeout-minutes {minutes} is outside 1..{MAX_TIMEOUT}" + return True, "" + return False, "" + + +def _timeout_minutes(keys, start: int, end: int, column: int) -> int | None: + """The minutes of the `timeout-minutes:` key at `column`, if it is a valid one.""" + for index, indent, name, value in keys: + if index < start or index >= end: + continue + if name != "timeout-minutes" or indent != column: + continue + match = TIMEOUT_RE.match(value) + if not match: + return None + minutes = int(match.group("value")) + if minutes < 1 or minutes > MAX_TIMEOUT: + return None + return minutes + return None + + +def check_text(text: str, label: str = "") -> list[str]: + """Every deadline missing from one workflow, as findings.""" + lines = text.splitlines() + keys, items = _structure(lines) + findings: list[str] = [] + + jobs = [key for key in keys if key[2] == "jobs"] + if not jobs: + return findings + jobs_index, jobs_indent = jobs[0][0], jobs[0][1] + + # The `jobs:` MAPPING ends at the next key at its OWN indent — a top-level + # `env:` or `on:` after it is a sibling of `jobs:`, not a job. Without this + # bound, a top-level `env:` child was read as a job and reported missing + # its deadline, which is a false positive on a perfectly good workflow. + jobs_end = len(lines) + for key in keys: + if key[0] > jobs_index and key[1] <= jobs_indent: + jobs_end = key[0] + break + + # A job is a MAPPING key under `jobs:`; the jobs sit at one column and + # each job's own keys sit one column deeper. + job_starts = [ + key for key in keys if jobs_index < key[0] < jobs_end and key[1] > jobs_indent + ] + if not job_starts: + return findings + job_indent = job_starts[0][1] + job_names = [ + key for key in job_starts if key[1] == job_indent + ] + for position, job in enumerate(job_names): + job_name = job[2] + job_start = job[0] + job_end = ( + job_names[position + 1][0] if position + 1 < len(job_names) else jobs_end + ) + # The job's own key column: the first key deeper than the job's name. + own = [key for key in keys if job_start < key[0] < job_end and key[1] > job_indent] + if not own: + continue + job_column = own[0][1] + bounded, why = _timeout_at(keys, job_start, job_end, job_column) + job_minutes = _timeout_minutes(keys, job_start, job_end, job_column) + if not bounded: + reason = why or "the job has no total deadline" + findings.append(f"{label}: job '{job_name}' {reason}") + + steps = [key for key in own if key[2] == "steps" and key[1] == job_column] + if not steps: + continue + steps_index, steps_indent = steps[0][0], steps[0][1] + # The step LIST ends at the next key at the job's own column: a job + # whose `env:` is a list of maps would otherwise have its list items + # read as steps. + steps_end = job_end + for key in keys: + if key[0] > steps_index and key[1] == job_column: + steps_end = key[0] + break + step_items = [ + item for item in items if steps_index < item[0] < steps_end and item[1] > steps_indent + ] + if not step_items: + continue + step_indent = step_items[0][1] + for number, item in enumerate(step_items, start=1): + if item[1] != step_indent: + continue + step_start = item[0] + # The step ends where the NEXT step begins, or where the job ends. + # It must not reach past the job: a later job's `timeout-minutes:` is + # a different step's key, and letting the search run to the end of + # the file would let it satisfy this step. + step_end = job_end + for later in step_items[number:]: + if later[1] == step_indent: + step_end = later[0] + break + # The step's key column: where its first key starts on the dash line. + step_column = item[1] + 2 + while step_column < len(lines[step_start]) and lines[step_start][step_column] == " ": + step_column += 1 + bounded, why = _timeout_at(keys, step_start, step_end, step_column) + if not bounded: + reason = why or "the step has no deadline" + findings.append( + f"{label}: step {number} ('{_label_of(item[2])}') in job '{job_name}' {reason}" + ) + # A step's deadline must be SHORTER than its job's, or the job's + # deadline fires first and the step's is dead: the stall is then + # reported against the job, which names no step, and the one + # property a step bound exists for — that the failure SAYS WHICH + # STEP died — is gone. The step's own message says so, because the + # step is what fails and names itself. + step_minutes = _timeout_minutes(keys, step_start, step_end, step_column) + if step_minutes is not None and job_minutes is not None and step_minutes >= job_minutes: + findings.append( + f"{label}: step {number} ('{_label_of(item[2])}') in job '{job_name}' " + f"its deadline ({step_minutes} min) is not shorter than its job's " + f"({job_minutes} min), so the job's deadline fires first and the step's is dead" + ) + return findings + + +def _label_of_path(path: Path) -> str: + """The repo-relative label of a path, or the path itself when it is outside.""" + try: + return str(path.relative_to(ROOT)) + except ValueError: + return str(path) + + +def partition(paths: list[Path], explicit: bool) -> tuple[list[Path], list[str]]: + """The paths this checker asserts, and a note for each one it does not. + + An EXPLICIT path is always asserted: a caller that points at a file wants + that file checked, and the tool must not narrow a request it was given. + The DEFAULT scan asserts this repo's own workflow files and REPORTS the + org-distributed ones, with the owner named -- a skip that says nothing is + a gap nobody can see. + """ + asserted: list[Path] = [] + notes: list[str] = [] + for path in paths: + label = _label_of_path(path) + reason = ORG_DISTRIBUTED.get(label) + if reason and not explicit: + notes.append(f"{label}: not asserted here - {reason}") + continue + asserted.append(path) + return asserted, notes + + +def check_files(paths: list[Path]) -> list[str]: + findings: list[str] = [] + for path in paths: + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + findings.append(f"{path}: cannot be read: {exc}") + continue + findings.extend(check_text(text, _label_of_path(path))) + return findings + + +def _workflow_paths(arguments: list[str]) -> list[Path]: + if not arguments: + return sorted(WORKFLOW_DIR.glob("*.yml")) + sorted(WORKFLOW_DIR.glob("*.yaml")) + paths: list[Path] = [] + for argument in arguments: + path = Path(argument) + if path.is_dir(): + paths.extend(sorted(path.glob("*.yml")) + sorted(path.glob("*.yaml"))) + else: + paths.append(path) + return paths + + +def self_test() -> None: + """Prove the checker on fixtures -- including the trap a grep falls into.""" + failures: list[str] = [] + + def expect(label: str, condition: bool, detail: str = "") -> None: + if not condition: + failures.append(f"{label}{': ' + detail if detail else ''}") + + bounded_workflow = """ +name: bounded +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: first + run: echo one + timeout-minutes: 5 + - uses: actions/checkout@v7 + timeout-minutes: 10 +""" + expect("a fully bounded workflow has no findings", check_text(bounded_workflow) == [], + str(check_text(bounded_workflow))) + + unbounded_step = """ +name: unbounded +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: first + run: echo one + - name: second + run: echo two + timeout-minutes: 5 +""" + findings = check_text(unbounded_step) + expect("an unbounded step is reported", len(findings) == 1, str(findings)) + expect("the finding names the step", "step 1 ('name: first')" in findings[0] if findings else False, + str(findings)) + expect("the finding names the job", "job 'deploy'" in findings[0] if findings else False, str(findings)) + + # The TRAP: the only `timeout-minutes:` in the step is inside the shell + # body. It is text to the shell, not a key -- a grep would accept it. + trap = """ +name: trap +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: first + run: | + # timeout-minutes: 5 + echo one +""" + findings = check_text(trap) + expect("a timeout inside a run block does NOT satisfy the rule", len(findings) == 1, str(findings)) + expect("the trap's finding names the step", "step 1" in findings[0] if findings else False, str(findings)) + + # A job-level bound must not satisfy a step's requirement. + job_only = """ +name: job-only +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: first + run: echo one +""" + findings = check_text(job_only) + expect("a job bound does not bound its steps", len(findings) == 1, str(findings)) + expect("the job-only finding names the step", "step 1" in findings[0] if findings else False, str(findings)) + + # A job with no bound of its own. + job_unbounded = """ +name: job-unbounded +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: first + run: echo one + timeout-minutes: 5 +""" + findings = check_text(job_unbounded) + expect("a job with no total deadline is reported", len(findings) == 1, str(findings)) + expect("the job finding is about the job", "job 'deploy'" in findings[0] if findings else False, + str(findings)) + + # A reusable-workflow job has no steps and still needs its own deadline. + reusable = """ +name: reusable +jobs: + tag: + uses: opencharly/.github/.github/workflows/tag-on-merge.yml@main + timeout-minutes: 20 +""" + expect("a bounded reusable-workflow job has no findings", check_text(reusable) == [], + str(check_text(reusable))) + + reusable_unbounded = """ +name: reusable-unbounded +jobs: + tag: + uses: opencharly/.github/.github/workflows/tag-on-merge.yml@main +""" + findings = check_text(reusable_unbounded) + expect("an unbounded reusable-workflow job is reported", len(findings) == 1, str(findings)) + + # An out-of-range value is not a deadline. + out_of_range = """ +name: out-of-range +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 0 + steps: + - name: first + run: echo one + timeout-minutes: 5 +""" + findings = check_text(out_of_range) + expect("a zero timeout is rejected", len(findings) == 1, str(findings)) + expect("the rejection names the range", "outside 1..360" in findings[0] if findings else False, + str(findings)) + + # A step with no `name:` is labelled by its `uses:`. + unnamed = """ +name: unnamed +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 +""" + findings = check_text(unnamed) + expect("an unnamed step is labelled by its uses", "uses: actions/checkout@v7" in findings[0] if findings else False, + str(findings)) + + # A top-level `env:` is a SIBLING of `jobs:`, not a job. A checker that + # reads the `jobs:` mapping as "everything until the end of the file" + # reports `env:`'s child as a job with no deadline — a false positive on a + # perfectly good workflow. + # + # The sibling must come AFTER the `jobs:` mapping: that is the only + # position where the end of the mapping is decided, and a sibling BEFORE + # `jobs:` is excluded by the mapping's own start. A fixture that put it + # before would pass whatever the end bound did, and prove nothing. + sibling_env = """ +name: sibling-env +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: only + run: echo one + timeout-minutes: 5 +env: + FOO: + BAR: baz +""" + expect("a top-level env: is not a job", check_text(sibling_env) == [], str(check_text(sibling_env))) + + # A block scalar's BODY is text, not structure. The body of a job-level + # `env:` map whose value is a block scalar lands on the STEP's own key + # column, so a reader that does not know the body is text finds + # "timeout-minutes" there and calls the step bounded. It is not: the step + # carries no deadline of its own, and it must be reported. + # + # This fixture is the one that earns the block-scalar marking. The trap + # above does not: a `run:` body is indented deeper than the step's keys, + # so the COLUMN test alone rejects it. This shape puts the body on the + # step's own column, where only the marking can reject it. + block_body = """ +name: block-body +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: only + run: echo one + env: + NOTE: | + timeout-minutes: 5 +""" + findings = check_text(block_body) + expect("a block scalar's body is not a deadline", len(findings) == 1, str(findings)) + expect("the block-body finding names the step", "step 1" in findings[0] if findings else False, + str(findings)) + + # A block scalar's body is not a job either: a `fake-job:` inside a + # top-level block scalar is text, and the workflow it sits in is not + # broken by it. + block_body_is_not_a_job = """ +name: block-body-is-not-a-job +x-note: | + fake-job: + name: not a job +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: only + run: echo one + timeout-minutes: 5 +""" + expect("a block scalar's body is not a job", check_text(block_body_is_not_a_job) == [], + str(check_text(block_body_is_not_a_job))) + + # A LATER job's step deadline must not satisfy an EARLIER job's last step. + # The step's search must stop at the end of its own job: a step key that + # belongs to the next job is a different step's key. (This is the shape + # that made this checker under-report, and it was found by cross-checking + # against a real YAML parser over the org's own workflows.) + cross_job = """ +name: cross-job +jobs: + first: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: only + run: echo one + second: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: later + run: echo two + timeout-minutes: 5 +""" + findings = check_text(cross_job) + expect("a later job's deadline does not satisfy an earlier job's step", len(findings) == 1, str(findings)) + expect("the cross-job finding names the first job's step", + "step 1 ('name: only') in job 'first'" in findings[0] if findings else False, str(findings)) + + # A step's deadline must be SHORTER than its job's. If it is not, the + # job's deadline fires first and the step's is dead: the stall is + # reported against the job, which names no step — and naming the step is + # the one property a step bound exists for. + dead_step_bound = """ +name: dead-step-bound +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: only + run: echo one + timeout-minutes: 10 +""" + findings = check_text(dead_step_bound) + expect("a step bound that is not shorter than its job's is reported", + len(findings) == 1, str(findings)) + expect("the dead-bound finding says the step's deadline is dead", + "is dead" in findings[0] if findings else False, str(findings)) + + # A step bound SHORTER than the job's is fine: the step is what fails, + # and it names itself. + live_step_bound = """ +name: live-step-bound +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: only + run: echo one + timeout-minutes: 10 +""" + expect("a step bound shorter than its job's is fine", + check_text(live_step_bound) == [], str(check_text(live_step_bound))) + + # The SCOPE. A workflow file the ORG distributes and reconciles is not this + # repo's to bound: the deadline belongs to the canonical template, and a + # deadline added to the copy would be erased by the org's own reconcile + # pass. The copy is therefore REPORTED by the default scan, and the report + # must name the file and its owner -- a skip that says nothing is a gap + # nobody can see. An EXPLICIT path is still asserted: a caller that points + # at a file wants that file checked. + org_caller = ROOT / ".github" / "workflows" / "tag-on-merge.yml" + asserted, notes = partition([org_caller], explicit=False) + expect("an org-distributed copy is not asserted by the default scan", + asserted == [], str(asserted)) + expect("the org-distributed copy is reported, not silently skipped", + len(notes) == 1, str(notes)) + expect("the report names the file", + "tag-on-merge.yml" in notes[0] if notes else False, str(notes)) + expect("the report names the owner", + "opencharly/.github" in notes[0] if notes else False, str(notes)) + asserted, notes = partition([org_caller], explicit=True) + expect("an explicit path is asserted even when it is org-distributed", + asserted == [org_caller], str(asserted)) + expect("an explicit path carries no skip note", notes == [], str(notes)) + own = ROOT / ".github" / "workflows" / "deploy.yml" + asserted, notes = partition([own], explicit=False) + expect("a repo-owned workflow is asserted by the default scan", + asserted == [own], str(asserted)) + expect("a repo-owned workflow carries no skip note", notes == [], str(notes)) + lookalike = ROOT / ".github" / "workflows" / "tag-on-merge-extra.yml" + asserted, notes = partition([lookalike], explicit=False) + expect("a file that merely resembles the org-distributed one is still asserted", + asserted == [lookalike], str(asserted)) + outside = Path("/etc/other-repo/workflows/deploy.yml") + asserted, notes = partition([outside], explicit=False) + expect("a path outside this repo is asserted", asserted == [outside], str(asserted)) + + if failures: + print("check-step-timeouts self-test: FAILED") + for failure in failures: + print(f"- {failure}") + raise SystemExit(1) + print("check-step-timeouts self-test: OK (unbounded steps, jobs and traps are all caught, and the org-distributed copy is reported)") + + +def main() -> int: + arguments = sys.argv[1:] + if arguments == ["--self-test"]: + self_test() + return 0 + paths = _workflow_paths(arguments) + if not paths: + print("check-step-timeouts: FAILED") + print("- no workflow files found") + return 1 + asserted, notes = partition(paths, explicit=bool(arguments)) + findings = check_files(asserted) + if findings: + print("check-step-timeouts: FAILED") + for finding in findings: + print(f"- {finding}") + for note in notes: + print(f"note: {note}") + return 1 + if not asserted: + print("check-step-timeouts: FAILED") + print("- no workflow file to assert") + for note in notes: + print(f"note: {note}") + return 1 + print( + f"check-step-timeouts: OK ({len(asserted)} workflow file(s) asserted; " + f"every job and every step carries a deadline)" + ) + for note in notes: + print(f"note: {note}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_check_step_timeouts.py b/.github/scripts/test_check_step_timeouts.py new file mode 100644 index 00000000..819782e4 --- /dev/null +++ b/.github/scripts/test_check_step_timeouts.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Prove that the step-deadline checker's SELF-TEST can fail — one mechanism at a time. + +WHY THIS EXISTS. `.github/scripts/check_step_timeouts.py` ships a `--self-test` +mode, and that mode is the committed coverage of the rule. A self-test is only +evidence if it can FAIL: a checker whose self-test passes whatever the code +does proves nothing, and that failure mode is SILENT — the gate stays green +while the rule it claims to enforce has stopped being enforced. + +So each mechanism of the checker is removed, one at a time, and this harness +asserts that the self-test then turns RED on the fixture that covers exactly +that mechanism. A mutant that still passes is a mechanism the self-test does +not cover, and the harness fails on it. + +The mutation is an ASSERTED single-string replacement: the anchor must occur +EXACTLY once in the committed source, and the harness fails loudly when it does +not, so a moved anchor can never masquerade as a passing mutation. + +WHAT IS PROVED. Each mutant removes ONE mechanism: + + mutA the block-scalar marking — a `timeout-minutes:` inside a `run:` + body is TEXT, not a key + mutB the cross-job step bound — a later job's deadline does not + satisfy an earlier job's last step + mutC the jobs-mapping end bound — a top-level `env:` is a sibling of + `jobs:`, not a job + mutE the dead-step-bound rule — a step bound at or above its job's is + dead code, and must be reported + mutF the org-distributed exclusion — a copy the org distributes is REPORTED + by the default scan, not asserted + +The mutant is written into a temporary tree that MIRRORS this repo's layout +(`.github/scripts/`), because the checker derives its repository root +from its own path — a mutant written anywhere else would compute a different +root and would turn the self-test red for the WRONG reason, which would let a +broken mutation pass unnoticed. + +usage: python3 .github/scripts/test_check_step_timeouts.py + python3 .github/scripts/test_check_step_timeouts.py --emit +exit: 0 = every mutant turned the self-test red; 1 = a mutant still passed. +""" + +from pathlib import Path +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parent.parent.parent +SOURCE = ROOT / ".github" / "scripts" / "check_step_timeouts.py" + +# (name, anchor, replacement, what the mutation removes, the self-test case +# that must turn red). +MUTANTS = [ + ( + "mutA-nomark", + " skip = [False] * len(lines)", + " skip = [False] * len(lines)\n return skip # MUTANT: the block-scalar marking disabled", + "the block-scalar marking", + "a block scalar's body is not a deadline", + ), + ( + "mutB-crossjob", + " step_end = job_end", + " step_end = len(lines) # MUTANT: the cross-job step bound removed", + "the cross-job step bound", + "a later job's deadline does not satisfy an earlier job's step", + ), + ( + "mutC-jobsend", + " jobs_end = len(lines)\n for key in keys:\n if key[0] > jobs_index and key[1] <= jobs_indent:\n jobs_end = key[0]\n break", + " jobs_end = len(lines) # MUTANT: the jobs-mapping end bound removed", + "the jobs-mapping end bound", + "a top-level env: is not a job", + ), + ( + "mutE-deadbound", + " if step_minutes is not None and job_minutes is not None and step_minutes >= job_minutes:", + " if False: # MUTANT: the dead-step-bound rule removed", + "the dead-step-bound rule", + "a step bound that is not shorter than its job's is reported", + ), + ( + "mutF-orgdistributed", + " reason = ORG_DISTRIBUTED.get(label)", + " reason = None # MUTANT: the org-distributed exclusion removed", + "the org-distributed exclusion", + "an org-distributed copy is not asserted by the default scan", + ), +] + + +def emit(name: str, destination: str) -> int: + """Write ONE mutant to a path, so another harness can use it as a control. + + The anchor assertion is the same one the harness makes: an anchor that has + moved must FAIL LOUDLY here too, because a mutant that is not the mutation + it claims to be is a control that proves nothing. + """ + text = SOURCE.read_text(encoding="utf-8") + for candidate, anchor, replacement, what, _case in MUTANTS: + if candidate != name: + continue + count = text.count(anchor) + if count != 1: + print( + f"FAIL: the anchor for {what} occurs {count} times, expected 1", + file=sys.stderr, + ) + return 1 + target = Path(destination) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text.replace(anchor, replacement), encoding="utf-8") + print(f"emitted {target} — {what} removed") + return 0 + print(f"FAIL: no mutant named {name!r}", file=sys.stderr) + return 1 + + +def main() -> int: + arguments = sys.argv[1:] + if arguments[:1] == ["--emit"]: + if len(arguments) != 3: + print("usage: test_check_step_timeouts.py --emit ", file=sys.stderr) + return 2 + return emit(arguments[1], arguments[2]) + + text = SOURCE.read_text(encoding="utf-8") + failures: list[str] = [] + passed = 0 + + for name, anchor, replacement, what, case in MUTANTS: + count = text.count(anchor) + if count != 1: + failures.append( + f"{name}: the anchor for {what} occurs {count} times, expected 1 — " + f"the mutation cannot be applied, so it would prove nothing" + ) + continue + with tempfile.TemporaryDirectory() as temporary: + scripts = Path(temporary) / ".github" / "scripts" + scripts.mkdir(parents=True) + mutant = scripts / f"{name}.py" + mutant.write_text(text.replace(anchor, replacement), encoding="utf-8") + completed = subprocess.run( + [sys.executable, str(mutant), "--self-test"], + capture_output=True, + text=True, + ) + output = completed.stdout + completed.stderr + if completed.returncode == 0: + failures.append( + f"{name}: the self-test still PASSED with {what} removed — " + f"the self-test does not cover that mechanism" + ) + continue + if case not in output: + failures.append( + f"{name}: the self-test failed, but not on the case that covers " + f"{what} (expected {case!r} in the output)" + ) + continue + passed += 1 + print(f"ok {name} — {what} removed, and the self-test turned red on it") + + if failures: + print(f"test_check_step_timeouts: FAIL ({len(failures)})") + for failure in failures: + print(f"- {failure}") + return 1 + print( + f"test_check_step_timeouts: OK ({passed} mutants; every one of them turns " + f"the self-test red)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c5b038f3..eadc654f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -54,8 +54,21 @@ permissions: jobs: deploy: runs-on: ubuntu-latest + # This job's TOTAL deadline. It is a backstop, not the discriminating + # bound: the step bounds below are what say WHICH step died, and this + # only fires when the whole job is stuck. Sized from the measured runs: + # the 56 successful runs of the FIXED window 2026-09-22..2026-10-24 + # (median 10.6 min), whose longest — 15.15 min, on 2026-09-22 — is also + # the longest this job has ever run in the retained history (583 + # successful runs, back to 2026-08-03). 60 min is 4.0x that maximum, + # and it stays UNDER the org validator's own 70-minute head-settle + # bound, so a stall in this workflow closes with a named failure — and + # closes BEFORE the validator's own wait gives up, which is the + # deadlock half of opencharly/docs#162. + timeout-minutes: 60 steps: - uses: actions/checkout@v7 + timeout-minutes: 10 with: # The pinned marketplace is the corpus the recipe pages project from — # fetched via the submodule below. charly is NOT a submodule anymore @@ -70,10 +83,49 @@ jobs: # ancestry — a depth-1 history cannot answer that question. submodules: recursive + # EVERY step in this workflow carries a deadline, and this is the control + # that keeps it that way. A step with no `timeout-minutes` has no bound: + # if the command it runs hangs — a clone that never finishes, a generator + # waiting on a lock — the step runs until the runner's own six-hour limit + # and the job never reports WHICH step died. This repo's workflow had no + # bound on any of its steps (opencharly/docs#162), so the rule is + # enforced here, on the workflows this repo OWNS. + # + # The check runs FIRST, before the expensive steps: a workflow that has + # lost a deadline is a defect in the workflow, and finding it before the + # regeneration says so. + # + # It reports, and does not assert, the one workflow file this repo does + # not own: `.github/workflows/tag-on-merge.yml` is a distributed copy of + # opencharly/.github's canonical template, and a deadline added to the + # copy here would be erased by the org's own reconcile pass — the + # deadline belongs to the template (opencharly/.github#256). + - name: Check that every workflow step carries a deadline + run: python3 .github/scripts/check_step_timeouts.py + timeout-minutes: 5 + + # A check that cannot fail is not a check. The self-test proves the + # checker rejects an unbounded step — and, the case a grep falls into, + # that a `timeout-minutes:` written inside a `run:` block is TEXT to the + # shell and does NOT satisfy the rule. + - name: Prove the step-deadline checker rejects an unbounded step + run: python3 .github/scripts/check_step_timeouts.py --self-test + timeout-minutes: 5 + + # And a self-test that cannot fail is not a self-test. Each mechanism of + # the checker is removed, one at a time, and the self-test must turn RED + # on the fixture that covers it: a mutation that still passes is a + # mechanism the coverage does not reach. + - name: Prove the step-deadline self-test can fail (mutation harness) + run: python3 .github/scripts/test_check_step_timeouts.py + timeout-minutes: 5 + - name: Test the corpus-currency report + timeout-minutes: 5 run: bash .github/docs-corpus/test_check_corpus_currency.sh - name: Report the corpus pin's currency + timeout-minutes: 10 # The corpus pin is advanced by a MANUAL act after the corpus lands, so the # site trails the corpus by however long nobody ran the regeneration — and # the drift gate further down CANNOT see it: the PINNED corpus generates its @@ -103,6 +155,7 @@ jobs: } >>"$GITHUB_STEP_SUMMARY" - name: Fetch the pinned charly source (CI-time temp checkout, recursive for box/*) + timeout-minutes: 15 run: | # Clone WITHOUT --recursive, then checkout the pinned commit, THEN init the box/* # submodules: --recursive at clone time pulls each box at its default-branch HEAD, @@ -116,6 +169,7 @@ jobs: git clone https://github.com/opencharly/opencharly.git /tmp/umbrella && git -C /tmp/umbrella checkout e631aacf32df4e26a66f2cb6a75fc9e85e212399 - uses: actions/setup-node@v7 + timeout-minutes: 10 with: node-version-file: .node-version cache: npm @@ -123,27 +177,42 @@ jobs: # setup-go hashes only files inside GITHUB_WORKSPACE. Keep the module it # actually compiles here, rather than keying its cache on /tmp/charly. - name: Fetch the pinned docs generator + timeout-minutes: 10 run: git clone --depth 1 --branch v2026.289.0814 https://github.com/opencharly/plugin-docs.git .docs-generator - uses: actions/setup-go@v7 + timeout-minutes: 10 with: go-version-file: .docs-generator/candy/plugin-docs/go.mod cache-dependency-path: .docs-generator/candy/plugin-docs/go.sum - name: Build the standalone plugin-docs CLI binary + timeout-minutes: 15 working-directory: .docs-generator/candy/plugin-docs run: go build -o /tmp/plugin-docs-bin ./cmd/serve - name: Test the generated-content drift gate + timeout-minutes: 10 run: python3 .github/docs-corpus/test_check_drift.py - name: Write the corpus wiring candy from the pins file + timeout-minutes: 5 run: | mkdir -p /tmp/charly/candy/docs-corpus cp .github/docs-corpus/build_wiring.py /tmp/build_wiring.py python3 /tmp/build_wiring.py .github/docs-corpus/pins.tsv /tmp/charly/candy/docs-corpus/charly.yml - name: Regenerate the generated content from the pinned charly + # 30 min, from the measured runs: this step is the whole cost of the + # job (3.3-10.8 min across the same 56 measured runs, longest + # 10.8 min), and it is the step opencharly/docs#162 observed still + # running while the PR's other check waited on it. It carried no + # deadline of its own, so a stall here had nothing to fail against + # and the run never said WHICH step was stuck. 30 min is 2.8x the + # observed maximum, so real work does not trip it, and it fires far + # sooner than the runner's own six-hour limit. It is under the job's + # 60-minute total, so the step's message is what the reader sees. + timeout-minutes: 30 # --root points at the pinned charly checkout (the catalog walk + landing page + the # skill/recipe reference pages); --narrative points at the umbrella checkout, which # holds the four root narratives (SOUL/VISION/GRIEVANCES/LIBERATION) at their new @@ -159,6 +228,7 @@ jobs: --out ${{ github.workspace }}/src/content/docs - name: Drift gate — the committed content must match the pinned charly + timeout-minutes: 10 # Regeneration on a clean tree is a no-op by contract. A diff here means a # docs PR must bump the charly pin (the CI-time commit) and commit the regenerated # pages in the same change. @@ -175,15 +245,18 @@ jobs: # package files and so stamps every deployment `commit_dirty: true`. Arguing for `npm ci` # and then installing a floating dependency two steps later gives the rationale away. - name: Install dependencies + timeout-minutes: 10 run: npm ci --no-audit --no-fund - name: Build + timeout-minutes: 15 run: npm run build # The site's own link-integrity gate ran in the generator step above (now available here, # since the pinned charly checkout carries it). What this job can still prove is that the # committed content builds cleanly from a clean install on a clean machine. - name: Verify the build produced a site + timeout-minutes: 5 run: | test -f dist/index.html echo "pages built: $(find dist -name '*.html' | wc -l)" @@ -193,6 +266,7 @@ jobs: # rather than skip, turning every fork contribution red for a reason the contributor # cannot fix. - name: Deploy to Cloudflare Pages + timeout-minutes: 20 if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: cloudflare/wrangler-action@v4 with: