From 3e02963e0a24af3ad27fcd7cabc675de82f57a63 Mon Sep 17 00:00:00 2001 From: Takwa Date: Fri, 30 Oct 2026 16:36:52 +0100 Subject: [PATCH] Apply cve.yml from CI too cve.yml puts the CVE-2026-98432 filter on sf and sf2, ipni/infra's indexers. It joins site.yml and platform.yml in CI: a dry run on pull requests, and on merge a dry run then the apply, once a required reviewer approves it in `production`. It runs on its own, in cve-main, since it needs nothing from the other two and they never touch these boxes. CI cannot sign in there as `ipni` (those boxes do not have it) or as the operators' shared `user`, whose sudo wants a password. So cve.yml now creates and keeps an account of its own for CI, ipni-ci: key only, with CI's existing key and nothing else, password locked, passwordless sudo. ci-playbook.sh makes CI sign in as it, reading the name from group_vars/infra_boxes. No new secret. CI only connects to pinned host keys, so sf's and sf2's go into inventory/known_hosts in ssh's [ip]:5222 form; the scanned key, the key file on each box and an operator's known_hosts agreed. CI cannot create its own account, so the first run on each box is an operator's, with -K. Until then cve.yml's CI dry runs fail to connect. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 92 ++++++++++++++++++++++++++++++--- AGENTS.md | 8 +-- README.md | 32 +++++++++--- cve.yml | 43 +++++++++++++-- group_vars/infra_boxes/main.yml | 14 +++-- inventory/known_hosts | 7 ++- scripts/ci-playbook.sh | 22 +++++--- scripts/ci-summary.py | 2 +- 8 files changed, 188 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbdbae4..e540273 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,9 @@ # unnoticed here before, which is why this exists. # # The exception is the playbooks that run from CI: site.yml (the boxes' base: -# packages, sysctls, SSH, firewall, k3s) and platform.yml (Argo CD and the -# monitoring stack's firewall, see the README). For each of them: +# packages, sysctls, SSH, firewall, k3s), platform.yml (Argo CD and the +# monitoring stack's firewall, see the README) and cve.yml (the CVE-2026-98432 +# filter on sf and sf2, ipni/infra's boxes). For each of them: # # pull request -> the checks, then a dry run (--check --diff) against the # boxes if the PR touches that playbook @@ -20,7 +21,8 @@ # # site.yml goes first: platform.yml relies on what it sets up, so when a merge # touches both, platform.yml's dry run starts only once site.yml's apply is -# done, and shows what is left to change after it. +# done, and shows what is left to change after it. cve.yml runs on other boxes +# and needs neither, so it goes on its own. # # One workflow, so a merge starts one run. Pull requests from forks get no # secrets: their dry runs are skipped. @@ -35,7 +37,7 @@ on: playbook: description: "Playbook to run against the boxes" type: choice - options: [site, platform] + options: [site, platform, cve] default: site run: description: "none (checks only), check (dry run) or apply (dry run, then the approved run)" @@ -211,7 +213,7 @@ jobs: shellcheck scripts/*.sh # --------------------------------------------------------------------------- - # Playbooks run from CI: site.yml, then platform.yml + # Playbooks run from CI: site.yml, then platform.yml; cve.yml alongside # --------------------------------------------------------------------------- # Whether this run touches what each playbook applies. On a pull request, @@ -222,6 +224,7 @@ jobs: outputs: site: ${{ steps.diff.outputs.site }} platform: ${{ steps.diff.outputs.platform }} + cve: ${{ steps.diff.outputs.cve }} steps: - uses: actions/checkout@v4 with: @@ -247,12 +250,13 @@ jobs: declare -A paths=( [site]="^(site\.yml|roles/(common|hardening|k3s|storage)/|host_vars/|keys/|$shared)" [platform]="^(platform\.yml|roles/(argocd|monitoring|ufw_allowlist)/|roles/route_origin/defaults/main\.yml|$shared)" + [cve]="^(cve\.yml|roles/hardening/(defaults/|tasks/cve_2026_98432\.yml|templates/cve-)|host_vars/sf2?/|keys/github-actions\.pub|$shared)" ) if [[ "$EVENT" != workflow_dispatch && "$RANGE" != 0000000000000000000000000000000000000000..* ]]; then changed=$(git diff --name-only "$RANGE") fi rows="" - for playbook in site platform; do + for playbook in site platform cve; do if [[ "$EVENT" == workflow_dispatch ]]; then hit=$([[ "$PLAYBOOK" == "$playbook" ]] && echo true || echo false) elif [[ "$RANGE" == 0000000000000000000000000000000000000000..* ]]; then @@ -301,7 +305,7 @@ jobs: echo "|---|---|---|" printf '%s' "$rows" echo - echo "Dry runs start once every check has passed, and site.yml goes before platform.yml." + echo "Dry runs start once every check has passed, and site.yml goes before platform.yml. cve.yml (sf and sf2) goes on its own." echo "Each dry run's summary below lists exactly what it would change." } >> "$GITHUB_STEP_SUMMARY" @@ -490,3 +494,77 @@ jobs: ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} LIMIT: ${{ inputs.limit }} run: scripts/ci-playbook.sh platform apply "$LIMIT" + + # --- cve.yml --------------------------------------------------------------- + + # As site-dry-run, for cve.yml: the CVE-2026-98432 filter on sf and sf2, + # ipni/infra's indexers, which no other playbook here touches. It signs in as + # CI's own account there (infra_ci_user), which an operator's first run of + # cve.yml creates; until then this fails to connect. It needs nothing from + # site.yml or platform.yml, so it waits for neither. + cve-dry-run: + needs: [ansible, manifests, generated, shell, changes] + outputs: + changes: ${{ steps.dry-run.outputs.changes }} + if: >- + needs.changes.outputs.cve == 'true' && ( + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + github.event_name == 'push' || + (github.event_name == 'workflow_dispatch' && inputs.run != 'none')) + runs-on: ubuntu-latest + concurrency: + group: ${{ github.event_name == 'pull_request' && format('cve-check-{0}', github.ref) || 'cve-main' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: + ANSIBLE_STDOUT_CALLBACK: default + ANSIBLE_FORCE_COLOR: "true" + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Dry run cve.yml + id: dry-run + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-playbook.sh cve check "$LIMIT" + + # As site-apply, for cve.yml. One box at a time (serial: 1 in the playbook), + # stopping at the first that fails. Loading the table restarts nothing. + cve-apply: + needs: [ansible, manifests, generated, shell, changes, cve-dry-run] + if: >- + needs.changes.outputs.cve == 'true' && ( + (github.event_name == 'push' && needs.cve-dry-run.outputs.changes == 'true') || + (github.event_name == 'workflow_dispatch' && inputs.run == 'apply')) + runs-on: ubuntu-latest + concurrency: + group: cve-main + cancel-in-progress: false + environment: production + env: + ANSIBLE_STDOUT_CALLBACK: default + ANSIBLE_FORCE_COLOR: "true" + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Run cve.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-playbook.sh cve apply "$LIMIT" diff --git a/AGENTS.md b/AGENTS.md index 510b7f6..9559ad7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,8 +14,10 @@ The inventory also has **sf** (103.232.206.165) and **sf2** (103.232.206.162), ipni/infra's indexers, in their own group `infra_boxes`. They are here for `cve.yml` and nothing else. Never add them to `ipfs_nodes` or run another playbook on them: `site.yml`'s hardening would swap their SSH -keys and turn on ufw. They take the shared `user` account on port 5222, and -sudo needs its password, so runs are an operator's with `-K`: +keys and turn on ufw. `cve.yml` applies from CI, as CI's own account there +(`ipni-ci`, which `cve.yml` creates and keeps). People take the shared `user` +account on port 5222, whose sudo needs its password, so an operator's run is +with `-K`: ```bash ansible-playbook cve.yml -K --check --diff -l sf # dry run first @@ -132,7 +134,7 @@ ansible-playbook routing.yml -l sing-1 # one box ansible-playbook routing.yml # one box at a time, warmed (default) ``` -`site.yml` and `platform.yml` apply from CI: a merge that touches one gets a +`site.yml`, `platform.yml` and `cve.yml` apply from CI: a merge that touches one gets a dry run on the run's summary page, then waits for a required reviewer to approve the real run in the `production` environment (README, "Running playbooks from CI"). Prefer that to running them from a controller, so what diff --git a/README.md b/README.md index f3d8de4..4dadb32 100644 --- a/README.md +++ b/README.md @@ -167,8 +167,10 @@ the table; set `off` to remove it for good once the fixed kernel is in. **sf and sf2** (ipni/infra's indexers) get the same filter from `cve.yml`, which runs only these tasks, on the `infra_boxes` group. Those boxes are not ours to harden: they stay out of `ipfs_nodes`, and nothing else here targets -them. Everyone signs in as their shared `user` account on port 5222, and sudo -there wants its password, so it is an operator's run with `-K`, never CI's: +them. It runs from CI (see "Running playbooks from CI"), as CI's own account +there, `ipni-ci`, which `cve.yml` creates and keeps. From an operator's +machine it signs in as their shared `user` account on port 5222, whose sudo +wants its password, so it takes `-K`: `ansible-playbook cve.yml -K --check --diff -l sf`, then without `--check`. Their mode is `cve_2026_98432_mode` in `group_vars/infra_boxes/main.yml`, rolled out the same way: `count`, then `enforce` one box at a time. To test @@ -1518,14 +1520,14 @@ in the vault: `ansible-playbook platform.yml --check --diff`. ## Running playbooks from CI -`site.yml` and `platform.yml` run from CI. The others (`routing.yml`, +`site.yml`, `platform.yml` and `cve.yml` run from CI. The others (`routing.yml`, `bootstrap.yml`, `content.yml`, `k3s-upgrade.yml`) deliberately stay on an operator's machine for now: they need the vault (for `bootstrap.yml` and `content.yml` that includes permanent identity keys), and a `routing.yml` run takes about an hour per box. `routing.yml`, `content.yml` and `bootstrap.yml` are meant to move to Helm charts later, deployed by Argo CD like the monitoring -stack, so CI is not given their secrets in the meantime. Both CI playbooks are -part of the repo's one workflow +stack, so CI is not given their secrets in the meantime. The CI playbooks are +all part of the repo's one workflow (`.github/workflows/ci.yml`), so a merge starts a single run: | When | What | @@ -1577,6 +1579,24 @@ If CI does lock itself out, an operator's key or the break-glass key (AGENTS.md) still gets in, unless the change removed those too, which the key check above stops after the first box. +**cve.yml.** The CVE-2026-98432 filter on sf and sf2 (`infra_boxes`), +ipni/infra's indexers. "Touching it" means `cve.yml`, the hardening role's +CVE tasks, templates and defaults, `host_vars/sf*/`, CI's public key, and the +shared paths above. It runs on its own, in `cve-main`: it needs nothing from +`site.yml` or `platform.yml`, and they never touch these boxes. + +CI signs in there with the same key, but not as `ipni`, which these boxes do +not have, nor as the operators' `user`, whose sudo wants a password. It uses +an account of its own, `ipni-ci` (`infra_ci_user` in +`group_vars/infra_boxes/main.yml`, read by `scripts/ci-playbook.sh`): key +only, password locked, passwordless sudo, so it is root on those boxes. +`cve.yml` creates and keeps it, with only CI's key installed. CI cannot +create it itself, so the **first run on each box is an operator's**, from the +branch that adds it: `ansible-playbook cve.yml -K -l sf`, then `-l sf2`. Until +then CI's dry runs fail to connect. To revoke CI's access there, remove the +account's tasks from `cve.yml`, then on each box `sudo userdel -r ipni-ci` and +`sudo rm /etc/sudoers.d/90-ipni-ci`. + ## Secrets **This repository is public, so no credentials are committed, not even @@ -1792,7 +1812,7 @@ dns.txt Cloudflare-importable bootstrap DNS records (generated dns-libp2p.txt bootstrap.libp2p.io records for the legacy identities (generated) site.yml base preparation playbook, run from CI k3s-upgrade.yml k3s upgrade, one minor version at a time, backup per step -cve.yml the CVE-2026-98432 filter alone, on sf and sf2 (infra_boxes), with -K +cve.yml the CVE-2026-98432 filter alone, on sf and sf2 (infra_boxes), run from CI platform.yml Argo CD (first install) and the monitoring stack's prerequisites, run from CI scripts/ci-playbook.sh CI's wrapper for site.yml and platform.yml: secrets to variables, SSH key, pinned host keys inventory/known_hosts the boxes' SSH host keys, pinned for CI diff --git a/cve.yml b/cve.yml index 1e6ff45..f47e9b4 100644 --- a/cve.yml +++ b/cve.yml @@ -4,18 +4,55 @@ # (roles/hardening/tasks/cve_2026_98432.yml), without the rest of that role. # The mode is cve_2026_98432_mode in group_vars/infra_boxes. # +# It runs from CI like site.yml (README, "Running playbooks from CI"), as CI's +# own account on these boxes, which this playbook also creates and keeps. From +# an operator's machine, as the shared `user` account, sudo wants its password: +# # ansible-playbook cve.yml -K --check --diff -l sf # dry run first # ansible-playbook cve.yml -K -l sf # one box # -# -K because sudo on these boxes wants the `user` account's password: an -# operator's run, never CI's. Loading the table restarts nothing, but these -# are the live indexers, so one box at a time, stopping at the first failure. +# The first run on a box has to be an operator's: until it has created the CI +# account, CI has no way in. Loading the table restarts nothing, but these are +# the live indexers, so one box at a time, stopping at the first failure. - name: Mitigate CVE-2026-98432 on the ipni/infra boxes hosts: infra_boxes become: true gather_facts: false serial: 1 tasks: + # CI's account: key only, password locked, passwordless sudo, which is + # what Ansible needs and makes it root on these boxes. It touches no other + # account, `user` included. To revoke CI's access, delete these tasks and + # the account (sudo userdel -r ipni-ci, sudo rm /etc/sudoers.d/90-ipni-ci). + - name: Create the CI account + ansible.builtin.user: + name: "{{ infra_ci_user }}" + shell: /bin/bash + create_home: true + password: "!" + state: present + register: infra_ci_account + + # exclusive: CI's key and nothing else, so a key added by hand goes on the + # next run. A dry run on a box without the account cannot look it up: the + # task above only reported it. + - name: Install CI's SSH key on the CI account + ansible.posix.authorized_key: + user: "{{ infra_ci_user }}" + key: "{{ lookup('ansible.builtin.file', infra_ci_pubkey_file) | trim }}" + state: present + exclusive: true + when: not (ansible_check_mode and infra_ci_account is changed) + + - name: Grant the CI account passwordless sudo + ansible.builtin.copy: + dest: "/etc/sudoers.d/90-{{ infra_ci_user }}" + mode: "0440" + owner: root + group: root + content: "{{ infra_ci_user }} ALL=(ALL) NOPASSWD:ALL\n" + validate: visudo -cf %s + - name: Run the hardening role's CVE-2026-98432 tasks ansible.builtin.include_role: name: hardening diff --git a/group_vars/infra_boxes/main.yml b/group_vars/infra_boxes/main.yml index c6ca393..07ad3ce 100644 --- a/group_vars/infra_boxes/main.yml +++ b/group_vars/infra_boxes/main.yml @@ -1,13 +1,19 @@ --- # sf and sf2: ipni/infra's boxes, here only for cve.yml (see inventory/hosts.yml). # -# Their gateway (a MikroTik) lets SSH through on 5222 only. Everyone signs in -# as the shared `user` account, whose sudo wants a password, so every run is an -# operator's with -K (--ask-become-pass); CI has neither the key nor the -# password, and must not get them. +# Their gateway (a MikroTik) lets SSH through on 5222 only. Operators sign in +# as the shared `user` account, whose sudo wants a password, so an operator's +# run takes -K (--ask-become-pass). ansible_user: user ansible_port: 5222 +# CI signs in as an account of its own, which cve.yml creates and keeps: key +# only (CI's key, the same one it uses on the ipfs_nodes boxes), passwordless +# sudo. scripts/ci-playbook.sh reads the name from here. An operator's -K run +# creates it the first time; CI cannot, since it has no way in until it exists. +infra_ci_user: ipni-ci +infra_ci_pubkey_file: "{{ playbook_dir }}/keys/github-actions.pub" + # The same switch as group_vars/ipfs_nodes (see its comment), set separately so # these boxes go through count before enforce on their own schedule. Override # per box in host_vars to enforce one before the other. diff --git a/inventory/known_hosts b/inventory/known_hosts index 7b7a4ff..2636219 100644 --- a/inventory/known_hosts +++ b/inventory/known_hosts @@ -1,4 +1,4 @@ -# SSH host keys of the boxes, for CI (.github/workflows/platform.yml), which +# SSH host keys of the boxes, for CI (.github/workflows/ci.yml), which # must refuse to connect to anything else. Public. Checked on 2026-10-12: the # scanned key, the key file on each box and an operator's known_hosts agreed. # Update when a box is reinstalled: ssh-keyscan -t ed25519 , then confirm @@ -6,3 +6,8 @@ 5.199.165.77 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK2+9yvOjU3tE3K7zEVHn5M30X3FHRA6mgkSvFjmH2uL 46.166.169.131 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBbz1MWGYfe2RzaH/SV+zBctwhUNWO0336K5N5x5ATOp 84.32.48.211 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGsaCZG1aU29AqxU84RhnxlMdYyvzz1f2hBhpaYu2gHR +# sf and sf2 (infra_boxes, cve.yml), on port 5222, so in ssh's [ip]:port form +# (ssh-keyscan -t ed25519 -p 5222 ). Checked the same three ways on +# 2026-10-30. +[103.232.206.165]:5222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL5ELQAJDOlCD2/LLr1GzjN6dQ3a5tR/avtk87AHOuAh +[103.232.206.162]:5222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEPvz6Hb9UV/XfJURDQauhTQ3xZvb1Hx0Xh7MQUCFQZX diff --git a/scripts/ci-playbook.sh b/scripts/ci-playbook.sh index d207ffc..aa952e6 100755 --- a/scripts/ci-playbook.sh +++ b/scripts/ci-playbook.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -# Run a playbook from CI (the site-* and platform-* jobs of +# Run a playbook from CI (the site-*, platform-* and cve-* jobs of # .github/workflows/ci.yml). # -# scripts/ci-playbook.sh site|platform check [limit] dry run: --check --diff -# scripts/ci-playbook.sh site|platform apply [limit] the real run +# scripts/ci-playbook.sh site|platform|cve check [limit] dry run: --check --diff +# scripts/ci-playbook.sh site|platform|cve apply [limit] the real run # # SSH as the admin user comes from ANSIBLE_SSH_PRIVATE_KEY, whose public half is -# keys/github-actions.pub. The vault files are gitignored, so CI has none: each +# keys/github-actions.pub. On sf and sf2 (cve.yml) the same key signs in as +# CI's own account there, infra_ci_user, instead of the operators' `user`. The vault files are gitignored, so CI has none: each # playbook's vault variables come from repository secrets instead, in the # environment. Copies of every secret are in 1Password (IPFS account, Compass # vault). @@ -19,12 +20,13 @@ # CLOUDFLARE_DNS_TOKEN -> vault_cloudflare_dns_token # ARGOCD_REPO_SSH_KEY -> vault_argocd_repo_ssh_key (optional: # only while ipni/infra is private) +# cve.yml none. Its account's sudo needs no password. set -euo pipefail -playbook=${1:?usage: $0 site|platform check|apply [limit]} -mode=${2:?usage: $0 site|platform check|apply [limit]} +playbook=${1:?usage: $0 site|platform|cve check|apply [limit]} +mode=${2:?usage: $0 site|platform|cve check|apply [limit]} limit=${3:-} -[[ $playbook == site || $playbook == platform ]] || { echo "playbook must be site or platform" >&2; exit 2; } +[[ $playbook == site || $playbook == platform || $playbook == cve ]] || { echo "playbook must be site, platform or cve" >&2; exit 2; } [[ $mode == check || $mode == apply ]] || { echo "mode must be check or apply" >&2; exit 2; } : "${ANSIBLE_SSH_PRIVATE_KEY:?not set}" if [[ $playbook == platform ]]; then @@ -58,6 +60,12 @@ elif playbook == "platform": values["vault_cloudflare_dns_token"] = os.environ["CLOUDFLARE_DNS_TOKEN"] if os.environ.get("ARGOCD_REPO_SSH_KEY"): values["vault_argocd_repo_ssh_key"] = os.environ["ARGOCD_REPO_SSH_KEY"] +elif playbook == "cve": + # Never the operators' `user`, whose sudo wants a password CI does not + # have. Extra vars outrank group_vars' ansible_user. + import yaml + with open("group_vars/infra_boxes/main.yml") as f: + values["ansible_user"] = yaml.safe_load(f)["infra_ci_user"] with open(path, "w") as f: json.dump(values, f) # JSON is YAML, which -e @file reads PY diff --git a/scripts/ci-summary.py b/scripts/ci-summary.py index a0e6d0c..2bba279 100644 --- a/scripts/ci-summary.py +++ b/scripts/ci-summary.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Turn an ansible-playbook log into the Markdown for the run's summary page. - scripts/ci-summary.py site|platform check|apply + scripts/ci-summary.py site|platform|cve check|apply Called by scripts/ci-playbook.sh in CI. For each box and task that changed (or would), it lists the loop items, for example the addresses a firewall rule