From 2349202af1291daac72c7c2b6ac62f80719321b5 Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 5 Nov 2026 17:23:48 +0100 Subject: [PATCH 1/4] Codify building Debian indexer boxes Add an indexers inventory group beside ipfs_nodes under a new managed_boxes parent, which now holds the shared settings. Existing boxes' variables are unchanged. site.yml can sign in to a new box as root by key, and refuses a box whose name or storage is undecided. Storage is per box, with a ZFS mode that never recreates a pool. Indexers get wider unattended-upgrades, no automatic reboots and a reboot-required metric. hardening checks sshd's effective settings. k3s-upgrade.yml fails a box if a hop replaced a container outside kube-system. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 + README.md | 95 +++++++++++++- group_vars/indexers/main.yml | 22 ++++ group_vars/ipfs_nodes/main.yml | 160 +---------------------- group_vars/managed_boxes/main.yml | 161 ++++++++++++++++++++++++ host_vars/idx-new-1/main.yml | 19 +++ inventory/hosts.yml | 22 ++++ k3s-upgrade.yml | 17 +-- roles/common/defaults/main.yml | 33 +++++ roles/common/tasks/main.yml | 45 ++++--- roles/common/tasks/reboot_reporting.yml | 84 +++++++++++++ roles/hardening/defaults/main.yml | 4 + roles/hardening/handlers/main.yml | 4 +- roles/hardening/tasks/main.yml | 28 ++++- roles/k3s/defaults/main.yml | 12 ++ roles/k3s/tasks/upgrade_hop.yml | 42 ++++++- roles/storage/defaults/main.yml | 18 +++ roles/storage/tasks/ext4_blank_nvme.yml | 86 +++++++++++++ roles/storage/tasks/main.yml | 101 +++------------ roles/storage/tasks/zfs.yml | 118 +++++++++++++++++ roles/storage/tasks/zfs_pool.yml | 79 ++++++++++++ site.yml | 72 +++++++++-- 22 files changed, 946 insertions(+), 282 deletions(-) create mode 100644 group_vars/indexers/main.yml create mode 100644 group_vars/managed_boxes/main.yml create mode 100644 host_vars/idx-new-1/main.yml create mode 100644 roles/common/tasks/reboot_reporting.yml create mode 100644 roles/k3s/defaults/main.yml create mode 100644 roles/storage/defaults/main.yml create mode 100644 roles/storage/tasks/ext4_blank_nvme.yml create mode 100644 roles/storage/tasks/zfs.yml create mode 100644 roles/storage/tasks/zfs_pool.yml diff --git a/AGENTS.md b/AGENTS.md index 9559ad7..15381ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,12 @@ ansible-playbook cve.yml -K --check --diff -l sf # dry run first ansible infra_boxes -m ping # no sudo needed ``` +**indexers** (`idx-new-1`, a placeholder name, 84.32.71.3) are built by this repo +but are not `ipfs_nodes`: Debian, `site.yml` and `k3s-upgrade.yml` only. Shared +settings are in `group_vars/managed_boxes/`. See "Indexer boxes" in the README. +Merge a new box only after it is built and its host key is pinned, or CI's +`site.yml` runs fail on it. + ## Getting in SSH is key-only as the hardened admin user `ipni`; root login is disabled and diff --git a/README.md b/README.md index 0712762..e9b1728 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,9 @@ The playbook is idempotent and safe to re-run. **Controller prerequisites.** `ansible-core` with the `community.general` and `ansible.posix` collections, and `kubectl` for the tunnel scripts. A **first** run against a freshly provisioned box also needs `sshpass` installed on the -controller: site.yml falls back to root password authentication when the admin -key does not work yet, and Ansible shells out to `sshpass` for that +controller: site.yml falls back to root when the admin key does not work yet, +by key from your ssh-agent if the box takes it (no `sshpass` needed), otherwise +with the root password, and Ansible shells out to `sshpass` for that (`ansible.cfg` keeps `password` in `PreferredAuthentications` for the same reason). Once hardening has run, every later connection is key-based and `sshpass` is no longer used. @@ -217,7 +218,7 @@ those with this box's own IP (the same machine under its old name) and any that are not Ready. A Ready node on another IP is never deleted. **Upgrading k3s.** Add the latest patch of each minor version you need, up to -the stable channel, to `k3s_upgrade_path` in `group_vars/ipfs_nodes/main.yml`, +the stable channel, to `k3s_upgrade_path` in `group_vars/managed_boxes/main.yml`, and set `k3s_version` there to the last entry — the playbook refuses to run unless the two agree, so the target and the route to it stay in step. Then run `k3s-upgrade.yml`. Look versions up at @@ -231,6 +232,87 @@ unless the two agree, so the target and the route to it stay in step. Then run k3s itself, it first flushes every `KUBE-*`/flannel iptables rule. - Each step waits for the node version, Ready, and every deployment. - Rolling back to an older minor needs the backup taken on that minor. +- Each step records every container ID first and compares afterwards. A + container replaced outside `kube-system` fails the box, so the next box is + not touched; one replaced inside it is reported. + +**Do containers restart on a k3s upgrade?** Normally not, from 1.31 on. This was +checked against k3s's docs and the source of k3s and Kubernetes, not measured on +a box, so the check above is what proves it on each hop. + +Why they survive: the unit has `KillMode=process`, so stopping k3s kills only +k3s itself. Its containerd child dies with it (`Pdeathsig`), but each container +has its own shim process, which keeps running and which the new containerd +reconnects to. A running container keeps its old shim and runc until it is +replaced. k3s's docs say containers continue while k3s is stopped. The kubelet +then decides from two checks, and neither looks at its own version or at the +pause image: the container hash (since 1.31 only the name and image, unchanged +through 1.37) and `PodSandboxChanged`. Readiness is carried over a kubelet +restart for containers that were already ready. + +Where containers **do** restart, or traffic is otherwise touched: + +| Case | What happens | +|---|---| +| Crossing 1.30 to 1.31 | The hash changed, so every container on the node restarts. Any box still below 1.31 hits this once (sf and sf2, on 1.28, would). | +| A later Kubernetes changes the hash again | The same, with no warning. Upstream only supports draining before a minor kubelet upgrade, and declined to treat this as a breaking change. | +| k3s's packaged components get a new image | Those pods roll: CoreDNS (1.36.4 to 1.37.2 moves it from 1.14.6 to 1.14.7) and, where enabled, Traefik, metrics-server, local-path and servicelb's `svclb` pods. | +| A new pause image | Existing pods keep theirs; only new pods get it. Not a restart. | +| A containerd bug | Has happened: k3s issue 6809 restarted non-host-network pods because CNI state was lost. Read the release notes. | +| The API server and kubelet are down for the backup and restart | Containers run on, but nothing is restarted if one crashes, and anything calling the API sees errors: Argo CD, operators, pods that watch. | +| A reboot or `k3s-killall.sh` | Everything restarts. | + +## Indexer boxes + +The `indexers` group holds indexers built by this repo: Debian stable, standalone +k3s, built by `site.yml` and upgraded by `k3s-upgrade.yml`. No other playbook +reaches them. They share the fleet's settings through `managed_boxes`, the parent +of `ipfs_nodes` and `indexers` (`group_vars/managed_boxes/main.yml`); a value in +`group_vars/indexers/` or `host_vars//` overrides it. What runs on the box +comes from ipni/infra through Argo CD. sf and sf2 are not part of this. + +**Storage** is per box, in `host_vars//main.yml`. `storage_mode` is +`ext4_blank_nvme` (as on the IPFS boxes) or `zfs` (the pools in +`storage_zfs_pools`, with disks by `/dev/disk/by-id`). The ZFS path never +recreates anything: an imported pool is left alone, an exported pool of that name +or a disk with any signature stops the run, `zpool create` never gets `-f`, and a +run from CI refuses to create a pool at all. k3s will not start until the datasets +are mounted, so pods cannot write to the OS disk after a failed import. + +**Building a box.** Add it to `indexers` with its `host_vars`, on a branch. With +the root key in your agent (`ssh-add`), and an admin key from `keys/` too: + +```bash +ansible-playbook site.yml -l +``` + +A dry run cannot get far on a box this new: the access gate needs the admin user +to exist and later tasks need packages installed. So check the box with read-only +commands first. Then pin its host key in `inventory/known_hosts` and merge. **Do +not merge before the build**: CI refuses an unhardened box, and its `site.yml` runs +would fail until the box is built and pinned. If root takes neither your key nor +`vault_root_password` (which only `ipfs_nodes` can see), the run stops there. + +### Package updates and reboots + +| | Policy on an indexer | +|---|---| +| What updates | `unattended-upgrades`, daily: Debian stable, its security archive and `-updates`. Never backports. | +| Reboots | Never automatic. | +| ZFS packages | Held out of unattended upgrades on a zfs box: a DKMS build that fails would only show at the next reboot, as a pool that does not import. Upgrade them by hand. | +| Services | Packages that restart their own service on upgrade still do (ssh, chrony, fail2ban). Established SSH sessions survive. k3s is not an apt package, so apt never restarts k3s or containers. `needrestart` only lists what is left running on old libraries. | +| Kernels | Installed but inactive until a reboot. | + +A pending reboot shows in three places: `/var/run/reboot-required`, the `reboot:` +line in every `site.yml` summary, and the `node_reboot_required` metric, which an +hourly timer writes to `/var/lib/node_exporter/textfile/reboot.prom`. The metric +reaches Prometheus only once ipni/infra's node-exporter is given that directory +(`--collector.textfile.directory`, with a host mount) and an alert is added. + +To reboot: on a zfs box run `dkms status` first and check ZFS is built for the new +kernel, then reboot one box at a time. Every container restarts. + +The IPFS boxes keep their existing update policy. They only gain the `reboot:` summary line. ## needle @@ -1772,8 +1854,11 @@ turn an unrelated pull request red. ``` ansible.cfg inventory, vault and SSH defaults -inventory/hosts.yml the three hosts -group_vars/ipfs_nodes/ tunables (admin user, firewall, k3s, sysctl) +inventory/hosts.yml the boxes and their groups +group_vars/managed_boxes/ what every built box shares (admin user, firewall, k3s, sysctl) +group_vars/ipfs_nodes/ the IPFS boxes' own settings (rollout, bootstrappers, Argo CD) +group_vars/indexers/ the indexers' own settings +host_vars//main.yml an indexer's site and storage host_vars//vault.yml encrypted per-box credentials (gitignored, see Secrets) roles/{common,storage,hardening,k3s}/ base preparation roles/needle/ needle firewall + deploy diff --git a/group_vars/indexers/main.yml b/group_vars/indexers/main.yml new file mode 100644 index 0000000..5d330f0 --- /dev/null +++ b/group_vars/indexers/main.yml @@ -0,0 +1,22 @@ +--- +# Indexer boxes. Everything not set here comes from group_vars/managed_boxes. + +# `debian` is the Debian cloud image's login account, the same uid 1000 risk as +# Ubuntu's (see roles/hardening/tasks/main.yml). +hardening_removed_accounts: + - ubuntu + - debian + +# Point releases as well as security fixes, so packages keep up with Debian +# stable and not only with its advisories. Debian's own defaults already allow +# the release itself and Debian-Security; this adds -updates. Backports are +# never matched. See "Package updates and reboots" in the README. +common_unattended_extra_origins: + - "origin=Debian,codename=${distro_codename}-updates" + +# needrestart reports and restarts nothing, and the reboot state goes to a +# file node-exporter can read. +common_reboot_reporting: true + +# To upgrade k3s here ahead of the fleet (a canary), set k3s_version and +# k3s_upgrade_path in this file: they override group_vars/managed_boxes. diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 6167c41..408b89a 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -1,70 +1,7 @@ --- -# --------------------------------------------------------------------------- -# Admin account -# --------------------------------------------------------------------------- -# The non-root account that replaces root SSH access. Its authorized_keys is -# the ONLY way in over the network once hardening completes; the provider -# KVM/web console is the designated out-of-band fallback. -# -# The role installs these keys exclusively: a key that is not listed here is -# REMOVED on the next run. So every key is a committed public key file in -# keys/, never "whoever runs the playbook" (~/.ssh/id_ed25519.pub), which would -# lock out every other operator. To add someone, commit their public key here -# BEFORE the run that would lock them out; to remove someone, delete their line -# and re-run the hardening role (site.yml --tags hardening, one box at a time). -admin_user: ipni -admin_pubkey_file: - - "{{ playbook_dir }}/keys/takgn.pub" - - "{{ playbook_dir }}/keys/nymd.pub" - - "{{ playbook_dir }}/keys/byo.pub" - # Break-glass: only for when no operator key works. Its private half exists - # only in 1Password (IPFS account, Compass vault); see AGENTS.md. Never - # remove it without putting another emergency key in its place. - - "{{ playbook_dir }}/keys/break-glass.pub" - # CI (.github/workflows/ci.yml, platform.yml). Its private key is the - # ANSIBLE_SSH_PRIVATE_KEY repository secret. - - "{{ playbook_dir }}/keys/github-actions.pub" - -# --------------------------------------------------------------------------- -# SSH hardening -# --------------------------------------------------------------------------- -ssh_port: 22 -ssh_permit_root_login: "no" -ssh_password_authentication: "no" -ssh_allow_users: "{{ admin_user }}" - -# --------------------------------------------------------------------------- -# Firewall (default deny inbound) -# --------------------------------------------------------------------------- -# Base-prep ports only. Service ports (IPFS swarm, HTTP ingress) get opened by -# the per-service playbooks in the next pass, so we are not exposing surface -# for workloads that do not exist yet. -# -# The k3s API server (6443) is deliberately absent from this list: it stays -# closed to the internet and remote kubectl goes over an SSH tunnel (see -# scripts/kubectl-tunnel.sh). Do not add it here - this list opens a port to -# every source. The one exception is narrower: roles/argocd opens 6443 on the -# other boxes to argocd_hub's address only, for Argo CD (see below). -firewall_allowed_tcp: - - "{{ ssh_port }}" -firewall_allowed_udp: [] - -# CVE-2026-98432: a remote bug in the kernel's TCP handshake, triggered by a SYN -# that carries IPv4 options. Until the boxes run a fixed kernel, an nft table -# of its own drops IPv4 TCP with IP options, and strips the MPTCP option from -# SYNs (roles/hardening/tasks/cve_2026_98432.yml). Nothing legitimate sends -# IP options on TCP. -# -# count match and count, change nothing: what to run first, to see what -# enforce would have hit (sudo nft list table inet cve_2026_98432) -# enforce drop and strip -# off remove the table and its unit -# -# Override per box in host_vars to enforce one box before the others. All -# three counted from 2026-10-30 13:20 UTC (no IP-options packets in the first -# hour), then sing-1 enforced alone from 14:48 before this became the default. -cve_2026_98432_mode: enforce - +# The IPFS boxes' own settings. What they share with every other box this repo +# builds (admin user, SSH, firewall, storage, k3s, sysctls) is in +# group_vars/managed_boxes/main.yml. # --------------------------------------------------------------------------- # Rollout mode # --------------------------------------------------------------------------- @@ -91,97 +28,6 @@ cve_2026_98432_mode: enforce # -e production_rollout=false only for a box that carries no traffic. production_rollout: true -# --------------------------------------------------------------------------- -# Storage -# --------------------------------------------------------------------------- -# Each host has a second, blank ~894 GiB NVMe. It becomes the dedicated data -# volume so IPFS blockstores never compete with the OS disk. The device name -# differs per host, so the storage role detects it by shape rather than name. -data_mount: /data - -# --------------------------------------------------------------------------- -# k3s -# --------------------------------------------------------------------------- -k3s_version: v1.36.4+k3s1 # stable channel, checked 2026-09-15; upgrade with k3s-upgrade.yml - -# The route k3s-upgrade.yml walks to reach k3s_version, one minor at a time: -# k3s and Kubernetes do not support skipping minors. Latest patch of each minor -# from https://update.k3s.io/v1-release/channels (checked 2026-09-15), ending -# at the stable channel. Kept here beside k3s_version so one file records both -# the target and the route to it; the last entry MUST equal k3s_version. -k3s_upgrade_path: - - v1.31.14+k3s1 - - v1.32.13+k3s1 - - v1.33.13+k3s2 - - v1.34.11+k3s1 - - v1.35.8+k3s1 - - v1.36.4+k3s1 - -# Traefik is disabled: we will choose an ingress deliberately when the services -# land, rather than inheriting k3s's default. servicelb/local-path are kept. -k3s_disable_components: - - traefik - -# --------------------------------------------------------------------------- -# Kernel tuning for IPFS/libp2p workloads -# --------------------------------------------------------------------------- -# libp2p nodes hold very large numbers of concurrent connections and rely -# heavily on UDP (QUIC), so the stock Ubuntu defaults are too small. -sysctl_tuning: - fs.file-max: 2097152 - fs.inotify.max_user_watches: 524288 - fs.inotify.max_user_instances: 8192 - net.core.rmem_max: 7500000 - net.core.wmem_max: 7500000 - net.core.somaxconn: 4096 - net.ipv4.tcp_max_syn_backlog: 8192 - net.ipv4.ip_local_port_range: "1024 65535" - # Host-network listeners that sit INSIDE the ephemeral range above. Without - # this, any outbound socket (needle's DHT crawl alone opens ~800) can be - # handed one of these numbers as its source port; a pod restarting under - # `Recreate` then fails to re-bind with "address already in use". - # Reserving only stops the kernel choosing them for outbound connections; - # explicit binds still work. Update when a host-network port is added: - # 443 Envoy (outside the range today, kept for safety) - # 4001 bootstrapper libp2p 4002 bootstrapper /ws (loopback) - # 4011 bootstrap-b libp2p 4012 bootstrap-b /ws (loopback) - # 4004-4007 needle libp2p (four instances) 4101 content kubo libp2p - # 4443 bootstrapper WSS (Envoy) 4453 bootstrap-b WSS (Envoy) - # 5011 bootstrapper API 5012 bootstrap-b API - # 5021 content kubo RPC API (loopback) - # 6443-6444 k3s API 8190-8193 needle APIs (loopback, four instances) - # 8000 Healthchecks (healthchecks_host only) - # 8443 monitoring Traefik HTTPS, 9800 its ping, 9801 its metrics (argocd_hub only, - # host network: ipni/infra kubernetes/manifests/eu/lith1/monitoring/values/traefik.yaml) - # 9100 node-exporter (every box, host network: the chart's default in - # ipni/infra kubernetes/platform/node-exporter) - # 9810 metrics proxy (Traefik), 9811 its ping (every box, host network: - # ipni/infra kubernetes/platform/metrics-proxy/values.yaml) - # 4317-4318 trace collector OTLP gRPC and HTTP, 9812 its HTTP server - # (loopback, every box, host network: ipni/infra - # kubernetes/platform/alloy-traces/values.yaml) - # 8472/udp flannel VXLAN - # 8888 content ipfs-cluster Prometheus metrics (loopback; reserved ahead of - # enabling them in k8s/content, scraped through the metrics proxy) - # 9094 content ipfs-cluster REST API (loopback) - # 9095 content ipfs-cluster IPFS proxy (loopback) - # 9096 content ipfs-cluster swarm - # 9097 content ipfs-cluster Pinning Service API (loopback) - # 9443 content pinning API over TLS (Envoy) 9904 its Envoy admin (loopback) - # 9901 route-origin Envoy admin 9902 bootstrap-wss Envoy admin - # 9903 bootstrap-b's bootstrap-wss Envoy admin - # 10010 containerd 10248-10259 kubelet/kube-proxy/k3s - net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4317-4318,4443,4453,5011-5012,5021,6443-6444,8000,8190-8193,8443,8472,8888,9094-9097,9100,9443,9800-9801,9810-9812,9901-9904,10010,10248-10259" - net.ipv4.tcp_fin_timeout: 15 - # Must equal what k3s's kube-proxy sets on every start: max(32768 * CPUs, - # 131072), i.e. 1048576 on these 32-core boxes. A fixed lower value both - # flapped against k3s on each restart and cut the conntrack table to a - # quarter of what Kubernetes itself chose for this hardware. - net.netfilter.nf_conntrack_max: "{{ [32768 * (ansible_processor_vcpus | int), 131072] | max }}" - vm.max_map_count: 262144 - -nofile_limit: 1048576 - # --------------------------------------------------------------------------- # Bootstrappers (bootstrap.yml) # --------------------------------------------------------------------------- diff --git a/group_vars/managed_boxes/main.yml b/group_vars/managed_boxes/main.yml new file mode 100644 index 0000000..d6e961f --- /dev/null +++ b/group_vars/managed_boxes/main.yml @@ -0,0 +1,161 @@ +--- +# What every box this repo builds has in common: ipfs_nodes and indexers, the +# children of managed_boxes (inventory/hosts.yml). A child group's group_vars +# override these, so group_vars/indexers can differ without touching the fleet. +# sf and sf2 (infra_boxes) are not in this group and never get these. +# --------------------------------------------------------------------------- +# Admin account +# --------------------------------------------------------------------------- +# The non-root account that replaces root SSH access. Its authorized_keys is +# the ONLY way in over the network once hardening completes; the provider +# KVM/web console is the designated out-of-band fallback. +# +# The role installs these keys exclusively: a key that is not listed here is +# REMOVED on the next run. So every key is a committed public key file in +# keys/, never "whoever runs the playbook" (~/.ssh/id_ed25519.pub), which would +# lock out every other operator. To add someone, commit their public key here +# BEFORE the run that would lock them out; to remove someone, delete their line +# and re-run the hardening role (site.yml --tags hardening, one box at a time). +admin_user: ipni +admin_pubkey_file: + - "{{ playbook_dir }}/keys/takgn.pub" + - "{{ playbook_dir }}/keys/nymd.pub" + - "{{ playbook_dir }}/keys/byo.pub" + # Break-glass: only for when no operator key works. Its private half exists + # only in 1Password (IPFS account, Compass vault); see AGENTS.md. Never + # remove it without putting another emergency key in its place. + - "{{ playbook_dir }}/keys/break-glass.pub" + # CI (.github/workflows/ci.yml, platform.yml). Its private key is the + # ANSIBLE_SSH_PRIVATE_KEY repository secret. + - "{{ playbook_dir }}/keys/github-actions.pub" + +# --------------------------------------------------------------------------- +# SSH hardening +# --------------------------------------------------------------------------- +ssh_port: 22 +ssh_permit_root_login: "no" +ssh_password_authentication: "no" +ssh_allow_users: "{{ admin_user }}" + +# --------------------------------------------------------------------------- +# Firewall (default deny inbound) +# --------------------------------------------------------------------------- +# Base-prep ports only. Service ports (IPFS swarm, HTTP ingress) get opened by +# the per-service playbooks in the next pass, so we are not exposing surface +# for workloads that do not exist yet. +# +# The k3s API server (6443) is deliberately absent from this list: it stays +# closed to the internet and remote kubectl goes over an SSH tunnel (see +# scripts/kubectl-tunnel.sh). Do not add it here - this list opens a port to +# every source. The one exception is narrower: roles/argocd opens 6443 on the +# other boxes to argocd_hub's address only, for Argo CD (see below). +firewall_allowed_tcp: + - "{{ ssh_port }}" +firewall_allowed_udp: [] + +# CVE-2026-98432: a remote bug in the kernel's TCP handshake, triggered by a SYN +# that carries IPv4 options. Until the boxes run a fixed kernel, an nft table +# of its own drops IPv4 TCP with IP options, and strips the MPTCP option from +# SYNs (roles/hardening/tasks/cve_2026_98432.yml). Nothing legitimate sends +# IP options on TCP. +# +# count match and count, change nothing: what to run first, to see what +# enforce would have hit (sudo nft list table inet cve_2026_98432) +# enforce drop and strip +# off remove the table and its unit +# +# Override per box in host_vars to enforce one box before the others. All +# three counted from 2026-10-30 13:20 UTC (no IP-options packets in the first +# hour), then sing-1 enforced alone from 14:48 before this became the default. +cve_2026_98432_mode: enforce + +# --------------------------------------------------------------------------- +# Storage +# --------------------------------------------------------------------------- +# On the IPFS hosts: a second, blank ~894 GiB NVMe. It becomes the dedicated data +# volume so IPFS blockstores never compete with the OS disk. The device name +# differs per host, so the storage role detects it by shape rather than name. +data_mount: /data + +# --------------------------------------------------------------------------- +# k3s +# --------------------------------------------------------------------------- +k3s_version: v1.36.4+k3s1 # stable channel, checked 2026-09-15; upgrade with k3s-upgrade.yml + +# The route k3s-upgrade.yml walks to reach k3s_version, one minor at a time: +# k3s and Kubernetes do not support skipping minors. Latest patch of each minor +# from https://update.k3s.io/v1-release/channels (checked 2026-09-15), ending +# at the stable channel. Kept here beside k3s_version so one file records both +# the target and the route to it; the last entry MUST equal k3s_version. +k3s_upgrade_path: + - v1.31.14+k3s1 + - v1.32.13+k3s1 + - v1.33.13+k3s2 + - v1.34.11+k3s1 + - v1.35.8+k3s1 + - v1.36.4+k3s1 + +# Traefik is disabled: we will choose an ingress deliberately when the services +# land, rather than inheriting k3s's default. servicelb/local-path are kept. +k3s_disable_components: + - traefik + +# --------------------------------------------------------------------------- +# Kernel tuning for IPFS/libp2p workloads +# --------------------------------------------------------------------------- +# libp2p nodes hold very large numbers of concurrent connections and rely +# heavily on UDP (QUIC), so the stock Ubuntu defaults are too small. +sysctl_tuning: + fs.file-max: 2097152 + fs.inotify.max_user_watches: 524288 + fs.inotify.max_user_instances: 8192 + net.core.rmem_max: 7500000 + net.core.wmem_max: 7500000 + net.core.somaxconn: 4096 + net.ipv4.tcp_max_syn_backlog: 8192 + net.ipv4.ip_local_port_range: "1024 65535" + # Host-network listeners that sit INSIDE the ephemeral range above. Without + # this, any outbound socket (needle's DHT crawl alone opens ~800) can be + # handed one of these numbers as its source port; a pod restarting under + # `Recreate` then fails to re-bind with "address already in use". + # Reserving only stops the kernel choosing them for outbound connections; + # explicit binds still work. Update when a host-network port is added: + # 443 Envoy (outside the range today, kept for safety) + # 4001 bootstrapper libp2p 4002 bootstrapper /ws (loopback) + # 4011 bootstrap-b libp2p 4012 bootstrap-b /ws (loopback) + # 4004-4007 needle libp2p (four instances) 4101 content kubo libp2p + # 4443 bootstrapper WSS (Envoy) 4453 bootstrap-b WSS (Envoy) + # 5011 bootstrapper API 5012 bootstrap-b API + # 5021 content kubo RPC API (loopback) + # 6443-6444 k3s API 8190-8193 needle APIs (loopback, four instances) + # 8000 Healthchecks (healthchecks_host only) + # 8443 monitoring Traefik HTTPS, 9800 its ping, 9801 its metrics (argocd_hub only, + # host network: ipni/infra kubernetes/manifests/eu/lith1/monitoring/values/traefik.yaml) + # 9100 node-exporter (every box, host network: the chart's default in + # ipni/infra kubernetes/platform/node-exporter) + # 9810 metrics proxy (Traefik), 9811 its ping (every box, host network: + # ipni/infra kubernetes/platform/metrics-proxy/values.yaml) + # 4317-4318 trace collector OTLP gRPC and HTTP, 9812 its HTTP server + # (loopback, every box, host network: ipni/infra + # kubernetes/platform/alloy-traces/values.yaml) + # 8472/udp flannel VXLAN + # 8888 content ipfs-cluster Prometheus metrics (loopback; reserved ahead of + # enabling them in k8s/content, scraped through the metrics proxy) + # 9094 content ipfs-cluster REST API (loopback) + # 9095 content ipfs-cluster IPFS proxy (loopback) + # 9096 content ipfs-cluster swarm + # 9097 content ipfs-cluster Pinning Service API (loopback) + # 9443 content pinning API over TLS (Envoy) 9904 its Envoy admin (loopback) + # 9901 route-origin Envoy admin 9902 bootstrap-wss Envoy admin + # 9903 bootstrap-b's bootstrap-wss Envoy admin + # 10010 containerd 10248-10259 kubelet/kube-proxy/k3s + net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4317-4318,4443,4453,5011-5012,5021,6443-6444,8000,8190-8193,8443,8472,8888,9094-9097,9100,9443,9800-9801,9810-9812,9901-9904,10010,10248-10259" + net.ipv4.tcp_fin_timeout: 15 + # Must equal what k3s's kube-proxy sets on every start: max(32768 * CPUs, + # 131072), i.e. 1048576 on these 32-core boxes. A fixed lower value both + # flapped against k3s on each restart and cut the conntrack table to a + # quarter of what Kubernetes itself chose for this hardware. + net.netfilter.nf_conntrack_max: "{{ [32768 * (ansible_processor_vcpus | int), 131072] | max }}" + vm.max_map_count: 262144 + +nofile_limit: 1048576 diff --git a/host_vars/idx-new-1/main.yml b/host_vars/idx-new-1/main.yml new file mode 100644 index 0000000..2d1411a --- /dev/null +++ b/host_vars/idx-new-1/main.yml @@ -0,0 +1,19 @@ +--- +# Placeholder box: site.yml refuses to build it until both of these are decided. +site: undecided # the location, for the site= node label + +# ext4_blank_nvme: the single blank NVMe, ext4 at data_mount, as on ipfs_nodes. +# zfs: the pools below. Disks by /dev/disk/by-id only. An existing pool is never +# recreated, and a disk carrying any signature stops the run. +storage_mode: undecided + +# Shape of a zfs box, to fill in from the box's real disks: +# +# storage_mode: zfs +# data_mount: /ipni # must be one of the mountpoints below; k3s's local-path lives here +# storage_zfs_pools: +# - name: tank +# vdevs: [mirror, /dev/disk/by-id/nvme-AAA, /dev/disk/by-id/nvme-BBB] +# datasets: +# - {name: ipni, properties: {mountpoint: /ipni}} +# - {name: ipni-mirror, properties: {mountpoint: /ipni-mirror}} diff --git a/inventory/hosts.yml b/inventory/hosts.yml index 22252cb..11ed25d 100644 --- a/inventory/hosts.yml +++ b/inventory/hosts.yml @@ -32,3 +32,25 @@ all: ansible_host: 103.232.206.165 sf2: ansible_host: 103.232.206.162 + + # Indexers built by this repo: Debian, site.yml and k3s-upgrade.yml only. + # Not ipfs_nodes, whose membership means bootstrap.yml, content.yml and + # routing.yml, and whose peers list one another. Not infra_boxes, which is + # for cve.yml alone. See "Indexer boxes" in the README. + # + # idx-new-1 is a placeholder name: the name becomes the hostname and k3s + # node name, so choose it before the first run. site.yml refuses to build + # it while host_vars/idx-new-1/main.yml says "undecided". + indexers: + hosts: + idx-new-1: + ansible_host: 84.32.71.3 + + # Every box site.yml and k3s-upgrade.yml manage, and where their shared + # settings live (group_vars/managed_boxes). Membership is listed here, not + # by nesting the groups above, because scripts read + # all.children.ipfs_nodes.hosts directly. + managed_boxes: + children: + ipfs_nodes: + indexers: diff --git a/k3s-upgrade.yml b/k3s-upgrade.yml index f69914e..4e6ed84 100644 --- a/k3s-upgrade.yml +++ b/k3s-upgrade.yml @@ -9,12 +9,15 @@ # ansible-playbook k3s-upgrade.yml -l chic-1 # # Every step backs up /var/lib/rancher/k3s/server to /var/lib/rancher/k3s-backups -# first. The path itself lives in group_vars/ipfs_nodes/main.yml beside -# k3s_version, so one file records both the target and the route to it; the -# last entry there must equal k3s_version. The assert below enforces that. +# first, and afterwards fails the box if a container outside kube-system was +# replaced (see "Upgrading k3s" in the README). The path itself lives in +# group_vars/managed_boxes/main.yml beside k3s_version, so one file records both +# the target and the route to it; the last entry there must equal k3s_version. +# The assert below enforces that. group_vars/indexers can override both, to +# upgrade the indexers first. - name: Upgrade k3s - hosts: ipfs_nodes + hosts: managed_boxes # serial is templated before any host is chosen, so inventory group_vars are # not visible here: read production_rollout from its group_vars file # directly. -e production_rollout and -e rollout_serial still override. @@ -22,7 +25,7 @@ {{ rollout_serial | default(1 if (production_rollout | default((lookup('ansible.builtin.file', playbook_dir ~ '/group_vars/ipfs_nodes/main.yml') | from_yaml).production_rollout | default(false)) | bool) - else (groups['ipfs_nodes'] | length)) }} + else (groups['managed_boxes'] | length)) }} remote_user: "{{ admin_user }}" become: true gather_facts: false @@ -34,9 +37,9 @@ that: k3s_upgrade_path[-1] == k3s_version fail_msg: >- k3s_upgrade_path ends at {{ k3s_upgrade_path[-1] }} but k3s_version is - {{ k3s_version }}; both are in group_vars/ipfs_nodes/main.yml. + {{ k3s_version }}; both are in group_vars/managed_boxes/main.yml, or + a group's override of it. quiet: true - run_once: true - name: Walk the upgrade path ansible.builtin.include_role: diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 5058427..1db15a9 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -20,3 +20,36 @@ # Security updates keep arriving on their own in the meantime; this switch is # about when a WHOLE-SYSTEM upgrade happens, not about whether patching happens. common_apt_upgrade: "{{ bootstrap_mode | default(false) }}" + +# Installed everywhere. Debian-only additions are in common_distro_packages. +common_base_packages: + - ca-certificates + - curl + - gnupg + - git + - jq + - htop + - iotop + - ncdu + - net-tools + - rsync + - unzip + - chrony + - unattended-upgrades + - apt-listchanges + - python3-kubernetes # lets Ansible drive k8s later without pip + +common_distro_packages: + Ubuntu: [dnsutils] + Debian: [sudo, bind9-dnsutils, apparmor] + +# Origins allowed on top of the distribution's own 50unattended-upgrades. +common_unattended_extra_origins: [] + +# needrestart in list-only mode, and a metric saying a reboot is pending. Off on +# the IPFS boxes, whose behaviour this must not change. +common_reboot_reporting: false + +# node-exporter reads *.prom from here once it is given +# --collector.textfile.directory (ipni/infra kubernetes/platform/node-exporter). +common_textfile_dir: /var/lib/node_exporter/textfile diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index da7f28c..fb803f3 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -54,23 +54,10 @@ - name: Install base packages ansible.builtin.apt: - name: - - ca-certificates - - curl - - gnupg - - git - - jq - - htop - - iotop - - ncdu - - dnsutils - - net-tools - - rsync - - unzip - - chrony - - unattended-upgrades - - apt-listchanges - - python3-kubernetes # lets Ansible drive k8s later without pip + # sudo: the admin user depends on it, and a minimal Debian may not have it. + # On Debian dnsutils is only a virtual package, provided by bind9-dnsutils, + # and k3s's containers need apparmor_parser while the kernel has AppArmor on. + name: "{{ common_base_packages + common_distro_packages[ansible_distribution] | default([]) }}" state: present - name: Enable and start chrony for time sync @@ -212,6 +199,8 @@ APT::Periodic::Unattended-Upgrade "1"; APT::Periodic::AutocleanInterval "7"; +# With the defaults this writes what it always has: the policy on the IPFS boxes +# is unchanged. An indexer widens the origins, and a zfs box adds the blocklist. - name: Configure unattended-upgrades to security only, no auto-reboot ansible.builtin.copy: dest: /etc/apt/apt.conf.d/51-ipfs-unattended @@ -223,3 +212,25 @@ Unattended-Upgrade::Automatic-Reboot "false"; Unattended-Upgrade::Remove-Unused-Kernel-Packages "true"; Unattended-Upgrade::Remove-Unused-Dependencies "true"; + {% if common_unattended_extra_origins %} + Unattended-Upgrade::Origins-Pattern { + {% for origin in common_unattended_extra_origins %} + "{{ origin }}"; + {% endfor %} + }; + {% endif %} + {% if storage_mode | default('') == 'zfs' %} + // A DKMS module that fails to build against a new ZFS would surface at + // the next reboot, as a pool that does not import. ZFS moves by hand. + Unattended-Upgrade::Package-Blacklist { + "zfs"; + "libzfs"; + "libzpool"; + "libnvpair"; + "libuutil"; + }; + {% endif %} + +- name: Report pending reboots and restarts + ansible.builtin.import_tasks: reboot_reporting.yml + when: common_reboot_reporting | bool diff --git a/roles/common/tasks/reboot_reporting.yml b/roles/common/tasks/reboot_reporting.yml new file mode 100644 index 0000000..c6b6854 --- /dev/null +++ b/roles/common/tasks/reboot_reporting.yml @@ -0,0 +1,84 @@ +--- +# Nothing here reboots or restarts anything. It makes "this box needs a reboot" +# something a person or an alert can see. + +- name: Install needrestart + ansible.builtin.apt: + name: needrestart + state: present + +# The default in a non-interactive apt run can be to restart services. `l` only +# lists them. Restarting k3s would be survivable, but this box decides nothing +# on its own. +- name: Keep needrestart to listing what needs a restart + ansible.builtin.copy: + dest: /etc/needrestart/conf.d/50-ipni.conf + mode: "0644" + content: | + # Managed by Ansible. + $nrconf{restart} = 'l'; + +- name: Create the node-exporter textfile directory + ansible.builtin.file: + path: "{{ common_textfile_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +# Two signals: the marker Debian's kernel hook leaves, and needrestart's verdict +# on the running kernel (KSTA: 1 current, 2 ABI change, 3 newer version). +- name: Install the reboot-required reporter + ansible.builtin.copy: + dest: /usr/local/sbin/ipni-reboot-metric + mode: "0755" + owner: root + group: root + content: | + #!/bin/sh + # Managed by Ansible. Writes node_reboot_required for node-exporter. + set -eu + required=0 + [ -e /var/run/reboot-required ] && required=1 + ksta=$(needrestart -b 2>/dev/null | sed -n 's/^NEEDRESTART-KSTA: //p') + case "${ksta:-1}" in 2|3) required=1 ;; esac + tmp="{{ common_textfile_dir }}/.reboot.prom.$$" + printf '# HELP node_reboot_required 1 when this box is waiting for a reboot.\n# TYPE node_reboot_required gauge\nnode_reboot_required %s\n' "$required" > "$tmp" + chmod 0644 "$tmp" + mv "$tmp" "{{ common_textfile_dir }}/reboot.prom" + +- name: Install the reboot-required reporter's unit + ansible.builtin.copy: + dest: /etc/systemd/system/ipni-reboot-metric.service + mode: "0644" + content: | + # Managed by Ansible. + [Unit] + Description=Report whether this box is waiting for a reboot + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/ipni-reboot-metric + +- name: Install the reboot-required reporter's timer + ansible.builtin.copy: + dest: /etc/systemd/system/ipni-reboot-metric.timer + mode: "0644" + content: | + # Managed by Ansible. + [Unit] + Description=Report hourly whether this box is waiting for a reboot + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1h + + [Install] + WantedBy=timers.target + +- name: Enable the reboot-required timer + ansible.builtin.systemd: + name: ipni-reboot-metric.timer + enabled: true + state: started + daemon_reload: true diff --git a/roles/hardening/defaults/main.yml b/roles/hardening/defaults/main.yml index 5d48070..c5dd4ee 100644 --- a/roles/hardening/defaults/main.yml +++ b/roles/hardening/defaults/main.yml @@ -3,3 +3,7 @@ # (tasks/cve_2026_98432.yml). A default rather than a var on the import, so # cve.yml can run those tasks on their own with tasks_from. cve_2026_98432_ruleset: /etc/nftables.d/cve-2026-98432.nft + +# Provider login accounts removed from the box; group_vars/indexers adds Debian's. +hardening_removed_accounts: + - ubuntu diff --git a/roles/hardening/handlers/main.yml b/roles/hardening/handlers/main.yml index 489c4c6..6643a45 100644 --- a/roles/hardening/handlers/main.yml +++ b/roles/hardening/handlers/main.yml @@ -1,5 +1,7 @@ --- -# Ubuntu 24.04 may run sshd under socket activation (ssh.socket), where +# The unit is `ssh` on Debian and Ubuntu alike. Debian runs it as a plain service, +# so the reload below is all it needs. Ubuntu 24.04 may run sshd under socket +# activation (ssh.socket), where # `systemctl reload ssh` is not necessarily valid. Try the reload, and fall # back to restarting the socket unit. Neither drops established sessions. # The post-hardening reachability check in tasks/main.yml is the real proof diff --git a/roles/hardening/tasks/main.yml b/roles/hardening/tasks/main.yml index d95fdc5..8b92b9a 100644 --- a/roles/hardening/tasks/main.yml +++ b/roles/hardening/tasks/main.yml @@ -69,11 +69,12 @@ # Removal is safe while containers run as uid 1000: userdel only refuses for # host processes (it skips processes whose root differs from its own, i.e. # containers), and a running container only ever sees the numeric uid. -- name: Remove the unused provider `ubuntu` account +- name: Remove the unused provider accounts ansible.builtin.user: - name: ubuntu + name: "{{ item }}" state: absent remove: true + loop: "{{ hardening_removed_accounts }}" # --------------------------------------------------------------------------- # 2. HARD GATE - prove the new access path before removing the old one @@ -288,6 +289,29 @@ until: "'POST_HARDENING_OK' in (post_check.stdout | default(''))" failed_when: "'POST_HARDENING_OK' not in (post_check.stdout | default(''))" +# sshd takes the FIRST value it reads for each setting, and the drop-ins load in +# name order, so a provider's 50-cloud-init.conf would beat 99-hardening.conf with +# PasswordAuthentication yes and nothing would say so. Ask sshd what it settled on. +- name: Read the settings sshd is actually using + ansible.builtin.command: + argv: [/usr/sbin/sshd, -T] + register: sshd_effective + changed_when: false + check_mode: false + when: not ansible_check_mode + +- name: Fail unless root and password logins really are off + ansible.builtin.assert: + that: + - ('permitrootlogin ' ~ ssh_permit_root_login) in sshd_effective.stdout_lines + - ('passwordauthentication ' ~ ssh_password_authentication) in sshd_effective.stdout_lines + fail_msg: >- + {{ inventory_hostname }}: another file under /etc/ssh overrides + 99-hardening.conf; sshd -T shows + {{ sshd_effective.stdout_lines | select('match', '^(permitrootlogin|passwordauthentication) ') | list }}. + quiet: true + when: not ansible_check_mode + # --------------------------------------------------------------------------- # 5. Hand the rest of the play over to the hardened credentials. # diff --git a/roles/k3s/defaults/main.yml b/roles/k3s/defaults/main.yml new file mode 100644 index 0000000..b524596 --- /dev/null +++ b/roles/k3s/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# Every running container as namespace/pod/container=id, one a line, which +# upgrade_hop.yml compares across a hop. +k3s_list_containers: + - k3s + - kubectl + - get + - pods + - --all-namespaces + - -o + - >- + go-template={{ '{{' }}range .items{{ '}}' }}{{ '{{' }}$p := .{{ '}}' }}{{ '{{' }}range .status.containerStatuses{{ '}}' }}{{ '{{' }}$p.metadata.namespace{{ '}}' }}/{{ '{{' }}$p.metadata.name{{ '}}' }}/{{ '{{' }}.name{{ '}}' }}={{ '{{' }}.containerID{{ '}}' }}{{ '{{' }}"\n"{{ '}}' }}{{ '{{' }}end{{ '}}' }}{{ '{{' }}end{{ '}}' }} diff --git a/roles/k3s/tasks/upgrade_hop.yml b/roles/k3s/tasks/upgrade_hop.yml index cf2fd54..0d0352e 100644 --- a/roles/k3s/tasks/upgrade_hop.yml +++ b/roles/k3s/tasks/upgrade_hop.yml @@ -44,8 +44,18 @@ group: root mode: "0700" + # What this hop must not disturb, compared at the end. A container ID only + # changes when the container is replaced, whatever the pod reports. + - name: Record the running containers before the upgrade + ansible.builtin.command: + argv: "{{ k3s_list_containers }}" + register: k3s_containers_before + changed_when: false + # Stopping k3s first gives a consistent copy of the SQLite database. - # Running containers are not stopped by this (k3s uses KillMode=process). + # Running containers are not stopped by this: KillMode=process kills only + # k3s itself. Its containerd child dies with it (Pdeathsig), but each + # container's shim is its own process and carries on. - name: Stop k3s for a consistent datastore backup ansible.builtin.systemd: name: k3s @@ -116,3 +126,33 @@ retries: 3 delay: 20 until: k3s_deployments.rc == 0 + + # Fails this box, so with production_rollout the next box is left alone. + # kube-system is only reported: its packaged components (CoreDNS and so on) + # roll when k3s ships them a new image. That is expected, and it is why a + # container there can change. Anything else being replaced breaks the + # promise this upgrade relies on. It also trips if something else replaced a + # pod during the hop, such as an Argo CD sync: read the list, then re-run, + # since this hop is already done. + - name: Read the running containers after the upgrade + ansible.builtin.command: + argv: "{{ k3s_list_containers }}" + register: k3s_containers_after + changed_when: false + + - name: Report containers this hop replaced + vars: + replaced: "{{ k3s_containers_before.stdout_lines | difference(k3s_containers_after.stdout_lines) }}" + ansible.builtin.debug: + msg: "{{ k3s_hop_version }}: {{ replaced | length }} of {{ k3s_containers_before.stdout_lines | length }} containers replaced: {{ replaced }}" + + - name: Fail if the hop replaced a container outside kube-system + vars: + replaced: "{{ k3s_containers_before.stdout_lines | difference(k3s_containers_after.stdout_lines) | reject('match', '^kube-system/') | list }}" + ansible.builtin.assert: + that: replaced | length == 0 + fail_msg: >- + {{ inventory_hostname }}: upgrading to {{ k3s_hop_version }} replaced + these containers, and the upgrade should not have: {{ replaced }}. + Stopping here so the next box is not touched. + quiet: true diff --git a/roles/storage/defaults/main.yml b/roles/storage/defaults/main.yml new file mode 100644 index 0000000..3fe5791 --- /dev/null +++ b/roles/storage/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# ext4_blank_nvme the one blank NVMe, ext4 at data_mount (the IPFS boxes) +# zfs the pools in storage_zfs_pools +storage_mode: ext4_blank_nvme + +# Each: name, vdevs (the zpool create layout: mirror, raidz..., and disks by +# /dev/disk/by-id), and datasets, each a name and its zfs properties. Listed per +# box in host_vars, because a disk layout is never guessed. See +# host_vars/idx-new-1/main.yml. +storage_zfs_pools: [] + +# Passed to zpool create only. Changing these does nothing to an existing pool. +storage_zfs_pool_options: + ashift: "12" +storage_zfs_fs_options: + compression: lz4 + atime: "off" + xattr: sa diff --git a/roles/storage/tasks/ext4_blank_nvme.yml b/roles/storage/tasks/ext4_blank_nvme.yml new file mode 100644 index 0000000..61974aa --- /dev/null +++ b/roles/storage/tasks/ext4_blank_nvme.yml @@ -0,0 +1,86 @@ +--- +# Each host has two ~894 GiB NVMe drives. One holds the OS; the second is +# blank. Device naming is NOT consistent across hosts (chic-1 has the OS on +# nvme1n1, sing-1 and lith-1 on nvme0n1), so the target is identified by +# shape, never by name. +# +# Every check below reads LIVE state (blkid/lsblk/findmnt) rather than +# ansible_devices/ansible_mounts. Facts are cached for an hour (ansible.cfg), +# and the cache for a host is written at the START of the run that creates +# /data - so a cached fact set reports the in-use data disk as unpartitioned +# and unmounted, i.e. as a blank candidate for reformatting. Only two +# downstream safeties (the filesystem module refusing to reformat, and mkfs +# refusing a mounted device) stood between that and data loss. The role's own +# gate must not depend on stale data. + +- name: Look for an existing data volume by label + ansible.builtin.command: + argv: [blkid, -L, ipfsdata] + register: existing_label + changed_when: false + failed_when: false + check_mode: false + +- name: Record whether the data volume has already been provisioned + ansible.builtin.set_fact: + data_volume_exists: "{{ existing_label.rc == 0 }}" + +- name: Read live block device topology + ansible.builtin.command: + argv: [lsblk, -J, -o, "NAME,TYPE,MOUNTPOINT,FSTYPE"] + register: lsblk_raw + changed_when: false + check_mode: false + +# A candidate must be a whole NVMe disk with no children (no partition table), +# no mountpoint, and no filesystem signature at all. Requiring fstype to be +# empty is deliberate: any disk carrying a filesystem is not blank, even if it +# happens to be unmounted right now. +- name: Identify genuinely blank NVMe disks + ansible.builtin.set_fact: + data_disk_candidates: >- + {{ + (lsblk_raw.stdout | from_json).blockdevices + | selectattr('type', 'eq', 'disk') + | selectattr('name', 'match', '^nvme') + | rejectattr('children', 'defined') + | selectattr('mountpoint', 'none') + | selectattr('fstype', 'none') + | map(attribute='name') + | list + }} + +- name: Show what was detected + ansible.builtin.debug: + msg: >- + {{ 'data volume already provisioned (' ~ existing_label.stdout | trim ~ ')' + if data_volume_exists + else 'blank NVMe candidates: ' ~ data_disk_candidates }} + +# Safety gate: refuse to guess. Skipped entirely once the volume exists. +- name: Fail if the blank disk cannot be identified unambiguously + ansible.builtin.fail: + msg: >- + Expected exactly one blank NVMe to use for {{ data_mount }}, found + {{ data_disk_candidates | length }}: {{ data_disk_candidates }}. + Refusing to format anything. Inspect the host manually. + when: + - not data_volume_exists + - data_disk_candidates | length != 1 + +- name: Create an ext4 filesystem on the blank NVMe + community.general.filesystem: + fstype: ext4 + dev: "/dev/{{ data_disk_candidates | first }}" + opts: -L ipfsdata -m 0 # no root reserve; this is data, not system space + when: not data_volume_exists + +# Mounting by label rather than UUID is deliberate: the label survives a +# deliberate reformat, so fstab does not need rewriting alongside it. +- name: Mount the data volume by label and persist it in fstab + ansible.posix.mount: + path: "{{ data_mount }}" + src: LABEL=ipfsdata + fstype: ext4 + opts: defaults,noatime + state: mounted diff --git a/roles/storage/tasks/main.yml b/roles/storage/tasks/main.yml index 3d8f7d1..cfeaeb7 100644 --- a/roles/storage/tasks/main.yml +++ b/roles/storage/tasks/main.yml @@ -1,89 +1,20 @@ --- -# Each host has two ~894 GiB NVMe drives. One holds the OS; the second is -# blank. Device naming is NOT consistent across hosts (chic-1 has the OS on -# nvme1n1, sing-1 and lith-1 on nvme0n1), so the target is identified by -# shape, never by name. -# -# Every check below reads LIVE state (blkid/lsblk/findmnt) rather than -# ansible_devices/ansible_mounts. Facts are cached for an hour (ansible.cfg), -# and the cache for a host is written at the START of the run that creates -# /data - so a cached fact set reports the in-use data disk as unpartitioned -# and unmounted, i.e. as a blank candidate for reformatting. Only two -# downstream safeties (the filesystem module refusing to reformat, and mkfs -# refusing a mounted device) stood between that and data loss. The role's own -# gate must not depend on stale data. - -- name: Look for an existing data volume by label - ansible.builtin.command: - argv: [blkid, -L, ipfsdata] - register: existing_label - changed_when: false - failed_when: false - check_mode: false - -- name: Record whether the data volume has already been provisioned - ansible.builtin.set_fact: - data_volume_exists: "{{ existing_label.rc == 0 }}" - -- name: Read live block device topology - ansible.builtin.command: - argv: [lsblk, -J, -o, "NAME,TYPE,MOUNTPOINT,FSTYPE"] - register: lsblk_raw - changed_when: false - check_mode: false - -# A candidate must be a whole NVMe disk with no children (no partition table), -# no mountpoint, and no filesystem signature at all. Requiring fstype to be -# empty is deliberate: any disk carrying a filesystem is not blank, even if it -# happens to be unmounted right now. -- name: Identify genuinely blank NVMe disks - ansible.builtin.set_fact: - data_disk_candidates: >- - {{ - (lsblk_raw.stdout | from_json).blockdevices - | selectattr('type', 'eq', 'disk') - | selectattr('name', 'match', '^nvme') - | rejectattr('children', 'defined') - | selectattr('mountpoint', 'none') - | selectattr('fstype', 'none') - | map(attribute='name') - | list - }} - -- name: Show what was detected - ansible.builtin.debug: - msg: >- - {{ 'data volume already provisioned (' ~ existing_label.stdout | trim ~ ')' - if data_volume_exists - else 'blank NVMe candidates: ' ~ data_disk_candidates }} - -# Safety gate: refuse to guess. Skipped entirely once the volume exists. -- name: Fail if the blank disk cannot be identified unambiguously - ansible.builtin.fail: - msg: >- - Expected exactly one blank NVMe to use for {{ data_mount }}, found - {{ data_disk_candidates | length }}: {{ data_disk_candidates }}. - Refusing to format anything. Inspect the host manually. - when: - - not data_volume_exists - - data_disk_candidates | length != 1 - -- name: Create an ext4 filesystem on the blank NVMe - community.general.filesystem: - fstype: ext4 - dev: "/dev/{{ data_disk_candidates | first }}" - opts: -L ipfsdata -m 0 # no root reserve; this is data, not system space - when: not data_volume_exists - -# Mounting by label rather than UUID is deliberate: the label survives a -# deliberate reformat, so fstab does not need rewriting alongside it. -- name: Mount the data volume by label and persist it in fstab - ansible.posix.mount: - path: "{{ data_mount }}" - src: LABEL=ipfsdata - fstype: ext4 - opts: defaults,noatime - state: mounted +# storage_mode picks how this box gets its data volume at data_mount (see +# defaults/main.yml). Imported rather than included, so `--tags storage` reaches +# the tasks inside. +- name: Check storage_mode is one this role knows + ansible.builtin.assert: + that: storage_mode in ['ext4_blank_nvme', 'zfs'] + fail_msg: "{{ inventory_hostname }}: unknown storage_mode {{ storage_mode }}" + quiet: true + +- name: Provision the blank NVMe as ext4 + ansible.builtin.import_tasks: ext4_blank_nvme.yml + when: storage_mode == 'ext4_blank_nvme' + +- name: Provision ZFS + ansible.builtin.import_tasks: zfs.yml + when: storage_mode == 'zfs' # local-path-provisioner resets its root to 0700 whenever it provisions a # volume. Matching that here stops Ansible and the provisioner flipping the diff --git a/roles/storage/tasks/zfs.yml b/roles/storage/tasks/zfs.yml new file mode 100644 index 0000000..c59df8b --- /dev/null +++ b/roles/storage/tasks/zfs.yml @@ -0,0 +1,118 @@ +--- +# Pools from storage_zfs_pools. This never recreates or forces anything: a pool +# that exists is left alone, `zpool create` runs without -f, and a disk with any +# signature on it stops the run. Reads live state, like the ext4 path. + +- name: Check the ZFS layout is complete + vars: + mountpoints: "{{ storage_zfs_pools | map(attribute='datasets') | flatten | map(attribute='properties') | map(attribute='mountpoint') | list }}" + disks: "{{ storage_zfs_pools | map(attribute='vdevs') | flatten | select('match', '^/') | list }}" + keywords: "{{ storage_zfs_pools | map(attribute='vdevs') | flatten | reject('match', '^/') | list }}" + ansible.builtin.assert: + that: + - storage_zfs_pools | length > 0 + - data_mount in mountpoints + - disks | length > 0 + - disks | reject('match', '^/dev/disk/by-id/') | list | length == 0 + - keywords | reject('match', '^(mirror|raidz[123]?|log|cache|spare|special)$') | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: storage_zfs_pools needs at least one pool, disks + as /dev/disk/by-id paths (kernel names move between boots), and a dataset + mounted at data_mount ({{ data_mount }}). + quiet: true + +# Ubuntu's kernel carries the module. Debian's cannot, for licence reasons: the +# module is built by DKMS from contrib. +- name: Enable contrib for ZFS on Debian + ansible.builtin.copy: + dest: /etc/apt/sources.list.d/ipni-contrib.sources + mode: "0644" + content: | + # Managed by Ansible. ZFS lives in contrib. + Types: deb + URIs: http://deb.debian.org/debian + Suites: {{ ansible_distribution_release }} {{ ansible_distribution_release }}-updates + Components: contrib + Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg + register: storage_contrib + when: ansible_distribution == 'Debian' + +- name: Refresh apt after enabling contrib + ansible.builtin.apt: + update_cache: true + changed_when: false + when: storage_contrib is changed # noqa: no-handler + +- name: Accept the ZFS licence note non-interactively + ansible.builtin.debconf: + name: zfs-dkms + question: zfs-dkms/note-incompatible-licenses + value: "true" + vtype: note + when: ansible_distribution == 'Debian' + +- name: Install ZFS + ansible.builtin.apt: + name: "{{ ['linux-headers-' ~ ansible_kernel, 'zfs-dkms', 'zfsutils-linux'] if ansible_distribution == 'Debian' else ['zfsutils-linux'] }}" + state: present + +- name: Look for the zpool command + ansible.builtin.stat: + path: /usr/sbin/zpool + register: storage_zpool_bin + +# In a dry run on a box without ZFS there is nothing yet to ask. +- name: Provision the pools + when: storage_zpool_bin.stat.exists + block: + - name: Load the ZFS module + community.general.modprobe: + name: zfs + state: present + + - name: Provision each pool + ansible.builtin.include_tasks: zfs_pool.yml + loop: "{{ storage_zfs_pools }}" + loop_control: + loop_var: storage_pool + label: "{{ storage_pool.name }}" + + - name: Import and mount the pools on boot + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + loop: + - zfs-import-cache.service + - zfs-mount.service + - zfs.target + +# Without this a pool that failed to import leaves empty directories, and k3s +# would start pods that write their data to the OS disk. After= alone, not +# Requires=: restarting a ZFS unit must not restart k3s. +- name: Create the k3s unit drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/k3s.service.d + state: directory + mode: "0755" + +- name: Hold k3s until the ZFS datasets are mounted + vars: + mountpoints: "{{ storage_zfs_pools | map(attribute='datasets') | flatten | map(attribute='properties') | map(attribute='mountpoint') | list }}" + ansible.builtin.copy: + dest: /etc/systemd/system/k3s.service.d/10-wait-for-zfs.conf + mode: "0644" + content: | + # Managed by Ansible. + [Unit] + After=zfs-mount.service + + [Service] + {% for path in mountpoints %} + ExecStartPre=/usr/bin/mountpoint -q {{ path }} + {% endfor %} + register: storage_k3s_dropin + +- name: Reload systemd for the k3s drop-in + ansible.builtin.systemd: + daemon_reload: true + when: storage_k3s_dropin is changed # noqa: no-handler diff --git a/roles/storage/tasks/zfs_pool.yml b/roles/storage/tasks/zfs_pool.yml new file mode 100644 index 0000000..acda82a --- /dev/null +++ b/roles/storage/tasks/zfs_pool.yml @@ -0,0 +1,79 @@ +--- +# One entry of storage_zfs_pools, as storage_pool. + +- name: "Look for pool {{ storage_pool.name }}" + ansible.builtin.command: + argv: [zpool, list, -H, -o, name, "{{ storage_pool.name }}"] + register: storage_pool_list + changed_when: false + failed_when: false + check_mode: false + +- name: "Create pool {{ storage_pool.name }} if it does not exist" + when: storage_pool_list.rc != 0 + block: + # CI only maintains built boxes. A pool missing there is an incident (a typo, + # a failed import, a dead disk), not something to build. + - name: Refuse to create a pool from a run that only maintains built boxes + ansible.builtin.assert: + that: not (site_require_hardened | default(false) | bool) + fail_msg: "{{ inventory_hostname }}: pool {{ storage_pool.name }} is not imported; look at the box." + quiet: true + + - name: List pools that exist on disk but are not imported + ansible.builtin.command: + argv: [zpool, import] + register: storage_importable + changed_when: false + failed_when: false + check_mode: false + + - name: Refuse to create over a pool that is only exported + ansible.builtin.assert: + that: "storage_importable.stdout is not search('pool: ' ~ storage_pool.name ~ '$', multiline=True)" + fail_msg: >- + {{ inventory_hostname }}: pool {{ storage_pool.name }} exists on disk but + is not imported. Import it by hand rather than creating a new one. + quiet: true + + # wipefs -n reads and reports only. + - name: Read the signatures on each disk + ansible.builtin.command: + argv: [wipefs, --no-act, "{{ item }}"] + loop: "{{ storage_pool.vdevs | select('match', '^/') | list }}" + register: storage_disk_signatures + changed_when: false + check_mode: false + + - name: Refuse to touch a disk that is not blank + ansible.builtin.assert: + that: storage_disk_signatures.results | selectattr('stdout', 'ne', '') | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: not blank: + {{ storage_disk_signatures.results | selectattr('stdout', 'ne', '') | map(attribute='item') | list }}. + Nothing was changed. Clearing a disk is an operator's decision. + quiet: true + + - name: "Create pool {{ storage_pool.name }}" + vars: + pool_args: >- + {%- set a = [] -%} + {%- for k, v in storage_zfs_pool_options.items() -%}{%- set _ = a.extend(['-o', k ~ '=' ~ v]) -%}{%- endfor -%} + {%- for k, v in storage_zfs_fs_options.items() -%}{%- set _ = a.extend(['-O', k ~ '=' ~ v]) -%}{%- endfor -%} + {{ a }} + ansible.builtin.command: + argv: "{{ ['zpool', 'create'] + pool_args + ['-m', 'none', storage_pool.name] + storage_pool.vdevs }}" + changed_when: true + register: storage_pool_created + when: not ansible_check_mode + +# In a dry run a missing pool stays missing, and zfs would have nothing to change. +- name: "Create the datasets on {{ storage_pool.name }}" + community.general.zfs: + name: "{{ storage_pool.name }}/{{ item.name }}" + state: present + extra_zfs_properties: "{{ item.properties }}" + loop: "{{ storage_pool.datasets }}" + loop_control: + label: "{{ item.name }}" + when: storage_pool_list.rc == 0 or not ansible_check_mode diff --git a/site.yml b/site.yml index fb3ffe6..61c67fc 100644 --- a/site.yml +++ b/site.yml @@ -1,5 +1,6 @@ --- -# Base preparation for the three IPFS/IPNI hosts. +# Base preparation for every box this repo builds (managed_boxes: the IPFS hosts +# and the indexers). # # ansible-playbook site.yml # all three # ansible-playbook site.yml -l chic-1 # one host @@ -16,7 +17,7 @@ # sets site_require_hardened: CI only maintains boxes that are already built. - name: Detect which credentials work for each host - hosts: ipfs_nodes + hosts: managed_boxes gather_facts: false connection: local tags: always @@ -26,6 +27,19 @@ # a Homebrew python on an Ubuntu box. ansible_python_interpreter: "{{ ansible_playbook_python }}" tasks: + # Before anything touches the box: the name becomes its hostname and k3s + # node name, and the storage choice decides what gets formatted. + - name: Refuse to build a box whose placeholders are still undecided + ansible.builtin.assert: + that: + - site != 'undecided' + - storage_mode | default('') != 'undecided' + fail_msg: >- + {{ inventory_hostname }}: set site and storage_mode in + host_vars/{{ inventory_hostname }}/main.yml, and rename the box if its + name is a placeholder, before building it. + quiet: true + - name: Probe key-based login as the admin user ansible.builtin.command: argv: @@ -67,6 +81,40 @@ - admin_probe.rc != 0 - site_require_hardened | default(false) | bool + # A new box may take root by key, from the operator's ssh-agent. That needs + # no vault password, and hardening disables it just the same. + - name: Probe key-based login as root on a box that is not hardened yet + ansible.builtin.command: + argv: + - ssh + - -o + - BatchMode=yes + - -o + - ConnectTimeout=10 + - -o + - StrictHostKeyChecking=accept-new + - -o + - PasswordAuthentication=no + - -p + - "{{ ssh_port }}" + - "root@{{ ansible_host }}" + - "true" + register: root_key_probe + check_mode: false + changed_when: false + failed_when: false + when: admin_probe.rc != 0 + + - name: Use root by key on a freshly provisioned host + ansible.builtin.set_fact: + ansible_user: root + ansible_become: false + bootstrap_mode: true + bootstrap_auth: key + when: + - admin_probe.rc != 0 + - root_key_probe.rc == 0 + # no_log: set_fact echoes its arguments, so at -v or through any callback # plugin the vaulted root password would be printed in cleartext. - name: Fall back to root password auth on a freshly provisioned host @@ -75,17 +123,20 @@ ansible_password: "{{ vault_root_password }}" ansible_become: false bootstrap_mode: true - when: admin_probe.rc != 0 + bootstrap_auth: password + when: + - admin_probe.rc != 0 + - root_key_probe.rc != 0 no_log: true - name: Report the connection mode ansible.builtin.debug: msg: >- - {{ inventory_hostname }}: {{ 'BOOTSTRAP (root + password)' + {{ inventory_hostname }}: {{ 'BOOTSTRAP (root + ' ~ bootstrap_auth ~ ')' if bootstrap_mode else 'HARDENED (admin key + sudo)' }} -- name: Prepare IPFS hosts - hosts: ipfs_nodes +- name: Prepare the boxes + hosts: managed_boxes gather_facts: true # serial is templated before any host is chosen, so inventory group_vars are # not visible here: read production_rollout from its group_vars file @@ -99,7 +150,7 @@ {{ rollout_serial | default(1 if (production_rollout | default((lookup('ansible.builtin.file', playbook_dir ~ '/group_vars/ipfs_nodes/main.yml') | from_yaml).production_rollout | default(false)) | bool) - else (groups['ipfs_nodes'] | length)) }} + else (groups['managed_boxes'] | length)) }} any_errors_fatal: false roles: - role: common @@ -201,6 +252,12 @@ changed_when: false failed_when: false + # Read-only, and printed in every run's summary, CI's included. + - name: Check whether this box is waiting for a reboot + ansible.builtin.stat: + path: /var/run/reboot-required + register: site_reboot_required + - name: Summarise the finished host ansible.builtin.debug: msg: @@ -208,5 +265,6 @@ - "site: {{ site }}" - "os: {{ ansible_distribution }} {{ ansible_distribution_version }}" - "data disk: {{ live_data_mount.stdout | trim | default('NOT MOUNTED', true) }}" + - "reboot: {{ 'PENDING (/var/run/reboot-required); reboot deliberately, see the README' if site_reboot_required.stat.exists else 'none pending' }}" - "ssh: key-only as {{ admin_user }}, root login disabled" - "kubeconfig: kubeconfigs/{{ inventory_hostname }}.yaml (via SSH tunnel)" From 88ba8e6e597d872c122170ec7514f11f2441d806 Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 5 Nov 2026 20:05:56 +0100 Subject: [PATCH 2/4] Add an md RAID + XFS storage mode for the indexer test box idx-new-1 becomes the test bed for the next indexer layout, meant to replace sf and sf2's later: the OS on a 500 GiB md RAID1 with ext4, installed by hand, and the rest of each disk as one md RAID5 with XFS at /ipni. storage_mode md_xfs creates the data partition in each disk's free space, the array and the filesystem, and like the zfs mode never recreates anything: existing arrays and filesystems are left alone, non-blank members stop the run, no force flags, and CI runs refuse to create an array. Indexers mount data at /ipni, the path ipni/infra's manifests use. Co-Authored-By: Claude Opus 5.5 --- README.md | 43 +++++- group_vars/indexers/main.yml | 4 + host_vars/idx-new-1/main.yml | 31 ++--- roles/storage/defaults/main.yml | 12 ++ roles/storage/tasks/main.yml | 6 +- roles/storage/tasks/md_xfs.yml | 234 ++++++++++++++++++++++++++++++++ site.yml | 3 +- 7 files changed, 309 insertions(+), 24 deletions(-) create mode 100644 roles/storage/tasks/md_xfs.yml diff --git a/README.md b/README.md index e9b1728..300b226 100644 --- a/README.md +++ b/README.md @@ -272,12 +272,43 @@ of `ipfs_nodes` and `indexers` (`group_vars/managed_boxes/main.yml`); a value in comes from ipni/infra through Argo CD. sf and sf2 are not part of this. **Storage** is per box, in `host_vars//main.yml`. `storage_mode` is -`ext4_blank_nvme` (as on the IPFS boxes) or `zfs` (the pools in -`storage_zfs_pools`, with disks by `/dev/disk/by-id`). The ZFS path never -recreates anything: an imported pool is left alone, an exported pool of that name -or a disk with any signature stops the run, `zpool create` never gets `-f`, and a -run from CI refuses to create a pool at all. k3s will not start until the datasets -are mounted, so pods cannot write to the OS disk after a failed import. +`ext4_blank_nvme` (as on the IPFS boxes), `zfs` (the pools in +`storage_zfs_pools`, with disks by `/dev/disk/by-id`) or `md_xfs` (one md array +over a partition on each disk, XFS on it, from `storage_md`). Indexers mount +their data at `/ipni`, the path ipni/infra's indexer manifests use. The `zfs` and +`md_xfs` modes never recreate anything: an existing pool, array or filesystem is left alone, a disk +or member with any signature stops the run, `zpool create` and `mdadm --create` +get no force flag, and a run from CI refuses to create a pool or array at all. +k3s will not start until the data volume is mounted, so pods cannot write to the +OS disk after a failed import or assembly. + +**The indexer test layout** (`idx-new-1`, three 15.36 TB NVMe), meant to become +the layout for sf and sf2 once it has proved itself: + +| Part | Where | Layout | Filesystem | +|------|-------|--------|------------| +| EFI | partition 1 on each disk | the installer's | vfat | +| OS, 500 GiB | partition 2 on each disk | md RAID1 over the three | ext4, `/` | +| Data, the rest (about 27 TiB) | partition 3 on each disk | md RAID5 over the three | XFS, `/ipni` | + +The OS part is the install: the provider's custom partitioning or a Debian +install from the console, with the rest of each disk left unallocated. Ansible +cannot repartition a running root. `site.yml` then does the data part: it creates +partition 3 after the last partition on each disk (only if every disk has at +least `storage_md_min_free_gib` free and a GPT table), checks the three are +blank, creates the array, records it in `mdadm.conf` and the initramfs, and +formats XFS with 4 KiB sectors, which `mkfs.xfs` aligns to the RAID stripe by +itself. The array's first resync runs for hours afterwards; it is usable +meanwhile. Check what the drives report before building, since a drive can +also be reformatted to 4 KiB sectors, which erases it: + +```bash +lsblk -d -o NAME,MODEL,PHY-SEC,LOG-SEC,MIN-IO,OPT-IO +nvme id-ns -H /dev/nvme0n1 | grep 'LBA Format' +``` + +To try a different data layout on the same box, stop k3s, unmount `/ipni`, and +stop and clear the array by hand; the role will not do that. **Building a box.** Add it to `indexers` with its `host_vars`, on a branch. With the root key in your agent (`ssh-add`), and an admin key from `keys/` too: diff --git a/group_vars/indexers/main.yml b/group_vars/indexers/main.yml index 5d330f0..cd88229 100644 --- a/group_vars/indexers/main.yml +++ b/group_vars/indexers/main.yml @@ -1,6 +1,10 @@ --- # Indexer boxes. Everything not set here comes from group_vars/managed_boxes. +# Where the data volume is mounted, as on sf and sf2: ipni/infra's indexer +# manifests use /ipni/indexer/... host paths. k3s and local-path live here too. +data_mount: /ipni + # `debian` is the Debian cloud image's login account, the same uid 1000 risk as # Ubuntu's (see roles/hardening/tasks/main.yml). hardening_removed_accounts: diff --git a/host_vars/idx-new-1/main.yml b/host_vars/idx-new-1/main.yml index 2d1411a..60dc57c 100644 --- a/host_vars/idx-new-1/main.yml +++ b/host_vars/idx-new-1/main.yml @@ -1,19 +1,18 @@ --- -# Placeholder box: site.yml refuses to build it until both of these are decided. +# Placeholder box: site.yml refuses to build it until site is decided and its +# disks are listed below. site: undecided # the location, for the site= node label -# ext4_blank_nvme: the single blank NVMe, ext4 at data_mount, as on ipfs_nodes. -# zfs: the pools below. Disks by /dev/disk/by-id only. An existing pool is never -# recreated, and a disk carrying any signature stops the run. -storage_mode: undecided - -# Shape of a zfs box, to fill in from the box's real disks: -# -# storage_mode: zfs -# data_mount: /ipni # must be one of the mountpoints below; k3s's local-path lives here -# storage_zfs_pools: -# - name: tank -# vdevs: [mirror, /dev/disk/by-id/nvme-AAA, /dev/disk/by-id/nvme-BBB] -# datasets: -# - {name: ipni, properties: {mountpoint: /ipni}} -# - {name: ipni-mirror, properties: {mountpoint: /ipni-mirror}} +# The test layout for the next indexers (README, "Indexer boxes"). Installed by +# hand first: on each of the three disks the provider's EFI partition, then a +# 500 GiB partition, the three in an md RAID1 with ext4 for the OS, and the rest +# left unallocated. site.yml then creates partition 3 in that space on each +# disk, one md RAID5 over the three, and XFS on it at data_mount (/ipni). +storage_mode: md_xfs +storage_md: + name: ipni + level: 5 + partition_number: 3 + # Whole disks, by /dev/disk/by-id (nvme-_, without -partN), + # from `ls -l /dev/disk/by-id/` on the rebuilt box. + disks: [] diff --git a/roles/storage/defaults/main.yml b/roles/storage/defaults/main.yml index 3fe5791..8951c4d 100644 --- a/roles/storage/defaults/main.yml +++ b/roles/storage/defaults/main.yml @@ -1,6 +1,7 @@ --- # ext4_blank_nvme the one blank NVMe, ext4 at data_mount (the IPFS boxes) # zfs the pools in storage_zfs_pools +# md_xfs an md array over a partition on each disk, XFS (storage_md) storage_mode: ext4_blank_nvme # Each: name, vdevs (the zpool create layout: mirror, raidz..., and disks by @@ -16,3 +17,14 @@ storage_zfs_fs_options: compression: lz4 atime: "off" xattr: sa + +# md_xfs: name, level (1, 5, 6 or 10), partition_number (the partition this +# creates in the free space after the OS's partitions) and disks (whole disks +# by /dev/disk/by-id). Listed per box in host_vars, like the pools. +storage_md: {} +# The filesystem label; fstab mounts by it. +storage_md_label: ipnidata +# Passed to mdadm --create only. +storage_md_chunk_kib: 512 +# A disk with less free space than this after its last partition stops the run. +storage_md_min_free_gib: 1024 diff --git a/roles/storage/tasks/main.yml b/roles/storage/tasks/main.yml index cfeaeb7..5115e28 100644 --- a/roles/storage/tasks/main.yml +++ b/roles/storage/tasks/main.yml @@ -4,7 +4,7 @@ # the tasks inside. - name: Check storage_mode is one this role knows ansible.builtin.assert: - that: storage_mode in ['ext4_blank_nvme', 'zfs'] + that: storage_mode in ['ext4_blank_nvme', 'zfs', 'md_xfs'] fail_msg: "{{ inventory_hostname }}: unknown storage_mode {{ storage_mode }}" quiet: true @@ -16,6 +16,10 @@ ansible.builtin.import_tasks: zfs.yml when: storage_mode == 'zfs' +- name: Provision an md array with XFS + ansible.builtin.import_tasks: md_xfs.yml + when: storage_mode == 'md_xfs' + # local-path-provisioner resets its root to 0700 whenever it provisions a # volume. Matching that here stops Ansible and the provisioner flipping the # mode back and forth on every run, and 0700 is the right value anyway: it diff --git a/roles/storage/tasks/md_xfs.yml b/roles/storage/tasks/md_xfs.yml new file mode 100644 index 0000000..e141a5d --- /dev/null +++ b/roles/storage/tasks/md_xfs.yml @@ -0,0 +1,234 @@ +--- +# storage_md: one md array built from a partition on each disk, XFS on it, at +# data_mount. The OS is installed first on its own small partitions and the +# rest of each disk is left unallocated; this fills that space with partition +# storage_md.partition_number. See host_vars/idx-new-1/main.yml. +# +# Like the other modes, this never recreates anything: an array or filesystem +# that exists is left alone, a member with any signature stops the run, mdadm +# gets no --force or --run (so a question it would ask aborts it), and a run from +# CI refuses to create an array at all. Live state only, never cached facts. + +- name: Check the md layout is complete + vars: + min_disks: "{{ {1: 2, 5: 3, 6: 4, 10: 4}[storage_md.level | default(0) | int] | default(99) }}" + ansible.builtin.assert: + that: + - storage_md.name | default('') is match('^[a-z0-9-]+$') + - storage_md.level | default(0) | int in [1, 5, 6, 10] + - storage_md.partition_number | default(0) | int > 0 + - storage_md.disks | default([]) | length >= min_disks | int + - storage_md.disks | reject('match', '^/dev/disk/by-id/') | list | length == 0 + - storage_md.disks | select('search', '-part[0-9]+$') | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: storage_md needs a name, a RAID level (1, 5, 6 or + 10) with enough disks for it, the partition number to use, and whole disks + as /dev/disk/by-id paths (kernel names move between boots). + quiet: true + +- name: Install mdadm, XFS and parted + ansible.builtin.apt: + name: [mdadm, xfsprogs, parted] + state: present + +- name: Name the array's device and members + ansible.builtin.set_fact: + storage_md_dev: "/dev/md/{{ storage_md.name }}" + storage_md_members: "{{ storage_md.disks | map('regex_replace', '$', '-part' ~ storage_md.partition_number) | list }}" + +- name: Look for the data filesystem by label + ansible.builtin.command: + argv: [blkid, -L, "{{ storage_md_label }}"] + register: storage_md_fs + changed_when: false + failed_when: false + check_mode: false + +- name: Look for the array + ansible.builtin.command: + argv: [mdadm, --detail, "{{ storage_md_dev }}"] + register: storage_md_detail + changed_when: false + failed_when: false + check_mode: false + +- name: Create the array if it does not exist + when: + - storage_md_detail.rc != 0 + - storage_md_fs.rc != 0 + block: + # CI only maintains built boxes. A missing array there is an incident (a + # failed assembly, a dead disk), not something to build. + - name: Refuse to create an array from a run that only maintains built boxes + ansible.builtin.assert: + that: not (site_require_hardened | default(false) | bool) + fail_msg: "{{ inventory_hostname }}: {{ storage_md_dev }} is not assembled; look at the box." + quiet: true + + - name: Read each disk's partition table + community.general.parted: + device: "{{ item }}" + unit: MiB + state: info + loop: "{{ storage_md.disks }}" + register: storage_md_tables + check_mode: false + + # parted would write a new, empty label over any other kind of table. + - name: Refuse a disk without a GPT table + ansible.builtin.assert: + that: storage_md_tables.results | rejectattr('disk.table', 'eq', 'gpt') | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: no GPT table on + {{ storage_md_tables.results | rejectattr('disk.table', 'eq', 'gpt') | map(attribute='item') | list }}. + Install the OS with the layout in the README first. + quiet: true + + # The new partition starts after the last one on the disk and must leave a + # real data volume: storage_md_min_free_gib of space after it. Every disk is + # checked before any is partitioned. + - name: Work out where each data partition goes + ansible.builtin.set_fact: + storage_md_plan: >- + {%- set plan = [] -%} + {%- for t in storage_md_tables.results -%} + {%- set ends = t.partitions | map(attribute='end') | map('float') | list -%} + {%- set start = ((ends | max) if ends else 1) | round(0, 'ceil') | int + 1 -%} + {%- set _ = plan.append({'disk': t.item, + 'exists': t.partitions | selectattr('num', 'eq', storage_md.partition_number | int) | list | length > 0, + 'start_mib': start, + 'free_gib': ((t.disk.size | float) - start) / 1024}) -%} + {%- endfor -%} + {{ plan }} + + - name: Check there is room for every data partition + ansible.builtin.assert: + that: storage_md_plan | rejectattr('exists') | selectattr('free_gib', 'lt', storage_md_min_free_gib) | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: under {{ storage_md_min_free_gib }} GiB free after + the last partition on + {{ storage_md_plan | rejectattr('exists') | selectattr('free_gib', 'lt', storage_md_min_free_gib) | map(attribute='disk') | list }}. + Nothing was changed. + quiet: true + + - name: Create the data partition in each disk's free space + community.general.parted: + device: "{{ item.disk }}" + label: gpt + number: "{{ storage_md.partition_number | int }}" + part_start: "{{ item.start_mib }}MiB" + part_end: "100%" + name: "{{ storage_md_label }}" + flags: [raid] + align: optimal + state: present + loop: "{{ storage_md_plan | rejectattr('exists') | list }}" + loop_control: + label: "{{ item.disk }} from {{ item.start_mib }} MiB" + + - name: Let udev create the partitions' by-id links + ansible.builtin.command: + argv: [udevadm, settle] + changed_when: false + when: not ansible_check_mode + + # wipefs -n reads and reports only. An empty answer means no filesystem, RAID + # or partition-table signature: a member that was in another array stops here. + - name: Read the signatures on each member + ansible.builtin.command: + argv: [wipefs, --no-act, "{{ item }}"] + loop: "{{ storage_md_members }}" + register: storage_md_signatures + changed_when: false + check_mode: false + when: not ansible_check_mode + + - name: Refuse to touch a member that is not blank + ansible.builtin.assert: + that: storage_md_signatures.results | selectattr('stdout', 'defined') | selectattr('stdout', 'ne', '') | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: not blank: + {{ storage_md_signatures.results | selectattr('stdout', 'defined') | selectattr('stdout', 'ne', '') | map(attribute='item') | list }}. + Nothing was changed. Clearing a disk is an operator's decision. + quiet: true + when: not ansible_check_mode + + # --homehost=any keeps the array's name from depending on the hostname, so + # renaming the box does not rename /dev/md/. The first resync of a + # large array takes hours; the array is usable meanwhile. + - name: "Create {{ storage_md_dev }}" + ansible.builtin.command: + argv: >- + {{ ['mdadm', '--create', storage_md_dev, '--name=' ~ storage_md.name, '--homehost=any', + '--metadata=1.2', '--level=' ~ storage_md.level, '--chunk=' ~ storage_md_chunk_kib, + '--bitmap=internal', '--raid-devices=' ~ (storage_md_members | length)] + storage_md_members }} + changed_when: true + when: not ansible_check_mode + +- name: Read the array's identity + ansible.builtin.command: + argv: [mdadm, --detail, --brief, "{{ storage_md_dev }}"] + register: storage_md_brief + changed_when: false + failed_when: false + check_mode: false + +# Without this line Debian assembles the array under a made-up md12x name. +- name: Record the array in mdadm.conf + ansible.builtin.lineinfile: + path: /etc/mdadm/mdadm.conf + regexp: "^ARRAY {{ storage_md_dev }} " + line: "{{ storage_md_brief.stdout | trim }}" + register: storage_md_conf + when: storage_md_brief.rc == 0 + +- name: Rebuild the initramfs for the new mdadm.conf + ansible.builtin.command: + argv: [update-initramfs, -u] + changed_when: true + when: storage_md_conf is changed # noqa: no-handler + +# The filesystem module never reformats: it refuses a device that already +# carries a filesystem. mkfs.xfs reads the RAID geometry from md itself; +# -s size=4096 keeps 4 KiB writes on drives that still report 512-byte sectors. +- name: Create the XFS filesystem on the array + community.general.filesystem: + fstype: xfs + dev: "{{ storage_md_dev }}" + opts: "-L {{ storage_md_label }} -s size=4096" + when: storage_md_brief.rc == 0 and storage_md_fs.rc != 0 + +# nofail: a box whose array fails to assemble still boots and can be reached +# over SSH. The k3s drop-in below then keeps pods off the OS disk. +- name: Mount the data volume by label and persist it in fstab + ansible.posix.mount: + path: "{{ data_mount }}" + src: "LABEL={{ storage_md_label }}" + fstype: xfs + opts: defaults,noatime,nofail + state: mounted + when: storage_md_brief.rc == 0 or storage_md_fs.rc == 0 + +- name: Create the k3s unit drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/k3s.service.d + state: directory + mode: "0755" + +- name: Hold k3s until the data volume is mounted + ansible.builtin.copy: + dest: /etc/systemd/system/k3s.service.d/10-wait-for-data.conf + mode: "0644" + content: | + # Managed by Ansible. + [Unit] + After=local-fs.target + + [Service] + ExecStartPre=/usr/bin/mountpoint -q {{ data_mount }} + register: storage_md_k3s_dropin + +- name: Reload systemd for the k3s drop-in + ansible.builtin.systemd: + daemon_reload: true + when: storage_md_k3s_dropin is changed # noqa: no-handler diff --git a/site.yml b/site.yml index 61c67fc..8a0da32 100644 --- a/site.yml +++ b/site.yml @@ -34,8 +34,9 @@ that: - site != 'undecided' - storage_mode | default('') != 'undecided' + - storage_mode | default('') != 'md_xfs' or storage_md.disks | default([]) | length > 0 fail_msg: >- - {{ inventory_hostname }}: set site and storage_mode in + {{ inventory_hostname }}: set site, storage_mode and its disks in host_vars/{{ inventory_hostname }}/main.yml, and rename the box if its name is a placeholder, before building it. quiet: true From 8a38fdfc339de107a606e3bffb31eb4a64b7c664 Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 5 Nov 2026 21:46:17 +0100 Subject: [PATCH 3/4] List the indexer test box's disks and switch them to 4 KiB sectors The box was reinstalled with a 300 GB md RAID1 for the OS (not 500 GiB) after its three NVMe drives were switched from 512- to 4096-byte sectors in Cherry's rescue system. host_vars now names the disks by ID, and the README has the format switch as a build step. Co-Authored-By: Claude Opus 5.5 --- README.md | 21 ++++++++++++++++----- host_vars/idx-new-1/main.yml | 10 +++++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 300b226..bda3571 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ the layout for sf and sf2 once it has proved itself: | Part | Where | Layout | Filesystem | |------|-------|--------|------------| | EFI | partition 1 on each disk | the installer's | vfat | -| OS, 500 GiB | partition 2 on each disk | md RAID1 over the three | ext4, `/` | +| OS, 300 GB | partition 2 on each disk | md RAID1 over the three | ext4, `/` | | Data, the rest (about 27 TiB) | partition 3 on each disk | md RAID5 over the three | XFS, `/ipni` | The OS part is the install: the provider's custom partitioning or a Debian @@ -299,14 +299,25 @@ least `storage_md_min_free_gib` free and a GPT table), checks the three are blank, creates the array, records it in `mdadm.conf` and the initramfs, and formats XFS with 4 KiB sectors, which `mkfs.xfs` aligns to the RAID stripe by itself. The array's first resync runs for hours afterwards; it is usable -meanwhile. Check what the drives report before building, since a drive can -also be reformatted to 4 KiB sectors, which erases it: +meanwhile. + +**Switch the drives to 4096-byte sectors before the install.** Drives often +ship on 512-byte sectors and offer 4096, which their makers rate faster (the +Micron 7450s here: 512 "Good", 4096 "Best"). Switching erases the drive, so it +is done from the provider's rescue system before the OS goes on, never from the +installed OS: ```bash -lsblk -d -o NAME,MODEL,PHY-SEC,LOG-SEC,MIN-IO,OPT-IO -nvme id-ns -H /dev/nvme0n1 | grep 'LBA Format' +nvme id-ns -H /dev/nvme0n1 | grep 'LBA Format' # which format number is 4096 +mdadm --stop --scan # if the rescue system assembled the old RAID +nvme format /dev/nvme0n1 --lbaf=1 # each drive; erases it +lsblk -d -o NAME,PHY-SEC,LOG-SEC # LOG-SEC 4096 on each ``` +Cherry's Debian 13 reinstall boots on 4096-byte drives (idx-new-1, 2026-11). A +`PHY-SEC` of 262144 on the 7450s is their preferred write size, not a sector +size; partitions on whole MiB and the 512 KiB chunk are multiples of it. + To try a different data layout on the same box, stop k3s, unmount `/ipni`, and stop and clear the array by hand; the role will not do that. diff --git a/host_vars/idx-new-1/main.yml b/host_vars/idx-new-1/main.yml index 60dc57c..f95f015 100644 --- a/host_vars/idx-new-1/main.yml +++ b/host_vars/idx-new-1/main.yml @@ -5,7 +5,7 @@ site: undecided # the location, for the site= node label # The test layout for the next indexers (README, "Indexer boxes"). Installed by # hand first: on each of the three disks the provider's EFI partition, then a -# 500 GiB partition, the three in an md RAID1 with ext4 for the OS, and the rest +# 300 GB partition, the three in an md RAID1 with ext4 for the OS, and the rest # left unallocated. site.yml then creates partition 3 in that space on each # disk, one md RAID5 over the three, and XFS on it at data_mount (/ipni). storage_mode: md_xfs @@ -14,5 +14,9 @@ storage_md: level: 5 partition_number: 3 # Whole disks, by /dev/disk/by-id (nvme-_, without -partN), - # from `ls -l /dev/disk/by-id/` on the rebuilt box. - disks: [] + # from `ls -l /dev/disk/by-id/` on the rebuilt box. All three were switched to + # 4096-byte sectors before the install (README, "Indexer boxes"). + disks: + - /dev/disk/by-id/nvme-Micron_7450_MTFDKCC15T3TFR_2217388B0293 + - /dev/disk/by-id/nvme-Micron_7450_MTFDKCC15T3TFR_2217388B02AA + - /dev/disk/by-id/nvme-MTFDKCC15T3TGP-1BK1DABYY_46241074EFA2 From b6d690440b652b141b4a184797b30cafec26f1c7 Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 5 Nov 2026 21:50:09 +0100 Subject: [PATCH 4/4] Name the indexer test box indexers-chic1, in Chicago Replaces the idx-new-1 placeholder with the name the box already carries, and sets its site label, so site.yml no longer refuses to build it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- README.md | 4 ++-- host_vars/{idx-new-1 => indexers-chic1}/main.yml | 5 ++--- inventory/hosts.yml | 6 +----- roles/storage/defaults/main.yml | 2 +- roles/storage/tasks/md_xfs.yml | 2 +- 6 files changed, 8 insertions(+), 13 deletions(-) rename host_vars/{idx-new-1 => indexers-chic1}/main.yml (84%) diff --git a/AGENTS.md b/AGENTS.md index 15381ac..0ea1545 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ ansible-playbook cve.yml -K --check --diff -l sf # dry run first ansible infra_boxes -m ping # no sudo needed ``` -**indexers** (`idx-new-1`, a placeholder name, 84.32.71.3) are built by this repo +**indexers** (`indexers-chic1`, 84.32.71.3, the indexer test box) are built by this repo but are not `ipfs_nodes`: Debian, `site.yml` and `k3s-upgrade.yml` only. Shared settings are in `group_vars/managed_boxes/`. See "Indexer boxes" in the README. Merge a new box only after it is built and its host key is pinned, or CI's diff --git a/README.md b/README.md index bda3571..6654083 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,7 @@ get no force flag, and a run from CI refuses to create a pool or array at all. k3s will not start until the data volume is mounted, so pods cannot write to the OS disk after a failed import or assembly. -**The indexer test layout** (`idx-new-1`, three 15.36 TB NVMe), meant to become +**The indexer test layout** (`indexers-chic1`, three 15.36 TB NVMe), meant to become the layout for sf and sf2 once it has proved itself: | Part | Where | Layout | Filesystem | @@ -314,7 +314,7 @@ nvme format /dev/nvme0n1 --lbaf=1 # each drive; erases it lsblk -d -o NAME,PHY-SEC,LOG-SEC # LOG-SEC 4096 on each ``` -Cherry's Debian 13 reinstall boots on 4096-byte drives (idx-new-1, 2026-11). A +Cherry's Debian 13 reinstall boots on 4096-byte drives (indexers-chic1, 2026-11). A `PHY-SEC` of 262144 on the 7450s is their preferred write size, not a sector size; partitions on whole MiB and the 512 KiB chunk are multiples of it. diff --git a/host_vars/idx-new-1/main.yml b/host_vars/indexers-chic1/main.yml similarity index 84% rename from host_vars/idx-new-1/main.yml rename to host_vars/indexers-chic1/main.yml index f95f015..84af10b 100644 --- a/host_vars/idx-new-1/main.yml +++ b/host_vars/indexers-chic1/main.yml @@ -1,7 +1,6 @@ --- -# Placeholder box: site.yml refuses to build it until site is decided and its -# disks are listed below. -site: undecided # the location, for the site= node label +# The indexer test box, Cherry Servers in Chicago. +site: chicago # the location, for the site= node label # The test layout for the next indexers (README, "Indexer boxes"). Installed by # hand first: on each of the three disks the provider's EFI partition, then a diff --git a/inventory/hosts.yml b/inventory/hosts.yml index 11ed25d..da1de5a 100644 --- a/inventory/hosts.yml +++ b/inventory/hosts.yml @@ -37,13 +37,9 @@ all: # Not ipfs_nodes, whose membership means bootstrap.yml, content.yml and # routing.yml, and whose peers list one another. Not infra_boxes, which is # for cve.yml alone. See "Indexer boxes" in the README. - # - # idx-new-1 is a placeholder name: the name becomes the hostname and k3s - # node name, so choose it before the first run. site.yml refuses to build - # it while host_vars/idx-new-1/main.yml says "undecided". indexers: hosts: - idx-new-1: + indexers-chic1: ansible_host: 84.32.71.3 # Every box site.yml and k3s-upgrade.yml manage, and where their shared diff --git a/roles/storage/defaults/main.yml b/roles/storage/defaults/main.yml index 8951c4d..52b5911 100644 --- a/roles/storage/defaults/main.yml +++ b/roles/storage/defaults/main.yml @@ -7,7 +7,7 @@ storage_mode: ext4_blank_nvme # Each: name, vdevs (the zpool create layout: mirror, raidz..., and disks by # /dev/disk/by-id), and datasets, each a name and its zfs properties. Listed per # box in host_vars, because a disk layout is never guessed. See -# host_vars/idx-new-1/main.yml. +# host_vars/indexers-chic1/main.yml. storage_zfs_pools: [] # Passed to zpool create only. Changing these does nothing to an existing pool. diff --git a/roles/storage/tasks/md_xfs.yml b/roles/storage/tasks/md_xfs.yml index e141a5d..67741c8 100644 --- a/roles/storage/tasks/md_xfs.yml +++ b/roles/storage/tasks/md_xfs.yml @@ -2,7 +2,7 @@ # storage_md: one md array built from a partition on each disk, XFS on it, at # data_mount. The OS is installed first on its own small partitions and the # rest of each disk is left unallocated; this fills that space with partition -# storage_md.partition_number. See host_vars/idx-new-1/main.yml. +# storage_md.partition_number. See host_vars/indexers-chic1/main.yml. # # Like the other modes, this never recreates anything: an array or filesystem # that exists is left alone, a member with any signature stops the run, mdadm