From b4faa6768491090a887216013e98599c41de8381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 12 Oct 2026 22:10:39 +0200 Subject: [PATCH 01/43] feat: add kubernetes runner prototype --- README.md | 28 ++++++ docs/TODO.md | 35 +++++++ k8s/.env.example | 5 + k8s/Dockerfile | 41 ++++++++ k8s/Makefile | 48 +++++++++ k8s/README.md | 60 ++++++++++++ k8s/manifests/namespace.yaml | 7 ++ k8s/manifests/runner.yaml | 95 ++++++++++++++++++ k8s/runner.sh | 183 +++++++++++++++++++++++++++++++++++ 9 files changed, 502 insertions(+) create mode 100644 docs/TODO.md create mode 100644 k8s/.env.example create mode 100644 k8s/Dockerfile create mode 100644 k8s/Makefile create mode 100644 k8s/README.md create mode 100644 k8s/manifests/namespace.yaml create mode 100644 k8s/manifests/runner.yaml create mode 100755 k8s/runner.sh diff --git a/README.md b/README.md index e8c0c6d..f335018 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ - [3.4 Restart](#34-restart) - [System diagnostics](#system-diagnostics) - [Managing `cgroupsv2`](#managing-cgroupsv2) +- [Kubernetes (prototype)](#kubernetes-prototype) - [Troubleshooting](#troubleshooting) - [Development](#development) @@ -187,6 +188,33 @@ To revert you can just run: ./main.sh cgroupsv2 enable ``` +## Kubernetes (prototype) + +> **Prototype only.** Not supported for customer use. + +[`k8s/`](k8s/README.md) runs a Private Runner as a Kubernetes pod: each +StatefulSet replica is one registered runner, with nested Docker, the SSM agent +and the ECS agent as containers, and `main.sh` sourced for registration and +network setup. Tested end to end on OrbStack Kubernetes against QA. See +[`k8s/README.md`](k8s/README.md) for usage. + +### Known gaps + +- **Privileged pod.** Nested Docker needs it; clusters enforcing restricted Pod + Security reject it. +- **No workflow network isolation.** Workflow containers can reach cluster DNS + and the Kubernetes API. Needs an egress NetworkPolicy (see + [`docs/TODO.md`](docs/TODO.md)). +- **Workspaces in the container layer.** `/var/tmp/workspace` isn't on a volume + yet (see [`docs/TODO.md`](docs/TODO.md)). +- **No task CPU/memory limits.** The agent can't create task cgroups from its + container; the pod's resources are the only cap. +- **No proxy support.** `--http-proxy` isn't wired up. +- **Scale-down doesn't deregister.** Removing a replica orphans its runner; + use `make nuke` (deregisters first). +- **`ecs-init` is bypassed.** Agent defaults are copied into `k8s/runner.sh` and + may drift with new ECS agent releases. + ## Troubleshooting - StackGuardian uses AWS SSM to setup connection between SG control plane and runners, you can diagnose SSM client using `ssm-cli get-diagnostics --output table` diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..eff9bac --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,35 @@ +# TODO + +## Kubernetes prototype (`k8s/`) + +### Workspace volume for the nested Docker daemon + +The run controller and every workflow step bind-mount `/var/tmp/workspace` from +the Docker daemon they run on. In the pod that daemon is the `dockerd` +container, and the path isn't backed by a volume, so workspaces land in the +container's writable layer: they count against the node's container storage +and are lost when the container restarts. + +- Mount a volume at `/var/tmp/workspace` in the `dockerd` container: an + `emptyDir` matches VM behavior (scratch space), a PVC only if runs must + survive a pod restart. +- Size it with `sizeLimit` / storage requests so a large run can't fill the node. + +### NetworkPolicy for workflow isolation + +Workflow containers can resolve cluster DNS and reach the Kubernetes API +(verified on OrbStack: `kubernetes.default.svc` answered with `401`). The +`DOCKER-USER` DROP rules from `main.sh` don't help: they target `sg-net`, and +the run controller and step containers run on the default bridge. + +Nested traffic leaves through the pod IP, so an egress NetworkPolicy on the +runner pod covers workflow containers and the agents alike: + +- Allow DNS to the cluster resolver only if workflows need it; otherwise + resolve via an external resolver. +- Deny the pod, service and node CIDRs (Kubernetes API included) and the + cloud metadata endpoint (`169.254.169.254`). +- Allow everything else (SG API, AWS endpoints, VCS, registries, workflow + targets). +- Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...). Check what + OrbStack's CNI does before testing there. diff --git a/k8s/.env.example b/k8s/.env.example new file mode 100644 index 0000000..2c843fc --- /dev/null +++ b/k8s/.env.example @@ -0,0 +1,5 @@ +# Copy to k8s/.env (gitignored) and fill in. Use a throwaway runner group. +SG_NODE_TOKEN= +SG_ORGANIZATION= +SG_RUNNER_GROUP= +SG_BASE_API=https://api.app.stackguardian.io/api/v1 diff --git a/k8s/Dockerfile b/k8s/Dockerfile new file mode 100644 index 0000000..8a7733a --- /dev/null +++ b/k8s/Dockerfile @@ -0,0 +1,41 @@ +# sg-runner Kubernetes prototype image: one image backs every container of +# the pod (see runner.sh). Build from the repo root: +# docker build -f k8s/Dockerfile -t sg-runner-k8s:dev . + +FROM amazon/amazon-ecs-agent:latest AS ecs-agent + +FROM debian:bookworm-slim + +ARG TARGETARCH +ARG DOCKER_VERSION=27.5.1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash ca-certificates curl jq iproute2 iptables procps xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# Docker engine + CLI (static build), run nested in the privileged container. +RUN arch="$([ "$TARGETARCH" = arm64 ] && echo aarch64 || echo x86_64)" \ + && curl -fsSL "https://download.docker.com/linux/static/stable/${arch}/docker-${DOCKER_VERSION}.tgz" \ + | tar -xz -C /usr/local/bin --strip-components=1 + +# SSM agent. Unpacked rather than installed: the postinst expects systemd. +# Staged first: extracting onto / would replace the merged-usr /lib symlink. +RUN curl -fsSL -o /tmp/ssm.deb \ + "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_${TARGETARCH}/amazon-ssm-agent.deb" \ + && mkdir /tmp/ssm \ + && dpkg-deb -x /tmp/ssm.deb /tmp/ssm \ + && cp /tmp/ssm/usr/bin/* /usr/bin/ \ + && cp -r /tmp/ssm/etc/amazon /etc/ \ + && cp /etc/amazon/ssm/seelog.xml.template /etc/amazon/ssm/seelog.xml \ + && rm -rf /tmp/ssm /tmp/ssm.deb + +# ECS agent, run directly instead of through ecs-init. +COPY --from=ecs-agent /agent /usr/local/bin/ecs-agent +COPY --from=ecs-agent /amazon-ecs-cni-plugins /amazon-ecs-cni-plugins +COPY --from=ecs-agent /images /images + +COPY main.sh /opt/sg-runner/main.sh +COPY k8s/runner.sh /opt/sg-runner/runner.sh + +ENTRYPOINT ["/opt/sg-runner/runner.sh"] diff --git a/k8s/Makefile b/k8s/Makefile new file mode 100644 index 0000000..920d588 --- /dev/null +++ b/k8s/Makefile @@ -0,0 +1,48 @@ +# sg-runner Kubernetes prototype. Run from k8s/. +# Every kubectl call is pinned to CONTEXT so this never touches a real cluster +# by accident. + +CONTEXT ?= orbstack +IMAGE ?= sg-runner-k8s:dev +NS := sg-runner +POD := sg-runner-0 +KUBECTL := kubectl --context $(CONTEXT) -n $(NS) + +.DEFAULT_GOAL := help + +.PHONY: help build secret up status logs deregister down nuke + +help: ## List the available targets + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "} {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}' + +build: ## Build the runner image (OrbStack k8s uses local images directly) + docker build -f Dockerfile -t $(IMAGE) .. + +secret: ## Create the SG credentials Secret from .env + @test -f .env || { echo "Copy .env.example to .env and fill it in first."; exit 1; } + kubectl --context $(CONTEXT) apply -f manifests/namespace.yaml + $(KUBECTL) create secret generic sg-runner-credentials --from-env-file=.env \ + --dry-run=client -o yaml | $(KUBECTL) apply -f - + +up: ## Deploy the runner StatefulSet (registers on first start) + kubectl --context $(CONTEXT) apply -f manifests/ + +status: ## Show pod, container states and agent health + $(KUBECTL) get pods,pvc -o wide + -$(KUBECTL) exec $(POD) -c ecs-agent -- curl -fsS localhost:51678/v1/metadata + +logs: ## Tail all runner containers + $(KUBECTL) logs $(POD) --all-containers --prefix -f --max-log-requests 10 + +deregister: ## Deregister from SG and wipe local identity (reads .env on stdin) + $(KUBECTL) exec -i $(POD) -c ecs-agent -- /opt/sg-runner/runner.sh deregister < .env + +down: ## Delete the StatefulSet, keep PVCs (runner identity survives) + -$(KUBECTL) delete statefulset sg-runner + +# deregister runs first and aborts nuke on failure, so a runner is never +# orphaned by deleting its identity. +nuke: deregister down ## Deregister, then delete everything including PVCs + -$(KUBECTL) delete pvc --all + -kubectl --context $(CONTEXT) delete namespace $(NS) diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..41ee5d5 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,60 @@ +# sg-runner on Kubernetes (prototype) + +> **Status: prototype.** Not supported, not documented for customers. + +Runs a Private Runner as a pod. Each StatefulSet replica is one registered +runner: it stands in for the VM that `main.sh register` normally prepares. + +## How it maps to a VM + +| VM (`main.sh` + `ecs-anywhere-install.sh`) | Pod | +|--------------------------------------------|----------------------------------------------------------| +| `systemd` + `dockerd` | `dockerd` container (privileged, nested Docker) | +| `amazon-ssm-agent` service | `ssm-agent` container | +| `ecs-init` → `ecs-agent` container | `ecs-agent` container, env loaded from `ecs.config` | +| `register` (API + SSM activation) | `register` init container, skipped if state exists | +| `/etc/ecs`, `/var/lib/ecs`, `/var/lib/amazon/ssm` | `state` PVC: runner identity survives pod restarts | +| `/root/.aws/credentials` (SSM → agent) | in-memory `emptyDir` shared by both agents | +| `configure_local_network` (iptables/sysctl) | same function, run in the pod's network namespace | + +All containers use one image (`Dockerfile`) and `runner.sh`, which sources +`main.sh` for the API, config and network helpers. + +## Usage (OrbStack Kubernetes) + +```sh +cd k8s +cp .env.example .env # fill in a throwaway runner group's credentials +make build # build sg-runner-k8s:dev +make secret # namespace + credentials Secret +make up # deploy; first start registers the runner +make logs # follow all containers +make status # pod state + ECS agent metadata + +make deregister # SG deregister + wipe identity (reads .env on stdin) +make nuke # deregister, then delete StatefulSet, PVCs, namespace +``` + +`make nuke` deregisters first and stops if that fails: deleting the PVC without +deregistering would leave a dead runner in the runner group. + +## Known gaps + +- **Privileged pod.** Needed for nested Docker; restricted PSA clusters reject it. +- **Network isolation.** The VM rule (`DOCKER-USER` DROP for + `169.254.169.254,10.0.0.0/24`) doesn't fit a cluster: job containers can + resolve cluster DNS and reach the Kubernetes API (verified). Needs a + NetworkPolicy on the runner pod and a CNI that enforces it + ([`docs/TODO.md`](../docs/TODO.md)). +- **Workspaces in the container layer.** `/var/tmp/workspace`, bind-mounted + by the run controller and steps, isn't on a volume yet + ([`docs/TODO.md`](../docs/TODO.md)). +- **Task cgroup limits off** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`): the agent + can't create task cgroups from its container. Size the pod's resources instead. +- **No proxy support** yet (`--http-proxy` path not wired). +- **Scale-down doesn't deregister.** Removing a replica orphans its runner; + deregistration is manual. +- **SSM Run Command** lands in the `ssm-agent` container (it has the Docker + CLI and socket), not on a host. Unverified whether SG relies on it. +- `ecs-init` is bypassed: agent defaults it sets are copied in `runner.sh` and + may drift with new agent releases. diff --git a/k8s/manifests/namespace.yaml b/k8s/manifests/namespace.yaml new file mode 100644 index 0000000..59e9de3 --- /dev/null +++ b/k8s/manifests/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sg-runner + labels: + # dockerd needs a privileged container. + pod-security.kubernetes.io/enforce: privileged diff --git a/k8s/manifests/runner.yaml b/k8s/manifests/runner.yaml new file mode 100644 index 0000000..7c05264 --- /dev/null +++ b/k8s/manifests/runner.yaml @@ -0,0 +1,95 @@ +# sg-runner Kubernetes prototype: each replica is one registered runner. +# Namespace and the SG credentials Secret come from `make secret`. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: sg-runner + namespace: sg-runner +spec: + serviceName: sg-runner + replicas: 1 + selector: + matchLabels: + app: sg-runner + template: + metadata: + labels: + app: sg-runner + spec: + terminationGracePeriodSeconds: 60 + initContainers: + - name: register + image: sg-runner-k8s:dev + imagePullPolicy: IfNotPresent + args: ["register"] + envFrom: + - secretRef: + name: sg-runner-credentials + volumeMounts: + - { name: state, mountPath: /etc/ecs, subPath: ecs-config } + - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } + - { name: state, mountPath: /var/log/registration, subPath: registration } + containers: + - name: dockerd + image: sg-runner-k8s:dev + imagePullPolicy: IfNotPresent + args: ["dockerd"] + securityContext: + privileged: true + readinessProbe: + exec: + command: ["docker", "network", "inspect", "sg-net"] + periodSeconds: 10 + volumeMounts: + - { name: docker-run, mountPath: /var/run } + - { name: docker, mountPath: /var/lib/docker } + # Same path the agent's ECS_HOST_DATA_DIR points at. + - { name: state, mountPath: /var/lib/ecs/data, subPath: ecs-data } + - name: ssm-agent + image: sg-runner-k8s:dev + imagePullPolicy: IfNotPresent + args: ["ssm-agent"] + volumeMounts: + - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } + - { name: aws-creds, mountPath: /root/.aws } + - { name: docker-run, mountPath: /var/run } + - name: ecs-agent + image: sg-runner-k8s:dev + imagePullPolicy: IfNotPresent + args: ["ecs-agent"] + livenessProbe: + exec: + command: ["ecs-agent", "--healthcheck"] + initialDelaySeconds: 120 + periodSeconds: 30 + volumeMounts: + - { name: state, mountPath: /etc/ecs, subPath: ecs-config } + - { name: state, mountPath: /data, subPath: ecs-data } + - { name: state, mountPath: /var/lib/ecs/data, subPath: ecs-data } + - { name: state, mountPath: /log, subPath: ecs-log } + - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } + - { name: aws-creds, mountPath: /rotatingcreds, readOnly: true } + - { name: docker-run, mountPath: /var/run } + volumes: + - name: docker-run + emptyDir: {} + - name: aws-creds + emptyDir: + medium: Memory + volumeClaimTemplates: + # Runner identity (ecs.config, SSM registration, agent state): survives + # pod restarts so the runner doesn't re-register. + - metadata: + name: state + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + - metadata: + name: docker + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 20Gi diff --git a/k8s/runner.sh b/k8s/runner.sh new file mode 100755 index 0000000..316b271 --- /dev/null +++ b/k8s/runner.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# +# Kubernetes entrypoint for the sg-runner pod (prototype). +# +# One pod plays the part of one registered runner host. Each subcommand backs +# one container of the StatefulSet, standing in for what systemd and +# ecs-anywhere-install.sh do on a VM: +# +# register init container: SG API registration + SSM hybrid activation +# dockerd privileged: nested Docker daemon + main.sh network setup +# ssm-agent amazon-ssm-agent; shares rotating creds via /root/.aws +# ecs-agent ECS agent binary, configured from ecs.config (replaces ecs-init) +# deregister run via `kubectl exec -i`; reads SG credentials from stdin +# +# main.sh is sourced for the API, config and network helpers; its +# BASH_SOURCE guard keeps main() from running. + +set -o pipefail + +# shellcheck source-path=SCRIPTDIR/.. source=main.sh +source /opt/sg-runner/main.sh + +readonly SSM_DIR="/var/lib/amazon/ssm" +readonly SSM_REGISTRATION_FILE="${SSM_DIR}/Vault/Store/RegistrationKey" +readonly AGENT_CREDS_FILE="/rotatingcreds/credentials" +readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API" + +CONTAINER_ORCHESTRATOR="docker" + +wait_for() { #{{{ + # Usage: wait_for + local what="$1" + shift + until "$@" >&/dev/null; do + info "Waiting for" "$what" + sleep 3 + done +} +#}}}: wait_for + +load_sg_credentials() { #{{{ + # Map the k8s-friendly names onto the globals main.sh expects. + SG_NODE_TOKEN="${SG_NODE_TOKEN:-}" + ORGANIZATION_ID="${SG_ORGANIZATION:-}" + RUNNER_GROUP_ID="${SG_RUNNER_GROUP:-}" + check_sg_args +} +#}}}: load_sg_credentials + +is_registered() { + [[ -s "$ECS_CONFIG_DIR/ecs.config" && -s "$SSM_REGISTRATION_FILE" ]] +} + +cmd_register() { #{{{ + if is_registered; then + info "Already registered, reusing state from" "$ECS_CONFIG_DIR/ecs.config" + return 0 + fi + if [[ -s "$ECS_CONFIG_DIR/ecs.config" || -s "$SSM_REGISTRATION_FILE" ]]; then + die "Partial registration state found." "Deregister and wipe the PVC before retrying." + fi + + load_sg_credentials + fetch_organization_info + + info "Registering SSM agent in" "$LOCAL_AWS_DEFAULT_REGION" + if ! amazon-ssm-agent -register -y \ + -code "$SSM_ACTIVATION_CODE" \ + -id "$SSM_ACTIVATION_ID" \ + -region "$LOCAL_AWS_DEFAULT_REGION" >>"$LOG_FILE" 2>&1; then + log_err + die "SSM registration failed." "See $LOG_FILE" + fi + info "SSM managed instance" "$(jq -r '.instanceID' "$SSM_REGISTRATION_FILE")" + + # Written last: its presence marks a complete registration. + configure_local_data + save_registration_details +} +#}}}: cmd_register + +cmd_dockerd() { #{{{ + # cgroup v2 nesting, as in moby's hack/dind: move our processes out of the + # root cgroup so controllers can be delegated to the inner containers. + if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then + mkdir -p /sys/fs/cgroup/init + xargs -rn1 /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || : + sed -e 's/ / +/g' -e 's/^/+/' /sys/fs/cgroup/cgroup.subtree_control + fi + + dockerd --host=unix:///var/run/docker.sock & + local dockerd_pid=$! + trap 'kill -TERM "$dockerd_pid"; wait "$dockerd_pid"' TERM INT + + wait_for "dockerd" docker info + # Same bridge, IMDS block and task-credential DNAT as on a VM. Rules land in + # the pod's network namespace, not the node's. + configure_local_network + info "Docker ready, network" "$SG_DOCKER_NETWORK" + + wait "$dockerd_pid" +} +#}}}: cmd_dockerd + +cmd_ssm_agent() { #{{{ + wait_for "SSM registration" test -s "$SSM_REGISTRATION_FILE" + exec amazon-ssm-agent +} +#}}}: cmd_ssm_agent + +cmd_ecs_agent() { #{{{ + wait_for "ecs.config" test -s "$ECS_CONFIG_DIR/ecs.config" + wait_for "dockerd" docker info + wait_for "SSM rotating credentials" test -s "$AGENT_CREDS_FILE" + + # ecs-init normally turns ecs.config into the agent's environment. Read it + # line by line: ECS_INSTANCE_ATTRIBUTES holds JSON that `source` would mangle. + local line + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" == \#* ]] && continue + export "${line%%=*}=${line#*=}" + done <"$ECS_CONFIG_DIR/ecs.config" + + # Defaults ecs-init would set (see ecs-init/docker/docker.go). + export ECS_AVAILABLE_LOGGING_DRIVERS='["json-file","syslog","awslogs","fluentd","none"]' + export ECS_UPDATES_ENABLED=false + # Where dockerd sees the agent's data dir, for bind-mounting task metadata. + export ECS_HOST_DATA_DIR=/var/lib/ecs + # The agent can't create task cgroups from an unprivileged container. + export ECS_ENABLE_TASK_CPU_MEM_LIMIT=false + + exec ecs-agent +} +#}}}: cmd_ecs_agent + +read_sg_credentials_from_stdin() { #{{{ + # KEY=VALUE lines (the k8s/.env format); only known keys are taken, so the + # token never has to live in the pod spec or on a command line. + local key value + while IFS='=' read -r key value || [[ -n "$key" ]]; do + [[ " $SG_CRED_KEYS " == *" $key "* ]] || continue + value="${value%\"}" + value="${value#\"}" + printf -v "$key" '%s' "$value" + done +} +#}}}: read_sg_credentials_from_stdin + +cmd_deregister() { #{{{ + [[ -t 0 ]] && die "Pipe SG credentials on stdin:" "kubectl exec -i ... deregister < k8s/.env" + read_sg_credentials_from_stdin + load_sg_credentials + + local instance_attrs url + [[ -s "$ECS_CONFIG_DIR/ecs.config" ]] || die "Not registered:" "$ECS_CONFIG_DIR/ecs.config missing" + instance_attrs="$(grep ECS_INSTANCE_ATTRIBUTES "$ECS_CONFIG_DIR/ecs.config" | cut -d "=" -f2-)" + RUNNER_ID="$(echo "$instance_attrs" | jq -r '.sg_runner_id')" + validate_runner_id "$RUNNER_ID" + + url="${SG_BASE_API}/orgs/${ORGANIZATION_ID}/runnergroups/${RUNNER_GROUP_ID}/deregister/" + if ! api_call "POST" "$url" "{ \"RunnerId\": \"${RUNNER_ID}\" }"; then + die "Could not deregister runner." "$status_code" "$message" + fi + info "Deregistered runner" "$RUNNER_ID" + + # Wipe the identity so a restarted pod can't come back as a ghost runner. + find "$ECS_CONFIG_DIR" "$ECS_DATA_DIR" "$SSM_DIR" -mindepth 1 -delete + info "Local state wiped." "Delete the StatefulSet and PVCs next." +} +#}}}: cmd_deregister + +case "${1:-}" in +register) cmd_register ;; +dockerd) cmd_dockerd ;; +ssm-agent) cmd_ssm_agent ;; +ecs-agent) cmd_ecs_agent ;; +deregister) cmd_deregister ;; +*) + echo "Usage: $(basename "$0") register|dockerd|ssm-agent|ecs-agent|deregister" >&2 + exit 2 + ;; +esac From 97b940383fb8d7d79795bf7236defa90f918a375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 12 Oct 2026 22:33:40 +0200 Subject: [PATCH 02/43] feat: size and gracefully drain k8s runners --- README.md | 11 +- docs/TODO.md | 38 ++++++ k8s/README.md | 53 +++++++- k8s/manifests/runner.yaml | 124 +++++++++++++++-- k8s/runner.sh | 278 +++++++++++++++++++++++++++++++++++--- 5 files changed, 466 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index f335018..768570f 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,8 @@ network setup. Tested end to end on OrbStack Kubernetes against QA. See ### Known gaps -- **Privileged pod.** Nested Docker needs it; clusters enforcing restricted Pod - Security reject it. +- **Privileged pod.** Nested Docker needs it, or an equivalent set of + capabilities; clusters enforcing restricted Pod Security reject either. - **No workflow network isolation.** Workflow containers can reach cluster DNS and the Kubernetes API. Needs an egress NetworkPolicy (see [`docs/TODO.md`](docs/TODO.md)). @@ -210,8 +210,11 @@ network setup. Tested end to end on OrbStack Kubernetes against QA. See - **No task CPU/memory limits.** The agent can't create task cgroups from its container; the pod's resources are the only cap. - **No proxy support.** `--http-proxy` isn't wired up. -- **Scale-down doesn't deregister.** Removing a replica orphans its runner; - use `make nuke` (deregisters first). +- **Shutdowns can't stop new placements.** Pods wait for running workflows + (and deregister on scale-down), but the runner role isn't allowed to set its + ECS container instance `DRAINING`, so a busy runner can hold its pod for the + full grace period (see [`docs/TODO.md`](docs/TODO.md)). +- **SG token in the pod.** Needed to deregister on scale-down. - **`ecs-init` is bypassed.** Agent defaults are copied into `k8s/runner.sh` and may drift with new ECS agent releases. diff --git a/docs/TODO.md b/docs/TODO.md index eff9bac..ad44c41 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -33,3 +33,41 @@ runner pod covers workflow containers and the agents alike: targets). - Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...). Check what OrbStack's CNI does before testing there. + +### Allow the runner to drain itself (platform IAM) + +On every shutdown the drain hook sets the runner's ECS container instance to +`DRAINING`, so no new task lands while running ones finish, and sets it `ACTIVE` +again when a restarted runner comes back. On QA both calls fail: + +``` +AccessDeniedException: User: arn:aws:sts:::assumed-role/ECSAnywhereRole/... +is not authorized to perform: ecs:UpdateContainerInstancesState on resource: +arn:aws:ecs:eu-central-1::container-instance/admin-workflow-qa/... +because no identity-based policy allows the ecs:UpdateContainerInstancesState action +``` + +`ECSAnywherePolicy` grants the action on `cluster/admin-workflow-qa` only, while +ECS authorizes it against the container-instance ARN. Without it, a busy runner +keeps receiving work while it drains and can hold its pod for the whole grace +period (1h), one pod at a time during a rollout. Prod's policy isn't checked. + +- Add `arn:aws:ecs:::container-instance//*` to that + statement (or use the `ecs:cluster` condition key). Platform change, not + something this repo controls. +- Until then the hook logs the failure and only waits for running workflows. + +### Drop privileged mode + +Spike results on OrbStack (k8s 1.36, cri-dockerd, arm64): + +| Option | Result | +|---|---| +| `hostUsers: false` (user namespaces) | Not testable: cri-dockerd doesn't support user namespaces | +| Capabilities only | Works with `SYS_ADMIN`, `NET_ADMIN`, `SYS_RESOURCE`, seccomp/AppArmor `Unconfined` and `mount -o remount,rw` of `/sys/fs/cgroup` and `/proc/sys`. Not a real gain: `CAP_SYS_ADMIN` in the host user namespace with seccomp off is close to privileged | +| Rootless dind | Containers fail to mount `/proc` without `procMount: Unmasked` (needs user namespaces); task-credential DNAT would target the wrong loopback | +| Sysbox | Not tried (needs a node install) | + +Next: test `hostUsers: false` (with `procMount: Unmasked`) on containerd 2.x +or CRI-O, kernel 6.3+, and Sysbox on a cluster where it can be installed. +Both keep root inside the pod unprivileged on the node. diff --git a/k8s/README.md b/k8s/README.md index 41ee5d5..6a6da8f 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -38,9 +38,51 @@ make nuke # deregister, then delete StatefulSet, PVCs, namespace `make nuke` deregisters first and stops if that fails: deleting the PVC without deregistering would leave a dead runner in the runner group. +## Resource sizing + +The ECS agent sizes the runner from the node, not the pod: CPU is +`runtime.NumCPU() x 1024` (CPU affinity, not the cgroup quota) and memory is +`MemTotal` from `/proc/meminfo`. Workflows run inside the `dockerd` container, +so its limits are what the runner should advertise. `runner.sh ecs-agent` +reads them through the downward API and: + +- sets `ECS_RESERVED_MEMORY` to node memory minus (limit - 256 MiB for dockerd, + `SG_DOCKERD_OVERHEAD_MIB`). The agent parses it as `uint16`; a reservation + above 65535 MiB would silently become 0, so the agent refuses to start instead. +- pins the agent to `limits.cpu` CPUs (rounded up) with `taskset`. + +Without limits on `dockerd`, the runner advertises the whole node. + +## Stopping and scaling down + +The `ecs-agent` container's `preStop` hook (`runner.sh drain`) first sets the +runner's ECS container instance `DRAINING`, so ECS stops placing new tasks on +it, then holds the pod until running workflows finish, up to +`terminationGracePeriodSeconds` (1h). Draining on every shutdown keeps a +busy runner from picking up new work during a restart or rollout, which would +otherwise hold each pod for the full hour. `dockerd` and `ssm-agent` wait for +the hook (`wait-drained`), so workflows keep their Docker daemon and the agent +keeps rotating credentials meanwhile. On start, the agent sets the container +instance `ACTIVE` again once it has registered. + +It tells a scale-down from any other termination by reading its own +StatefulSet (a Role allows `get` on it only; the token is projected into +`ecs-agent` alone): the pod's ordinal no longer fits under `spec.replicas`. + +| Termination | Drain behavior | +|---|---| +| Scale-down (`kubectl scale`) | `DRAINING`, wait for workflows, deregister from SG, wipe identity | +| Restart, rollout, eviction, StatefulSet deleted | `DRAINING`, wait for workflows, keep the identity; `ACTIVE` again on start | + +PVCs are kept on scale-down: if deregistration fails, the identity is still +there for the next scale-up. + ## Known gaps - **Privileged pod.** Needed for nested Docker; restricted PSA clusters reject it. + Non-privileged works with `SYS_ADMIN`, `NET_ADMIN`, `SYS_RESOURCE`, Unconfined + seccomp and read-write remounts of `/sys/fs/cgroup` and `/proc/sys` (verified on + OrbStack), but that's no real isolation gain. See [`docs/TODO.md`](../docs/TODO.md). - **Network isolation.** The VM rule (`DOCKER-USER` DROP for `169.254.169.254,10.0.0.0/24`) doesn't fit a cluster: job containers can resolve cluster DNS and reach the Kubernetes API (verified). Needs a @@ -52,8 +94,15 @@ deregistering would leave a dead runner in the runner group. - **Task cgroup limits off** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`): the agent can't create task cgroups from its container. Size the pod's resources instead. - **No proxy support** yet (`--http-proxy` path not wired). -- **Scale-down doesn't deregister.** Removing a replica orphans its runner; - deregistration is manual. +- **`DRAINING`/`ACTIVE` are denied on QA** (`AccessDeniedException` on the + container-instance ARN; `ECSAnywherePolicy` only names the cluster ARN). Shutdowns + still wait for running workflows, but ECS may place new tasks meanwhile, so a + busy runner can hold its pod for the full grace period + ([`docs/TODO.md`](../docs/TODO.md)). +- **SG token in the pod.** Deregistering on scale-down needs it, so the + credentials Secret is mounted (read-only) into `ecs-agent`. +- **Pods end in `Error`.** The containers exit non-zero on SIGTERM after a + clean drain; cosmetic. - **SSM Run Command** lands in the `ssm-agent` container (it has the Docker CLI and socket), not on a host. Unverified whether SG relies on it. - `ecs-init` is bypassed: agent defaults it sets are copied in `runner.sh` and diff --git a/k8s/manifests/runner.yaml b/k8s/manifests/runner.yaml index 7c05264..c67cd0a 100644 --- a/k8s/manifests/runner.yaml +++ b/k8s/manifests/runner.yaml @@ -1,5 +1,38 @@ # sg-runner Kubernetes prototype: each replica is one registered runner. # Namespace and the SG credentials Secret come from `make secret`. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sg-runner + namespace: sg-runner +automountServiceAccountToken: false +--- +# The drain hook reads its own StatefulSet to tell a scale-down from a restart. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sg-runner + namespace: sg-runner +rules: + - apiGroups: ['apps'] + resources: ['statefulsets'] + resourceNames: ['sg-runner'] + verbs: ['get'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sg-runner + namespace: sg-runner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sg-runner +subjects: + - kind: ServiceAccount + name: sg-runner + namespace: sg-runner +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -16,29 +49,46 @@ spec: labels: app: sg-runner spec: - terminationGracePeriodSeconds: 60 + serviceAccountName: sg-runner + automountServiceAccountToken: false + # Upper bound on how long a terminating runner waits for running + # workflows (the ecs-agent preStop hook) before it is killed. + terminationGracePeriodSeconds: 3600 initContainers: - name: register image: sg-runner-k8s:dev imagePullPolicy: IfNotPresent - args: ["register"] + args: ['register'] envFrom: - secretRef: name: sg-runner-credentials volumeMounts: - { name: state, mountPath: /etc/ecs, subPath: ecs-config } - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } - - { name: state, mountPath: /var/log/registration, subPath: registration } + - { + name: state, + mountPath: /var/log/registration, + subPath: registration, + } containers: - name: dockerd image: sg-runner-k8s:dev imagePullPolicy: IfNotPresent - args: ["dockerd"] + args: ['dockerd'] securityContext: privileged: true + # Workflow containers run inside this container, so its limits are + # what the ECS agent advertises (see SG_TASK_* on ecs-agent). + resources: + requests: { cpu: '1', memory: 2Gi } + limits: { cpu: '2', memory: 4Gi } + lifecycle: + preStop: + exec: + command: ['/opt/sg-runner/runner.sh', 'wait-drained'] readinessProbe: exec: - command: ["docker", "network", "inspect", "sg-net"] + command: ['docker', 'network', 'inspect', 'sg-net'] periodSeconds: 10 volumeMounts: - { name: docker-run, mountPath: /var/run } @@ -48,7 +98,12 @@ spec: - name: ssm-agent image: sg-runner-k8s:dev imagePullPolicy: IfNotPresent - args: ["ssm-agent"] + args: ['ssm-agent'] + # Keeps rotating the agent's credentials while a drain runs. + lifecycle: + preStop: + exec: + command: ['/opt/sg-runner/runner.sh', 'wait-drained'] volumeMounts: - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } - { name: aws-creds, mountPath: /root/.aws } @@ -56,10 +111,30 @@ spec: - name: ecs-agent image: sg-runner-k8s:dev imagePullPolicy: IfNotPresent - args: ["ecs-agent"] + args: ['ecs-agent'] + env: + - name: SG_STATEFULSET + value: sg-runner + # Rounded up to whole CPUs. + - name: SG_TASK_CPU_LIMIT + valueFrom: + resourceFieldRef: + { containerName: dockerd, resource: limits.cpu, divisor: '1' } + - name: SG_TASK_MEMORY_LIMIT_MIB + valueFrom: + resourceFieldRef: + { + containerName: dockerd, + resource: limits.memory, + divisor: 1Mi, + } + lifecycle: + preStop: + exec: + command: ['/opt/sg-runner/runner.sh', 'drain'] livenessProbe: exec: - command: ["ecs-agent", "--healthcheck"] + command: ['ecs-agent', '--healthcheck'] initialDelaySeconds: 120 periodSeconds: 30 volumeMounts: @@ -70,26 +145,53 @@ spec: - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } - { name: aws-creds, mountPath: /rotatingcreds, readOnly: true } - { name: docker-run, mountPath: /var/run } + # Deregistration on scale-down needs the SG credentials. + - { + name: sg-credentials, + mountPath: /etc/sg-runner/credentials, + readOnly: true, + } + - { name: kube-api, mountPath: /etc/sg-runner/kube, readOnly: true } volumes: - name: docker-run emptyDir: {} - name: aws-creds emptyDir: medium: Memory + - name: sg-credentials + secret: + secretName: sg-runner-credentials + defaultMode: 0400 + # Service account token for the drain hook only; not automounted + # into the other containers. + - name: kube-api + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: [{ key: ca.crt, path: ca.crt }] + - downwardAPI: + items: + - path: namespace + fieldRef: { fieldPath: metadata.namespace } volumeClaimTemplates: # Runner identity (ecs.config, SSM registration, agent state): survives - # pod restarts so the runner doesn't re-register. + # pod restarts so the runner doesn't re-register. Kept on scale-down too: + # a failed deregistration then leaves the identity for the next scale-up. - metadata: name: state spec: - accessModes: ["ReadWriteOnce"] + accessModes: ['ReadWriteOnce'] resources: requests: storage: 1Gi - metadata: name: docker spec: - accessModes: ["ReadWriteOnce"] + accessModes: ['ReadWriteOnce'] resources: requests: storage: 20Gi diff --git a/k8s/runner.sh b/k8s/runner.sh index 316b271..d3a9e35 100755 --- a/k8s/runner.sh +++ b/k8s/runner.sh @@ -10,6 +10,8 @@ # dockerd privileged: nested Docker daemon + main.sh network setup # ssm-agent amazon-ssm-agent; shares rotating creds via /root/.aws # ecs-agent ECS agent binary, configured from ecs.config (replaces ecs-init) +# drain ecs-agent preStop: DRAINING, wait for workflows; deregister on scale-down +# wait-drained dockerd/ssm-agent preStop: outlive the drain # deregister run via `kubectl exec -i`; reads SG credentials from stdin # # main.sh is sourced for the API, config and network helpers; its @@ -24,6 +26,15 @@ readonly SSM_DIR="/var/lib/amazon/ssm" readonly SSM_REGISTRATION_FILE="${SSM_DIR}/Vault/Store/RegistrationKey" readonly AGENT_CREDS_FILE="/rotatingcreds/credentials" readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API" +readonly SG_CREDS_DIR="/etc/sg-runner/credentials" +readonly K8S_TOKEN_DIR="/etc/sg-runner/kube" +# On the emptyDir shared by every container: tells the others the drain is over. +readonly DRAINED_MARKER="/var/run/sg-runner-drained" +readonly AGENT_API="http://localhost:51678/v1" + +# Memory kept back for dockerd/containerd themselves, out of the dockerd +# container's limit, when sizing what the agent advertises. +SG_DOCKERD_OVERHEAD_MIB="${SG_DOCKERD_OVERHEAD_MIB:-256}" CONTAINER_ORCHESTRATOR="docker" @@ -109,11 +120,7 @@ cmd_ssm_agent() { #{{{ } #}}}: cmd_ssm_agent -cmd_ecs_agent() { #{{{ - wait_for "ecs.config" test -s "$ECS_CONFIG_DIR/ecs.config" - wait_for "dockerd" docker info - wait_for "SSM rotating credentials" test -s "$AGENT_CREDS_FILE" - +load_ecs_config() { #{{{ # ecs-init normally turns ecs.config into the agent's environment. Read it # line by line: ECS_INSTANCE_ATTRIBUTES holds JSON that `source` would mangle. local line @@ -121,6 +128,57 @@ cmd_ecs_agent() { #{{{ [[ -z "$line" || "$line" == \#* ]] && continue export "${line%%=*}=${line#*=}" done <"$ECS_CONFIG_DIR/ecs.config" +} +#}}}: load_ecs_config + +size_agent_memory() { #{{{ + # The agent advertises MemTotal from /proc/meminfo, which is the whole node. + # Tasks run inside the dockerd container, so advertise its limit (downward + # API) minus dockerd's own overhead, by reserving the difference. + local limit="${SG_TASK_MEMORY_LIMIT_MIB:-}" total available reserved + [[ "$limit" =~ ^[0-9]+$ ]] || return 0 + + total=$(($(awk '/^MemTotal:/ {print $2}' /proc/meminfo) / 1024)) + available=$((limit - SG_DOCKERD_OVERHEAD_MIB)) + reserved=$((total > available ? total - available : 0)) + # The agent parses this as uint16: anything larger silently becomes 0, which + # would advertise the whole node again. + if ((reserved > 65535)); then + die "Can't size agent memory:" "reservation of ${reserved}MiB exceeds 65535MiB" + fi + export ECS_RESERVED_MEMORY="$reserved" + info "Advertising memory (MiB):" "$((total - reserved)) of $total" +} +#}}}: size_agent_memory + +cpu_affinity_list() { #{{{ + # Usage: cpu_affinity_list + # Prints the first n CPUs this process may run on as a taskset list, or + # nothing if n already covers all of them. + local n="$1" range first last cpu allowed=0 cpus=() ranges=() + IFS=, read -ra ranges <<<"$(awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status)" + for range in "${ranges[@]}"; do + first="${range%-*}" + last="${range#*-}" + for ((cpu = first; cpu <= last; cpu++)); do + allowed=$((allowed + 1)) + ((${#cpus[@]} < n)) && cpus+=("$cpu") + done + done + ((n >= allowed)) && return 0 + ( + IFS=, + echo "${cpus[*]}" + ) +} +#}}}: cpu_affinity_list + +cmd_ecs_agent() { #{{{ + wait_for "ecs.config" test -s "$ECS_CONFIG_DIR/ecs.config" + wait_for "dockerd" docker info + wait_for "SSM rotating credentials" test -s "$AGENT_CREDS_FILE" + + load_ecs_config # Defaults ecs-init would set (see ecs-init/docker/docker.go). export ECS_AVAILABLE_LOGGING_DRIVERS='["json-file","syslog","awslogs","fluentd","none"]' @@ -130,6 +188,21 @@ cmd_ecs_agent() { #{{{ # The agent can't create task cgroups from an unprivileged container. export ECS_ENABLE_TASK_CPU_MEM_LIMIT=false + size_agent_memory + + # The agent advertises runtime.NumCPU() x 1024 units. NumCPU follows CPU + # affinity, not the cgroup quota, so pin the agent to as many CPUs as the + # dockerd container's limit allows. + local cpu_list="" + if [[ "${SG_TASK_CPU_LIMIT:-}" =~ ^[0-9]+$ ]]; then + cpu_list="$(cpu_affinity_list "$SG_TASK_CPU_LIMIT")" + fi + reactivate_when_registered & + + if [[ -n "$cpu_list" ]]; then + info "Advertising CPUs:" "$SG_TASK_CPU_LIMIT (agent pinned to $cpu_list)" + exec taskset -c "$cpu_list" ecs-agent + fi exec ecs-agent } #}}}: cmd_ecs_agent @@ -147,12 +220,19 @@ read_sg_credentials_from_stdin() { #{{{ } #}}}: read_sg_credentials_from_stdin -cmd_deregister() { #{{{ - [[ -t 0 ]] && die "Pipe SG credentials on stdin:" "kubectl exec -i ... deregister < k8s/.env" - read_sg_credentials_from_stdin - load_sg_credentials +load_sg_credentials_from_dir() { #{{{ + # The credentials Secret, mounted as one file per key. + local key + for key in $SG_CRED_KEYS; do + [[ -r "$SG_CREDS_DIR/$key" ]] && printf -v "$key" '%s' "$(<"$SG_CREDS_DIR/$key")" + done +} +#}}}: load_sg_credentials_from_dir +deregister_runner() { #{{{ local instance_attrs url + load_sg_credentials + [[ -s "$ECS_CONFIG_DIR/ecs.config" ]] || die "Not registered:" "$ECS_CONFIG_DIR/ecs.config missing" instance_attrs="$(grep ECS_INSTANCE_ATTRIBUTES "$ECS_CONFIG_DIR/ecs.config" | cut -d "=" -f2-)" RUNNER_ID="$(echo "$instance_attrs" | jq -r '.sg_runner_id')" @@ -166,18 +246,174 @@ cmd_deregister() { #{{{ # Wipe the identity so a restarted pod can't come back as a ghost runner. find "$ECS_CONFIG_DIR" "$ECS_DATA_DIR" "$SSM_DIR" -mindepth 1 -delete - info "Local state wiped." "Delete the StatefulSet and PVCs next." + info "Local state wiped." +} +#}}}: deregister_runner + +cmd_deregister() { #{{{ + [[ -t 0 ]] && die "Pipe SG credentials on stdin:" "kubectl exec -i ... deregister < k8s/.env" + read_sg_credentials_from_stdin + deregister_runner } #}}}: cmd_deregister -case "${1:-}" in -register) cmd_register ;; -dockerd) cmd_dockerd ;; -ssm-agent) cmd_ssm_agent ;; -ecs-agent) cmd_ecs_agent ;; -deregister) cmd_deregister ;; -*) - echo "Usage: $(basename "$0") register|dockerd|ssm-agent|ecs-agent|deregister" >&2 - exit 2 - ;; -esac +k8s_api() { #{{{ + # Usage: k8s_api ; GET with the pod's projected service account token. + curl -fsS --max-time 10 \ + --cacert "$K8S_TOKEN_DIR/ca.crt" \ + -H "Authorization: Bearer $(<"$K8S_TOKEN_DIR/token")" \ + "https://kubernetes.default.svc$1" +} +#}}}: k8s_api + +is_scaling_down() { #{{{ + # A StatefulSet scale-down removes the highest ordinals, so this pod is going + # away for good only if its ordinal no longer fits under spec.replicas. + # Rollouts, evictions and deleting the StatefulSet keep the identity. + local ns replicas ordinal="${HOSTNAME##*-}" + ns="$(<"$K8S_TOKEN_DIR/namespace")" + replicas="$(k8s_api "/apis/apps/v1/namespaces/${ns}/statefulsets/${SG_STATEFULSET}" | + jq -r '.spec.replicas')" || return 1 + [[ "$replicas" =~ ^[0-9]+$ ]] && ((ordinal >= replicas)) +} +#}}}: is_scaling_down + +ecs_api() { #{{{ + # Usage: ecs_api + # Signed with the instance credentials the SSM agent rotates for the ECS + # agent; the key pair goes through a curl config file, not the command line. + local action="$1" payload="$2" curl_config response code rc + curl_config=$(mktemp) + chmod 600 "$curl_config" + awk -F' *= *' ' + $1 == "aws_access_key_id" { key = $2 } + $1 == "aws_secret_access_key" { secret = $2 } + $1 == "aws_session_token" { token = $2 } + END { + printf "user = \"%s:%s\"\n", key, secret + printf "header = \"x-amz-security-token: %s\"\n", token + }' "$AGENT_CREDS_FILE" >"$curl_config" + + response="$(curl -sS --max-time 15 -K "$curl_config" \ + --aws-sigv4 "aws:amz:${AWS_DEFAULT_REGION}:ecs" \ + -H "Content-Type: application/x-amz-json-1.1" \ + -H "X-Amz-Target: AmazonEC2ContainerServiceV20141113.${action}" \ + -w '\n%{http_code}' \ + -d "$payload" \ + "https://ecs.${AWS_DEFAULT_REGION}.amazonaws.com/")" + rc=$? + rm -f "$curl_config" + ((rc == 0)) || return $rc + + code="${response##*$'\n'}" + response="${response%$'\n'*}" + if [[ "$code" != 2* ]]; then + # e.g. AccessDeniedException: ... not authorized to perform ... + # stderr: callers capture stdout for the response. + info "ECS $action failed ($code):" \ + "$(echo "$response" | jq -r '"\(.__type // "") \(.message // .Message // "")"' 2>/dev/null)" >&2 + return 1 + fi + echo "$response" +} +#}}}: ecs_api + +set_container_instance_state() { #{{{ + # Usage: set_container_instance_state DRAINING|ACTIVE + # DRAINING stops ECS from placing new tasks on this runner; running ones + # carry on. + local metadata payload response + metadata="$(curl -fsS --max-time 5 "$AGENT_API/metadata")" || return 1 + payload="$(echo "$metadata" | jq -c --arg status "$1" \ + '{cluster: .Cluster, containerInstances: [.ContainerInstanceArn], status: $status}')" + response="$(ecs_api UpdateContainerInstancesState "$payload")" || return 1 + if [[ "$(echo "$response" | jq '.failures | length')" != 0 ]]; then + info "ECS UpdateContainerInstancesState failures:" "$(echo "$response" | jq -c '.failures')" >&2 + return 1 + fi +} +#}}}: set_container_instance_state + +reactivate_when_registered() { #{{{ + # Every shutdown drains the runner (cmd_drain). After a restart it comes back + # with the same container instance, still DRAINING, so set it ACTIVE again as + # soon as the agent has registered. Harmless if it already is. + local attempt + for ((attempt = 0; attempt < 120; attempt++)); do + if curl -fsS --max-time 5 "$AGENT_API/metadata" 2>/dev/null | + jq -e '.ContainerInstanceArn // empty' >/dev/null; then + if set_container_instance_state ACTIVE; then + info "Container instance" "ACTIVE" + else + info "Could not set ACTIVE;" "a restarted runner may stay DRAINING" + fi + return + fi + sleep 5 + done +} +#}}}: reactivate_when_registered + +cmd_drain() { #{{{ + # preStop hook of the ecs-agent container. Every shutdown first sets the + # runner DRAINING, so it waits only for workflows already running (bounded + # by terminationGracePeriodSeconds) instead of picking up new ones. A + # scale-down also deregisters at the end; any other shutdown keeps the + # identity and reactivates on start (reactivate_when_registered). + # Hook output isn't logged by the kubelet, so write to the container's. + exec >/proc/1/fd/1 2>&1 + # Release the other containers whatever happens below. + trap 'touch "$DRAINED_MARKER"' EXIT + + local scale_down=false + if is_scaling_down; then + scale_down=true + fi + info "Shutting down:" "$([[ "$scale_down" == true ]] && echo scale-down || echo restart)" + + # Already gone after a manual deregister (make nuke). + if [[ -s "$ECS_CONFIG_DIR/ecs.config" ]]; then + load_ecs_config + if set_container_instance_state DRAINING; then + info "Container instance" "DRAINING" + else + info "Could not set DRAINING;" "new tasks may still land" + fi + fi + + until [[ -z "$(docker ps -q 2>/dev/null)" ]]; do + info "Waiting for running workflow containers.." + sleep 10 + done + info "No workflow containers running." + + if [[ "$scale_down" == true ]]; then + load_sg_credentials_from_dir + deregister_runner + fi +} +#}}}: cmd_drain + +cmd_wait_drained() { #{{{ + until [[ -e "$DRAINED_MARKER" ]]; do + sleep 5 + done +} +#}}}: cmd_wait_drained + +# Only dispatch when executed directly; sourcing exposes the functions (tests). +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + case "${1:-}" in + register) cmd_register ;; + dockerd) cmd_dockerd ;; + ssm-agent) cmd_ssm_agent ;; + ecs-agent) cmd_ecs_agent ;; + drain) cmd_drain ;; + wait-drained) cmd_wait_drained ;; + deregister) cmd_deregister ;; + *) + echo "Usage: $(basename "$0") register|dockerd|ssm-agent|ecs-agent|drain|wait-drained|deregister" >&2 + exit 2 + ;; + esac +fi From 02c7bd8c35a4d5d612cadb445653654e2c692a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 13:41:13 +0200 Subject: [PATCH 03/43] feat: park k8s runners and drain them through sg --- README.md | 14 ++- docs/TODO.md | 34 +++---- k8s/.env.example | 3 + k8s/Makefile | 17 +++- k8s/README.md | 78 ++++++++++------ k8s/manifests/runner.yaml | 74 ++++----------- k8s/runner.sh | 190 ++++++++++++++++++-------------------- 7 files changed, 194 insertions(+), 216 deletions(-) diff --git a/README.md b/README.md index 768570f..e14e303 100644 --- a/README.md +++ b/README.md @@ -210,11 +210,15 @@ network setup. Tested end to end on OrbStack Kubernetes against QA. See - **No task CPU/memory limits.** The agent can't create task cgroups from its container; the pod's resources are the only cap. - **No proxy support.** `--http-proxy` isn't wired up. -- **Shutdowns can't stop new placements.** Pods wait for running workflows - (and deregister on scale-down), but the runner role isn't allowed to set its - ECS container instance `DRAINING`, so a busy runner can hold its pod for the - full grace period (see [`docs/TODO.md`](docs/TODO.md)). -- **SG token in the pod.** Needed to deregister on scale-down. +- **Draining needs an SG API key.** Pods set themselves `DRAINING` through + SG's `runner_status` API on shutdown, which needs `SG_API_KEY`; without it, + a busy runner can keep receiving work and hold its pod for the full grace + period. +- **Parked runners stay registered.** Scaling down parks runners (stopped, + registration kept, woken in seconds); nothing deregisters them automatically + yet. Long parking periods are untested. +- **SG credentials in the pod.** Mounted into the `ecs-agent` container to set + the runner's status. - **`ecs-init` is bypassed.** Agent defaults are copied into `k8s/runner.sh` and may drift with new ECS agent releases. diff --git a/docs/TODO.md b/docs/TODO.md index ad44c41..1c634fb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -34,28 +34,18 @@ runner pod covers workflow containers and the agents alike: - Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...). Check what OrbStack's CNI does before testing there. -### Allow the runner to drain itself (platform IAM) - -On every shutdown the drain hook sets the runner's ECS container instance to -`DRAINING`, so no new task lands while running ones finish, and sets it `ACTIVE` -again when a restarted runner comes back. On QA both calls fail: - -``` -AccessDeniedException: User: arn:aws:sts:::assumed-role/ECSAnywhereRole/... -is not authorized to perform: ecs:UpdateContainerInstancesState on resource: -arn:aws:ecs:eu-central-1::container-instance/admin-workflow-qa/... -because no identity-based policy allows the ecs:UpdateContainerInstancesState action -``` - -`ECSAnywherePolicy` grants the action on `cluster/admin-workflow-qa` only, while -ECS authorizes it against the container-instance ARN. Without it, a busy runner -keeps receiving work while it drains and can hold its pod for the whole grace -period (1h), one pod at a time during a rollout. Prod's policy isn't checked. - -- Add `arn:aws:ecs:::container-instance//*` to that - statement (or use the `ecs:cluster` condition key). Platform change, not - something this repo controls. -- Until then the hook logs the failure and only waits for running workflows. +### Drain through SG: confirm the credentials + +The drain hook sets the runner `DRAINING`/`ACTIVE` through SG's +`POST /orgs/{org}/runnergroups/{group}/runner_status/` (as +`sg-runner-autoscaler` does), authenticated with `SG_API_KEY`. The runner +never calls ECS itself: on QA the runner's instance role was denied +`ecs:UpdateContainerInstancesState` anyway, and a customer's runner has no ECS +permissions of its own. + +- Verify end to end on QA (DRAINING on shutdown, ACTIVE after restart). +- Check whether the node token is accepted too; if so, drop the extra key. +- Otherwise, document the least-privileged API key role that can call it. ### Drop privileged mode diff --git a/k8s/.env.example b/k8s/.env.example index 2c843fc..e709c6d 100644 --- a/k8s/.env.example +++ b/k8s/.env.example @@ -3,3 +3,6 @@ SG_NODE_TOKEN= SG_ORGANIZATION= SG_RUNNER_GROUP= SG_BASE_API=https://api.app.stackguardian.io/api/v1 +# Optional: SG API key allowed to manage this runner group. Lets the pod set +# itself DRAINING/ACTIVE through SG (runner_status API) on shutdown/restart. +SG_API_KEY= diff --git a/k8s/Makefile b/k8s/Makefile index 920d588..97f5932 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -28,15 +28,24 @@ secret: ## Create the SG credentials Secret from .env up: ## Deploy the runner StatefulSet (registers on first start) kubectl --context $(CONTEXT) apply -f manifests/ -status: ## Show pod, container states and agent health - $(KUBECTL) get pods,pvc -o wide +status: ## Show runner pods and state PVCs (a PVC without a pod is a parked runner) + $(KUBECTL) get statefulset,pods,pvc -o wide -$(KUBECTL) exec $(POD) -c ecs-agent -- curl -fsS localhost:51678/v1/metadata logs: ## Tail all runner containers $(KUBECTL) logs $(POD) --all-containers --prefix -f --max-log-requests 10 -deregister: ## Deregister from SG and wipe local identity (reads .env on stdin) - $(KUBECTL) exec -i $(POD) -c ecs-agent -- /opt/sg-runner/runner.sh deregister < .env +# Parked runners are still registered but have no pod to run from, so wake +# every runner that has a state PVC first, then deregister them one by one. +deregister: ## Wake parked runners, deregister all from SG, wipe identities (reads .env on stdin) + @n=$$($(KUBECTL) get pvc -o name | grep -c '/state-sg-runner-' || true); \ + if [ "$$n" -eq 0 ]; then echo "No runners to deregister."; exit 0; fi; \ + $(KUBECTL) scale statefulset sg-runner --replicas=$$n && \ + for i in $$(seq 0 $$((n - 1))); do \ + $(KUBECTL) wait --for=condition=Ready pod/sg-runner-$$i --timeout=300s && \ + $(KUBECTL) exec -i sg-runner-$$i -c ecs-agent -- /opt/sg-runner/runner.sh deregister < .env \ + || exit 1; \ + done down: ## Delete the StatefulSet, keep PVCs (runner identity survives) -$(KUBECTL) delete statefulset sg-runner diff --git a/k8s/README.md b/k8s/README.md index 6a6da8f..a52fcce 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -31,7 +31,9 @@ make up # deploy; first start registers the runner make logs # follow all containers make status # pod state + ECS agent metadata -make deregister # SG deregister + wipe identity (reads .env on stdin) +kubectl -n sg-runner scale sts sg-runner --replicas=N # park / wake runners + +make deregister # wake parked runners, deregister all (reads .env on stdin) make nuke # deregister, then delete StatefulSet, PVCs, namespace ``` @@ -53,29 +55,45 @@ reads them through the downward API and: Without limits on `dockerd`, the runner advertises the whole node. -## Stopping and scaling down - -The `ecs-agent` container's `preStop` hook (`runner.sh drain`) first sets the -runner's ECS container instance `DRAINING`, so ECS stops placing new tasks on -it, then holds the pod until running workflows finish, up to -`terminationGracePeriodSeconds` (1h). Draining on every shutdown keeps a -busy runner from picking up new work during a restart or rollout, which would -otherwise hold each pod for the full hour. `dockerd` and `ssm-agent` wait for -the hook (`wait-drained`), so workflows keep their Docker daemon and the agent -keeps rotating credentials meanwhile. On start, the agent sets the container -instance `ACTIVE` again once it has registered. +## Parking: stopping and scaling down -It tells a scale-down from any other termination by reading its own -StatefulSet (a Role allows `get` on it only; the token is projected into -`ecs-agent` alone): the pod's ordinal no longer fits under `spec.replicas`. +Scaling down doesn't deregister a runner, it **parks** it: the pod stops, the +registration and Docker image cache stay on its PVCs, and scaling back up +wakes the same runner without registering again. -| Termination | Drain behavior | +| Measured on QA | Time to ECS `ACTIVE` + connected | |---|---| -| Scale-down (`kubectl scale`) | `DRAINING`, wait for workflows, deregister from SG, wipe identity | -| Restart, rollout, eviction, StatefulSet deleted | `DRAINING`, wait for workflows, keep the identity; `ACTIVE` again on start | - -PVCs are kept on scale-down: if deregistration fails, the identity is still -there for the next scale-up. +| Wake a parked runner (`kubectl scale` up) | **8s**, same container instance, image cache kept | +| Fresh runner (new ordinal, full registration) | 20s, plus image pulls on its first workflow | +| Park a runner (`kubectl scale` down, idle) | 14s | + +Every shutdown (scale-down, restart, rollout, eviction) parks the same way. +The `ecs-agent` container's `preStop` hook (`runner.sh drain`): + +1. sets the runner `DRAINING` through SG's `runner_status` API (the same call + `sg-runner-autoscaler` uses), so no new workflows are placed on it, and + leaves a marker on the state PVC; +2. waits for running workflows, up to `terminationGracePeriodSeconds` (1h). + `dockerd` and `ssm-agent` wait for the hook (`wait-drained`), so workflows + keep their Docker daemon and the agent keeps rotating credentials. + +On wake the runner sets itself `ACTIVE` again once the agent has reconnected, +only if the marker is there. A runner that was already `DRAINING` when it +parked, drained on purpose from the SG UI or by an autoscaler, stays drained. + +A parked runner shows in SG as registered but disconnected and `DRAINING`; +it still counts against the runner group's `MaxNumberOfRunners`. Pods are +managed in parallel (`podManagementPolicy: Parallel`), so several runners park +or wake at once. + +Setting the status needs `SG_API_KEY` in `.env`: an SG API key allowed to +manage the runner group. The runner never calls ECS itself; a customer's +runner has no ECS permissions. Without the key it skips the status change and +only waits for running workflows. + +Deregistering is explicit: `make deregister` wakes every parked runner (each +state PVC) and deregisters them all; `make nuke` does that and deletes +everything. Reaping runners parked for too long is up to the scaler (planned). ## Known gaps @@ -94,13 +112,17 @@ there for the next scale-up. - **Task cgroup limits off** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`): the agent can't create task cgroups from its container. Size the pod's resources instead. - **No proxy support** yet (`--http-proxy` path not wired). -- **`DRAINING`/`ACTIVE` are denied on QA** (`AccessDeniedException` on the - container-instance ARN; `ECSAnywherePolicy` only names the cluster ARN). Shutdowns - still wait for running workflows, but ECS may place new tasks meanwhile, so a - busy runner can hold its pod for the full grace period - ([`docs/TODO.md`](../docs/TODO.md)). -- **SG token in the pod.** Deregistering on scale-down needs it, so the - credentials Secret is mounted (read-only) into `ecs-agent`. +- **Draining needs an SG API key.** Without `SG_API_KEY`, shutdowns wait for + running workflows but new ones may still be placed, so a busy runner can + hold its pod for the full grace period. The key also lives in the pod. +- **No automatic deregistration yet.** Parked runners stay registered until + `make deregister`; a TTL reaper belongs in the scaler. +- **Unverified: long parking.** Waking was tested after seconds. Whether SSM + and ECS reconnect after hours or days parked isn't tested yet. +- **Unverified: hard kills.** A pod killed without its `preStop` hook (node + crash) parks as `ACTIVE` but disconnected; how ECS treats that isn't tested. +- **SG credentials in the pod.** The credentials Secret is mounted + (read-only) into `ecs-agent` for the status changes. - **Pods end in `Error`.** The containers exit non-zero on SIGTERM after a clean drain; cosmetic. - **SSM Run Command** lands in the `ssm-agent` container (it has the Docker diff --git a/k8s/manifests/runner.yaml b/k8s/manifests/runner.yaml index c67cd0a..d5259e6 100644 --- a/k8s/manifests/runner.yaml +++ b/k8s/manifests/runner.yaml @@ -1,38 +1,5 @@ # sg-runner Kubernetes prototype: each replica is one registered runner. # Namespace and the SG credentials Secret come from `make secret`. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sg-runner - namespace: sg-runner -automountServiceAccountToken: false ---- -# The drain hook reads its own StatefulSet to tell a scale-down from a restart. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: sg-runner - namespace: sg-runner -rules: - - apiGroups: ['apps'] - resources: ['statefulsets'] - resourceNames: ['sg-runner'] - verbs: ['get'] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: sg-runner - namespace: sg-runner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: sg-runner -subjects: - - kind: ServiceAccount - name: sg-runner - namespace: sg-runner ---- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -41,6 +8,9 @@ metadata: spec: serviceName: sg-runner replicas: 1 + # Park and wake runners side by side, not one at a time (each park can wait + # for running workflows). + podManagementPolicy: Parallel selector: matchLabels: app: sg-runner @@ -49,7 +19,6 @@ spec: labels: app: sg-runner spec: - serviceAccountName: sg-runner automountServiceAccountToken: false # Upper bound on how long a terminating runner waits for running # workflows (the ecs-agent preStop hook) before it is killed. @@ -59,9 +28,20 @@ spec: image: sg-runner-k8s:dev imagePullPolicy: IfNotPresent args: ['register'] - envFrom: - - secretRef: - name: sg-runner-credentials + # Only what registration needs; SG_API_KEY stays out of it. + env: + - name: SG_NODE_TOKEN + valueFrom: + secretKeyRef: { name: sg-runner-credentials, key: SG_NODE_TOKEN } + - name: SG_ORGANIZATION + valueFrom: + secretKeyRef: { name: sg-runner-credentials, key: SG_ORGANIZATION } + - name: SG_RUNNER_GROUP + valueFrom: + secretKeyRef: { name: sg-runner-credentials, key: SG_RUNNER_GROUP } + - name: SG_BASE_API + valueFrom: + secretKeyRef: { name: sg-runner-credentials, key: SG_BASE_API, optional: true } volumeMounts: - { name: state, mountPath: /etc/ecs, subPath: ecs-config } - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } @@ -113,8 +93,6 @@ spec: imagePullPolicy: IfNotPresent args: ['ecs-agent'] env: - - name: SG_STATEFULSET - value: sg-runner # Rounded up to whole CPUs. - name: SG_TASK_CPU_LIMIT valueFrom: @@ -145,13 +123,12 @@ spec: - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } - { name: aws-creds, mountPath: /rotatingcreds, readOnly: true } - { name: docker-run, mountPath: /var/run } - # Deregistration on scale-down needs the SG credentials. + # Setting the runner's status through SG needs the SG credentials. - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true, } - - { name: kube-api, mountPath: /etc/sg-runner/kube, readOnly: true } volumes: - name: docker-run emptyDir: {} @@ -162,21 +139,6 @@ spec: secret: secretName: sg-runner-credentials defaultMode: 0400 - # Service account token for the drain hook only; not automounted - # into the other containers. - - name: kube-api - projected: - sources: - - serviceAccountToken: - path: token - expirationSeconds: 3600 - - configMap: - name: kube-root-ca.crt - items: [{ key: ca.crt, path: ca.crt }] - - downwardAPI: - items: - - path: namespace - fieldRef: { fieldPath: metadata.namespace } volumeClaimTemplates: # Runner identity (ecs.config, SSM registration, agent state): survives # pod restarts so the runner doesn't re-register. Kept on scale-down too: diff --git a/k8s/runner.sh b/k8s/runner.sh index d3a9e35..6fdf3f3 100755 --- a/k8s/runner.sh +++ b/k8s/runner.sh @@ -10,7 +10,7 @@ # dockerd privileged: nested Docker daemon + main.sh network setup # ssm-agent amazon-ssm-agent; shares rotating creds via /root/.aws # ecs-agent ECS agent binary, configured from ecs.config (replaces ecs-init) -# drain ecs-agent preStop: DRAINING, wait for workflows; deregister on scale-down +# drain ecs-agent preStop: DRAINING, wait for workflows, park (keep registration) # wait-drained dockerd/ssm-agent preStop: outlive the drain # deregister run via `kubectl exec -i`; reads SG credentials from stdin # @@ -25,11 +25,13 @@ source /opt/sg-runner/main.sh readonly SSM_DIR="/var/lib/amazon/ssm" readonly SSM_REGISTRATION_FILE="${SSM_DIR}/Vault/Store/RegistrationKey" readonly AGENT_CREDS_FILE="/rotatingcreds/credentials" -readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API" +readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API SG_API_KEY" readonly SG_CREDS_DIR="/etc/sg-runner/credentials" -readonly K8S_TOKEN_DIR="/etc/sg-runner/kube" # On the emptyDir shared by every container: tells the others the drain is over. readonly DRAINED_MARKER="/var/run/sg-runner-drained" +# On the state PVC: this pod drained the runner when it parked, so the next +# start may set it ACTIVE again. Absent when someone else drained it on purpose. +readonly SELF_DRAIN_MARKER="$ECS_CONFIG_DIR/.parked" readonly AGENT_API="http://localhost:51678/v1" # Memory kept back for dockerd/containerd themselves, out of the dockerd @@ -229,13 +231,19 @@ load_sg_credentials_from_dir() { #{{{ } #}}}: load_sg_credentials_from_dir +configured_runner_id() { #{{{ + # Prints the SG runner ID recorded in ecs.config at registration. + [[ -s "$ECS_CONFIG_DIR/ecs.config" ]] || die "Not registered:" "$ECS_CONFIG_DIR/ecs.config missing" + grep ECS_INSTANCE_ATTRIBUTES "$ECS_CONFIG_DIR/ecs.config" | cut -d "=" -f2- | + jq -r '.sg_runner_id' +} +#}}}: configured_runner_id + deregister_runner() { #{{{ - local instance_attrs url + local url load_sg_credentials - [[ -s "$ECS_CONFIG_DIR/ecs.config" ]] || die "Not registered:" "$ECS_CONFIG_DIR/ecs.config missing" - instance_attrs="$(grep ECS_INSTANCE_ATTRIBUTES "$ECS_CONFIG_DIR/ecs.config" | cut -d "=" -f2-)" - RUNNER_ID="$(echo "$instance_attrs" | jq -r '.sg_runner_id')" + RUNNER_ID="$(configured_runner_id)" validate_runner_id "$RUNNER_ID" url="${SG_BASE_API}/orgs/${ORGANIZATION_ID}/runnergroups/${RUNNER_GROUP_ID}/deregister/" @@ -257,93 +265,77 @@ cmd_deregister() { #{{{ } #}}}: cmd_deregister -k8s_api() { #{{{ - # Usage: k8s_api ; GET with the pod's projected service account token. - curl -fsS --max-time 10 \ - --cacert "$K8S_TOKEN_DIR/ca.crt" \ - -H "Authorization: Bearer $(<"$K8S_TOKEN_DIR/token")" \ - "https://kubernetes.default.svc$1" + + +sg_api() { #{{{ + # Usage: sg_api [payload] + # Calls the runner group's SG API with SG_API_KEY and prints the response. + # api_call authenticates with SG_NODE_TOKEN: swap in the key, in a subshell + # so the node token stays untouched for deregistration. + local method="$1" path="$2" payload="${3:-}" + ( + SG_NODE_TOKEN="$SG_API_KEY" + if ! api_call "$method" \ + "${SG_BASE_API}/orgs/${SG_ORGANIZATION}/runnergroups/${SG_RUNNER_GROUP}/${path}" \ + "$payload"; then + info "SG API $method $path failed ($status_code):" "$message" >&2 + exit 1 + fi + printf '%s' "$response" + ) } -#}}}: k8s_api - -is_scaling_down() { #{{{ - # A StatefulSet scale-down removes the highest ordinals, so this pod is going - # away for good only if its ordinal no longer fits under spec.replicas. - # Rollouts, evictions and deleting the StatefulSet keep the identity. - local ns replicas ordinal="${HOSTNAME##*-}" - ns="$(<"$K8S_TOKEN_DIR/namespace")" - replicas="$(k8s_api "/apis/apps/v1/namespaces/${ns}/statefulsets/${SG_STATEFULSET}" | - jq -r '.spec.replicas')" || return 1 - [[ "$replicas" =~ ^[0-9]+$ ]] && ((ordinal >= replicas)) +#}}}: sg_api + +set_sg_runner_status() { #{{{ + # Usage: set_sg_runner_status DRAINING|ACTIVE + # SG's runner_status API, as used by sg-runner-autoscaler: SG changes the + # ECS container instance state on its side, with its own permissions. + local runner_id + runner_id="$(configured_runner_id)" + validate_runner_id "$runner_id" + sg_api POST runner_status/ \ + "$(jq -nc --arg s "$1" --arg r "$runner_id" '{Status: $s, RunnerId: $r}')" >/dev/null } -#}}}: is_scaling_down - -ecs_api() { #{{{ - # Usage: ecs_api - # Signed with the instance credentials the SSM agent rotates for the ECS - # agent; the key pair goes through a curl config file, not the command line. - local action="$1" payload="$2" curl_config response code rc - curl_config=$(mktemp) - chmod 600 "$curl_config" - awk -F' *= *' ' - $1 == "aws_access_key_id" { key = $2 } - $1 == "aws_secret_access_key" { secret = $2 } - $1 == "aws_session_token" { token = $2 } - END { - printf "user = \"%s:%s\"\n", key, secret - printf "header = \"x-amz-security-token: %s\"\n", token - }' "$AGENT_CREDS_FILE" >"$curl_config" - - response="$(curl -sS --max-time 15 -K "$curl_config" \ - --aws-sigv4 "aws:amz:${AWS_DEFAULT_REGION}:ecs" \ - -H "Content-Type: application/x-amz-json-1.1" \ - -H "X-Amz-Target: AmazonEC2ContainerServiceV20141113.${action}" \ - -w '\n%{http_code}' \ - -d "$payload" \ - "https://ecs.${AWS_DEFAULT_REGION}.amazonaws.com/")" - rc=$? - rm -f "$curl_config" - ((rc == 0)) || return $rc - - code="${response##*$'\n'}" - response="${response%$'\n'*}" - if [[ "$code" != 2* ]]; then - # e.g. AccessDeniedException: ... not authorized to perform ... - # stderr: callers capture stdout for the response. - info "ECS $action failed ($code):" \ - "$(echo "$response" | jq -r '"\(.__type // "") \(.message // .Message // "")"' 2>/dev/null)" >&2 +#}}}: set_sg_runner_status + +set_runner_status() { #{{{ + # Usage: set_runner_status DRAINING|ACTIVE + # Only through SG: a customer's runner has no ECS permissions of its own. + # Without SG_API_KEY, shutdowns still wait for running workflows but can't + # stop new ones from being placed meanwhile. + load_sg_credentials_from_dir + if [[ -z "${SG_API_KEY:-}" ]]; then + info "SG_API_KEY not set:" "skipping $1" return 1 fi - echo "$response" + set_sg_runner_status "$1" } -#}}}: ecs_api - -set_container_instance_state() { #{{{ - # Usage: set_container_instance_state DRAINING|ACTIVE - # DRAINING stops ECS from placing new tasks on this runner; running ones - # carry on. - local metadata payload response - metadata="$(curl -fsS --max-time 5 "$AGENT_API/metadata")" || return 1 - payload="$(echo "$metadata" | jq -c --arg status "$1" \ - '{cluster: .Cluster, containerInstances: [.ContainerInstanceArn], status: $status}')" - response="$(ecs_api UpdateContainerInstancesState "$payload")" || return 1 - if [[ "$(echo "$response" | jq '.failures | length')" != 0 ]]; then - info "ECS UpdateContainerInstancesState failures:" "$(echo "$response" | jq -c '.failures')" >&2 - return 1 - fi +#}}}: set_runner_status + +current_runner_status() { #{{{ + # Prints this runner's status as SG reports it (ACTIVE, DRAINING, ...). + # The response holds secrets; only the status leaves this function. + local runner_id + load_sg_credentials_from_dir + [[ -n "${SG_API_KEY:-}" ]] || return 1 + runner_id="$(configured_runner_id)" + sg_api GET "" | jq -r --arg r "$runner_id" \ + '.msg.ContainerInstances[]? | select(.runnerId == $r) | .status' } -#}}}: set_container_instance_state +#}}}: current_runner_status reactivate_when_registered() { #{{{ - # Every shutdown drains the runner (cmd_drain). After a restart it comes back - # with the same container instance, still DRAINING, so set it ACTIVE again as - # soon as the agent has registered. Harmless if it already is. + # Wake a parked runner: undo the drain this pod did on its way down + # (cmd_drain); it comes back with the same container instance. A drain anyone else + # started (SG UI, an autoscaler) leaves no marker and is kept. + [[ -e "$SELF_DRAIN_MARKER" ]] || return 0 local attempt for ((attempt = 0; attempt < 120; attempt++)); do if curl -fsS --max-time 5 "$AGENT_API/metadata" 2>/dev/null | jq -e '.ContainerInstanceArn // empty' >/dev/null; then - if set_container_instance_state ACTIVE; then - info "Container instance" "ACTIVE" + if set_runner_status ACTIVE; then + rm -f "$SELF_DRAIN_MARKER" + info "Runner" "ACTIVE" else info "Could not set ACTIVE;" "a restarted runner may stay DRAINING" fi @@ -355,27 +347,28 @@ reactivate_when_registered() { #{{{ #}}}: reactivate_when_registered cmd_drain() { #{{{ - # preStop hook of the ecs-agent container. Every shutdown first sets the - # runner DRAINING, so it waits only for workflows already running (bounded - # by terminationGracePeriodSeconds) instead of picking up new ones. A - # scale-down also deregisters at the end; any other shutdown keeps the - # identity and reactivates on start (reactivate_when_registered). + # preStop hook of the ecs-agent container: park the runner. Every shutdown + # (scale-down, restart, rollout, eviction) sets it DRAINING so no new + # workflows land, waits for the running ones (bounded by + # terminationGracePeriodSeconds), and keeps the registration on the state + # PVC. Scaling back up wakes the same runner in seconds + # (reactivate_when_registered); deregistering parked runners is left to + # `make deregister` / the scaler. # Hook output isn't logged by the kubelet, so write to the container's. exec >/proc/1/fd/1 2>&1 # Release the other containers whatever happens below. trap 'touch "$DRAINED_MARKER"' EXIT - local scale_down=false - if is_scaling_down; then - scale_down=true - fi - info "Shutting down:" "$([[ "$scale_down" == true ]] && echo scale-down || echo restart)" - + info "Parking runner.." # Already gone after a manual deregister (make nuke). if [[ -s "$ECS_CONFIG_DIR/ecs.config" ]]; then load_ecs_config - if set_container_instance_state DRAINING; then - info "Container instance" "DRAINING" + if [[ "$(current_runner_status)" == "DRAINING" ]]; then + # Drained on purpose by someone else: keep it that way on wake. + info "Runner already" "DRAINING" + elif set_runner_status DRAINING; then + info "Runner" "DRAINING" + touch "$SELF_DRAIN_MARKER" else info "Could not set DRAINING;" "new tasks may still land" fi @@ -385,12 +378,7 @@ cmd_drain() { #{{{ info "Waiting for running workflow containers.." sleep 10 done - info "No workflow containers running." - - if [[ "$scale_down" == true ]]; then - load_sg_credentials_from_dir - deregister_runner - fi + info "Parked:" "no workflow containers running" } #}}}: cmd_drain From 071969e24cfc02c992f668e8c61f24ef860aa775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 14:44:15 +0200 Subject: [PATCH 04/43] feat: autoscale k8s runners with a scaler --- README.md | 4 - docs/TODO.md | 13 -- k8s/.env.example | 3 - k8s/Dockerfile | 1 + k8s/README.md | 44 ++++- k8s/manifests/runner.yaml | 5 +- k8s/manifests/scaler.yaml | 85 ++++++++++ k8s/runner.sh | 27 +-- k8s/scaler.sh | 334 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 472 insertions(+), 44 deletions(-) create mode 100644 k8s/manifests/scaler.yaml create mode 100755 k8s/scaler.sh diff --git a/README.md b/README.md index e14e303..26b3479 100644 --- a/README.md +++ b/README.md @@ -210,10 +210,6 @@ network setup. Tested end to end on OrbStack Kubernetes against QA. See - **No task CPU/memory limits.** The agent can't create task cgroups from its container; the pod's resources are the only cap. - **No proxy support.** `--http-proxy` isn't wired up. -- **Draining needs an SG API key.** Pods set themselves `DRAINING` through - SG's `runner_status` API on shutdown, which needs `SG_API_KEY`; without it, - a busy runner can keep receiving work and hold its pod for the full grace - period. - **Parked runners stay registered.** Scaling down parks runners (stopped, registration kept, woken in seconds); nothing deregisters them automatically yet. Long parking periods are untested. diff --git a/docs/TODO.md b/docs/TODO.md index 1c634fb..8a93d65 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -34,19 +34,6 @@ runner pod covers workflow containers and the agents alike: - Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...). Check what OrbStack's CNI does before testing there. -### Drain through SG: confirm the credentials - -The drain hook sets the runner `DRAINING`/`ACTIVE` through SG's -`POST /orgs/{org}/runnergroups/{group}/runner_status/` (as -`sg-runner-autoscaler` does), authenticated with `SG_API_KEY`. The runner -never calls ECS itself: on QA the runner's instance role was denied -`ecs:UpdateContainerInstancesState` anyway, and a customer's runner has no ECS -permissions of its own. - -- Verify end to end on QA (DRAINING on shutdown, ACTIVE after restart). -- Check whether the node token is accepted too; if so, drop the extra key. -- Otherwise, document the least-privileged API key role that can call it. - ### Drop privileged mode Spike results on OrbStack (k8s 1.36, cri-dockerd, arm64): diff --git a/k8s/.env.example b/k8s/.env.example index e709c6d..2c843fc 100644 --- a/k8s/.env.example +++ b/k8s/.env.example @@ -3,6 +3,3 @@ SG_NODE_TOKEN= SG_ORGANIZATION= SG_RUNNER_GROUP= SG_BASE_API=https://api.app.stackguardian.io/api/v1 -# Optional: SG API key allowed to manage this runner group. Lets the pod set -# itself DRAINING/ACTIVE through SG (runner_status API) on shutdown/restart. -SG_API_KEY= diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 8a7733a..a856a47 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -37,5 +37,6 @@ COPY --from=ecs-agent /images /images COPY main.sh /opt/sg-runner/main.sh COPY k8s/runner.sh /opt/sg-runner/runner.sh +COPY k8s/scaler.sh /opt/sg-runner/scaler.sh ENTRYPOINT ["/opt/sg-runner/runner.sh"] diff --git a/k8s/README.md b/k8s/README.md index a52fcce..c3c64bc 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -86,15 +86,37 @@ it still counts against the runner group's `MaxNumberOfRunners`. Pods are managed in parallel (`podManagementPolicy: Parallel`), so several runners park or wake at once. -Setting the status needs `SG_API_KEY` in `.env`: an SG API key allowed to -manage the runner group. The runner never calls ECS itself; a customer's -runner has no ECS permissions. Without the key it skips the status change and -only waits for running workflows. +Status changes use the runner group's node token, the same credential the +runner registers with; no extra key is needed. The runner never calls ECS +itself: a customer's runner has no ECS permissions. Deregistering is explicit: `make deregister` wakes every parked runner (each state PVC) and deregisters them all; `make nuke` does that and deletes everything. Reaping runners parked for too long is up to the scaler (planned). +## Autoscaling + +`manifests/scaler.yaml` runs `scaler.sh`, a small loop that owns the runner +StatefulSet's `replicas`. Every `SCALER_INTERVAL` (15s) it reads the runner +group from SG with the node token (`?getActiveWorkflows=true`) and: + +| Rule | When | Action | +|---|---|---| +| Scale out | workflows queued and no awake runner fully idle | +1 runner: wakes a parked one, or registers a new one | +| Headroom (optional) | less than `SCALER_HEADROOM_PCT` of awake runners' CPU or memory free (0 = off, the default) | +1 runner, kept as a warm spare | +| Scale in | nothing queued, highest awake runner idle for `SCALER_IDLE_SECONDS` (300), the rest still keeps the headroom | park it (-1) | +| Stuck queue | workflows queued while an awake runner sits fully idle for `SCALER_STUCK_SECONDS` (180) | warn, ignore the queue (it needs more than a runner pod offers) | +| Reap | runner parked longer than `SCALER_PARK_TTL` (24h) | Job deregisters it from its PVC, then its PVCs are deleted | + +Replicas stay within `SCALER_MIN` (≥ 1: SG pauses workflows without an +`ACTIVE` runner) and the lower of `SCALER_MAX` and the runner group's +`MaxNumberOfRunners`. Runners map to ordinals by hostname (the pod's FQDN). +While an awake runner hasn't connected yet, the scaler waits instead of +deciding again, for up to `SCALER_SETTLE_TIMEOUT` (300s); a runner that still +isn't connected by then (failed registration, crash loop) is left out of the +decision and reported every loop, so one broken pod can't freeze scaling. Its Role covers only this StatefulSet's scale, PVCs and reap +Jobs; runner pods have no Kubernetes API access. + ## Known gaps - **Privileged pod.** Needed for nested Docker; restricted PSA clusters reject it. @@ -112,11 +134,15 @@ everything. Reaping runners parked for too long is up to the scaler (planned). - **Task cgroup limits off** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`): the agent can't create task cgroups from its container. Size the pod's resources instead. - **No proxy support** yet (`--http-proxy` path not wired). -- **Draining needs an SG API key.** Without `SG_API_KEY`, shutdowns wait for - running workflows but new ones may still be placed, so a busy runner can - hold its pod for the full grace period. The key also lives in the pod. -- **No automatic deregistration yet.** Parked runners stay registered until - `make deregister`; a TTL reaper belongs in the scaler. +- **Workflows bigger than a runner pod never run.** A workflow reserves + 0.5 vCPU / 512 MiB plus its own resource setting; if that exceeds the + `dockerd` container's limits it stays queued. The scaler detects it and warns + instead of waking every runner, but the pod size must cover the largest + workflow. +- **Restarting a single runner pauses new workflows** for the ~10s it takes: + SG pauses workflows submitted while no runner is `ACTIVE`, and a StatefulSet + stops the old pod before starting the new one. Use `SCALER_MIN=2` where that + matters. - **Unverified: long parking.** Waking was tested after seconds. Whether SSM and ECS reconnect after hours or days parked isn't tested yet. - **Unverified: hard kills.** A pod killed without its `preStop` hook (node diff --git a/k8s/manifests/runner.yaml b/k8s/manifests/runner.yaml index d5259e6..b65952e 100644 --- a/k8s/manifests/runner.yaml +++ b/k8s/manifests/runner.yaml @@ -7,7 +7,8 @@ metadata: namespace: sg-runner spec: serviceName: sg-runner - replicas: 1 + # No replicas here: the scaler (scaler.yaml) owns them, and re-applying this + # manifest must not reset them. A new StatefulSet starts with 1. # Park and wake runners side by side, not one at a time (each park can wait # for running workflows). podManagementPolicy: Parallel @@ -28,7 +29,7 @@ spec: image: sg-runner-k8s:dev imagePullPolicy: IfNotPresent args: ['register'] - # Only what registration needs; SG_API_KEY stays out of it. + # Only what registration needs. env: - name: SG_NODE_TOKEN valueFrom: diff --git a/k8s/manifests/scaler.yaml b/k8s/manifests/scaler.yaml new file mode 100644 index 0000000..0205f79 --- /dev/null +++ b/k8s/manifests/scaler.yaml @@ -0,0 +1,85 @@ +# Scaler for the runner StatefulSet (see scaler.sh). It owns spec.replicas. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sg-runner-scaler + namespace: sg-runner +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sg-runner-scaler + namespace: sg-runner +rules: + - apiGroups: ["apps"] + resources: ["statefulsets"] + resourceNames: ["sg-runner"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["statefulsets/scale"] + resourceNames: ["sg-runner"] + verbs: ["get", "patch"] + # Parked-at stamps, and deleting a reaped runner's PVCs. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "get", "patch", "delete"] + # Reap Jobs deregister long-parked runners from their PVC. + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sg-runner-scaler + namespace: sg-runner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sg-runner-scaler +subjects: + - kind: ServiceAccount + name: sg-runner-scaler + namespace: sg-runner +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sg-runner-scaler + namespace: sg-runner +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: sg-runner-scaler + template: + metadata: + labels: + app: sg-runner-scaler + spec: + serviceAccountName: sg-runner-scaler + containers: + - name: scaler + image: sg-runner-k8s:dev + imagePullPolicy: IfNotPresent + command: ["/opt/sg-runner/scaler.sh"] + env: + - { name: SCALER_MIN, value: "1" } + - { name: SCALER_MAX, value: "10" } + # % of awake runners' capacity to keep free (warm spare); 0 = off. + - { name: SCALER_HEADROOM_PCT, value: "0" } + - { name: SCALER_IDLE_SECONDS, value: "300" } + - { name: SCALER_PARK_TTL, value: "86400" } + - { name: SCALER_IMAGE, value: sg-runner-k8s:dev } + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { memory: 128Mi } + volumeMounts: + - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true } + volumes: + - name: sg-credentials + secret: + secretName: sg-runner-credentials + defaultMode: 0400 diff --git a/k8s/runner.sh b/k8s/runner.sh index 6fdf3f3..64f8dfe 100755 --- a/k8s/runner.sh +++ b/k8s/runner.sh @@ -13,6 +13,7 @@ # drain ecs-agent preStop: DRAINING, wait for workflows, park (keep registration) # wait-drained dockerd/ssm-agent preStop: outlive the drain # deregister run via `kubectl exec -i`; reads SG credentials from stdin +# reap scaler Job: deregister a runner parked too long, from its PVC # # main.sh is sourced for the API, config and network helpers; its # BASH_SOURCE guard keeps main() from running. @@ -25,7 +26,7 @@ source /opt/sg-runner/main.sh readonly SSM_DIR="/var/lib/amazon/ssm" readonly SSM_REGISTRATION_FILE="${SSM_DIR}/Vault/Store/RegistrationKey" readonly AGENT_CREDS_FILE="/rotatingcreds/credentials" -readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API SG_API_KEY" +readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API" readonly SG_CREDS_DIR="/etc/sg-runner/credentials" # On the emptyDir shared by every container: tells the others the drain is over. readonly DRAINED_MARKER="/var/run/sg-runner-drained" @@ -265,16 +266,22 @@ cmd_deregister() { #{{{ } #}}}: cmd_deregister +cmd_reap() { #{{{ + # One-off Job the scaler starts for a runner parked longer than its TTL: the + # pod is gone, so deregister using the identity on the mounted state PVC. + load_sg_credentials_from_dir + deregister_runner +} +#}}}: cmd_reap + sg_api() { #{{{ # Usage: sg_api [payload] - # Calls the runner group's SG API with SG_API_KEY and prints the response. - # api_call authenticates with SG_NODE_TOKEN: swap in the key, in a subshell - # so the node token stays untouched for deregistration. + # Calls the runner group's SG API with the node token and prints the + # response. Subshell: api_call exits on an empty response. local method="$1" path="$2" payload="${3:-}" ( - SG_NODE_TOKEN="$SG_API_KEY" if ! api_call "$method" \ "${SG_BASE_API}/orgs/${SG_ORGANIZATION}/runnergroups/${SG_RUNNER_GROUP}/${path}" \ "$payload"; then @@ -301,13 +308,7 @@ set_sg_runner_status() { #{{{ set_runner_status() { #{{{ # Usage: set_runner_status DRAINING|ACTIVE # Only through SG: a customer's runner has no ECS permissions of its own. - # Without SG_API_KEY, shutdowns still wait for running workflows but can't - # stop new ones from being placed meanwhile. load_sg_credentials_from_dir - if [[ -z "${SG_API_KEY:-}" ]]; then - info "SG_API_KEY not set:" "skipping $1" - return 1 - fi set_sg_runner_status "$1" } #}}}: set_runner_status @@ -317,7 +318,6 @@ current_runner_status() { #{{{ # The response holds secrets; only the status leaves this function. local runner_id load_sg_credentials_from_dir - [[ -n "${SG_API_KEY:-}" ]] || return 1 runner_id="$(configured_runner_id)" sg_api GET "" | jq -r --arg r "$runner_id" \ '.msg.ContainerInstances[]? | select(.runnerId == $r) | .status' @@ -399,8 +399,9 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then drain) cmd_drain ;; wait-drained) cmd_wait_drained ;; deregister) cmd_deregister ;; + reap) cmd_reap ;; *) - echo "Usage: $(basename "$0") register|dockerd|ssm-agent|ecs-agent|drain|wait-drained|deregister" >&2 + echo "Usage: $(basename "$0") register|dockerd|ssm-agent|ecs-agent|drain|wait-drained|deregister|reap" >&2 exit 2 ;; esac diff --git a/k8s/scaler.sh b/k8s/scaler.sh new file mode 100755 index 0000000..589e323 --- /dev/null +++ b/k8s/scaler.sh @@ -0,0 +1,334 @@ +#!/bin/bash +# +# sg-runner scaler (prototype): sizes the runner StatefulSet from the runner +# group's state in SG, and deregisters runners parked for too long. +# +# Every SCALER_INTERVAL seconds: +# out workflows queued and no awake runner fully idle → one more runner +# (wakes a parked one if its PVC is there, registers a new one +# otherwise). With a runner fully idle, SG's next retry places queued +# work there if it fits; another runner of the same size can't help. +# Optional headroom: with SCALER_HEADROOM_PCT > 0, also scale out when +# less than that share of the awake runners' CPU or memory is free, +# so the next workflow finds a warm runner instead of queueing. +# in nothing queued and the highest awake runner idle for +# SCALER_IDLE_SECONDS, and the rest keeps the headroom → park it (StatefulSets always remove the +# highest ordinal, so that's the only one worth checking) +# reap runner parked longer than SCALER_PARK_TTL → deregister it from its +# PVC (a Job running `runner.sh reap`), then delete its PVCs +# stuck workflows queued while an awake runner sat fully idle for +# SCALER_STUCK_SECONDS → what's queued fits no runner (bigger than a +# runner pod), so more runners of the same size won't help: ignore +# the queue (no scale out for it, park as if empty) and warn +# +# Replicas stay within [SCALER_MIN, max], where max is the lower of SCALER_MAX +# and the runner group's MaxNumberOfRunners. SCALER_MIN >= 1: SG pauses +# workflows when no runner is ACTIVE and connected. +# +# All SG calls use the runner group's node token. + +set -o pipefail + +# shellcheck source-path=SCRIPTDIR source=runner.sh +source /opt/sg-runner/runner.sh + +SCALER_INTERVAL="${SCALER_INTERVAL:-15}" +SCALER_MIN="${SCALER_MIN:-1}" +SCALER_MAX="${SCALER_MAX:-10}" +# Share of awake capacity to keep free; 0 = off (scale out on the queue only). +SCALER_HEADROOM_PCT="${SCALER_HEADROOM_PCT:-0}" +SCALER_IDLE_SECONDS="${SCALER_IDLE_SECONDS:-300}" +SCALER_PARK_TTL="${SCALER_PARK_TTL:-86400}" +# Longer than SG's retry delay for queued workflows. +SCALER_STUCK_SECONDS="${SCALER_STUCK_SECONDS:-180}" +# How long to wait for an awake runner to connect before deciding without it. +SCALER_SETTLE_TIMEOUT="${SCALER_SETTLE_TIMEOUT:-300}" +SCALER_STATEFULSET="${SCALER_STATEFULSET:-sg-runner}" +SCALER_IMAGE="${SCALER_IMAGE:-sg-runner-k8s:dev}" + +readonly SA_DIR="/var/run/secrets/kubernetes.io/serviceaccount" +readonly PARKED_AT_ANNOTATION="sg-runner.stackguardian.io/parked-at" + +# Ordinal of the highest awake runner -> epoch it was first seen idle. +declare -A IDLE_SINCE=() +# Since when workflows have been queued while an awake runner was fully idle. +STUCK_SINCE="" +# Ordinal of an awake runner not connected to SG -> epoch first seen so. +declare -A UNCONNECTED_SINCE=() + +log() { + printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" +} + +kube() { #{{{ + # Usage: kube [body] + # Kubernetes API call with the scaler's service account. PATCH bodies are + # merge patches, POST bodies plain JSON. + local method="$1" path="$2" body="${3:-}" content_type="application/json" + [[ "$method" == PATCH ]] && content_type="application/merge-patch+json" + local args=(-fsS --max-time 10 -X "$method" + --cacert "$SA_DIR/ca.crt" + -H "Authorization: Bearer $(<"$SA_DIR/token")") + [[ -n "$body" ]] && args+=(-H "Content-Type: $content_type" -d "$body") + curl "${args[@]}" "https://kubernetes.default.svc/${path#/}" +} +#}}}: kube + +sg_runner_group() { #{{{ + # Prints the runner group as SG reports it, with active workflow counts. + # It holds secrets (the group's runner token): only extract fields from it. + # Subshell: api_call exits on an empty response, the loop must not. + ( + api_call GET \ + "${SG_BASE_API}/orgs/${SG_ORGANIZATION}/runnergroups/${SG_RUNNER_GROUP}/?getActiveWorkflows=true" \ + >/dev/null 2>&1 && + printf '%s' "$response" + ) +} +#}}}: sg_runner_group + +summarize_group() { #{{{ + # stdin: runner group JSON. Prints what scaling needs, with each SG runner + # mapped to its StatefulSet ordinal through its hostname (the pod's FQDN). + jq -c --arg sts "$SCALER_STATEFULSET" ' + def res($list; $name): [($list // [])[] | select(.name == $name) | .integerValue][0] // 0; + .msg | { + queued: (.QueuedWorkflowsCount // 0), + max_runners: .MaxNumberOfRunners, + runners: [(.ContainerInstances // [])[] | { + ordinal: (try (.instanceDetails[0].ComputerName + | capture("^" + $sts + "-(?[0-9]+)\\.").n | tonumber) catch null), + status, + connected: (.agentConnected == true), + busy: ((.runningTasksCount // 0) + (.pendingTasksCount // 0)), + cpu: res(.registeredResources; "CPU"), + cpu_free: res(.remainingResources; "CPU"), + mem: res(.registeredResources; "MEMORY"), + mem_free: res(.remainingResources; "MEMORY") + } | select(.ordinal != null)] + }' +} +#}}}: summarize_group + +reserved_pct() { #{{{ + # Usage: reserved_pct + # Percent of CPU or memory (whichever is higher) reserved on runners with an + # ordinal below the given one that are ACTIVE and connected. 100 if none. + jq -r --argjson below "$2" ' + [.runners[] | select(.ordinal < $below and .status == "ACTIVE" and .connected)] as $r + | if ($r | length) == 0 then 100 else + ([$r[].cpu] | add) as $cpu | ([$r[].mem] | add) as $mem + | [(($cpu - ([$r[].cpu_free] | add)) * 100 / ([$cpu, 1] | max)), + (($mem - ([$r[].mem_free] | add)) * 100 / ([$mem, 1] | max))] + | max | floor + end' <<<"$1" +} +#}}}: reserved_pct + +set_replicas() { #{{{ + kube PATCH "apis/apps/v1/namespaces/${NS}/statefulsets/${SCALER_STATEFULSET}/scale" \ + "{\"spec\":{\"replicas\":$1}}" >/dev/null +} +#}}}: set_replicas + +annotate_pvc() { #{{{ + # Usage: annotate_pvc + local value="null" + [[ "$2" != null ]] && value="\"$2\"" + kube PATCH "api/v1/namespaces/${NS}/persistentvolumeclaims/$1" \ + "{\"metadata\":{\"annotations\":{\"${PARKED_AT_ANNOTATION}\":${value}}}}" >/dev/null +} +#}}}: annotate_pvc + +reap_job() { #{{{ + # Usage: reap_job ; prints the Job manifest. + local n="$1" + jq -n --arg n "$n" --arg sts "$SCALER_STATEFULSET" --arg image "$SCALER_IMAGE" '{ + apiVersion: "batch/v1", kind: "Job", + metadata: {name: "\($sts)-reap-\($n)", labels: {app: "\($sts)-reaper"}}, + spec: {backoffLimit: 0, template: {spec: { + restartPolicy: "Never", + automountServiceAccountToken: false, + containers: [{ + name: "reap", image: $image, imagePullPolicy: "IfNotPresent", args: ["reap"], + volumeMounts: [ + {name: "state", mountPath: "/etc/ecs", subPath: "ecs-config"}, + {name: "state", mountPath: "/var/lib/ecs/data", subPath: "ecs-data"}, + {name: "state", mountPath: "/var/lib/amazon/ssm", subPath: "ssm"}, + {name: "sg-credentials", mountPath: "/etc/sg-runner/credentials", readOnly: true} + ] + }], + volumes: [ + {name: "state", persistentVolumeClaim: {claimName: "state-\($sts)-\($n)"}}, + {name: "sg-credentials", secret: {secretName: "sg-runner-credentials", defaultMode: 256}} + ] + }}} + }' +} +#}}}: reap_job + +reap_parked() { #{{{ + # Usage: reap_parked + # Stamps newly parked runners and deregisters those parked past the TTL. + local replicas="$1" now pvcs pvc n parked_at job job_state + now="$(date +%s)" + pvcs="$(kube GET "api/v1/namespaces/${NS}/persistentvolumeclaims")" || return + + while IFS=$'\t' read -r pvc parked_at; do + n="${pvc##*-}" + [[ "$n" =~ ^[0-9]+$ ]] || continue + + if ((n < replicas)); then + # Awake again: clear a stale stamp. + [[ -n "$parked_at" ]] && annotate_pvc "$pvc" null + continue + fi + if [[ -z "$parked_at" ]]; then + annotate_pvc "$pvc" "$now" + log "runner $n parked" + continue + fi + ((now - parked_at >= SCALER_PARK_TTL)) || continue + + job="${SCALER_STATEFULSET}-reap-${n}" + job_state="$(kube GET "apis/batch/v1/namespaces/${NS}/jobs/${job}" 2>/dev/null | + jq -r 'if .status.succeeded then "done" elif .status.failed then "failed" else "running" end')" + case "$job_state" in + "") + log "runner $n parked for $((now - parked_at))s: deregistering" + kube POST "apis/batch/v1/namespaces/${NS}/jobs" "$(reap_job "$n")" >/dev/null + ;; + done) + log "runner $n deregistered: deleting its PVCs" + kube DELETE "api/v1/namespaces/${NS}/persistentvolumeclaims/${pvc}" >/dev/null + kube DELETE "api/v1/namespaces/${NS}/persistentvolumeclaims/docker-${SCALER_STATEFULSET}-${n}" >/dev/null + kube DELETE "apis/batch/v1/namespaces/${NS}/jobs/${job}?propagationPolicy=Background" >/dev/null + ;; + failed) + # Keep the identity; retry on a later loop. + log "runner $n: deregistration failed, retrying (see job/${job})" + kube DELETE "apis/batch/v1/namespaces/${NS}/jobs/${job}?propagationPolicy=Background" >/dev/null + ;; + esac + done < <(jq -r --arg prefix "state-${SCALER_STATEFULSET}-" --arg a "$PARKED_AT_ANNOTATION" ' + .items[] | select(.metadata.name | startswith($prefix)) + | [.metadata.name, (.metadata.annotations[$a] // "")] | @tsv' <<<"$pvcs") +} +#}}}: reap_parked + +reaping() { #{{{ + # Usage: reaping ; true while a reap Job holds that runner's PVC. + kube GET "apis/batch/v1/namespaces/${NS}/jobs/${SCALER_STATEFULSET}-reap-$1" >/dev/null 2>&1 +} +#}}}: reaping + +scale() { #{{{ + local group summary replicas queued max top top_runner pct free free_without_top now settling + group="$(sg_runner_group)" || { + log "can't read the runner group from SG" + return + } + summary="$(summarize_group <<<"$group")" || return + replicas="$(kube GET "apis/apps/v1/namespaces/${NS}/statefulsets/${SCALER_STATEFULSET}" | + jq -r '.spec.replicas')" || return + + queued="$(jq -r '.queued' <<<"$summary")" + max="$(jq -r --argjson max "$SCALER_MAX" '[.max_runners // $max, $max] | min' <<<"$summary")" + pct="$(reserved_pct "$summary" "$replicas")" + free=$((100 - pct)) + # A runner that is awake but not yet connected: let it settle before + # deciding again, or one queued workflow wakes several runners. One that + # never connects (failed registration, crash loop) mustn't freeze scaling: + # past SCALER_SETTLE_TIMEOUT it's left out and reported. + local unconnected n + unconnected="$(jq -r --argjson r "$replicas" ' + [range(0; $r)] - [.runners[] | select(.connected) | .ordinal] | .[]' <<<"$summary")" + now="$(date +%s)" + for n in "${!UNCONNECTED_SINCE[@]}"; do + grep -qx "$n" <<<"$unconnected" || unset "UNCONNECTED_SINCE[$n]" + done + settling=0 + for n in $unconnected; do + : "${UNCONNECTED_SINCE[$n]:=$now}" + if ((now - UNCONNECTED_SINCE[$n] < SCALER_SETTLE_TIMEOUT)); then + settling=$((settling + 1)) + else + log "WARNING: runner $n awake but not connected to SG for $((now - UNCONNECTED_SINCE[$n]))s;" \ + "scaling without it (check pod/${SCALER_STATEFULSET}-$n)" + fi + done + + # Queued while a runner sits fully idle: once that lasts past SG's retry + # delay, the queued workflows don't fit on a runner of this size. + local stuck=false idle_runner + idle_runner="$(jq -r --argjson r "$replicas" '[.runners[] | select(.ordinal < $r + and .status == "ACTIVE" and .connected and .busy == 0 and .cpu_free == .cpu + and .mem_free == .mem)] | first | if . then "\(.cpu)/\(.mem)" else "" end' <<<"$summary")" + if ((queued > 0)) && [[ -n "$idle_runner" ]]; then + : "${STUCK_SINCE:=$now}" + if ((now - STUCK_SINCE >= SCALER_STUCK_SECONDS)); then + stuck=true + log "WARNING: $queued workflow(s) queued for $((now - STUCK_SINCE))s while a runner" \ + "(CPU/MiB $idle_runner) sits idle: they need more than one runner pod offers." \ + "Ignoring them; raise the dockerd container's limits to run them." + fi + else + STUCK_SINCE="" + fi + + log "replicas=$replicas max=$max queued=$queued reserved=${pct}% free=${free}% settling=$settling stuck=$stuck" + + if ((settling > 0)); then + : + elif ((replicas < max)) && { { ((queued > 0)) && [[ -z "$idle_runner" ]]; } || + ((free < SCALER_HEADROOM_PCT)); }; then + if reaping "$replicas"; then + log "scale out waits: runner $replicas is being deregistered" + else + log "scale out: $replicas -> $((replicas + 1))" + set_replicas $((replicas + 1)) + IDLE_SINCE=() + # The new runner starts idle: don't mistake it for a stuck queue. + STUCK_SINCE="" + fi + elif ((replicas > SCALER_MIN)) && { ((queued == 0)) || [[ "$stuck" == true ]]; }; then + top=$((replicas - 1)) + top_runner="$(jq -c --argjson n "$top" '.runners[] | select(.ordinal == $n)' <<<"$summary")" + if [[ "$(jq -r '.busy' <<<"$top_runner")" != 0 ]]; then + unset "IDLE_SINCE[$top]" + else + : "${IDLE_SINCE[$top]:=$now}" + # Don't park into a scale out: the rest must keep the headroom. + free_without_top=$((100 - $(reserved_pct "$summary" "$top"))) + if ((now - IDLE_SINCE[$top] >= SCALER_IDLE_SECONDS && free_without_top >= SCALER_HEADROOM_PCT)); then + log "scale in: runner $top idle for $((now - IDLE_SINCE[$top]))s, parking ($replicas -> $top)" + set_replicas "$top" + unset "IDLE_SINCE[$top]" + fi + fi + fi + + reap_parked "$(kube GET "apis/apps/v1/namespaces/${NS}/statefulsets/${SCALER_STATEFULSET}" | + jq -r '.spec.replicas')" +} +#}}}: scale + +main() { + if ((SCALER_MIN < 1)); then + log "SCALER_MIN=$SCALER_MIN raised to 1: SG pauses workflows without an ACTIVE runner" + SCALER_MIN=1 + fi + NS="$(<"$SA_DIR/namespace")" + load_sg_credentials_from_dir + log "scaling statefulset/${SCALER_STATEFULSET}: min=$SCALER_MIN max=$SCALER_MAX" \ + "headroom=${SCALER_HEADROOM_PCT}% idle=${SCALER_IDLE_SECONDS}s park-ttl=${SCALER_PARK_TTL}s" \ + "stuck-after=${SCALER_STUCK_SECONDS}s settle-timeout=${SCALER_SETTLE_TIMEOUT}s" + while :; do + scale + sleep "$SCALER_INTERVAL" + done +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi From 753aaabe9b242a9bbd61df26cf077d3cda2b19e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:30:24 +0200 Subject: [PATCH 05/43] feat: park idle runners after 1m, deregister after 12h --- k8s/README.md | 4 ++-- k8s/manifests/scaler.yaml | 5 +++-- k8s/scaler.sh | 11 +++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index c3c64bc..0190540 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -104,9 +104,9 @@ group from SG with the node token (`?getActiveWorkflows=true`) and: |---|---|---| | Scale out | workflows queued and no awake runner fully idle | +1 runner: wakes a parked one, or registers a new one | | Headroom (optional) | less than `SCALER_HEADROOM_PCT` of awake runners' CPU or memory free (0 = off, the default) | +1 runner, kept as a warm spare | -| Scale in | nothing queued, highest awake runner idle for `SCALER_IDLE_SECONDS` (300), the rest still keeps the headroom | park it (-1) | +| Scale in | nothing queued, highest awake runner idle for `SCALER_IDLE_SECONDS` (60), the rest still keeps the headroom | park it (-1) | | Stuck queue | workflows queued while an awake runner sits fully idle for `SCALER_STUCK_SECONDS` (180) | warn, ignore the queue (it needs more than a runner pod offers) | -| Reap | runner parked longer than `SCALER_PARK_TTL` (24h) | Job deregisters it from its PVC, then its PVCs are deleted | +| Reap | runner parked longer than `SCALER_PARK_TTL` (12h) | Job deregisters it from its PVC, then its PVCs are deleted | Replicas stay within `SCALER_MIN` (≥ 1: SG pauses workflows without an `ACTIVE` runner) and the lower of `SCALER_MAX` and the runner group's diff --git a/k8s/manifests/scaler.yaml b/k8s/manifests/scaler.yaml index 0205f79..91deea2 100644 --- a/k8s/manifests/scaler.yaml +++ b/k8s/manifests/scaler.yaml @@ -70,9 +70,10 @@ spec: - { name: SCALER_MAX, value: "10" } # % of awake runners' capacity to keep free (warm spare); 0 = off. - { name: SCALER_HEADROOM_PCT, value: "0" } - - { name: SCALER_IDLE_SECONDS, value: "300" } - - { name: SCALER_PARK_TTL, value: "86400" } + - { name: SCALER_IDLE_SECONDS, value: "60" } + - { name: SCALER_PARK_TTL, value: "43200" } - { name: SCALER_IMAGE, value: sg-runner-k8s:dev } + - { name: SCALER_CREDENTIALS_SECRET, value: sg-runner-credentials } resources: requests: { cpu: 10m, memory: 32Mi } limits: { memory: 128Mi } diff --git a/k8s/scaler.sh b/k8s/scaler.sh index 589e323..a6a6d54 100755 --- a/k8s/scaler.sh +++ b/k8s/scaler.sh @@ -37,14 +37,16 @@ SCALER_MIN="${SCALER_MIN:-1}" SCALER_MAX="${SCALER_MAX:-10}" # Share of awake capacity to keep free; 0 = off (scale out on the queue only). SCALER_HEADROOM_PCT="${SCALER_HEADROOM_PCT:-0}" -SCALER_IDLE_SECONDS="${SCALER_IDLE_SECONDS:-300}" -SCALER_PARK_TTL="${SCALER_PARK_TTL:-86400}" +SCALER_IDLE_SECONDS="${SCALER_IDLE_SECONDS:-60}" +SCALER_PARK_TTL="${SCALER_PARK_TTL:-43200}" # Longer than SG's retry delay for queued workflows. SCALER_STUCK_SECONDS="${SCALER_STUCK_SECONDS:-180}" # How long to wait for an awake runner to connect before deciding without it. SCALER_SETTLE_TIMEOUT="${SCALER_SETTLE_TIMEOUT:-300}" SCALER_STATEFULSET="${SCALER_STATEFULSET:-sg-runner}" SCALER_IMAGE="${SCALER_IMAGE:-sg-runner-k8s:dev}" +# The runner group's SG credentials, mounted by reap Jobs. +SCALER_CREDENTIALS_SECRET="${SCALER_CREDENTIALS_SECRET:-sg-runner-credentials}" readonly SA_DIR="/var/run/secrets/kubernetes.io/serviceaccount" readonly PARKED_AT_ANNOTATION="sg-runner.stackguardian.io/parked-at" @@ -143,7 +145,8 @@ annotate_pvc() { #{{{ reap_job() { #{{{ # Usage: reap_job ; prints the Job manifest. local n="$1" - jq -n --arg n "$n" --arg sts "$SCALER_STATEFULSET" --arg image "$SCALER_IMAGE" '{ + jq -n --arg n "$n" --arg sts "$SCALER_STATEFULSET" --arg image "$SCALER_IMAGE" \ + --arg secret "$SCALER_CREDENTIALS_SECRET" '{ apiVersion: "batch/v1", kind: "Job", metadata: {name: "\($sts)-reap-\($n)", labels: {app: "\($sts)-reaper"}}, spec: {backoffLimit: 0, template: {spec: { @@ -160,7 +163,7 @@ reap_job() { #{{{ }], volumes: [ {name: "state", persistentVolumeClaim: {claimName: "state-\($sts)-\($n)"}}, - {name: "sg-credentials", secret: {secretName: "sg-runner-credentials", defaultMode: 256}} + {name: "sg-credentials", secret: {secretName: $secret, defaultMode: 256}} ] }}} }' From 2f28ea6714fd97f8c4797b2be2f8fc7d42271a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:34:15 +0200 Subject: [PATCH 06/43] fix: detect the private ip from the outbound route --- main.sh | 25 +++++++++++- test/README.md | 3 +- test/mocks/bin/hostname | 8 ++++ test/mocks/bin/ip | 26 +++++++++---- test/unit/detect_private_ip.bats | 65 ++++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 10 deletions(-) create mode 100755 test/mocks/bin/hostname create mode 100644 test/unit/detect_private_ip.bats diff --git a/main.sh b/main.sh index ac8fadf..5a1e1e6 100755 --- a/main.sh +++ b/main.sh @@ -239,7 +239,7 @@ print_details() { #{{{ echo details_frame "Host Information" details_item "Hostname" "$HOSTNAME" - details_item "Private IP Address" "$(ip route | grep default | cut -d" " -f9)" + details_item "Private IP Address" "$(detect_private_ip)" details_item "Public IP Address" "$(curl -fSs ifconfig.me)" details_item "HTTP PROXY" "$HTTP_PROXY" echo @@ -334,7 +334,7 @@ update_runner_group() { #{{{ err_msg=$(printf '%s' "$1" | tr -cd "[:print:]") debug "Error message ${err_msg}" - ip_addr=$(ip route | grep default | cut -d" " -f9) + ip_addr=$(detect_private_ip) payload=$(jq -n \ --arg ip "$ip_addr" \ @@ -594,6 +594,27 @@ clean_local_setup() { #{{{ } #}}}: clean_local_setup +detect_private_ip() { #{{{ + local ip_addr + # Ask the kernel which source address it would use for outbound traffic. + # This is a local route lookup (no packet is sent) and works whether the + # default route is dhcp, static or a pod's (no "src" on the default route). + # Pick the token after "src" by name: its position varies between setups. + ip_addr=$(ip -4 route get 1.1.1.1 2>/dev/null | + awk '{for (i = 1; i < NF; i++) if ($i == "src") {print $(i + 1); exit}}') + # No usable route (e.g. no default gateway): take the first global IPv4. + if [[ -z "$ip_addr" ]]; then + ip_addr=$(ip -4 -o addr show scope global 2>/dev/null | + awk '{split($4, addr, "/"); print addr[1]; exit}') + fi + # Last resort for hosts where the ip queries fail; not every hostname has -I. + if [[ -z "$ip_addr" ]] && command -v hostname >/dev/null 2>&1; then + ip_addr=$(hostname -I 2>/dev/null | awk '{print $1}') + fi + printf '%s' "$ip_addr" +} +#}}}: detect_private_ip + check_variable_value() { #{{{ local variable_name=$1 [[ -z "${!variable_name}" ]] && diff --git a/test/README.md b/test/README.md index f27dca9..036c846 100644 --- a/test/README.md +++ b/test/README.md @@ -140,7 +140,8 @@ Set the override env vars **before** calling the function under test. Use | `curl` | `MOCK_CURL_OUTPUT` (empty) | `MOCK_CURL_EXIT` (`0`) | | `crontab` | `MOCK_CRONTAB_LIST` (empty; used by `crontab -l`) | `MOCK_CRONTAB_EXIT` (`0`) | | `iptables` | (no stdout); `MOCK_IPTABLES_CHECK_EXIT` (`1`) is the exit for `-C` rule checks | `MOCK_IPTABLES_EXIT` (`0`) | -| `ip` | `MOCK_IP_ROUTE_OUTPUT` (default route, field 9 = `10.0.0.1`) | `MOCK_IP_EXIT` (`0`) | +| `ip` | `MOCK_IP_ROUTE_GET_OUTPUT` (`route get` reply, src `10.0.0.1`), `MOCK_IP_ADDR_OUTPUT` (one global `10.0.0.1/24`), `MOCK_IP_ROUTE_OUTPUT` (dhcp default route) | `MOCK_IP_EXIT` (`0`); per query `MOCK_IP_ROUTE_GET_EXIT`, `MOCK_IP_ADDR_EXIT` | +| `hostname` | `MOCK_HOSTNAME_OUTPUT` (`10.0.0.1`; used by `hostname -I`) | `MOCK_HOSTNAME_EXIT` (`0`) | | `free` | `MOCK_FREE_OUTPUT` (a minimal `free -h` table) | `MOCK_FREE_EXIT` (`0`) | | `nproc` | `MOCK_NPROC_OUTPUT` (`4`) | `MOCK_NPROC_EXIT` (`0`) | | `uptime` | `MOCK_UPTIME_OUTPUT` (a typical `uptime` line with load average) | `MOCK_UPTIME_EXIT` (`0`) | diff --git a/test/mocks/bin/hostname b/test/mocks/bin/hostname new file mode 100755 index 0000000..5574d0c --- /dev/null +++ b/test/mocks/bin/hostname @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Mock: hostname +# Records the call. detect_private_ip falls back to `hostname -I`. +# $MOCK_HOSTNAME_OUTPUT (default: 10.0.0.1) +# $MOCK_HOSTNAME_EXIT (default: 0) +printf '%s\n' "hostname $*" >>"${MOCK_CALL_LOG}" +echo "${MOCK_HOSTNAME_OUTPUT-10.0.0.1}" +exit "${MOCK_HOSTNAME_EXIT:-0}" diff --git a/test/mocks/bin/ip b/test/mocks/bin/ip index 9a44818..8ec1331 100755 --- a/test/mocks/bin/ip +++ b/test/mocks/bin/ip @@ -1,14 +1,26 @@ #!/usr/bin/env bash # Mock: ip -# Records the call. main.sh extracts the node IP with: -# ip route | grep default | cut -d" " -f9 -# so the default route line below is shaped to put 10.0.0.1 in field 9 -# (default via dev proto

src 10.0.0.1 metric ). -# $MOCK_IP_ROUTE_OUTPUT (default: a default route whose field 9 is 10.0.0.1) -# $MOCK_IP_EXIT (default: 0) +# Records the call. main.sh's detect_private_ip queries, in order: +# ip -4 route get 1.1.1.1 -> $MOCK_IP_ROUTE_GET_OUTPUT +# ip -4 -o addr show scope global -> $MOCK_IP_ADDR_OUTPUT +# Each query prints its output and exits with its own exit override, so a +# test can fail the route lookup and still exercise the addr fallback. +# $MOCK_IP_ROUTE_OUTPUT (default: a dhcp default route with src 10.0.0.1) +# $MOCK_IP_ROUTE_GET_OUTPUT (default: a route get reply with src 10.0.0.1) +# $MOCK_IP_ROUTE_GET_EXIT (default: $MOCK_IP_EXIT) +# $MOCK_IP_ADDR_OUTPUT (default: one global eth0 address, 10.0.0.1/24) +# $MOCK_IP_ADDR_EXIT (default: $MOCK_IP_EXIT) +# $MOCK_IP_EXIT (default: 0) printf '%s\n' "ip $*" >>"${MOCK_CALL_LOG}" -if [[ "$1" == "route" ]]; then +args=" $* " +if [[ "$args" == *" route get "* ]]; then + echo "${MOCK_IP_ROUTE_GET_OUTPUT-1.1.1.1 via 10.0.0.254 dev eth0 src 10.0.0.1 uid 0}" + exit "${MOCK_IP_ROUTE_GET_EXIT:-${MOCK_IP_EXIT:-0}}" +elif [[ "$args" == *" addr "* ]]; then + echo "${MOCK_IP_ADDR_OUTPUT-2: eth0 inet 10.0.0.1/24 brd 10.0.0.255 scope global eth0\ valid_lft forever preferred_lft forever}" + exit "${MOCK_IP_ADDR_EXIT:-${MOCK_IP_EXIT:-0}}" +elif [[ "$args" == *" route "* ]]; then echo "${MOCK_IP_ROUTE_OUTPUT:-default via 10.0.0.254 dev eth0 proto dhcp src 10.0.0.1 metric 100}" fi diff --git a/test/unit/detect_private_ip.bats b/test/unit/detect_private_ip.bats new file mode 100644 index 0000000..6464c80 --- /dev/null +++ b/test/unit/detect_private_ip.bats @@ -0,0 +1,65 @@ +#!/usr/bin/env bats +# +# Unit tests for detect_private_ip - reads the source address of the outbound +# route by name, then falls back to the first global IPv4, then hostname -I. + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" +} + +@test "detect_private_ip reads src from a dhcp-style route" { + export MOCK_IP_ROUTE_GET_OUTPUT="1.1.1.1 via 10.0.0.254 dev eth0 proto dhcp src 10.0.0.1 metric 100 uid 0" + load_main + run detect_private_ip + assert_success + assert_output "10.0.0.1" + assert_called ip "-4 route get 1.1.1.1" +} + +@test "detect_private_ip reads src behind a static default route" { + export MOCK_IP_ROUTE_GET_OUTPUT="1.1.1.1 via 10.0.0.1 dev eth0 src 10.0.0.5 uid 0" + load_main + run detect_private_ip + assert_success + assert_output "10.0.0.5" +} + +@test "detect_private_ip reads src in a Kubernetes pod" { + export MOCK_IP_ROUTE_GET_OUTPUT="1.1.1.1 via 192.168.194.1 dev eth0 src 192.168.194.8 uid 0" + load_main + run detect_private_ip + assert_success + assert_output "192.168.194.8" +} + +@test "detect_private_ip falls back to the first global address when route get fails" { + export MOCK_IP_ROUTE_GET_OUTPUT="" + export MOCK_IP_ROUTE_GET_EXIT=2 + export MOCK_IP_ADDR_OUTPUT="2: eth0 inet 172.16.4.20/16 brd 172.16.255.255 scope global eth0 +3: eth1 inet 172.17.0.1/16 scope global eth1" + load_main + run detect_private_ip + assert_success + assert_output "172.16.4.20" + assert_called ip "-4 -o addr show scope global" + refute_called hostname +} + +@test "detect_private_ip falls back to hostname -I when both ip queries fail" { + export MOCK_IP_EXIT=1 MOCK_IP_ROUTE_GET_OUTPUT="" MOCK_IP_ADDR_OUTPUT="" + export MOCK_HOSTNAME_OUTPUT="10.1.2.3 172.17.0.1" + load_main + run detect_private_ip + assert_success + assert_output "10.1.2.3" + assert_called hostname "-I" +} + +@test "detect_private_ip prints nothing when every source fails" { + export MOCK_IP_EXIT=1 MOCK_IP_ROUTE_GET_OUTPUT="" MOCK_IP_ADDR_OUTPUT="" + export MOCK_HOSTNAME_OUTPUT="" MOCK_HOSTNAME_EXIT=1 + load_main + run detect_private_ip + assert_success + assert_output "" +} From e8c21e1f0702891c924b3c4e5ce13d5186cca0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:40:48 +0200 Subject: [PATCH 07/43] feat: pin k8s image versions and add a multi-arch build --- k8s/Dockerfile | 10 ++++++++-- k8s/Makefile | 26 +++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/k8s/Dockerfile b/k8s/Dockerfile index a856a47..98cf1ac 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -2,12 +2,18 @@ # the pod (see runner.sh). Build from the repo root: # docker build -f k8s/Dockerfile -t sg-runner-k8s:dev . -FROM amazon/amazon-ecs-agent:latest AS ecs-agent +# Pinned for reproducible builds; each exists for linux/amd64 and linux/arm64. +# ECS agent: Docker Hub tags (https://hub.docker.com/r/amazon/amazon-ecs-agent/tags). +ARG ECS_AGENT_VERSION=v1.108.0 + +FROM amazon/amazon-ecs-agent:${ECS_AGENT_VERSION} AS ecs-agent FROM debian:bookworm-slim ARG TARGETARCH ARG DOCKER_VERSION=27.5.1 +# SSM agent: .../SSMAgent/latest/VERSION names the current release. +ARG SSM_AGENT_VERSION=3.3.5390.0 RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -22,7 +28,7 @@ RUN arch="$([ "$TARGETARCH" = arm64 ] && echo aarch64 || echo x86_64)" \ # SSM agent. Unpacked rather than installed: the postinst expects systemd. # Staged first: extracting onto / would replace the merged-usr /lib symlink. RUN curl -fsSL -o /tmp/ssm.deb \ - "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_${TARGETARCH}/amazon-ssm-agent.deb" \ + "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/${SSM_AGENT_VERSION}/debian_${TARGETARCH}/amazon-ssm-agent.deb" \ && mkdir /tmp/ssm \ && dpkg-deb -x /tmp/ssm.deb /tmp/ssm \ && cp /tmp/ssm/usr/bin/* /usr/bin/ \ diff --git a/k8s/Makefile b/k8s/Makefile index 97f5932..6714c31 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -8,9 +8,19 @@ NS := sg-runner POD := sg-runner-0 KUBECTL := kubectl --context $(CONTEXT) -n $(NS) +# build-multiarch: builds every platform and keeps only the build cache unless +# PUSH=1, which pushes $(REGISTRY)/$(IMAGE) (REGISTRY required). The default +# docker driver can't build multi-platform without the containerd image store, +# so it runs on its own docker-container builder, created on first use and +# never made the default. +PLATFORMS ?= linux/amd64,linux/arm64 +BUILDER ?= sg-runner-multiarch +PUSH ?= 0 +REGISTRY ?= + .DEFAULT_GOAL := help -.PHONY: help build secret up status logs deregister down nuke +.PHONY: help build build-multiarch secret up status logs deregister down nuke help: ## List the available targets @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ @@ -19,6 +29,20 @@ help: ## List the available targets build: ## Build the runner image (OrbStack k8s uses local images directly) docker build -f Dockerfile -t $(IMAGE) .. +build-multiarch: ## Build amd64 + arm64 (PLATFORMS); cache only, or PUSH=1 REGISTRY=... to push +ifeq ($(PUSH),1) + @test -n "$(REGISTRY)" || { echo "PUSH=1 needs REGISTRY=."; exit 1; } +endif + @docker buildx inspect $(BUILDER) >/dev/null 2>&1 \ + || docker buildx create --name $(BUILDER) --driver docker-container +ifeq ($(PUSH),1) + docker buildx build --builder $(BUILDER) -f Dockerfile --platform $(PLATFORMS) \ + -t $(REGISTRY)/$(IMAGE) --push .. +else + docker buildx build --builder $(BUILDER) -f Dockerfile --platform $(PLATFORMS) \ + --output type=cacheonly .. +endif + secret: ## Create the SG credentials Secret from .env @test -f .env || { echo "Copy .env.example to .env and fill it in first."; exit 1; } kubectl --context $(CONTEXT) apply -f manifests/namespace.yaml From a6d7d836b02c2659792a9f967e3012017687d7b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:40:49 +0200 Subject: [PATCH 08/43] feat: prune the nested docker from the k8s dockerd container --- k8s/README.md | 24 ++++++++++ k8s/runner.sh | 119 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 137 insertions(+), 6 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index 0190540..f2e7f7f 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -40,6 +40,30 @@ make nuke # deregister, then delete StatefulSet, PVCs, namespace `make nuke` deregisters first and stops if that fails: deleting the PVC without deregistering would leave a dead runner in the runner group. +Image versions are pinned as build args in `Dockerfile` (`ECS_AGENT_VERSION`, +`SSM_AGENT_VERSION`, `DOCKER_VERSION`). `make build-multiarch` builds +`linux/amd64` and `linux/arm64` on a dedicated buildx builder and keeps only +the build cache; `PUSH=1 REGISTRY=` pushes the result. + +## Docker cleanup + +On a VM, `main.sh prune` runs from cron. In the pod, the `dockerd` container +prunes the nested Docker itself: once at start (a runner may park before a +full interval) and then every `SG_PRUNE_INTERVAL_SECONDS`. + +| Env (on `dockerd`) | Default | | +|---|---|---| +| `SG_PRUNE_INTERVAL_SECONDS` | `14400` (4h, as the VM cron) | `0` turns cleanup off | +| `SG_PRUNE_UNTIL` | `24h` | minimum age of stopped containers, images and build cache it removes | +| `SG_PRUNE_IMAGES_ABOVE_PCT` | `70` | unused images go only once the Docker volume is this full (`0`: every run) | + +Every sweep that can remove a container is age-guarded: an unguarded prune +deletes containers the ECS agent created but hasn't started yet. Volumes are +pruned on their own (`docker volume prune`). Unused images are the wake-up +cache, so they're kept until the volume fills up; Docker filters images by +creation time, not last use, so a prune under pressure removes every unused +image older than `SG_PRUNE_UNTIL`. + ## Resource sizing The ECS agent sizes the runner from the node, not the pod: CPU is diff --git a/k8s/runner.sh b/k8s/runner.sh index 64f8dfe..e8f6b3e 100755 --- a/k8s/runner.sh +++ b/k8s/runner.sh @@ -20,8 +20,16 @@ set -o pipefail -# shellcheck source-path=SCRIPTDIR/.. source=main.sh -source /opt/sg-runner/main.sh +# Next to this script in the image (/opt/sg-runner), one level up in the repo +# (k8s/runner.sh, main.sh), where the tests source it from. +RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -r "$RUNNER_DIR/main.sh" ]]; then + # shellcheck source-path=SCRIPTDIR/.. source=main.sh + source "$RUNNER_DIR/main.sh" +else + # shellcheck source-path=SCRIPTDIR/.. source=main.sh + source "$RUNNER_DIR/../main.sh" +fi readonly SSM_DIR="/var/lib/amazon/ssm" readonly SSM_REGISTRATION_FILE="${SSM_DIR}/Vault/Store/RegistrationKey" @@ -39,6 +47,20 @@ readonly AGENT_API="http://localhost:51678/v1" # container's limit, when sizing what the agent advertises. SG_DOCKERD_OVERHEAD_MIB="${SG_DOCKERD_OVERHEAD_MIB:-256}" +# Nested Docker cleanup (prune_loop), in place of the VM's `main.sh prune` +# cron. Every interval (0 = off), remove stopped containers, dangling images +# and build cache older than SG_PRUNE_UNTIL, and unused volumes. Unused images +# are the wake-up cache, so they only go once the Docker volume is +# SG_PRUNE_IMAGES_ABOVE_PCT full (0 = every run). +SG_PRUNE_INTERVAL_SECONDS="${SG_PRUNE_INTERVAL_SECONDS:-14400}" +SG_PRUNE_UNTIL="${SG_PRUNE_UNTIL:-24h}" +SG_PRUNE_IMAGES_ABOVE_PCT="${SG_PRUNE_IMAGES_ABOVE_PCT:-70}" +DOCKER_DATA_ROOT="${DOCKER_DATA_ROOT:-/var/lib/docker}" + +# What the agent sizing reads; overridable so tests can feed fake files. +PROC_STATUS_FILE="${PROC_STATUS_FILE:-/proc/self/status}" +MEMINFO_FILE="${MEMINFO_FILE:-/proc/meminfo}" + CONTAINER_ORCHESTRATOR="docker" wait_for() { #{{{ @@ -104,8 +126,8 @@ cmd_dockerd() { #{{{ fi dockerd --host=unix:///var/run/docker.sock & - local dockerd_pid=$! - trap 'kill -TERM "$dockerd_pid"; wait "$dockerd_pid"' TERM INT + local dockerd_pid=$! prune_pid="" + trap 'kill -TERM $prune_pid "$dockerd_pid" 2>/dev/null; wait "$dockerd_pid"' TERM INT wait_for "dockerd" docker info # Same bridge, IMDS block and task-credential DNAT as on a VM. Rules land in @@ -113,10 +135,95 @@ cmd_dockerd() { #{{{ configure_local_network info "Docker ready, network" "$SG_DOCKER_NETWORK" + if prune_enabled; then + prune_loop & + prune_pid=$! + else + info "Docker prune disabled:" "SG_PRUNE_INTERVAL_SECONDS=$SG_PRUNE_INTERVAL_SECONDS" + fi + wait "$dockerd_pid" } #}}}: cmd_dockerd +prune_enabled() { + [[ "$SG_PRUNE_INTERVAL_SECONDS" =~ ^[0-9]+$ ]] && ((SG_PRUNE_INTERVAL_SECONDS > 0)) +} + +docker_disk_used_pct() { #{{{ + # Prints how full the filesystem holding Docker's data is, as a bare number. + df -P "$DOCKER_DATA_ROOT" 2>/dev/null | awk 'NR == 2 {sub(/%/, "", $5); print $5}' +} +#}}}: docker_disk_used_pct + +should_prune_images() { #{{{ + # Usage: should_prune_images + # Unused images are kept as the wake-up cache until the disk is filling up. + # An unknown usage counts as full: better a cold cache than a full PVC. + local used="$1" threshold="$SG_PRUNE_IMAGES_ABOVE_PCT" + [[ "$threshold" =~ ^[0-9]+$ ]] || threshold=70 + [[ "$used" =~ ^[0-9]+$ ]] || return 0 + ((threshold == 0 || used >= threshold)) +} +#}}}: should_prune_images + +docker_prune() { #{{{ + # Usage: docker_prune + # Runs one prune and prints the space it reclaimed (0B when docker reports + # none: `builder prune` prints "Total:", the others "Total reclaimed space:"). + local out space + if ! out="$(docker "$@" 2>&1)"; then + info "docker $* failed:" "$(tail -n1 <<<"$out")" >&2 + printf 'failed' + return 1 + fi + space="$(sed -nE 's/^Total( reclaimed space)?:[[:space:]]*//p' <<<"$out" | tail -n1)" + printf '%s' "${space:-0B}" +} +#}}}: docker_prune + +prune_docker() { #{{{ + # One cleanup pass over the nested Docker. Never a bare `system prune`: it + # removes every non-running container regardless of age, including ones the + # ECS agent has created but not started yet (see main.sh prune()). Every + # container-touching sweep carries the until= age guard. + local filter="until=$SG_PRUNE_UNTIL" used all=() containers images build volumes + containers="$(docker_prune container prune -f --filter "$filter")" + + used="$(docker_disk_used_pct)" + # -a: unused images too, not just dangling ones. `until` is the image's + # creation time, not when it was pulled or last used, so it's no LRU: under + # pressure, every unused image older than that goes. + should_prune_images "$used" && all=(-a) + images="$(docker_prune image prune -f "${all[@]}" --filter "$filter")" + + build="$(docker_prune builder prune -f --filter "$filter")" + # Anonymous volumes no container (in any state) refers to. Separate from the + # container sweep on purpose: this one can't remove a container. + volumes="$(docker_prune volume prune -f)" + + info "Docker prune ($filter, disk ${used:-?}% used${all[*]:+, unused images too}) reclaimed:" \ + "containers $containers, images $images, build cache $build, volumes $volumes" +} +#}}}: prune_docker + +prune_loop() { #{{{ + # Runs once right away, then every interval: a pod may park within minutes + # of waking and never see a full interval, so its first pass can't wait for + # one. Each pass runs in a subshell so a failing docker call or a stray exit + # can't end the loop. + local sleep_pid="" + trap 'kill $sleep_pid 2>/dev/null; exit 0' TERM INT + info "Docker prune every" "${SG_PRUNE_INTERVAL_SECONDS}s, keeping anything newer than $SG_PRUNE_UNTIL" + while :; do + (prune_docker) || info "Docker prune failed;" "retrying in ${SG_PRUNE_INTERVAL_SECONDS}s" + sleep "$SG_PRUNE_INTERVAL_SECONDS" & + sleep_pid=$! + wait "$sleep_pid" + done +} +#}}}: prune_loop + cmd_ssm_agent() { #{{{ wait_for "SSM registration" test -s "$SSM_REGISTRATION_FILE" exec amazon-ssm-agent @@ -141,7 +248,7 @@ size_agent_memory() { #{{{ local limit="${SG_TASK_MEMORY_LIMIT_MIB:-}" total available reserved [[ "$limit" =~ ^[0-9]+$ ]] || return 0 - total=$(($(awk '/^MemTotal:/ {print $2}' /proc/meminfo) / 1024)) + total=$(($(awk '/^MemTotal:/ {print $2}' "$MEMINFO_FILE") / 1024)) available=$((limit - SG_DOCKERD_OVERHEAD_MIB)) reserved=$((total > available ? total - available : 0)) # The agent parses this as uint16: anything larger silently becomes 0, which @@ -159,7 +266,7 @@ cpu_affinity_list() { #{{{ # Prints the first n CPUs this process may run on as a taskset list, or # nothing if n already covers all of them. local n="$1" range first last cpu allowed=0 cpus=() ranges=() - IFS=, read -ra ranges <<<"$(awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status)" + IFS=, read -ra ranges <<<"$(awk '/^Cpus_allowed_list:/ {print $2}' "$PROC_STATUS_FILE")" for range in "${ranges[@]}"; do first="${range%-*}" last="${range#*-}" From 40e05edd0f295ce438104ecf9ab07233922ff723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:40:49 +0200 Subject: [PATCH 09/43] feat: test and lint k8s runner.sh --- Makefile | 4 +- test/helpers/k8s.bash | 33 ++++++++++ test/unit/k8s_prune.bats | 135 ++++++++++++++++++++++++++++++++++++++ test/unit/k8s_runner.bats | 124 ++++++++++++++++++++++++++++++++++ 4 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 test/helpers/k8s.bash create mode 100644 test/unit/k8s_prune.bats create mode 100644 test/unit/k8s_runner.bats diff --git a/Makefile b/Makefile index 7e5c748..0c5f9b8 100644 --- a/Makefile +++ b/Makefile @@ -23,11 +23,13 @@ test-unit: ## Run the unit tier (pure functions, no I/O) test-smoke: ## Run the smoke tier (subprocess CLI-contract) $(BATS) --recursive test/smoke -SHELL_SOURCES := main.sh test/mocks/bin/* test/helpers/*.bash +SHELL_SOURCES := main.sh k8s/runner.sh test/mocks/bin/* test/helpers/*.bash lint: ## shellcheck correctness + modern-idiom gate (config in .shellcheckrc) # General correctness (warnings and up); SC2034 is disabled in .shellcheckrc. shellcheck --severity=warning main.sh + # -x: follows runner.sh's source of main.sh. + shellcheck -x --severity=warning k8s/runner.sh shellcheck test/mocks/bin/* shellcheck test/helpers/*.bash # Idiom gate: enforce [[ ]] over [ ] (SC2292) and command substitution over diff --git a/test/helpers/k8s.bash b/test/helpers/k8s.bash new file mode 100644 index 0000000..804357a --- /dev/null +++ b/test/helpers/k8s.bash @@ -0,0 +1,33 @@ +# shellcheck shell=bash +# +# k8s.bash - extra setup for tests of k8s/runner.sh. Load after load.bash: +# load "${BATS_TEST_DIRNAME}/../helpers/load.bash" +# load "${BATS_TEST_DIRNAME}/../helpers/k8s.bash" + +SG_RUNNER_SH="${SG_REPO_ROOT}/k8s/runner.sh" +export SG_RUNNER_SH + +# Fake /proc files the agent sizing reads, so no test depends on the host. +export PROC_STATUS_FILE="${SG_TEST_TMPDIR}/proc-self-status" +export MEMINFO_FILE="${SG_TEST_TMPDIR}/proc-meminfo" + +# Source runner.sh (which sources main.sh from the repo root), exposing its +# functions. Its dispatcher only runs when executed, so nothing starts. +load_runner() { + # shellcheck source=/dev/null + if ! source "${SG_RUNNER_SH}"; then + echo "load_runner: failed to source ${SG_RUNNER_SH}" >&2 + return 1 + fi +} + +# Usage: fake_cpus_allowed , e.g. "0-3,8-11" +fake_cpus_allowed() { + printf 'Name:\tawk\nCpus_allowed:\tff\nCpus_allowed_list:\t%s\nMems_allowed_list:\t0\n' \ + "$1" >"${PROC_STATUS_FILE}" +} + +# Usage: fake_mem_total_mib +fake_mem_total_mib() { + printf 'MemTotal: %d kB\nMemFree: 1024 kB\n' "$(($1 * 1024))" >"${MEMINFO_FILE}" +} diff --git a/test/unit/k8s_prune.bats b/test/unit/k8s_prune.bats new file mode 100644 index 0000000..04530d4 --- /dev/null +++ b/test/unit/k8s_prune.bats @@ -0,0 +1,135 @@ +#!/usr/bin/env bats +# +# Unit tests for the nested Docker cleanup in k8s/runner.sh (prune_loop in +# the dockerd container), which replaces the VM's `main.sh prune` cron. +# +# Same rule as test/unit/prune.bats: no sweep that can remove a container may +# run without the until= age guard, or it races the ECS agent between +# docker create and docker start (CannotStartContainerError). + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" + load "${BATS_TEST_DIRNAME}/../helpers/k8s.bash" + load_runner +} + +# Fake `df -P` output for the Docker data root at % used. +disk_at() { + export MOCK_DF_OUTPUT="Filesystem 1024-blocks Used Available Capacity Mounted on +/dev/vdb 20511312 0 0 $1% /var/lib/docker" +} + +prune_calls() { + mock_calls docker | grep -F "prune" || true +} + +#{{{ prune_enabled + +@test "prune_enabled defaults to on, every 4 hours" { + assert_equal "$SG_PRUNE_INTERVAL_SECONDS" "14400" + run prune_enabled + assert_success +} + +@test "prune_enabled is off at interval 0 or a non-number" { + SG_PRUNE_INTERVAL_SECONDS=0 + run prune_enabled + assert_failure + + SG_PRUNE_INTERVAL_SECONDS=4h + run prune_enabled + assert_failure +} + +#}}}: prune_enabled + +#{{{ should_prune_images + +@test "should_prune_images keeps the image cache below the threshold" { + SG_PRUNE_IMAGES_ABOVE_PCT=70 + run should_prune_images 69 + assert_failure + run should_prune_images 70 + assert_success +} + +@test "should_prune_images always prunes at threshold 0" { + SG_PRUNE_IMAGES_ABOVE_PCT=0 + run should_prune_images 1 + assert_success +} + +@test "should_prune_images prunes when disk usage is unknown" { + run should_prune_images "" + assert_success +} + +@test "docker_disk_used_pct reads Capacity from df" { + disk_at 42 + run docker_disk_used_pct + assert_output "42" + assert_called df "-P /var/lib/docker" +} + +#}}}: should_prune_images + +#{{{ prune_docker + +@test "prune_docker age-guards every sweep that can remove a container" { + run prune_docker + assert_success + + assert_called docker "container prune -f --filter until=24h" + refute [ -n "$(prune_calls | grep -F "system prune")" ] + refute [ -n "$(prune_calls | grep -E "(container|image|builder) prune" | grep -vF -- "--filter until=")" ] +} + +@test "prune_docker reclaims volumes with a volume-only prune" { + run prune_docker + assert_success + assert_called docker "volume prune -f" +} + +@test "prune_docker honours SG_PRUNE_UNTIL" { + SG_PRUNE_UNTIL=6h + run prune_docker + assert_called docker "container prune -f --filter until=6h" + assert_called docker "image prune -f --filter until=6h" + assert_called docker "builder prune -f --filter until=6h" +} + +@test "prune_docker keeps unused images while the disk has room" { + disk_at 30 + run prune_docker + assert_called docker "image prune -f --filter until=24h" + refute [ -n "$(prune_calls | grep -F "image prune -f -a")" ] +} + +@test "prune_docker prunes unused images once the disk fills up" { + disk_at 85 + run prune_docker + assert_called docker "image prune -f -a --filter until=24h" + assert_output --partial "unused images too" +} + +@test "prune_docker logs what each sweep reclaimed" { + export MOCK_DOCKER_VOLUME_OUTPUT="Deleted Volumes: +abc + +Total reclaimed space: 1.5GB" + run prune_docker + assert_success + assert_output --partial "volumes 1.5GB" + assert_output --partial "containers 0B" +} + +@test "prune_docker carries on and reports a failing sweep" { + export MOCK_DOCKER_EXIT=1 + run prune_docker + assert_success + assert_output --partial "containers failed" + assert_output --partial "volumes failed" + assert_called docker "volume prune -f" +} + +#}}}: prune_docker diff --git a/test/unit/k8s_runner.bats b/test/unit/k8s_runner.bats new file mode 100644 index 0000000..d07095e --- /dev/null +++ b/test/unit/k8s_runner.bats @@ -0,0 +1,124 @@ +#!/usr/bin/env bats +# +# Unit tests for the pure helpers of k8s/runner.sh: agent sizing (CPU +# affinity, reserved memory) and reading the registration in ecs.config. + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" + load "${BATS_TEST_DIRNAME}/../helpers/k8s.bash" + load_runner + mkdir -p "$ECS_CONFIG_DIR" +} + +#{{{ cpu_affinity_list + +@test "cpu_affinity_list takes the first n CPUs of a range" { + fake_cpus_allowed "0-7" + run cpu_affinity_list 2 + assert_success + assert_output "0,1" +} + +@test "cpu_affinity_list walks split ranges and single CPUs" { + fake_cpus_allowed "2-3,6,9-11" + run cpu_affinity_list 5 + assert_success + assert_output "2,3,6,9,10" +} + +@test "cpu_affinity_list prints nothing when n covers every allowed CPU" { + fake_cpus_allowed "0-3" + run cpu_affinity_list 4 + assert_success + assert_output "" + + run cpu_affinity_list 16 + assert_success + assert_output "" +} + +#}}}: cpu_affinity_list + +#{{{ size_agent_memory + +@test "size_agent_memory leaves the agent alone without a memory limit" { + fake_mem_total_mib 16384 + unset SG_TASK_MEMORY_LIMIT_MIB ECS_RESERVED_MEMORY + size_agent_memory + assert [ -z "${ECS_RESERVED_MEMORY+set}" ] +} + +@test "size_agent_memory ignores a non-numeric limit" { + fake_mem_total_mib 16384 + unset ECS_RESERVED_MEMORY + SG_TASK_MEMORY_LIMIT_MIB="4Gi" size_agent_memory + assert [ -z "${ECS_RESERVED_MEMORY+set}" ] +} + +@test "size_agent_memory reserves everything but the limit minus dockerd overhead" { + fake_mem_total_mib 16384 + SG_TASK_MEMORY_LIMIT_MIB=4096 + size_agent_memory >/dev/null + # 16384 - (4096 - 256) + assert_equal "$ECS_RESERVED_MEMORY" "12544" + + run size_agent_memory + assert_output --partial "3840 of 16384" +} + +@test "size_agent_memory reserves nothing when the limit exceeds the node" { + fake_mem_total_mib 2048 + SG_TASK_MEMORY_LIMIT_MIB=4096 + size_agent_memory >/dev/null + assert_equal "$ECS_RESERVED_MEMORY" "0" +} + +@test "size_agent_memory refuses a reservation the agent would read as 0" { + # 128 GiB node, 4 GiB limit: 127232 MiB doesn't fit the agent's uint16. + fake_mem_total_mib 131072 + SG_TASK_MEMORY_LIMIT_MIB=4096 + run size_agent_memory + assert_failure + assert_output --partial "exceeds 65535MiB" +} + +#}}}: size_agent_memory + +#{{{ load_ecs_config / configured_runner_id + +@test "load_ecs_config exports values with spaces, quotes and = intact" { + cat >"$ECS_CONFIG_DIR/ecs.config" <<'CFG' +# written by configure_local_data +ECS_CLUSTER=sg-cluster + +ECS_INSTANCE_ATTRIBUTES={"sg_runner_id":"runner 1","note":"say \"hi\" a=b"} +CFG + # No trailing newline on the last line. + printf 'ECS_LOGLEVEL=info' >>"$ECS_CONFIG_DIR/ecs.config" + + load_ecs_config + + assert_equal "$ECS_CLUSTER" "sg-cluster" + assert_equal "$ECS_INSTANCE_ATTRIBUTES" '{"sg_runner_id":"runner 1","note":"say \"hi\" a=b"}' + assert_equal "$(jq -r .note <<<"$ECS_INSTANCE_ATTRIBUTES")" 'say "hi" a=b' + assert_equal "$ECS_LOGLEVEL" "info" + assert_equal "$(bash -c 'printf %s "$ECS_CLUSTER"')" "sg-cluster" +} + +@test "configured_runner_id reads sg_runner_id from the instance attributes" { + cat >"$ECS_CONFIG_DIR/ecs.config" <<'CFG' +ECS_CLUSTER=sg-cluster +ECS_INSTANCE_ATTRIBUTES={"sg_organization":"org","sg_runner_id":"abc123-runner"} +CFG + run configured_runner_id + assert_success + assert_output "abc123-runner" +} + +@test "configured_runner_id dies when the runner isn't registered" { + run configured_runner_id + assert_failure + assert_output --partial "Not registered" +} + +#}}}: load_ecs_config / configured_runner_id From d668ac0144724b7ec456663781f76a2b9bc54f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:41:15 +0200 Subject: [PATCH 10/43] feat: deploy k8s runners with a helm chart --- k8s/Makefile | 48 +++-- k8s/README.md | 13 +- k8s/chart/sg-runner/.helmignore | 23 +++ k8s/chart/sg-runner/Chart.yaml | 13 ++ k8s/chart/sg-runner/README.md | 107 ++++++++++ k8s/chart/sg-runner/templates/NOTES.txt | 37 ++++ k8s/chart/sg-runner/templates/_helpers.tpl | 131 ++++++++++++ .../templates/credentials-secret.yaml | 20 ++ .../sg-runner/templates/networkpolicy.yaml | 48 +++++ .../templates/poddisruptionbudget.yaml | 16 ++ .../sg-runner/templates/runner-service.yaml | 14 ++ .../templates/runner-statefulset.yaml} | 130 +++++++----- .../templates/scaler-deployment.yaml | 97 +++++++++ .../sg-runner/templates/scaler-metrics.yaml | 34 ++++ .../sg-runner/templates/scaler-rbac.yaml | 57 ++++++ k8s/chart/sg-runner/values.schema.json | 161 +++++++++++++++ k8s/chart/sg-runner/values.yaml | 192 ++++++++++++++++++ k8s/manifests/namespace.yaml | 7 - k8s/manifests/scaler.yaml | 86 -------- 19 files changed, 1074 insertions(+), 160 deletions(-) create mode 100644 k8s/chart/sg-runner/.helmignore create mode 100644 k8s/chart/sg-runner/Chart.yaml create mode 100644 k8s/chart/sg-runner/README.md create mode 100644 k8s/chart/sg-runner/templates/NOTES.txt create mode 100644 k8s/chart/sg-runner/templates/_helpers.tpl create mode 100644 k8s/chart/sg-runner/templates/credentials-secret.yaml create mode 100644 k8s/chart/sg-runner/templates/networkpolicy.yaml create mode 100644 k8s/chart/sg-runner/templates/poddisruptionbudget.yaml create mode 100644 k8s/chart/sg-runner/templates/runner-service.yaml rename k8s/{manifests/runner.yaml => chart/sg-runner/templates/runner-statefulset.yaml} (51%) create mode 100644 k8s/chart/sg-runner/templates/scaler-deployment.yaml create mode 100644 k8s/chart/sg-runner/templates/scaler-metrics.yaml create mode 100644 k8s/chart/sg-runner/templates/scaler-rbac.yaml create mode 100644 k8s/chart/sg-runner/values.schema.json create mode 100644 k8s/chart/sg-runner/values.yaml delete mode 100644 k8s/manifests/namespace.yaml delete mode 100644 k8s/manifests/scaler.yaml diff --git a/k8s/Makefile b/k8s/Makefile index 97f5932..fbadaa7 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -5,8 +5,15 @@ CONTEXT ?= orbstack IMAGE ?= sg-runner-k8s:dev NS := sg-runner -POD := sg-runner-0 +# Release name = chart name, so the StatefulSet (and its pods and PVCs) is +# named sg-runner too. +RELEASE := sg-runner +STS := $(RELEASE) +POD := $(STS)-0 +CHART := chart/sg-runner +SECRET := sg-runner-credentials KUBECTL := kubectl --context $(CONTEXT) -n $(NS) +HELM := helm --kube-context $(CONTEXT) -n $(NS) .DEFAULT_GOAL := help @@ -19,17 +26,25 @@ help: ## List the available targets build: ## Build the runner image (OrbStack k8s uses local images directly) docker build -f Dockerfile -t $(IMAGE) .. -secret: ## Create the SG credentials Secret from .env +# The namespace must allow privileged pods (dockerd); the chart doesn't create it. +secret: ## Create the namespace and the SG credentials Secret from .env @test -f .env || { echo "Copy .env.example to .env and fill it in first."; exit 1; } - kubectl --context $(CONTEXT) apply -f manifests/namespace.yaml - $(KUBECTL) create secret generic sg-runner-credentials --from-env-file=.env \ + kubectl --context $(CONTEXT) create namespace $(NS) --dry-run=client -o yaml \ + | kubectl --context $(CONTEXT) apply -f - + kubectl --context $(CONTEXT) label namespace $(NS) --overwrite \ + pod-security.kubernetes.io/enforce=privileged + $(KUBECTL) create secret generic $(SECRET) --from-env-file=.env \ --dry-run=client -o yaml | $(KUBECTL) apply -f - -up: ## Deploy the runner StatefulSet (registers on first start) - kubectl --context $(CONTEXT) apply -f manifests/ +# IMAGE is split into repository and tag for the chart. +up: ## Install or upgrade the chart (registers on first start) + $(HELM) upgrade --install $(RELEASE) $(CHART) \ + --set credentials.existingSecret=$(SECRET) \ + --set image.repository=$(firstword $(subst :, ,$(IMAGE))) \ + --set image.tag=$(lastword $(subst :, ,$(IMAGE))) status: ## Show runner pods and state PVCs (a PVC without a pod is a parked runner) - $(KUBECTL) get statefulset,pods,pvc -o wide + $(KUBECTL) get statefulset,deployment,pods,pvc -o wide -$(KUBECTL) exec $(POD) -c ecs-agent -- curl -fsS localhost:51678/v1/metadata logs: ## Tail all runner containers @@ -37,21 +52,26 @@ logs: ## Tail all runner containers # Parked runners are still registered but have no pod to run from, so wake # every runner that has a state PVC first, then deregister them one by one. +# The scaler is stopped first: it would park the woken runners again. deregister: ## Wake parked runners, deregister all from SG, wipe identities (reads .env on stdin) - @n=$$($(KUBECTL) get pvc -o name | grep -c '/state-sg-runner-' || true); \ + @n=$$($(KUBECTL) get pvc -o name | grep -c '/state-$(STS)-' || true); \ if [ "$$n" -eq 0 ]; then echo "No runners to deregister."; exit 0; fi; \ - $(KUBECTL) scale statefulset sg-runner --replicas=$$n && \ + $(KUBECTL) scale deployment $(STS)-scaler --replicas=0 2>/dev/null; \ + $(KUBECTL) scale statefulset $(STS) --replicas=$$n && \ for i in $$(seq 0 $$((n - 1))); do \ - $(KUBECTL) wait --for=condition=Ready pod/sg-runner-$$i --timeout=300s && \ - $(KUBECTL) exec -i sg-runner-$$i -c ecs-agent -- /opt/sg-runner/runner.sh deregister < .env \ + $(KUBECTL) wait --for=condition=Ready pod/$(STS)-$$i --timeout=300s && \ + $(KUBECTL) exec -i $(STS)-$$i -c ecs-agent -- /opt/sg-runner/runner.sh deregister < .env \ || exit 1; \ done -down: ## Delete the StatefulSet, keep PVCs (runner identity survives) - -$(KUBECTL) delete statefulset sg-runner +# The runners' PVCs aren't part of the release: uninstalling parks every +# runner (drain on the way out) and leaves it registered in SG. With the scaler +# gone nothing reaps them; `make up` wakes them again. +down: ## Uninstall the chart, keep PVCs (runners stay registered, parked) + -$(HELM) uninstall $(RELEASE) # deregister runs first and aborts nuke on failure, so a runner is never # orphaned by deleting its identity. -nuke: deregister down ## Deregister, then delete everything including PVCs +nuke: deregister down ## Deregister, then uninstall and delete PVCs and namespace -$(KUBECTL) delete pvc --all -kubectl --context $(CONTEXT) delete namespace $(NS) diff --git a/k8s/README.md b/k8s/README.md index 0190540..13e849a 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -27,14 +27,15 @@ cd k8s cp .env.example .env # fill in a throwaway runner group's credentials make build # build sg-runner-k8s:dev make secret # namespace + credentials Secret -make up # deploy; first start registers the runner +make up # helm upgrade --install chart/sg-runner; first start registers the runner make logs # follow all containers make status # pod state + ECS agent metadata kubectl -n sg-runner scale sts sg-runner --replicas=N # park / wake runners make deregister # wake parked runners, deregister all (reads .env on stdin) -make nuke # deregister, then delete StatefulSet, PVCs, namespace +make down # uninstall, keep PVCs (runners stay registered, parked) +make nuke # deregister, then uninstall, delete PVCs, namespace ``` `make nuke` deregisters first and stops if that fails: deleting the PVC without @@ -96,7 +97,7 @@ everything. Reaping runners parked for too long is up to the scaler (planned). ## Autoscaling -`manifests/scaler.yaml` runs `scaler.sh`, a small loop that owns the runner +The chart's scaler Deployment ([`chart/sg-runner`](chart/sg-runner/README.md)) runs `scaler.sh`, a small loop that owns the runner StatefulSet's `replicas`. Every `SCALER_INTERVAL` (15s) it reads the runner group from SG with the node token (`?getActiveWorkflows=true`) and: @@ -128,9 +129,9 @@ Jobs; runner pods have no Kubernetes API access. resolve cluster DNS and reach the Kubernetes API (verified). Needs a NetworkPolicy on the runner pod and a CNI that enforces it ([`docs/TODO.md`](../docs/TODO.md)). -- **Workspaces in the container layer.** `/var/tmp/workspace`, bind-mounted - by the run controller and steps, isn't on a volume yet - ([`docs/TODO.md`](../docs/TODO.md)). +- **Workspaces are scratch.** `/var/tmp/workspace` is an `emptyDir` on + `dockerd` (`runner.workspace.sizeLimit`, default 20Gi): a run doesn't + survive a pod restart, and a workspace past the limit gets the pod evicted. - **Task cgroup limits off** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`): the agent can't create task cgroups from its container. Size the pod's resources instead. - **No proxy support** yet (`--http-proxy` path not wired). diff --git a/k8s/chart/sg-runner/.helmignore b/k8s/chart/sg-runner/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s/chart/sg-runner/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/chart/sg-runner/Chart.yaml b/k8s/chart/sg-runner/Chart.yaml new file mode 100644 index 0000000..c852322 --- /dev/null +++ b/k8s/chart/sg-runner/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: sg-runner +description: StackGuardian Private Runner on Kubernetes (prototype). Each StatefulSet replica is one registered runner. +type: application +# Chart version, bumped on every chart change. +version: 0.1.0 +# The prototype image is built locally (`make build`), there is no released +# tag yet. Used as the image tag when image.tag is empty. +appVersion: "dev" +keywords: + - stackguardian + - runner +home: https://github.com/StackGuardian/sg-runner diff --git a/k8s/chart/sg-runner/README.md b/k8s/chart/sg-runner/README.md new file mode 100644 index 0000000..0fceb9d --- /dev/null +++ b/k8s/chart/sg-runner/README.md @@ -0,0 +1,107 @@ +# sg-runner Helm chart (prototype) + +> **Status: prototype.** Not supported, not documented for customers. + +Runs StackGuardian Private Runners on Kubernetes. Each replica of the runner +StatefulSet is one registered runner; a scaler Deployment wakes, parks and reaps +runners from the runner group's queue. Design and known gaps: [`k8s/README.md`](../../README.md). + +## Install + +The namespace must allow privileged pods (nested Docker). The chart doesn't +create it: + +```sh +kubectl create namespace sg-runner +kubectl label namespace sg-runner pod-security.kubernetes.io/enforce=privileged +kubectl -n sg-runner create secret generic sg-runner-credentials \ + --from-literal=SG_NODE_TOKEN=... --from-literal=SG_ORGANIZATION=... \ + --from-literal=SG_RUNNER_GROUP=... +helm -n sg-runner upgrade --install sg-runner ./k8s/chart/sg-runner \ + --set credentials.existingSecret=sg-runner-credentials --set runner.size=medium +``` + +From `k8s/`, `make secret up` does the same against OrbStack. + +Release `sg-runner` names the StatefulSet `sg-runner`, so pods `sg-runner-N` and +PVCs `state-sg-runner-N` / `docker-sg-runner-N`. Other release names get +`-sg-runner`. Don't rename an existing install: the runners +registered from the old PVCs would be orphaned. + +## Uninstall + +PVCs come from `volumeClaimTemplates` and aren't part of the release: +`helm uninstall` parks every runner, and they stay registered in SG (counting +against `MaxNumberOfRunners`). Deregister them first (`make deregister`), then +delete the PVCs. + +## Runner size + +`runner.size` sizes the `dockerd` container, where workflows run. Its limits +are what the runner advertises: CPU = `limits.cpu`, memory = `limits.memory` +minus 256 MiB for dockerd. + +| Size | Requests | Limits (advertised) | Default workflows | +|---|---|---|---| +| `small` | 1 CPU / 2Gi | 2 CPU / 4Gi | 2 | +| `medium` | 2 CPU / 4Gi | 4 CPU / 8Gi | 4 | +| `large` | 4 CPU / 8Gi | 8 CPU / 16Gi | 8 | +| `custom` | `runner.resources` | `runner.resources` (limits required) | | + +A workflow reserves 0.5 vCPU / 512 MiB for its run controller plus its own +setting (default 0.5 vCPU / 1 GiB): 1 vCPU / 1.5 GiB by default. The runner +must be at least the largest workflow + 0.5 vCPU / 512 MiB, else that workflow +stays queued (the scaler warns). + +## Values + +| Key | Default | Description | +|---|---|---| +| `nameOverride` / `fullnameOverride` | `""` | Rename resources; `fullnameOverride` also renames the StatefulSet (max 52 chars). | +| `image.repository` / `.tag` / `.pullPolicy` | `sg-runner-k8s` / `dev` / `IfNotPresent` | Runner image; also used by reap Jobs. Empty tag = `appVersion`. | +| `imagePullSecrets` | `[]` | For runner and scaler pods (not reap Jobs). | +| `credentials.existingSecret` | `""` | Secret with `SG_NODE_TOKEN`, `SG_ORGANIZATION`, `SG_RUNNER_GROUP`, optional `SG_BASE_API`. | +| `credentials.create` | `false` | Create the Secret from `nodeToken`, `organization`, `runnerGroup`, `apiUrl`. Exactly one of this or `existingSecret`. | +| `runner.replicas` | `1` | Only with `scaler.enabled=false`. | +| `runner.size` | `small` | `small`, `medium`, `large` or `custom`. | +| `runner.resources` | `{}` | dockerd resources for `size=custom`. | +| `runner.workspace.sizeLimit` | `20Gi` | `emptyDir` at `/var/tmp/workspace` on dockerd; exceeding it evicts the pod. | +| `runner.workspace.request` | `2Gi` | `ephemeral-storage` request on dockerd. Empty = none. | +| `runner.storage.state.size` / `.storageClassName` | `1Gi` / `""` | Runner identity PVC. Empty class = cluster default. | +| `runner.storage.docker.size` / `.storageClassName` | `20Gi` / `""` | Docker data root PVC (image cache). | +| `runner.dockerd.securityContext` | `{privileged: true}` | Nested Docker needs privileged for now. | +| `runner.terminationGracePeriodSeconds` | `3600` | Max wait for running workflows when a runner stops. | +| `runner.podAnnotations` / `podLabels` / `nodeSelector` / `tolerations` / `affinity` | empty | Runner pod scheduling and metadata. | +| `scaler.enabled` | `true` | Scaler Deployment + RBAC. When on, the chart leaves the StatefulSet's `replicas` to it. | +| `scaler.image.repository` / `.tag` / `.pullPolicy` | `""` / `""` / `IfNotPresent` | Empty repository = the runner image. | +| `scaler.command` | `["/opt/sg-runner/scaler.sh"]` | Scaler entrypoint. | +| `scaler.min` | `1` | `SCALER_MIN`, ≥ 1 (SG pauses workflows without an ACTIVE runner). | +| `scaler.max` | `10` | `SCALER_MAX`, also capped by `MaxNumberOfRunners`. | +| `scaler.headroomPct` | `0` | `SCALER_HEADROOM_PCT`: capacity to keep free, 0 = off. | +| `scaler.idleSeconds` | `60` | `SCALER_IDLE_SECONDS`: idle time before parking. | +| `scaler.parkTTL` | `43200` | `SCALER_PARK_TTL`: parked time before reaping (deregister + delete PVCs). | +| `scaler.stuckSeconds` | `180` | `SCALER_STUCK_SECONDS`: queue + idle runner for this long = workflow too big. | +| `scaler.settleTimeout` | `300` | `SCALER_SETTLE_TIMEOUT`: wait for a runner to connect. | +| `scaler.interval` | `15` | `SCALER_INTERVAL`: seconds between decisions. | +| `scaler.extraEnv` | `[]` | Extra env vars for the scaler. | +| `scaler.metrics.enabled` / `.port` | `false` / `9090` | Container port `metrics` and a Service. | +| `scaler.metrics.serviceMonitor.enabled` / `.interval` / `.labels` | `false` / `30s` / `{}` | ServiceMonitor, only if `monitoring.coreos.com/v1` is served. | +| `scaler.resources` | 10m / 32Mi, limit 128Mi | | +| `scaler.podSecurityContext` / `securityContext` / scheduling | empty | | +| `networkPolicy.enabled` | `false` | Egress isolation for runner pods; needs an enforcing CNI. | +| `networkPolicy.clusterCIDRs` | `[]` | Pod, service, node CIDRs (+ API endpoint). Required when enabled. | +| `networkPolicy.blockMetadata` | `true` | Deny `169.254.169.254/32`. | +| `networkPolicy.allowClusterDNS` / `dnsSelector` | `true` / kube-system `k8s-app: kube-dns` | Allow DNS (53/UDP+TCP) to the cluster resolver. | +| `networkPolicy.extraEgress` | `[]` | Extra egress rules, as-is. | +| `podDisruptionBudget.enabled` / `minAvailable` | `false` / `1` | Keep drains from evicting the last runner; with one awake runner it blocks drains, so pair with `scaler.min: 2`. | + +## Notes + +- **Replicas and the scaler.** With the scaler on, Helm upgrades keep the live + replica count. Turning the scaler on for an install that had it off drops + `replicas` from the manifest, which resets it to 1 (parks the rest). +- **Storage values are fixed after install**: `volumeClaimTemplates` are + immutable on a StatefulSet. +- **NetworkPolicy** covers runner pods, so workflow containers and agents + alike (nested traffic leaves through the pod IP). It doesn't cover reap Job + pods: `scaler.sh` labels the Job, not its pods. IPv4 only. diff --git a/k8s/chart/sg-runner/templates/NOTES.txt b/k8s/chart/sg-runner/templates/NOTES.txt new file mode 100644 index 0000000..8aea43d --- /dev/null +++ b/k8s/chart/sg-runner/templates/NOTES.txt @@ -0,0 +1,37 @@ +{{- $sts := include "sg-runner.runnerName" . -}} +{{- $ns := .Release.Namespace -}} +sg-runner {{ .Chart.Version }} installed as StatefulSet {{ $ns }}/{{ $sts }} (size: {{ .Values.runner.size }}). + +The namespace must allow privileged pods (nested Docker); the chart doesn't +create or label it: + + kubectl label namespace {{ $ns }} pod-security.kubernetes.io/enforce=privileged + +Each replica registers one runner in the runner group on its first start. + +Runners: + kubectl -n {{ $ns }} get statefulset,pods -l app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=runner + +Parked runners are state PVCs without a pod (still registered in SG, shown as +DRAINING and disconnected): + kubectl -n {{ $ns }} get pvc | grep '^state-{{ $sts }}-' +{{- if .Values.scaler.enabled }} + +The scaler owns the replica count ({{ .Values.scaler.min }}..{{ .Values.scaler.max }}). Watch its decisions: + kubectl -n {{ $ns }} logs -f deploy/{{ include "sg-runner.scalerName" . }} + +Don't `kubectl scale` the StatefulSet by hand; the scaler overrides it. +{{- else }} + +Scaler disabled: the StatefulSet runs {{ .Values.runner.replicas }} replica(s). Scaling down parks runners, +scaling up wakes them: + kubectl -n {{ $ns }} scale statefulset {{ $sts }} --replicas=N +{{- end }} + +Uninstalling keeps the PVCs: runners stay registered in SG as parked. To +remove them, deregister each runner first (see the chart README), then delete +the PVCs. +{{- if .Values.networkPolicy.enabled }} + +NetworkPolicy {{ $sts }} only takes effect with a CNI that enforces it. +{{- end }} diff --git a/k8s/chart/sg-runner/templates/_helpers.tpl b/k8s/chart/sg-runner/templates/_helpers.tpl new file mode 100644 index 0000000..441fe65 --- /dev/null +++ b/k8s/chart/sg-runner/templates/_helpers.tpl @@ -0,0 +1,131 @@ +{{/* +Chart name. +*/}} +{{- define "sg-runner.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified release name; also the runner StatefulSet's name. Kept to 52 +characters: the StatefulSet controller adds a `controller-revision-hash` label +of - that must fit in 63. +*/}} +{{- define "sg-runner.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 52 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 52 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 52 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Runner StatefulSet (and headless Service) name. Pods are -, PVCs +state-- and docker--; the scaler maps SG runners +to ordinals through that name. +*/}} +{{- define "sg-runner.runnerName" -}} +{{- include "sg-runner.fullname" . }} +{{- end }} + +{{- define "sg-runner.scalerName" -}} +{{- printf "%s-scaler" (include "sg-runner.fullname" .) }} +{{- end }} + +{{- define "sg-runner.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. Pass a dict with "ctx" (root context) and "component". +*/}} +{{- define "sg-runner.labels" -}} +helm.sh/chart: {{ include "sg-runner.chart" .ctx }} +{{ include "sg-runner.selectorLabels" . }} +{{- if .ctx.Chart.AppVersion }} +app.kubernetes.io/version: {{ .ctx.Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .ctx.Release.Service }} +app.kubernetes.io/part-of: sg-runner +{{- end }} + +{{/* +Selector labels: runner and scaler pods share name/instance, the component +tells them apart. +*/}} +{{- define "sg-runner.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sg-runner.name" .ctx }} +app.kubernetes.io/instance: {{ .ctx.Release.Name }} +app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{/* +Runner image reference; the scaler also uses it for reap Jobs. +*/}} +{{- define "sg-runner.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag | toString) }} +{{- end }} + +{{/* +Scaler image: its own repository/tag when set, the runner image otherwise. +*/}} +{{- define "sg-runner.scalerImage" -}} +{{- if .Values.scaler.image.repository }} +{{- printf "%s:%s" .Values.scaler.image.repository (default .Chart.AppVersion .Values.scaler.image.tag | toString) }} +{{- else }} +{{- include "sg-runner.image" . }} +{{- end }} +{{- end }} + +{{/* +Name of the SG credentials Secret. Fails rendering when neither an existing +Secret nor credentials.create is configured, or both are. +*/}} +{{- define "sg-runner.credentialsSecretName" -}} +{{- $c := .Values.credentials }} +{{- if and $c.existingSecret $c.create }} +{{- fail "credentials: set either credentials.existingSecret or credentials.create=true, not both" }} +{{- else if $c.existingSecret }} +{{- $c.existingSecret }} +{{- else if $c.create }} +{{- printf "%s-credentials" (include "sg-runner.fullname" .) }} +{{- else }} +{{- fail "credentials: set credentials.existingSecret (a Secret with SG_NODE_TOKEN, SG_ORGANIZATION, SG_RUNNER_GROUP and optionally SG_BASE_API) or credentials.create=true with credentials.nodeToken, credentials.organization and credentials.runnerGroup" }} +{{- end }} +{{- end }} + +{{/* +Resources of the dockerd container: a size preset or runner.resources, plus the +workspace's ephemeral-storage request. Workflows run inside dockerd, and the ECS +agent advertises these limits (SG_TASK_* on ecs-agent). +*/}} +{{- define "sg-runner.dockerdResources" -}} +{{- $presets := dict + "small" (dict "requests" (dict "cpu" "1" "memory" "2Gi") "limits" (dict "cpu" "2" "memory" "4Gi")) + "medium" (dict "requests" (dict "cpu" "2" "memory" "4Gi") "limits" (dict "cpu" "4" "memory" "8Gi")) + "large" (dict "requests" (dict "cpu" "4" "memory" "8Gi") "limits" (dict "cpu" "8" "memory" "16Gi")) +}} +{{- $size := .Values.runner.size }} +{{- $res := dict }} +{{- if eq $size "custom" }} +{{- $res = deepCopy (default dict .Values.runner.resources) }} +{{- $limits := default dict $res.limits }} +{{- if not (and $limits.cpu $limits.memory) }} +{{- fail "runner.size=custom needs runner.resources.limits.cpu and .memory: without limits the runner advertises the whole node to SG" }} +{{- end }} +{{- else if hasKey $presets $size }} +{{- $res = deepCopy (get $presets $size) }} +{{- else }} +{{- fail (printf "runner.size must be small, medium, large or custom, got %q" $size) }} +{{- end }} +{{- with .Values.runner.workspace.request }} +{{- $requests := default dict $res.requests }} +{{- $_ := set $requests "ephemeral-storage" . }} +{{- $_ := set $res "requests" $requests }} +{{- end }} +{{- toYaml $res }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/credentials-secret.yaml b/k8s/chart/sg-runner/templates/credentials-secret.yaml new file mode 100644 index 0000000..c7a2ab3 --- /dev/null +++ b/k8s/chart/sg-runner/templates/credentials-secret.yaml @@ -0,0 +1,20 @@ +{{- $name := include "sg-runner.credentialsSecretName" . }} +{{- if .Values.credentials.create }} +{{- $c := .Values.credentials }} +# SG credentials, one file per key when mounted: runner.sh and scaler.sh read +# these names. +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" (dict "ctx" . "component" "runner") | nindent 4 }} +type: Opaque +stringData: + SG_NODE_TOKEN: {{ required "credentials.nodeToken is required with credentials.create=true" $c.nodeToken | quote }} + SG_ORGANIZATION: {{ required "credentials.organization is required with credentials.create=true" $c.organization | quote }} + SG_RUNNER_GROUP: {{ required "credentials.runnerGroup is required with credentials.create=true" $c.runnerGroup | quote }} + {{- with $c.apiUrl }} + SG_BASE_API: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/networkpolicy.yaml b/k8s/chart/sg-runner/templates/networkpolicy.yaml new file mode 100644 index 0000000..a0728ab --- /dev/null +++ b/k8s/chart/sg-runner/templates/networkpolicy.yaml @@ -0,0 +1,48 @@ +{{- if .Values.networkPolicy.enabled }} +{{- $np := .Values.networkPolicy }} +{{- $component := dict "ctx" . "component" "runner" }} +{{- if not $np.clusterCIDRs }} +{{- fail "networkPolicy.clusterCIDRs is required with networkPolicy.enabled=true: list the pod, service and node CIDRs (and the API server endpoint if outside them)" }} +{{- end }} +# Workflow isolation. Nested workflow traffic leaves through the runner pod's +# IP, so this policy covers workflow containers and the agents alike: no +# cluster-internal destinations, no cloud metadata, everything else (SG API, +# AWS endpoints, VCS, registries, workflow targets) allowed. +# Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...); others ignore +# it without warning. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "sg-runner.runnerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + policyTypes: ["Ingress", "Egress"] + # No ingress rules: runners only dial out. + ingress: [] + egress: + {{- if $np.allowClusterDNS }} + # Cluster DNS only; the CIDR rule below excludes it otherwise. + - to: + - {{ toYaml $np.dnsSelector | nindent 10 | trim }} + ports: + - { protocol: UDP, port: 53 } + - { protocol: TCP, port: 53 } + {{- end }} + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + {{- range $np.clusterCIDRs }} + - {{ . }} + {{- end }} + {{- if $np.blockMetadata }} + - 169.254.169.254/32 + {{- end }} + {{- with $np.extraEgress }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/poddisruptionbudget.yaml b/k8s/chart/sg-runner/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..7726b62 --- /dev/null +++ b/k8s/chart/sg-runner/templates/poddisruptionbudget.yaml @@ -0,0 +1,16 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{- $component := dict "ctx" . "component" "runner" }} +# Keeps voluntary disruptions (node drains) from evicting the last runner: SG +# pauses workflows while no runner is ACTIVE. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "sg-runner.runnerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/runner-service.yaml b/k8s/chart/sg-runner/templates/runner-service.yaml new file mode 100644 index 0000000..d605930 --- /dev/null +++ b/k8s/chart/sg-runner/templates/runner-service.yaml @@ -0,0 +1,14 @@ +{{- $component := dict "ctx" . "component" "runner" }} +# Headless Service backing the StatefulSet's serviceName. Runners need no +# inbound traffic; it gives pods their . FQDN, which the scaler +# uses to map SG runners (by hostname) to ordinals. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sg-runner.runnerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + clusterIP: None + selector: + {{- include "sg-runner.selectorLabels" $component | nindent 4 }} diff --git a/k8s/manifests/runner.yaml b/k8s/chart/sg-runner/templates/runner-statefulset.yaml similarity index 51% rename from k8s/manifests/runner.yaml rename to k8s/chart/sg-runner/templates/runner-statefulset.yaml index b65952e..ecdff85 100644 --- a/k8s/manifests/runner.yaml +++ b/k8s/chart/sg-runner/templates/runner-statefulset.yaml @@ -1,68 +1,85 @@ -# sg-runner Kubernetes prototype: each replica is one registered runner. -# Namespace and the SG credentials Secret come from `make secret`. +{{- $name := include "sg-runner.runnerName" . }} +{{- $secret := include "sg-runner.credentialsSecretName" . }} +{{- $image := include "sg-runner.image" . }} +{{- $component := dict "ctx" . "component" "runner" }} +# Each replica is one registered runner. apiVersion: apps/v1 kind: StatefulSet metadata: - name: sg-runner - namespace: sg-runner + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} spec: - serviceName: sg-runner - # No replicas here: the scaler (scaler.yaml) owns them, and re-applying this - # manifest must not reset them. A new StatefulSet starts with 1. + serviceName: {{ $name }} + {{- if not .Values.scaler.enabled }} + replicas: {{ .Values.runner.replicas }} + {{- end }} + {{- /* With the scaler, no replicas: it owns them, and Helm's 3-way merge + keeps the live value on upgrade. A new StatefulSet starts with 1. */}} # Park and wake runners side by side, not one at a time (each park can wait # for running workflows). podManagementPolicy: Parallel selector: matchLabels: - app: sg-runner + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} template: metadata: + # Selector labels only: a chart version label here would roll (drain) + # every runner on each chart upgrade. labels: - app: sg-runner + {{- include "sg-runner.selectorLabels" $component | nindent 8 }} + {{- with .Values.runner.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runner.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} # Upper bound on how long a terminating runner waits for running # workflows (the ecs-agent preStop hook) before it is killed. - terminationGracePeriodSeconds: 3600 + terminationGracePeriodSeconds: {{ .Values.runner.terminationGracePeriodSeconds }} initContainers: - name: register - image: sg-runner-k8s:dev - imagePullPolicy: IfNotPresent + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: ['register'] # Only what registration needs. env: - name: SG_NODE_TOKEN valueFrom: - secretKeyRef: { name: sg-runner-credentials, key: SG_NODE_TOKEN } + secretKeyRef: { name: {{ $secret }}, key: SG_NODE_TOKEN } - name: SG_ORGANIZATION valueFrom: - secretKeyRef: { name: sg-runner-credentials, key: SG_ORGANIZATION } + secretKeyRef: { name: {{ $secret }}, key: SG_ORGANIZATION } - name: SG_RUNNER_GROUP valueFrom: - secretKeyRef: { name: sg-runner-credentials, key: SG_RUNNER_GROUP } + secretKeyRef: { name: {{ $secret }}, key: SG_RUNNER_GROUP } - name: SG_BASE_API valueFrom: - secretKeyRef: { name: sg-runner-credentials, key: SG_BASE_API, optional: true } + secretKeyRef: { name: {{ $secret }}, key: SG_BASE_API, optional: true } volumeMounts: - { name: state, mountPath: /etc/ecs, subPath: ecs-config } - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } - - { - name: state, - mountPath: /var/log/registration, - subPath: registration, - } + - { name: state, mountPath: /var/log/registration, subPath: registration } containers: - name: dockerd - image: sg-runner-k8s:dev - imagePullPolicy: IfNotPresent + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: ['dockerd'] + {{- with .Values.runner.dockerd.securityContext }} securityContext: - privileged: true + {{- toYaml . | nindent 12 }} + {{- end }} # Workflow containers run inside this container, so its limits are # what the ECS agent advertises (see SG_TASK_* on ecs-agent). resources: - requests: { cpu: '1', memory: 2Gi } - limits: { cpu: '2', memory: 4Gi } + {{- include "sg-runner.dockerdResources" . | nindent 12 }} lifecycle: preStop: exec: @@ -76,9 +93,12 @@ spec: - { name: docker, mountPath: /var/lib/docker } # Same path the agent's ECS_HOST_DATA_DIR points at. - { name: state, mountPath: /var/lib/ecs/data, subPath: ecs-data } + # Run controller and step containers bind-mount this path from + # this daemon's filesystem; keep it off the writable layer. + - { name: workspace, mountPath: /var/tmp/workspace } - name: ssm-agent - image: sg-runner-k8s:dev - imagePullPolicy: IfNotPresent + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: ['ssm-agent'] # Keeps rotating the agent's credentials while a drain runs. lifecycle: @@ -90,23 +110,17 @@ spec: - { name: aws-creds, mountPath: /root/.aws } - { name: docker-run, mountPath: /var/run } - name: ecs-agent - image: sg-runner-k8s:dev - imagePullPolicy: IfNotPresent + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: ['ecs-agent'] env: # Rounded up to whole CPUs. - name: SG_TASK_CPU_LIMIT valueFrom: - resourceFieldRef: - { containerName: dockerd, resource: limits.cpu, divisor: '1' } + resourceFieldRef: { containerName: dockerd, resource: limits.cpu, divisor: '1' } - name: SG_TASK_MEMORY_LIMIT_MIB valueFrom: - resourceFieldRef: - { - containerName: dockerd, - resource: limits.memory, - divisor: 1Mi, - } + resourceFieldRef: { containerName: dockerd, resource: limits.memory, divisor: 1Mi } lifecycle: preStop: exec: @@ -125,36 +139,58 @@ spec: - { name: aws-creds, mountPath: /rotatingcreds, readOnly: true } - { name: docker-run, mountPath: /var/run } # Setting the runner's status through SG needs the SG credentials. - - { - name: sg-credentials, - mountPath: /etc/sg-runner/credentials, - readOnly: true, - } + - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true } volumes: - name: docker-run emptyDir: {} - name: aws-creds emptyDir: medium: Memory + - name: workspace + {{- if .Values.runner.workspace.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.runner.workspace.sizeLimit }} + {{- else }} + emptyDir: {} + {{- end }} - name: sg-credentials secret: - secretName: sg-runner-credentials + secretName: {{ $secret }} defaultMode: 0400 + {{- with .Values.runner.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runner.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runner.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} volumeClaimTemplates: # Runner identity (ecs.config, SSM registration, agent state): survives # pod restarts so the runner doesn't re-register. Kept on scale-down too: - # a failed deregistration then leaves the identity for the next scale-up. + # a parked runner wakes from it. Not managed by Helm, so uninstall keeps it. - metadata: name: state spec: accessModes: ['ReadWriteOnce'] + {{- with .Values.runner.storage.state.storageClassName }} + storageClassName: {{ . }} + {{- end }} resources: requests: - storage: 1Gi + storage: {{ .Values.runner.storage.state.size }} + # Docker data root: the image cache survives parking. - metadata: name: docker spec: accessModes: ['ReadWriteOnce'] + {{- with .Values.runner.storage.docker.storageClassName }} + storageClassName: {{ . }} + {{- end }} resources: requests: - storage: 20Gi + storage: {{ .Values.runner.storage.docker.size }} diff --git a/k8s/chart/sg-runner/templates/scaler-deployment.yaml b/k8s/chart/sg-runner/templates/scaler-deployment.yaml new file mode 100644 index 0000000..8c4757b --- /dev/null +++ b/k8s/chart/sg-runner/templates/scaler-deployment.yaml @@ -0,0 +1,97 @@ +{{- if .Values.scaler.enabled }} +{{- $s := .Values.scaler }} +{{- $name := include "sg-runner.scalerName" . }} +{{- $component := dict "ctx" . "component" "scaler" }} +# Scaler for the runner StatefulSet (see k8s/scaler.sh). It owns spec.replicas. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + # One decision-maker: two scalers would fight over replicas. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + template: + metadata: + labels: + {{- include "sg-runner.selectorLabels" $component | nindent 8 }} + {{- with $s.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ $name }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: scaler + image: {{ include "sg-runner.scalerImage" . }} + imagePullPolicy: {{ $s.image.pullPolicy }} + {{- with $s.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $s.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - { name: SCALER_STATEFULSET, value: {{ include "sg-runner.runnerName" . | quote }} } + - { name: SCALER_MIN, value: {{ $s.min | quote }} } + - { name: SCALER_MAX, value: {{ $s.max | quote }} } + # % of awake runners' capacity to keep free (warm spare); 0 = off. + - { name: SCALER_HEADROOM_PCT, value: {{ $s.headroomPct | quote }} } + - { name: SCALER_IDLE_SECONDS, value: {{ $s.idleSeconds | quote }} } + - { name: SCALER_PARK_TTL, value: {{ $s.parkTTL | quote }} } + - { name: SCALER_STUCK_SECONDS, value: {{ $s.stuckSeconds | quote }} } + - { name: SCALER_SETTLE_TIMEOUT, value: {{ $s.settleTimeout | quote }} } + - { name: SCALER_INTERVAL, value: {{ $s.interval | quote }} } + # Reap Jobs run the runner image with the credentials Secret. + - { name: SCALER_IMAGE, value: {{ include "sg-runner.image" . | quote }} } + - { name: SCALER_CREDENTIALS_SECRET, value: {{ include "sg-runner.credentialsSecretName" . | quote }} } + {{- with $s.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if $s.metrics.enabled }} + ports: + - { name: metrics, containerPort: {{ $s.metrics.port }}, protocol: TCP } + {{- end }} + {{- with $s.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true } + volumes: + - name: sg-credentials + secret: + secretName: {{ include "sg-runner.credentialsSecretName" . }} + defaultMode: 0400 + {{- with $s.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/scaler-metrics.yaml b/k8s/chart/sg-runner/templates/scaler-metrics.yaml new file mode 100644 index 0000000..b8078b1 --- /dev/null +++ b/k8s/chart/sg-runner/templates/scaler-metrics.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.scaler.enabled .Values.scaler.metrics.enabled }} +{{- $m := .Values.scaler.metrics }} +{{- $component := dict "ctx" . "component" "scaler" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sg-runner.scalerName" . }}-metrics + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + selector: + {{- include "sg-runner.selectorLabels" $component | nindent 4 }} + ports: + - { name: metrics, port: {{ $m.port }}, targetPort: metrics, protocol: TCP } +{{- if and $m.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "sg-runner.scalerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} + {{- with $m.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + endpoints: + - port: metrics + interval: {{ $m.serviceMonitor.interval }} +{{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/scaler-rbac.yaml b/k8s/chart/sg-runner/templates/scaler-rbac.yaml new file mode 100644 index 0000000..a18a64a --- /dev/null +++ b/k8s/chart/sg-runner/templates/scaler-rbac.yaml @@ -0,0 +1,57 @@ +{{- if .Values.scaler.enabled }} +{{- $name := include "sg-runner.scalerName" . }} +{{- $sts := include "sg-runner.runnerName" . }} +{{- $component := dict "ctx" . "component" "scaler" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +--- +# Only the runner StatefulSet's scale, PVCs and reap Jobs. Runner pods get no +# Kubernetes API access at all. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +rules: + # Read replicas; patch for the scaler's timer annotations. + - apiGroups: ["apps"] + resources: ["statefulsets"] + resourceNames: [{{ $sts | quote }}] + verbs: ["get", "patch"] + - apiGroups: ["apps"] + resources: ["statefulsets/scale"] + resourceNames: [{{ $sts | quote }}] + verbs: ["get", "patch"] + # Parked-at stamps, and deleting a reaped runner's PVCs. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "get", "patch", "delete"] + # Reap Jobs deregister long-parked runners from their PVC. + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "delete"] + # Scaling decisions as Kubernetes events. + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/k8s/chart/sg-runner/values.schema.json b/k8s/chart/sg-runner/values.schema.json new file mode 100644 index 0000000..7c5a4f7 --- /dev/null +++ b/k8s/chart/sg-runner/values.schema.json @@ -0,0 +1,161 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "sg-runner chart values", + "type": "object", + "definitions": { + "quantity": { + "type": ["string", "integer"], + "pattern": "^[0-9]+(\\.[0-9]+)?(m|k|Ki|M|Mi|G|Gi|T|Ti|P|Pi|E|Ei)?$" + }, + "storage": { + "type": "object", + "required": ["size"], + "properties": { + "size": { "$ref": "#/definitions/quantity" }, + "storageClassName": { "type": "string" } + } + }, + "image": { + "type": "object", + "properties": { + "repository": { "type": "string" }, + "tag": { "type": ["string", "number"] }, + "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] } + } + }, + "positiveInt": { "type": "integer", "minimum": 1 } + }, + "properties": { + "nameOverride": { "type": "string" }, + "fullnameOverride": { "type": "string", "maxLength": 52 }, + "image": { + "allOf": [ + { "$ref": "#/definitions/image" }, + { "required": ["repository"], "properties": { "repository": { "minLength": 1 } } } + ] + }, + "imagePullSecrets": { "type": "array" }, + "credentials": { + "type": "object", + "properties": { + "existingSecret": { "type": "string" }, + "create": { "type": "boolean" }, + "nodeToken": { "type": "string" }, + "organization": { "type": "string" }, + "runnerGroup": { "type": "string" }, + "apiUrl": { "type": "string", "pattern": "^(https?://.+)?$" } + } + }, + "runner": { + "type": "object", + "required": ["size"], + "properties": { + "replicas": { "type": "integer", "minimum": 0 }, + "size": { "enum": ["small", "medium", "large", "custom"] }, + "resources": { + "type": "object", + "properties": { + "requests": { "type": "object", "additionalProperties": { "$ref": "#/definitions/quantity" } }, + "limits": { "type": "object", "additionalProperties": { "$ref": "#/definitions/quantity" } } + } + }, + "workspace": { + "type": "object", + "properties": { + "sizeLimit": { "anyOf": [{ "$ref": "#/definitions/quantity" }, { "const": "" }, { "type": "null" }] }, + "request": { "anyOf": [{ "$ref": "#/definitions/quantity" }, { "const": "" }, { "type": "null" }] } + } + }, + "storage": { + "type": "object", + "properties": { + "state": { "$ref": "#/definitions/storage" }, + "docker": { "$ref": "#/definitions/storage" } + } + }, + "dockerd": { + "type": "object", + "properties": { "securityContext": { "type": "object" } } + }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 0 }, + "podAnnotations": { "type": "object" }, + "podLabels": { "type": "object" }, + "nodeSelector": { "type": "object" }, + "tolerations": { "type": "array" }, + "affinity": { "type": "object" } + }, + "if": { "properties": { "size": { "const": "custom" } } }, + "then": { + "properties": { + "resources": { + "required": ["limits"], + "properties": { "limits": { "required": ["cpu", "memory"] } } + } + } + } + }, + "scaler": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "image": { "$ref": "#/definitions/image" }, + "command": { "type": "array", "items": { "type": "string" } }, + "min": { "$ref": "#/definitions/positiveInt" }, + "max": { "$ref": "#/definitions/positiveInt" }, + "headroomPct": { "type": "integer", "minimum": 0, "maximum": 100 }, + "idleSeconds": { "type": "integer", "minimum": 0 }, + "parkTTL": { "$ref": "#/definitions/positiveInt" }, + "stuckSeconds": { "type": "integer", "minimum": 0 }, + "settleTimeout": { "type": "integer", "minimum": 0 }, + "interval": { "$ref": "#/definitions/positiveInt" }, + "extraEnv": { "type": "array" }, + "metrics": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "serviceMonitor": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "interval": { "type": "string" }, + "labels": { "type": "object" } + } + } + } + }, + "resources": { "type": "object" }, + "podAnnotations": { "type": "object" }, + "podLabels": { "type": "object" }, + "podSecurityContext": { "type": "object" }, + "securityContext": { "type": "object" }, + "nodeSelector": { "type": "object" }, + "tolerations": { "type": "array" }, + "affinity": { "type": "object" } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "clusterCIDRs": { + "type": "array", + "items": { "type": "string", "pattern": "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" } + }, + "blockMetadata": { "type": "boolean" }, + "allowClusterDNS": { "type": "boolean" }, + "dnsSelector": { "type": "object" }, + "extraEgress": { "type": "array" } + }, + "if": { "properties": { "enabled": { "const": true } }, "required": ["enabled"] }, + "then": { "properties": { "clusterCIDRs": { "minItems": 1 } } } + }, + "podDisruptionBudget": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "minAvailable": { "type": ["integer", "string"] } + } + } + } +} diff --git a/k8s/chart/sg-runner/values.yaml b/k8s/chart/sg-runner/values.yaml new file mode 100644 index 0000000..b50ddd4 --- /dev/null +++ b/k8s/chart/sg-runner/values.yaml @@ -0,0 +1,192 @@ +# sg-runner chart (prototype). Each runner StatefulSet replica is one +# registered StackGuardian Private Runner; the scaler owns the replica count. +# +# The namespace must allow privileged pods (nested Docker): +# pod-security.kubernetes.io/enforce: privileged +# The chart doesn't create the namespace. + +nameOverride: "" +# Also renames the runner StatefulSet, so its pods and PVCs. Changing it on an +# existing install orphans the runners registered from the old PVCs. +fullnameOverride: "" + +# Runner image (register, dockerd, ssm-agent, ecs-agent; reap Jobs too). +image: + repository: sg-runner-k8s + # Empty = the chart's appVersion. + tag: dev + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +# SG credentials: the runner group's node token, used to register, set runner +# status and (scaler) read the runner group. Set exactly one of existingSecret +# or create=true. +credentials: + # Secret with keys SG_NODE_TOKEN, SG_ORGANIZATION, SG_RUNNER_GROUP and + # optionally SG_BASE_API (runner.sh and scaler.sh read these file names). + existingSecret: "" + # Let the chart create the Secret from the values below. They end up in the + # Helm release (a Secret in the namespace); prefer existingSecret. + create: false + nodeToken: "" + organization: "" + runnerGroup: "" + # Empty = the runner's default (https://api.app.stackguardian.io/api/v1). + apiUrl: "" + +runner: + # Only used with scaler.enabled=false. With the scaler the chart leaves + # spec.replicas out: the scaler owns it, and upgrades keep the live value. + replicas: 1 + + # Size of the dockerd container, where workflows run. Its limits are what the + # runner advertises to SG: CPU = limits.cpu, memory = limits.memory - 256 MiB + # (SG_DOCKERD_OVERHEAD_MIB, left for dockerd itself). + # + # preset requests limits (advertised) default workflows + # small 1 CPU / 2Gi 2 CPU / 4Gi 2 + # medium 2 CPU / 4Gi 4 CPU / 8Gi 4 + # large 4 CPU / 8Gi 8 CPU / 16Gi 8 + # custom runner.resources (limits.cpu and limits.memory required) + # + # A workflow reserves 0.5 vCPU / 512 MiB for its run controller plus its own + # resource setting (default 0.5 vCPU / 1 GiB), so a default workflow takes + # 1 vCPU / 1.5 GiB; the presets are CPU-bound. The runner must be at least + # the largest workflow + 0.5 vCPU / 512 MiB, else that workflow stays queued + # forever (the scaler warns about it). + # + # Requests are half the limits, so nodes can be overcommitted; set equal + # requests and limits with `custom` to guarantee the advertised capacity. + size: small + # Used with size=custom, e.g.: + # requests: { cpu: "3", memory: 6Gi } + # limits: { cpu: "3", memory: 6Gi } + resources: {} + + # Scratch space for workflow runs. The run controller and every step + # bind-mount /var/tmp/workspace from the Docker daemon they run on, i.e. the + # dockerd container. Without a volume it lands in the container's writable + # layer. An emptyDir matches a VM (scratch, gone when the pod goes). + workspace: + # The kubelet evicts the pod (and with it running workflows) when the + # workspace grows past this. + sizeLimit: 20Gi + # ephemeral-storage request on dockerd, so the scheduler places runners on + # nodes with room for workspaces. Empty = no request. + request: 2Gi + + storage: + # Runner identity (ecs.config, SSM registration, agent state). Survives + # restarts and parking so a runner doesn't register again. + state: + size: 1Gi + # Empty = the cluster's default StorageClass. + storageClassName: "" + # Docker data root (/var/lib/docker): images cached across parking. + docker: + size: 20Gi + storageClassName: "" + + dockerd: + # Nested Docker needs a privileged container for now (see docs/TODO.md). + securityContext: + privileged: true + + # Upper bound on how long a terminating runner waits for running workflows + # (the ecs-agent preStop drain) before it is killed. + terminationGracePeriodSeconds: 3600 + + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +scaler: + # Deployment that owns the runner StatefulSet's replicas: wakes parked + # runners or adds new ones when workflows queue, parks idle ones, and reaps + # runners parked longer than parkTTL. + enabled: true + + # Defaults to the runner image (scaler.sh ships in it). Set when the scaler + # ships as its own image. + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + command: ["/opt/sg-runner/scaler.sh"] + + # Runners kept awake. At least 1: SG pauses workflows while no runner is + # ACTIVE. 2 avoids that pause while a single runner restarts. + min: 1 + # Also capped by the runner group's MaxNumberOfRunners. + max: 10 + # % of awake runners' CPU or memory to keep free (warm spare); 0 = off. + headroomPct: 0 + # Park the highest awake runner after it has been idle this long. + idleSeconds: 60 + # Deregister runners parked longer than this (12h) and delete their PVCs. + parkTTL: 43200 + # Queued workflows with an idle awake runner for this long are treated as + # too big for any runner: warn and stop scaling out for them. + stuckSeconds: 180 + # How long to wait for a woken or new runner to connect before deciding + # without it. + settleTimeout: 300 + # Seconds between scaling decisions. + interval: 15 + extraEnv: [] + + # Prometheus metrics (being added to the scaler): containerPort 9090 named + # `metrics` and a Service. + metrics: + enabled: false + port: 9090 + # Needs the Prometheus Operator CRDs; skipped when the API isn't there. + serviceMonitor: + enabled: false + interval: 30s + labels: {} + + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { memory: 128Mi } + podAnnotations: {} + podLabels: {} + podSecurityContext: {} + securityContext: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +# Egress isolation for runner pods. Nested workflow traffic leaves through the +# pod IP, so this covers workflow containers and the agents alike. Needs a CNI +# that enforces NetworkPolicy (Calico, Cilium, ...); others silently ignore it. +# Reap Job pods aren't covered (they carry no runner labels). +networkPolicy: + enabled: false + # Pod, service and node CIDRs, plus the Kubernetes API endpoint if it's + # outside them. Required when enabled: workflows must not reach the cluster. + clusterCIDRs: [] + # Block the cloud metadata endpoint (169.254.169.254). + blockMetadata: true + # Allow DNS to the cluster resolver. Off = workflows and agents must resolve + # through an external resolver. + allowClusterDNS: true + dnsSelector: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + # Extra egress rules (NetworkPolicyEgressRule list), added as-is. + extraEgress: [] + +# Keeps node drains from evicting the last runner: SG pauses workflows while no +# runner is ACTIVE. With a single awake runner it blocks drains until the +# scaler wakes another; use scaler.min >= 2 alongside it. +podDisruptionBudget: + enabled: false + minAvailable: 1 diff --git a/k8s/manifests/namespace.yaml b/k8s/manifests/namespace.yaml deleted file mode 100644 index 59e9de3..0000000 --- a/k8s/manifests/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: sg-runner - labels: - # dockerd needs a privileged container. - pod-security.kubernetes.io/enforce: privileged diff --git a/k8s/manifests/scaler.yaml b/k8s/manifests/scaler.yaml deleted file mode 100644 index 91deea2..0000000 --- a/k8s/manifests/scaler.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Scaler for the runner StatefulSet (see scaler.sh). It owns spec.replicas. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sg-runner-scaler - namespace: sg-runner ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: sg-runner-scaler - namespace: sg-runner -rules: - - apiGroups: ["apps"] - resources: ["statefulsets"] - resourceNames: ["sg-runner"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["statefulsets/scale"] - resourceNames: ["sg-runner"] - verbs: ["get", "patch"] - # Parked-at stamps, and deleting a reaped runner's PVCs. - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["list", "get", "patch", "delete"] - # Reap Jobs deregister long-parked runners from their PVC. - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create", "get", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: sg-runner-scaler - namespace: sg-runner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: sg-runner-scaler -subjects: - - kind: ServiceAccount - name: sg-runner-scaler - namespace: sg-runner ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sg-runner-scaler - namespace: sg-runner -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: sg-runner-scaler - template: - metadata: - labels: - app: sg-runner-scaler - spec: - serviceAccountName: sg-runner-scaler - containers: - - name: scaler - image: sg-runner-k8s:dev - imagePullPolicy: IfNotPresent - command: ["/opt/sg-runner/scaler.sh"] - env: - - { name: SCALER_MIN, value: "1" } - - { name: SCALER_MAX, value: "10" } - # % of awake runners' capacity to keep free (warm spare); 0 = off. - - { name: SCALER_HEADROOM_PCT, value: "0" } - - { name: SCALER_IDLE_SECONDS, value: "60" } - - { name: SCALER_PARK_TTL, value: "43200" } - - { name: SCALER_IMAGE, value: sg-runner-k8s:dev } - - { name: SCALER_CREDENTIALS_SECRET, value: sg-runner-credentials } - resources: - requests: { cpu: 10m, memory: 32Mi } - limits: { memory: 128Mi } - volumeMounts: - - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true } - volumes: - - name: sg-credentials - secret: - secretName: sg-runner-credentials - defaultMode: 0400 From 7c33c48baccd7689c9d7a58ca0d0bf470a0b6c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:45:15 +0200 Subject: [PATCH 11/43] feat: shape the chart's scaler for its own non-root image --- k8s/chart/sg-runner/README.md | 13 +- k8s/chart/sg-runner/templates/NOTES.txt | 4 + .../templates/scaler-deployment.yaml | 36 +- .../sg-runner/templates/scaler-rbac.yaml | 15 +- k8s/chart/sg-runner/values.schema.json | 393 ++++++++++++++---- k8s/chart/sg-runner/values.yaml | 44 +- 6 files changed, 409 insertions(+), 96 deletions(-) diff --git a/k8s/chart/sg-runner/README.md b/k8s/chart/sg-runner/README.md index 0fceb9d..efbe2c7 100644 --- a/k8s/chart/sg-runner/README.md +++ b/k8s/chart/sg-runner/README.md @@ -73,8 +73,10 @@ stays queued (the scaler warns). | `runner.terminationGracePeriodSeconds` | `3600` | Max wait for running workflows when a runner stops. | | `runner.podAnnotations` / `podLabels` / `nodeSelector` / `tolerations` / `affinity` | empty | Runner pod scheduling and metadata. | | `scaler.enabled` | `true` | Scaler Deployment + RBAC. When on, the chart leaves the StatefulSet's `replicas` to it. | -| `scaler.image.repository` / `.tag` / `.pullPolicy` | `""` / `""` / `IfNotPresent` | Empty repository = the runner image. | -| `scaler.command` | `["/opt/sg-runner/scaler.sh"]` | Scaler entrypoint. | +| `scaler.image.repository` / `.tag` / `.pullPolicy` | `sg-runner-scaler` / `dev` / `IfNotPresent` | Scaler image (Go, distroless, non-root). Empty repository = the runner image. | +| `scaler.command` | `[]` | Empty = image entrypoint. `["/opt/sg-runner/scaler.sh"]` runs the shell scaler from the runner image. | +| `scaler.replicas` | `1` | 2 is safe with leader election; > 1 without it fails rendering. | +| `scaler.leaderElection.enabled` | `true` | `SCALER_LEADER_ELECT`; lease `-scaler`. Also switches the rollout to `RollingUpdate`. | | `scaler.min` | `1` | `SCALER_MIN`, ≥ 1 (SG pauses workflows without an ACTIVE runner). | | `scaler.max` | `10` | `SCALER_MAX`, also capped by `MaxNumberOfRunners`. | | `scaler.headroomPct` | `0` | `SCALER_HEADROOM_PCT`: capacity to keep free, 0 = off. | @@ -84,10 +86,13 @@ stays queued (the scaler warns). | `scaler.settleTimeout` | `300` | `SCALER_SETTLE_TIMEOUT`: wait for a runner to connect. | | `scaler.interval` | `15` | `SCALER_INTERVAL`: seconds between decisions. | | `scaler.extraEnv` | `[]` | Extra env vars for the scaler. | -| `scaler.metrics.enabled` / `.port` | `false` / `9090` | Container port `metrics` and a Service. | +| `scaler.metrics.port` | `9090` | Container port `metrics`, always on: `/metrics`, `/healthz` (liveness), `/readyz` (readiness). | +| `scaler.metrics.enabled` | `false` | Adds a metrics Service. | | `scaler.metrics.serviceMonitor.enabled` / `.interval` / `.labels` | `false` / `30s` / `{}` | ServiceMonitor, only if `monitoring.coreos.com/v1` is served. | | `scaler.resources` | 10m / 32Mi, limit 128Mi | | -| `scaler.podSecurityContext` / `securityContext` / scheduling | empty | | +| `scaler.podSecurityContext` | non-root 65532, `fsGroup` 65532, seccomp `RuntimeDefault` | `fsGroup` + Secret mode `0440` let the non-root scaler read the credentials. | +| `scaler.securityContext` | read-only root FS, no privilege escalation, drop `ALL` | | +| `scaler.podAnnotations` / `podLabels` / scheduling | empty | | | `networkPolicy.enabled` | `false` | Egress isolation for runner pods; needs an enforcing CNI. | | `networkPolicy.clusterCIDRs` | `[]` | Pod, service, node CIDRs (+ API endpoint). Required when enabled. | | `networkPolicy.blockMetadata` | `true` | Deny `169.254.169.254/32`. | diff --git a/k8s/chart/sg-runner/templates/NOTES.txt b/k8s/chart/sg-runner/templates/NOTES.txt index 8aea43d..463fb86 100644 --- a/k8s/chart/sg-runner/templates/NOTES.txt +++ b/k8s/chart/sg-runner/templates/NOTES.txt @@ -19,6 +19,10 @@ DRAINING and disconnected): The scaler owns the replica count ({{ .Values.scaler.min }}..{{ .Values.scaler.max }}). Watch its decisions: kubectl -n {{ $ns }} logs -f deploy/{{ include "sg-runner.scalerName" . }} +{{- if .Values.scaler.leaderElection.enabled }} +With several scaler replicas only the lease holder decides: + kubectl -n {{ $ns }} get lease {{ include "sg-runner.runnerName" . }}-scaler +{{- end }} Don't `kubectl scale` the StatefulSet by hand; the scaler overrides it. {{- else }} diff --git a/k8s/chart/sg-runner/templates/scaler-deployment.yaml b/k8s/chart/sg-runner/templates/scaler-deployment.yaml index 8c4757b..edb654f 100644 --- a/k8s/chart/sg-runner/templates/scaler-deployment.yaml +++ b/k8s/chart/sg-runner/templates/scaler-deployment.yaml @@ -2,7 +2,10 @@ {{- $s := .Values.scaler }} {{- $name := include "sg-runner.scalerName" . }} {{- $component := dict "ctx" . "component" "scaler" }} -# Scaler for the runner StatefulSet (see k8s/scaler.sh). It owns spec.replicas. +{{- if and (gt (int $s.replicas) 1) (not $s.leaderElection.enabled) }} +{{- fail "scaler.replicas > 1 needs scaler.leaderElection.enabled: two scalers without a lease would fight over the runner StatefulSet's replicas" }} +{{- end }} +# Scaler for the runner StatefulSet. It owns spec.replicas. apiVersion: apps/v1 kind: Deployment metadata: @@ -10,10 +13,15 @@ metadata: labels: {{- include "sg-runner.labels" $component | nindent 4 }} spec: - # One decision-maker: two scalers would fight over replicas. - replicas: 1 + replicas: {{ $s.replicas }} strategy: + {{- if $s.leaderElection.enabled }} + # The lease keeps a single active scaler while old and new pods overlap. + type: RollingUpdate + {{- else }} + # No lease: never run two scalers at once. type: Recreate + {{- end }} selector: matchLabels: {{- include "sg-runner.selectorLabels" $component | nindent 6 }} @@ -51,7 +59,11 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: { fieldPath: metadata.namespace } - { name: SCALER_STATEFULSET, value: {{ include "sg-runner.runnerName" . | quote }} } + - { name: SCALER_LEADER_ELECT, value: {{ $s.leaderElection.enabled | quote }} } - { name: SCALER_MIN, value: {{ $s.min | quote }} } - { name: SCALER_MAX, value: {{ $s.max | quote }} } # % of awake runners' capacity to keep free (warm spare); 0 = off. @@ -61,16 +73,23 @@ spec: - { name: SCALER_STUCK_SECONDS, value: {{ $s.stuckSeconds | quote }} } - { name: SCALER_SETTLE_TIMEOUT, value: {{ $s.settleTimeout | quote }} } - { name: SCALER_INTERVAL, value: {{ $s.interval | quote }} } - # Reap Jobs run the runner image with the credentials Secret. + # Reap Jobs run the runner image (not the scaler's) with the + # credentials Secret. - { name: SCALER_IMAGE, value: {{ include "sg-runner.image" . | quote }} } - { name: SCALER_CREDENTIALS_SECRET, value: {{ include "sg-runner.credentialsSecretName" . | quote }} } {{- with $s.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} - {{- if $s.metrics.enabled }} + # Serves /metrics, /healthz and /readyz. ports: - { name: metrics, containerPort: {{ $s.metrics.port }}, protocol: TCP } - {{- end }} + livenessProbe: + httpGet: { path: /healthz, port: metrics } + periodSeconds: 20 + failureThreshold: 3 + readinessProbe: + httpGet: { path: /readyz, port: metrics } + periodSeconds: 10 {{- with $s.resources }} resources: {{- toYaml . | nindent 12 }} @@ -81,7 +100,10 @@ spec: - name: sg-credentials secret: secretName: {{ include "sg-runner.credentialsSecretName" . }} - defaultMode: 0400 + # 0440, not the runners' 0400: the files are owned by root, and the + # scaler runs as a non-root UID. The pod's fsGroup makes them + # group-owned by its GID, so group read is what lets it read them. + defaultMode: 0440 {{- with $s.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/k8s/chart/sg-runner/templates/scaler-rbac.yaml b/k8s/chart/sg-runner/templates/scaler-rbac.yaml index a18a64a..f32d3c4 100644 --- a/k8s/chart/sg-runner/templates/scaler-rbac.yaml +++ b/k8s/chart/sg-runner/templates/scaler-rbac.yaml @@ -9,8 +9,8 @@ metadata: labels: {{- include "sg-runner.labels" $component | nindent 4 }} --- -# Only the runner StatefulSet's scale, PVCs and reap Jobs. Runner pods get no -# Kubernetes API access at all. +# Only the runner StatefulSet and its scale, PVCs, reap Jobs, its own lease and +# events. Runner pods get no Kubernetes API access at all. apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -35,6 +35,17 @@ rules: - apiGroups: ["batch"] resources: ["jobs"] verbs: ["create", "get", "delete"] + {{- if .Values.scaler.leaderElection.enabled }} + # Leader election: one active scaler per StatefulSet. create can't be + # restricted by name (the object doesn't exist yet), get/update can. + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + resourceNames: [{{ printf "%s-scaler" $sts | quote }}] + verbs: ["get", "update"] + {{- end }} # Scaling decisions as Kubernetes events. - apiGroups: ["", "events.k8s.io"] resources: ["events"] diff --git a/k8s/chart/sg-runner/values.schema.json b/k8s/chart/sg-runner/values.schema.json index 7c5a4f7..6a15c31 100644 --- a/k8s/chart/sg-runner/values.schema.json +++ b/k8s/chart/sg-runner/values.schema.json @@ -4,92 +4,230 @@ "type": "object", "definitions": { "quantity": { - "type": ["string", "integer"], + "type": [ + "string", + "integer" + ], "pattern": "^[0-9]+(\\.[0-9]+)?(m|k|Ki|M|Mi|G|Gi|T|Ti|P|Pi|E|Ei)?$" }, "storage": { "type": "object", - "required": ["size"], + "required": [ + "size" + ], "properties": { - "size": { "$ref": "#/definitions/quantity" }, - "storageClassName": { "type": "string" } + "size": { + "$ref": "#/definitions/quantity" + }, + "storageClassName": { + "type": "string" + } } }, "image": { "type": "object", "properties": { - "repository": { "type": "string" }, - "tag": { "type": ["string", "number"] }, - "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] } + "repository": { + "type": "string" + }, + "tag": { + "type": [ + "string", + "number" + ] + }, + "pullPolicy": { + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } } }, - "positiveInt": { "type": "integer", "minimum": 1 } + "positiveInt": { + "type": "integer", + "minimum": 1 + } }, "properties": { - "nameOverride": { "type": "string" }, - "fullnameOverride": { "type": "string", "maxLength": 52 }, + "nameOverride": { + "type": "string" + }, + "fullnameOverride": { + "type": "string", + "maxLength": 52 + }, "image": { "allOf": [ - { "$ref": "#/definitions/image" }, - { "required": ["repository"], "properties": { "repository": { "minLength": 1 } } } + { + "$ref": "#/definitions/image" + }, + { + "required": [ + "repository" + ], + "properties": { + "repository": { + "minLength": 1 + } + } + } ] }, - "imagePullSecrets": { "type": "array" }, + "imagePullSecrets": { + "type": "array" + }, "credentials": { "type": "object", "properties": { - "existingSecret": { "type": "string" }, - "create": { "type": "boolean" }, - "nodeToken": { "type": "string" }, - "organization": { "type": "string" }, - "runnerGroup": { "type": "string" }, - "apiUrl": { "type": "string", "pattern": "^(https?://.+)?$" } + "existingSecret": { + "type": "string" + }, + "create": { + "type": "boolean" + }, + "nodeToken": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "runnerGroup": { + "type": "string" + }, + "apiUrl": { + "type": "string", + "pattern": "^(https?://.+)?$" + } } }, "runner": { "type": "object", - "required": ["size"], + "required": [ + "size" + ], "properties": { - "replicas": { "type": "integer", "minimum": 0 }, - "size": { "enum": ["small", "medium", "large", "custom"] }, + "replicas": { + "type": "integer", + "minimum": 0 + }, + "size": { + "enum": [ + "small", + "medium", + "large", + "custom" + ] + }, "resources": { "type": "object", "properties": { - "requests": { "type": "object", "additionalProperties": { "$ref": "#/definitions/quantity" } }, - "limits": { "type": "object", "additionalProperties": { "$ref": "#/definitions/quantity" } } + "requests": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/quantity" + } + }, + "limits": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/quantity" + } + } } }, "workspace": { "type": "object", "properties": { - "sizeLimit": { "anyOf": [{ "$ref": "#/definitions/quantity" }, { "const": "" }, { "type": "null" }] }, - "request": { "anyOf": [{ "$ref": "#/definitions/quantity" }, { "const": "" }, { "type": "null" }] } + "sizeLimit": { + "anyOf": [ + { + "$ref": "#/definitions/quantity" + }, + { + "const": "" + }, + { + "type": "null" + } + ] + }, + "request": { + "anyOf": [ + { + "$ref": "#/definitions/quantity" + }, + { + "const": "" + }, + { + "type": "null" + } + ] + } } }, "storage": { "type": "object", "properties": { - "state": { "$ref": "#/definitions/storage" }, - "docker": { "$ref": "#/definitions/storage" } + "state": { + "$ref": "#/definitions/storage" + }, + "docker": { + "$ref": "#/definitions/storage" + } } }, "dockerd": { "type": "object", - "properties": { "securityContext": { "type": "object" } } - }, - "terminationGracePeriodSeconds": { "type": "integer", "minimum": 0 }, - "podAnnotations": { "type": "object" }, - "podLabels": { "type": "object" }, - "nodeSelector": { "type": "object" }, - "tolerations": { "type": "array" }, - "affinity": { "type": "object" } + "properties": { + "securityContext": { + "type": "object" + } + } + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + } + }, + "if": { + "properties": { + "size": { + "const": "custom" + } + } }, - "if": { "properties": { "size": { "const": "custom" } } }, "then": { "properties": { "resources": { - "required": ["limits"], - "properties": { "limits": { "required": ["cpu", "memory"] } } + "required": [ + "limits" + ], + "properties": { + "limits": { + "required": [ + "cpu", + "memory" + ] + } + } } } } @@ -97,64 +235,171 @@ "scaler": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, - "image": { "$ref": "#/definitions/image" }, - "command": { "type": "array", "items": { "type": "string" } }, - "min": { "$ref": "#/definitions/positiveInt" }, - "max": { "$ref": "#/definitions/positiveInt" }, - "headroomPct": { "type": "integer", "minimum": 0, "maximum": 100 }, - "idleSeconds": { "type": "integer", "minimum": 0 }, - "parkTTL": { "$ref": "#/definitions/positiveInt" }, - "stuckSeconds": { "type": "integer", "minimum": 0 }, - "settleTimeout": { "type": "integer", "minimum": 0 }, - "interval": { "$ref": "#/definitions/positiveInt" }, - "extraEnv": { "type": "array" }, + "enabled": { + "type": "boolean" + }, + "image": { + "$ref": "#/definitions/image" + }, + "command": { + "type": "array", + "items": { + "type": "string" + } + }, + "min": { + "$ref": "#/definitions/positiveInt" + }, + "max": { + "$ref": "#/definitions/positiveInt" + }, + "headroomPct": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "idleSeconds": { + "type": "integer", + "minimum": 0 + }, + "parkTTL": { + "$ref": "#/definitions/positiveInt" + }, + "stuckSeconds": { + "type": "integer", + "minimum": 0 + }, + "settleTimeout": { + "type": "integer", + "minimum": 0 + }, + "interval": { + "$ref": "#/definitions/positiveInt" + }, + "extraEnv": { + "type": "array" + }, "metrics": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, - "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "enabled": { + "type": "boolean" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, "serviceMonitor": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, - "interval": { "type": "string" }, - "labels": { "type": "object" } + "enabled": { + "type": "boolean" + }, + "interval": { + "type": "string" + }, + "labels": { + "type": "object" + } } } } }, - "resources": { "type": "object" }, - "podAnnotations": { "type": "object" }, - "podLabels": { "type": "object" }, - "podSecurityContext": { "type": "object" }, - "securityContext": { "type": "object" }, - "nodeSelector": { "type": "object" }, - "tolerations": { "type": "array" }, - "affinity": { "type": "object" } + "resources": { + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "replicas": { + "type": "integer", + "minimum": 1 + }, + "leaderElection": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + } } }, "networkPolicy": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, + "enabled": { + "type": "boolean" + }, "clusterCIDRs": { "type": "array", - "items": { "type": "string", "pattern": "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" } + "items": { + "type": "string", + "pattern": "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" + } + }, + "blockMetadata": { + "type": "boolean" + }, + "allowClusterDNS": { + "type": "boolean" + }, + "dnsSelector": { + "type": "object" + }, + "extraEgress": { + "type": "array" + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } }, - "blockMetadata": { "type": "boolean" }, - "allowClusterDNS": { "type": "boolean" }, - "dnsSelector": { "type": "object" }, - "extraEgress": { "type": "array" } + "required": [ + "enabled" + ] }, - "if": { "properties": { "enabled": { "const": true } }, "required": ["enabled"] }, - "then": { "properties": { "clusterCIDRs": { "minItems": 1 } } } + "then": { + "properties": { + "clusterCIDRs": { + "minItems": 1 + } + } + } }, "podDisruptionBudget": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, - "minAvailable": { "type": ["integer", "string"] } + "enabled": { + "type": "boolean" + }, + "minAvailable": { + "type": [ + "integer", + "string" + ] + } } } } diff --git a/k8s/chart/sg-runner/values.yaml b/k8s/chart/sg-runner/values.yaml index b50ddd4..822514b 100644 --- a/k8s/chart/sg-runner/values.yaml +++ b/k8s/chart/sg-runner/values.yaml @@ -109,13 +109,24 @@ scaler: # runners parked longer than parkTTL. enabled: true - # Defaults to the runner image (scaler.sh ships in it). Set when the scaler - # ships as its own image. + # The scaler's own image (Go, distroless, non-root). Empty repository = the + # runner image, which still ships scaler.sh (then set + # command: ["/opt/sg-runner/scaler.sh"] and relax the securityContext). image: - repository: "" - tag: "" + repository: sg-runner-scaler + # Empty = the chart's appVersion. + tag: dev pullPolicy: IfNotPresent - command: ["/opt/sg-runner/scaler.sh"] + # Empty = the image's entrypoint. + command: [] + + # Two replicas are safe with leader election: only the lease holder scales, + # the other takes over when it goes away. Without it, keep 1 (two scalers + # would fight over replicas). + replicas: 1 + leaderElection: + # Lease -scaler in the release namespace. + enabled: true # Runners kept awake. At least 1: SG pauses workflows while no runner is # ACTIVE. 2 avoids that pause while a single runner restarts. @@ -138,8 +149,9 @@ scaler: interval: 15 extraEnv: [] - # Prometheus metrics (being added to the scaler): containerPort 9090 named - # `metrics` and a Service. + # The scaler always listens on this port (container port `metrics`): + # /metrics, plus /healthz and /readyz for the probes. + # metrics.enabled only adds a Service (and optionally a ServiceMonitor). metrics: enabled: false port: 9090 @@ -154,8 +166,22 @@ scaler: limits: { memory: 128Mi } podAnnotations: {} podLabels: {} - podSecurityContext: {} - securityContext: {} + # The Go image is distroless and runs as non-root. runAsUser is explicit: + # the kubelet can't verify runAsNonRoot against a named USER (distroless + # `nonroot` = 65532). fsGroup makes the credentials Secret's files group-owned + # by that GID, so they're readable with mode 0440 (see the Deployment). + podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] nodeSelector: {} tolerations: [] affinity: {} From 66f809b5200b6e7623156a62f25943e57f9d1de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:46:52 +0200 Subject: [PATCH 12/43] feat: pass reap job pod settings and metrics address to the scaler --- k8s/chart/sg-runner/README.md | 44 +++++++++++++++---- k8s/chart/sg-runner/templates/_helpers.tpl | 19 ++++++++ .../sg-runner/templates/networkpolicy.yaml | 33 +++++++++++--- .../templates/scaler-deployment.yaml | 13 ++++++ k8s/chart/sg-runner/values.schema.json | 12 ++++- k8s/chart/sg-runner/values.yaml | 14 ++++-- 6 files changed, 117 insertions(+), 18 deletions(-) diff --git a/k8s/chart/sg-runner/README.md b/k8s/chart/sg-runner/README.md index efbe2c7..dc9749d 100644 --- a/k8s/chart/sg-runner/README.md +++ b/k8s/chart/sg-runner/README.md @@ -59,7 +59,7 @@ stays queued (the scaler warns). |---|---|---| | `nameOverride` / `fullnameOverride` | `""` | Rename resources; `fullnameOverride` also renames the StatefulSet (max 52 chars). | | `image.repository` / `.tag` / `.pullPolicy` | `sg-runner-k8s` / `dev` / `IfNotPresent` | Runner image; also used by reap Jobs. Empty tag = `appVersion`. | -| `imagePullSecrets` | `[]` | For runner and scaler pods (not reap Jobs). | +| `imagePullSecrets` | `[]` | For runner, scaler and reap Job pods. | | `credentials.existingSecret` | `""` | Secret with `SG_NODE_TOKEN`, `SG_ORGANIZATION`, `SG_RUNNER_GROUP`, optional `SG_BASE_API`. | | `credentials.create` | `false` | Create the Secret from `nodeToken`, `organization`, `runnerGroup`, `apiUrl`. Exactly one of this or `existingSecret`. | | `runner.replicas` | `1` | Only with `scaler.enabled=false`. | @@ -71,7 +71,7 @@ stays queued (the scaler warns). | `runner.storage.docker.size` / `.storageClassName` | `20Gi` / `""` | Docker data root PVC (image cache). | | `runner.dockerd.securityContext` | `{privileged: true}` | Nested Docker needs privileged for now. | | `runner.terminationGracePeriodSeconds` | `3600` | Max wait for running workflows when a runner stops. | -| `runner.podAnnotations` / `podLabels` / `nodeSelector` / `tolerations` / `affinity` | empty | Runner pod scheduling and metadata. | +| `runner.podAnnotations` / `podLabels` / `nodeSelector` / `tolerations` / `affinity` | empty | Runner pod scheduling and metadata; `nodeSelector` and `tolerations` also apply to reap Jobs. | | `scaler.enabled` | `true` | Scaler Deployment + RBAC. When on, the chart leaves the StatefulSet's `replicas` to it. | | `scaler.image.repository` / `.tag` / `.pullPolicy` | `sg-runner-scaler` / `dev` / `IfNotPresent` | Scaler image (Go, distroless, non-root). Empty repository = the runner image. | | `scaler.command` | `[]` | Empty = image entrypoint. `["/opt/sg-runner/scaler.sh"]` runs the shell scaler from the runner image. | @@ -86,7 +86,7 @@ stays queued (the scaler warns). | `scaler.settleTimeout` | `300` | `SCALER_SETTLE_TIMEOUT`: wait for a runner to connect. | | `scaler.interval` | `15` | `SCALER_INTERVAL`: seconds between decisions. | | `scaler.extraEnv` | `[]` | Extra env vars for the scaler. | -| `scaler.metrics.port` | `9090` | Container port `metrics`, always on: `/metrics`, `/healthz` (liveness), `/readyz` (readiness). | +| `scaler.metrics.port` | `9090` | `SCALER_METRICS_ADDR=:`, container port `metrics`, always on: `/metrics`, `/healthz` (liveness), `/readyz` (readiness). | | `scaler.metrics.enabled` | `false` | Adds a metrics Service. | | `scaler.metrics.serviceMonitor.enabled` / `.interval` / `.labels` | `false` / `30s` / `{}` | ServiceMonitor, only if `monitoring.coreos.com/v1` is served. | | `scaler.resources` | 10m / 32Mi, limit 128Mi | | @@ -94,19 +94,45 @@ stays queued (the scaler warns). | `scaler.securityContext` | read-only root FS, no privilege escalation, drop `ALL` | | | `scaler.podAnnotations` / `podLabels` / scheduling | empty | | | `networkPolicy.enabled` | `false` | Egress isolation for runner pods; needs an enforcing CNI. | -| `networkPolicy.clusterCIDRs` | `[]` | Pod, service, node CIDRs (+ API endpoint). Required when enabled. | +| `networkPolicy.clusterCIDRs` | `[]` | Pod, service, node CIDRs (+ API endpoint). IPv4 ones required when enabled; IPv6 ones open IPv6 egress except them. | | `networkPolicy.blockMetadata` | `true` | Deny `169.254.169.254/32`. | +| `networkPolicy.blockMetadataIPv6` | `false` | Deny `fd00:ec2::254/128`; only matters with IPv6 egress open. | | `networkPolicy.allowClusterDNS` / `dnsSelector` | `true` / kube-system `k8s-app: kube-dns` | Allow DNS (53/UDP+TCP) to the cluster resolver. | | `networkPolicy.extraEgress` | `[]` | Extra egress rules, as-is. | | `podDisruptionBudget.enabled` / `minAvailable` | `false` / `1` | Keep drains from evicting the last runner; with one awake runner it blocks drains, so pair with `scaler.min: 2`. | ## Notes -- **Replicas and the scaler.** With the scaler on, Helm upgrades keep the live - replica count. Turning the scaler on for an install that had it off drops - `replicas` from the manifest, which resets it to 1 (parks the rest). +- **Replicas and the scaler.** With the scaler on, the chart leaves the + StatefulSet's `replicas` out and Helm upgrades keep the live count. Turning + the scaler on for an install that had it off removes the field Helm managed, + which resets the StatefulSet to 1 replica: every other runner parks until the + scaler wakes them for queued work. - **Storage values are fixed after install**: `volumeClaimTemplates` are immutable on a StatefulSet. - **NetworkPolicy** covers runner pods, so workflow containers and agents - alike (nested traffic leaves through the pod IP). It doesn't cover reap Job - pods: `scaler.sh` labels the Job, not its pods. IPv4 only. + alike (nested traffic leaves through the pod IP), and reap Job pods, which + the scaler labels `app.kubernetes.io/component: reaper` (`SCALER_REAP_POD_LABELS`; + the shell `scaler.sh` doesn't, only the Go scaler). +- **IPv6 gap.** The policy is IPv4: it allows no IPv6 egress unless IPv6 + CIDRs are listed in `clusterCIDRs` (dual stack), which opens `::/0` except + those and, with `blockMetadataIPv6`, the AWS IPv6 metadata endpoint. Other + clouds' IPv6 metadata addresses aren't covered. + +## Migrating from the raw manifests + +The old `k8s/manifests/` used `app: sg-runner` selectors. A StatefulSet's +selector is immutable and Helm won't adopt objects it didn't create, so the +chart can't take over in place. The PVCs carry the runner identities and keep +their names (`state-sg-runner-N`, `docker-sg-runner-N`) as long as the release +is named `sg-runner`, so: + +1. Note the current replica count: `kubectl -n sg-runner get sts sg-runner`. +2. Delete the old scaler, then the StatefulSet (runners drain and park; PVCs + stay): `kubectl -n sg-runner delete deploy/sg-runner-scaler sts/sg-runner`, + and the old RBAC (`sa`, `role`, `rolebinding` named `sg-runner-scaler`). +3. Install the chart as release `sg-runner` in the same namespace with + `credentials.existingSecret=sg-runner-credentials` (`make up`). +4. The StatefulSet starts at 1 replica; the scaler wakes the other parked + runners as work queues (or reaps them after `parkTTL`). Without the + scaler, set `runner.replicas` to the noted count. diff --git a/k8s/chart/sg-runner/templates/_helpers.tpl b/k8s/chart/sg-runner/templates/_helpers.tpl index 441fe65..6dcfaf7 100644 --- a/k8s/chart/sg-runner/templates/_helpers.tpl +++ b/k8s/chart/sg-runner/templates/_helpers.tpl @@ -63,6 +63,25 @@ app.kubernetes.io/instance: {{ .ctx.Release.Name }} app.kubernetes.io/component: {{ .component }} {{- end }} +{{/* +Labels of reap Job pods: the runner's selector labels with component reaper, +so the NetworkPolicy covers them too. +*/}} +{{- define "sg-runner.reaperLabels" -}} +{{- include "sg-runner.selectorLabels" (dict "ctx" . "component" "reaper") }} +{{- end }} + +{{/* +imagePullSecrets names, comma-separated. +*/}} +{{- define "sg-runner.pullSecretNames" -}} +{{- $names := list }} +{{- range .Values.imagePullSecrets }} +{{- $names = append $names (kindIs "string" . | ternary . .name) }} +{{- end }} +{{- join "," $names }} +{{- end }} + {{/* Runner image reference; the scaler also uses it for reap Jobs. */}} diff --git a/k8s/chart/sg-runner/templates/networkpolicy.yaml b/k8s/chart/sg-runner/templates/networkpolicy.yaml index a0728ab..6c2a8ca 100644 --- a/k8s/chart/sg-runner/templates/networkpolicy.yaml +++ b/k8s/chart/sg-runner/templates/networkpolicy.yaml @@ -1,11 +1,17 @@ {{- if .Values.networkPolicy.enabled }} {{- $np := .Values.networkPolicy }} {{- $component := dict "ctx" . "component" "runner" }} -{{- if not $np.clusterCIDRs }} -{{- fail "networkPolicy.clusterCIDRs is required with networkPolicy.enabled=true: list the pod, service and node CIDRs (and the API server endpoint if outside them)" }} +{{- $v4 := list }} +{{- $v6 := list }} +{{- range $np.clusterCIDRs }} +{{- if contains ":" . }}{{ $v6 = append $v6 . }}{{ else }}{{ $v4 = append $v4 . }}{{ end }} +{{- end }} +{{- if not $v4 }} +{{- fail "networkPolicy.clusterCIDRs needs the IPv4 pod, service and node CIDRs (and the API server endpoint if outside them) with networkPolicy.enabled=true" }} {{- end }} # Workflow isolation. Nested workflow traffic leaves through the runner pod's -# IP, so this policy covers workflow containers and the agents alike: no +# IP, so this policy covers workflow containers and the agents alike (and reap +# Job pods, labelled component=reaper by the scaler): no # cluster-internal destinations, no cloud metadata, everything else (SG API, # AWS endpoints, VCS, registries, workflow targets) allowed. # Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...); others ignore @@ -19,7 +25,10 @@ metadata: spec: podSelector: matchLabels: - {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + app.kubernetes.io/name: {{ include "sg-runner.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + matchExpressions: + - { key: app.kubernetes.io/component, operator: In, values: [runner, reaper] } policyTypes: ["Ingress", "Egress"] # No ingress rules: runners only dial out. ingress: [] @@ -36,12 +45,26 @@ spec: - ipBlock: cidr: 0.0.0.0/0 except: - {{- range $np.clusterCIDRs }} + {{- range $v4 }} - {{ . }} {{- end }} {{- if $np.blockMetadata }} - 169.254.169.254/32 {{- end }} + {{- /* IPv6 egress only when the cluster's IPv6 CIDRs are listed (dual + stack); otherwise this policy allows no IPv6 at all. */}} + {{- if $v6 }} + - to: + - ipBlock: + cidr: ::/0 + except: + {{- range $v6 }} + - {{ . }} + {{- end }} + {{- if $np.blockMetadataIPv6 }} + - fd00:ec2::254/128 + {{- end }} + {{- end }} {{- with $np.extraEgress }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/k8s/chart/sg-runner/templates/scaler-deployment.yaml b/k8s/chart/sg-runner/templates/scaler-deployment.yaml index edb654f..130bae7 100644 --- a/k8s/chart/sg-runner/templates/scaler-deployment.yaml +++ b/k8s/chart/sg-runner/templates/scaler-deployment.yaml @@ -77,6 +77,19 @@ spec: # credentials Secret. - { name: SCALER_IMAGE, value: {{ include "sg-runner.image" . | quote }} } - { name: SCALER_CREDENTIALS_SECRET, value: {{ include "sg-runner.credentialsSecretName" . | quote }} } + # Reap Job pods: labelled like runners (component reaper) so the + # NetworkPolicy covers them, scheduled like runners (they mount a + # runner's state PVC, which may be bound to a node or zone). + - name: SCALER_REAP_POD_LABELS + value: {{ include "sg-runner.reaperLabels" . | fromYaml | toJson | quote }} + - name: SCALER_REAP_IMAGE_PULL_SECRETS + value: {{ include "sg-runner.pullSecretNames" . | quote }} + - name: SCALER_REAP_NODE_SELECTOR + value: {{ default dict .Values.runner.nodeSelector | toJson | quote }} + - name: SCALER_REAP_TOLERATIONS + value: {{ default list .Values.runner.tolerations | toJson | quote }} + # Serves /metrics, /healthz and /readyz. + - { name: SCALER_METRICS_ADDR, value: {{ printf ":%v" $s.metrics.port | quote }} } {{- with $s.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/k8s/chart/sg-runner/values.schema.json b/k8s/chart/sg-runner/values.schema.json index 6a15c31..05dd201 100644 --- a/k8s/chart/sg-runner/values.schema.json +++ b/k8s/chart/sg-runner/values.schema.json @@ -354,7 +354,14 @@ "type": "array", "items": { "type": "string", - "pattern": "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" + "anyOf": [ + { + "pattern": "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" + }, + { + "pattern": "^[0-9a-fA-F:]+/[0-9]{1,3}$" + } + ] } }, "blockMetadata": { @@ -368,6 +375,9 @@ }, "extraEgress": { "type": "array" + }, + "blockMetadataIPv6": { + "type": "boolean" } }, "if": { diff --git a/k8s/chart/sg-runner/values.yaml b/k8s/chart/sg-runner/values.yaml index 822514b..a4b3aaf 100644 --- a/k8s/chart/sg-runner/values.yaml +++ b/k8s/chart/sg-runner/values.yaml @@ -149,8 +149,9 @@ scaler: interval: 15 extraEnv: [] - # The scaler always listens on this port (container port `metrics`): - # /metrics, plus /healthz and /readyz for the probes. + # The scaler always listens on this port (container port `metrics`, + # SCALER_METRICS_ADDR=:): /metrics, plus /healthz and /readyz for the + # probes. # metrics.enabled only adds a Service (and optionally a ServiceMonitor). metrics: enabled: false @@ -189,7 +190,11 @@ scaler: # Egress isolation for runner pods. Nested workflow traffic leaves through the # pod IP, so this covers workflow containers and the agents alike. Needs a CNI # that enforces NetworkPolicy (Calico, Cilium, ...); others silently ignore it. -# Reap Job pods aren't covered (they carry no runner labels). +# Also covers reap Job pods (component=reaper, set by the scaler). +# +# IPv4 by default: no IPv6 egress is allowed at all. On dual-stack clusters, +# list the IPv6 pod/service/node CIDRs in clusterCIDRs too, which opens IPv6 +# egress except those. networkPolicy: enabled: false # Pod, service and node CIDRs, plus the Kubernetes API endpoint if it's @@ -197,6 +202,9 @@ networkPolicy: clusterCIDRs: [] # Block the cloud metadata endpoint (169.254.169.254). blockMetadata: true + # Also block AWS's IPv6 metadata endpoint (fd00:ec2::254). Only matters when + # IPv6 egress is open (IPv6 entries in clusterCIDRs). + blockMetadataIPv6: false # Allow DNS to the cluster resolver. Off = workflows and agents must resolve # through an external resolver. allowClusterDNS: true From b491f5f111f9f3722432e77abee56d0eea0f439f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:47:28 +0200 Subject: [PATCH 13/43] feat: port the scaler rules to go, with an sg api client --- k8s/scaler/go.mod | 3 + k8s/scaler/internal/decide/decide.go | 467 +++++++++++++++++++++ k8s/scaler/internal/decide/decide_test.go | 468 ++++++++++++++++++++++ k8s/scaler/internal/decide/state.go | 77 ++++ k8s/scaler/internal/sg/sg.go | 208 ++++++++++ k8s/scaler/internal/sg/sg_test.go | 181 +++++++++ 6 files changed, 1404 insertions(+) create mode 100644 k8s/scaler/go.mod create mode 100644 k8s/scaler/internal/decide/decide.go create mode 100644 k8s/scaler/internal/decide/decide_test.go create mode 100644 k8s/scaler/internal/decide/state.go create mode 100644 k8s/scaler/internal/sg/sg.go create mode 100644 k8s/scaler/internal/sg/sg_test.go diff --git a/k8s/scaler/go.mod b/k8s/scaler/go.mod new file mode 100644 index 0000000..4a4d10b --- /dev/null +++ b/k8s/scaler/go.mod @@ -0,0 +1,3 @@ +module github.com/StackGuardian/sg-runner/k8s/scaler + +go 1.24 diff --git a/k8s/scaler/internal/decide/decide.go b/k8s/scaler/internal/decide/decide.go new file mode 100644 index 0000000..4666af7 --- /dev/null +++ b/k8s/scaler/internal/decide/decide.go @@ -0,0 +1,467 @@ +// Package decide holds the scaler's rules as a pure function: an observation +// of SG and the cluster, the timers kept between loops and the configuration +// go in; the replicas to set, runner status changes, reap steps, warnings and +// the updated timers come out. Nothing here does I/O. +package decide + +import ( + "fmt" + "math" + "sort" + "time" +) + +// Runner statuses as SG reports them. +const ( + StatusActive = "ACTIVE" + StatusDraining = "DRAINING" +) + +// Event types, as Kubernetes names them. +const ( + EventNormal = "Normal" + EventWarning = "Warning" +) + +// Config holds the scaling rules' knobs. +type Config struct { + Min int + Max int + HeadroomPct int + IdleAfter time.Duration + StuckAfter time.Duration + SettleTimeout time.Duration + ParkTTL time.Duration +} + +// Runner is one SG container instance mapped to a StatefulSet ordinal. +type Runner struct { + Ordinal int + ID string + Status string + Connected bool + // Busy is running plus pending tasks. + Busy int + CPU float64 + CPUFree float64 + Mem float64 + MemFree float64 +} + +// FullyIdle reports a runner with nothing running and nothing reserved. +func (r Runner) FullyIdle() bool { + return r.Busy == 0 && r.CPUFree == r.CPU && r.MemFree == r.Mem +} + +// Size is the runner's registered CPU units / MiB, for messages. +func (r Runner) Size() string { + return fmt.Sprintf("%g/%g", r.CPU, r.Mem) +} + +// JobState is the state of a runner's reap Job. +type JobState int + +// Reap Job states. +const ( + JobNone JobState = iota + JobRunning + JobSucceeded + JobFailed +) + +// Claim is a runner's state PVC. +type Claim struct { + Name string + // ParkedAt is the parked-at stamp; zero when absent or unreadable. + ParkedAt time.Time + // Stamped reports a parked-at annotation present, readable or not. + Stamped bool + // DrainedByScaler reports that the scaler set this runner DRAINING. + DrainedByScaler bool +} + +// Observation is what one loop sees. +type Observation struct { + Now time.Time + Replicas int + Queued int + // MaxRunners is the runner group's MaxNumberOfRunners; 0 when unset. + MaxRunners int + // Runners are the SG runners that map to an ordinal of the StatefulSet. + Runners []Runner + // Claims are the state PVCs by ordinal. + Claims map[int]Claim + // ReapJobs are the reap Jobs by ordinal. + ReapJobs map[int]JobState +} + +// State is what the scaler remembers between loops. +type State struct { + // IdleSince: highest awake ordinal -> when it was first seen idle. + IdleSince map[int]time.Time + // StuckSince: since when workflows were queued while an awake runner + // sat fully idle; zero when not. + StuckSince time.Time + // UnconnectedSince: awake ordinal not connected to SG -> first seen so. + UnconnectedSince map[int]time.Time +} + +// Clone returns a deep copy. +func (s State) Clone() State { + c := State{StuckSince: s.StuckSince, IdleSince: map[int]time.Time{}, UnconnectedSince: map[int]time.Time{}} + for k, v := range s.IdleSince { + c.IdleSince[k] = v + } + for k, v := range s.UnconnectedSince { + c.UnconnectedSince[k] = v + } + return c +} + +// Scale is a replicas change. +type Scale int + +// Scale directions. +const ( + ScaleNone Scale = iota + ScaleOut + ScaleIn +) + +func (s Scale) String() string { + switch s { + case ScaleOut: + return "out" + case ScaleIn: + return "in" + } + return "none" +} + +// StatusChange sets a runner's status through SG. +type StatusChange struct { + Ordinal int + RunnerID string + // Claim is the runner's state PVC, which records drains by the scaler. + Claim string + Status string + // Reason is DrainedParkedRunner, DrainedUnconnectedRunner or Reactivated. + Reason string +} + +// ReapKind is one step of reaping parked runners. +type ReapKind int + +// Reap steps. +const ( + // ReapStamp stamps parked-at on a newly parked runner's PVC. + ReapStamp ReapKind = iota + // ReapClearStamp clears parked-at on a runner awake again. + ReapClearStamp + // ReapStartJob starts the Job deregistering a runner parked past the TTL. + ReapStartJob + // ReapCleanup deletes a deregistered runner's PVCs and its Job. + ReapCleanup + // ReapRetry deletes a failed Job so a later loop retries. + ReapRetry +) + +// ReapAction is one reap step for one ordinal. +type ReapAction struct { + Kind ReapKind + Ordinal int + Claim string + ParkedFor time.Duration +} + +// Event is something worth reporting as a Kubernetes Event. Key identifies +// repeats of the same condition so they can be rate-limited. +type Event struct { + Type string + Reason string + Key string + Message string +} + +// Decision is the outcome of one loop. +type Decision struct { + State State + + Max int + ReservedPct int + FreePct int + Awake int + Settling int + // Unconnected are the awake ordinals not connected to SG. + Unconnected []int + Stuck bool + + // Replicas to set; equal to the observed replicas when Scale is ScaleNone. + Replicas int + Scale Scale + // Why explains the scale decision (or why a scale out waits). + Why string + + StatusChanges []StatusChange + // ClearDrainMarks are state PVCs whose drained-by-scaler mark is stale. + ClearDrainMarks []string + Reap []ReapAction + Events []Event +} + +// ReservedPct is the share (0-100, floored) of CPU or memory, whichever is +// higher, reserved on ACTIVE and connected runners with an ordinal below +// the given one. 100 when there are none. +func ReservedPct(runners []Runner, below int) int { + var cpu, cpuFree, mem, memFree float64 + n := 0 + for _, r := range runners { + if r.Ordinal < below && r.Status == StatusActive && r.Connected { + n++ + cpu += r.CPU + cpuFree += r.CPUFree + mem += r.Mem + memFree += r.MemFree + } + } + if n == 0 { + return 100 + } + cpuPct := (cpu - cpuFree) * 100 / math.Max(cpu, 1) + memPct := (mem - memFree) * 100 / math.Max(mem, 1) + return int(math.Floor(math.Max(cpuPct, memPct))) +} + +// Decide applies the scaling rules to one observation. +func Decide(cfg Config, obs Observation, prev State) Decision { + now := obs.Now + st := prev.Clone() + d := Decision{State: st, Replicas: obs.Replicas} + replicas := obs.Replicas + + d.Max = cfg.Max + if obs.MaxRunners > 0 && obs.MaxRunners < d.Max { + d.Max = obs.MaxRunners + } + d.ReservedPct = ReservedPct(obs.Runners, replicas) + d.FreePct = 100 - d.ReservedPct + d.Awake = replicas + + byOrdinal := map[int][]Runner{} + for _, r := range obs.Runners { + byOrdinal[r.Ordinal] = append(byOrdinal[r.Ordinal], r) + } + connected := func(n int) bool { + for _, r := range byOrdinal[n] { + if r.Connected { + return true + } + } + return false + } + + // Settling: an awake runner not connected yet. Wait for it, or one queued + // workflow wakes several runners; past the settle timeout, decide without + // it, report it and drain it so SG places nothing there. + unconnected := map[int]bool{} + for n := 0; n < replicas; n++ { + if !connected(n) { + unconnected[n] = true + d.Unconnected = append(d.Unconnected, n) + } + } + for n := range st.UnconnectedSince { + if !unconnected[n] { + delete(st.UnconnectedSince, n) + } + } + for _, n := range d.Unconnected { + since, ok := st.UnconnectedSince[n] + if !ok { + since = now + st.UnconnectedSince[n] = now + } + waited := now.Sub(since) + if waited < cfg.SettleTimeout { + d.Settling++ + continue + } + d.Events = append(d.Events, Event{ + Type: EventWarning, Reason: "RunnerNotConnected", Key: fmt.Sprintf("RunnerNotConnected/%d", n), + Message: fmt.Sprintf("runner %d awake but not connected to SG for %s; scaling without it (check its pod)", + n, waited.Round(time.Second)), + }) + claim, hasClaim := obs.Claims[n] + for _, r := range byOrdinal[n] { + if r.Status == StatusActive && r.ID != "" && hasClaim { + d.StatusChanges = append(d.StatusChanges, StatusChange{ + Ordinal: n, RunnerID: r.ID, Claim: claim.Name, Status: StatusDraining, + Reason: "DrainedUnconnectedRunner", + }) + } + } + } + + // Stuck queue: queued while a runner sits fully idle past SG's retry + // delay means what's queued fits no runner of this size. + var idle *Runner + for i, r := range obs.Runners { + if r.Ordinal < replicas && r.Status == StatusActive && r.Connected && r.FullyIdle() { + idle = &obs.Runners[i] + break + } + } + if obs.Queued > 0 && idle != nil { + if st.StuckSince.IsZero() { + st.StuckSince = now + } + if waited := now.Sub(st.StuckSince); waited >= cfg.StuckAfter { + d.Stuck = true + d.Events = append(d.Events, Event{ + Type: EventWarning, Reason: "StuckQueue", Key: "StuckQueue", + Message: fmt.Sprintf("%d workflow(s) queued for %s while runner %d (CPU units/MiB %s) sits idle: "+ + "they need more than one runner pod offers. Ignoring them; raise the runner's resources to run them.", + obs.Queued, waited.Round(time.Second), idle.Ordinal, idle.Size()), + }) + } + } else { + st.StuckSince = time.Time{} + } + + // Scaling. + queueWantsOut := obs.Queued > 0 && idle == nil + headroomWantsOut := d.FreePct < cfg.HeadroomPct + belowMin := replicas < cfg.Min + switch { + case d.Settling > 0: + d.Why = fmt.Sprintf("waiting for %d runner(s) to connect", d.Settling) + case replicas < d.Max && (queueWantsOut || headroomWantsOut || belowMin): + switch { + case queueWantsOut: + d.Why = fmt.Sprintf("%d workflow(s) queued and no runner idle", obs.Queued) + case headroomWantsOut: + d.Why = fmt.Sprintf("%d%% free, below the %d%% headroom", d.FreePct, cfg.HeadroomPct) + default: + d.Why = fmt.Sprintf("below the minimum of %d", cfg.Min) + } + if obs.ReapJobs[replicas] != JobNone { + d.Why = fmt.Sprintf("scale out waits: runner %d is being deregistered", replicas) + break + } + d.Scale = ScaleOut + d.Replicas = replicas + 1 + st.IdleSince = map[int]time.Time{} + // The new runner starts idle: don't mistake it for a stuck queue. + st.StuckSince = time.Time{} + case replicas > cfg.Min && (obs.Queued == 0 || d.Stuck || replicas > d.Max): + top := replicas - 1 + for n := range st.IdleSince { + if n != top { + delete(st.IdleSince, n) + } + } + busy, seen := 0, len(byOrdinal[top]) > 0 + for _, r := range byOrdinal[top] { + busy += r.Busy + } + if !seen || busy > 0 { + delete(st.IdleSince, top) + break + } + since, ok := st.IdleSince[top] + if !ok { + since = now + st.IdleSince[top] = now + } + // Don't park into a scale out: the rest must keep the headroom. + freeWithoutTop := 100 - ReservedPct(obs.Runners, top) + if idleFor := now.Sub(since); idleFor >= cfg.IdleAfter && freeWithoutTop >= cfg.HeadroomPct { + d.Scale = ScaleIn + d.Replicas = top + d.Why = fmt.Sprintf("runner %d idle for %s", top, idleFor.Round(time.Second)) + delete(st.IdleSince, top) + } + } + + // Parked runners SG still reports ACTIVE: the pod went away without its + // preStop drain (node loss, force delete). Drain them so SG places + // nothing there, and mark the drain as the scaler's own. + for _, r := range obs.Runners { + if r.Ordinal < replicas || r.Status != StatusActive || r.Connected || r.ID == "" { + continue + } + claim, ok := obs.Claims[r.Ordinal] + if !ok { + continue + } + d.StatusChanges = append(d.StatusChanges, StatusChange{ + Ordinal: r.Ordinal, RunnerID: r.ID, Claim: claim.Name, Status: StatusDraining, + Reason: "DrainedParkedRunner", + }) + } + + // Undo the scaler's own drains once the runner is awake and connected + // again; drains by anyone else (no mark) stick. + for _, r := range obs.Runners { + claim, ok := obs.Claims[r.Ordinal] + if !ok || !claim.DrainedByScaler || r.Ordinal >= d.Replicas || !r.Connected || r.ID == "" { + continue + } + switch r.Status { + case StatusDraining: + d.StatusChanges = append(d.StatusChanges, StatusChange{ + Ordinal: r.Ordinal, RunnerID: r.ID, Claim: claim.Name, Status: StatusActive, + Reason: "Reactivated", + }) + case StatusActive: + // Active again (by someone, or a drain that failed): the mark is stale. + d.ClearDrainMarks = append(d.ClearDrainMarks, claim.Name) + } + } + + d.Reap = PlanReap(obs.Claims, obs.ReapJobs, d.Replicas, now, cfg.ParkTTL) + d.State = st + return d +} + +// PlanReap stamps newly parked runners, clears stamps of runners awake +// again and deregisters runners parked longer than ttl. +func PlanReap(claims map[int]Claim, jobs map[int]JobState, replicas int, now time.Time, ttl time.Duration) []ReapAction { + ordinals := make([]int, 0, len(claims)) + for n := range claims { + ordinals = append(ordinals, n) + } + sort.Ints(ordinals) + + var out []ReapAction + for _, n := range ordinals { + c := claims[n] + if n < replicas { + if c.Stamped { + out = append(out, ReapAction{Kind: ReapClearStamp, Ordinal: n, Claim: c.Name}) + } + continue + } + if c.ParkedAt.IsZero() { + out = append(out, ReapAction{Kind: ReapStamp, Ordinal: n, Claim: c.Name}) + continue + } + parkedFor := now.Sub(c.ParkedAt) + if parkedFor < ttl { + continue + } + a := ReapAction{Ordinal: n, Claim: c.Name, ParkedFor: parkedFor} + switch jobs[n] { + case JobNone: + a.Kind = ReapStartJob + case JobSucceeded: + a.Kind = ReapCleanup + case JobFailed: + a.Kind = ReapRetry + default: + continue + } + out = append(out, a) + } + return out +} diff --git a/k8s/scaler/internal/decide/decide_test.go b/k8s/scaler/internal/decide/decide_test.go new file mode 100644 index 0000000..8661d18 --- /dev/null +++ b/k8s/scaler/internal/decide/decide_test.go @@ -0,0 +1,468 @@ +package decide + +import ( + "reflect" + "strings" + "testing" + "time" +) + +var t0 = time.Unix(1_700_000_000, 0) + +func cfg() Config { + return Config{ + Min: 1, Max: 10, HeadroomPct: 0, + IdleAfter: 60 * time.Second, StuckAfter: 180 * time.Second, + SettleTimeout: 300 * time.Second, ParkTTL: 12 * time.Hour, + } +} + +// Runner sizes as SG registers them: CPU units and MiB. +const cpu, mem = 2048, 4096 + +func idle(n int) Runner { + return Runner{Ordinal: n, ID: id(n), Status: StatusActive, Connected: true, CPU: cpu, CPUFree: cpu, Mem: mem, MemFree: mem} +} + +// full is a runner one workflow fills completely. +func full(n int) Runner { + r := idle(n) + r.Busy, r.CPUFree, r.MemFree = 1, 0, 0 + return r +} + +// half is a runner one workflow fills halfway. +func half(n int) Runner { + r := idle(n) + r.Busy, r.CPUFree, r.MemFree = 1, cpu/2, mem/2 + return r +} + +func unconnected(n int, status string) Runner { + r := idle(n) + r.Connected, r.Status = false, status + return r +} + +func id(n int) string { return "runner-id-" + string(rune('a'+n)) } + +func claims(ordinals ...int) map[int]Claim { + m := map[int]Claim{} + for _, n := range ordinals { + m[n] = Claim{Name: claimName(n)} + } + return m +} + +func claimName(n int) string { return "state-sg-runner-" + string(rune('0'+n)) } + +func obs(at time.Duration, replicas, queued int, runners ...Runner) Observation { + cl := map[int]Claim{} + for n := 0; n < replicas; n++ { + cl[n] = Claim{Name: claimName(n)} + } + return Observation{Now: t0.Add(at), Replicas: replicas, Queued: queued, Runners: runners, Claims: cl} +} + +func hasEvent(d Decision, reason string) bool { + for _, e := range d.Events { + if e.Reason == reason { + return true + } + } + return false +} + +func TestReservedPct(t *testing.T) { + tests := []struct { + name string + runners []Runner + below int + want int + }{ + {"none awake", nil, 3, 100}, + {"only unconnected", []Runner{unconnected(0, StatusActive)}, 1, 100}, + {"only draining", []Runner{func() Runner { r := idle(0); r.Status = StatusDraining; return r }()}, 1, 100}, + {"idle", []Runner{idle(0)}, 1, 0}, + {"full", []Runner{full(0)}, 1, 100}, + {"half of two", []Runner{full(0), idle(1)}, 2, 50}, + {"ordinal at or above excluded", []Runner{idle(0), full(1)}, 1, 0}, + {"memory higher than cpu", []Runner{func() Runner { r := idle(0); r.MemFree = mem / 4; return r }()}, 1, 75}, + {"floored", []Runner{func() Runner { r := idle(0); r.CPUFree = cpu * 2 / 3; return r }()}, 1, 33}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ReservedPct(tt.runners, tt.below); got != tt.want { + t.Errorf("ReservedPct = %d, want %d", got, tt.want) + } + }) + } +} + +// Single loops, each rule on its own. +func TestDecideRules(t *testing.T) { + tests := []struct { + name string + cfg func(*Config) + obs Observation + state State + wantScale Scale + wantRepl int + wantEvent string + }{ + {name: "queued, runner busy: out", obs: obs(0, 1, 1, full(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "queued, runner fully idle: wait for SG's retry", obs: obs(0, 1, 1, idle(0)), wantRepl: 1}, + {name: "queued, half-busy runner counts as not idle: out", obs: obs(0, 1, 1, half(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "nothing queued, no headroom: nothing", obs: obs(0, 1, 0, full(0)), wantRepl: 1}, + {name: "headroom: free below it: out", cfg: func(c *Config) { c.HeadroomPct = 60 }, obs: obs(0, 1, 0, half(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "headroom: free at it: nothing", cfg: func(c *Config) { c.HeadroomPct = 50 }, obs: obs(0, 1, 0, half(0)), wantRepl: 1}, + {name: "at SCALER_MAX: no out", cfg: func(c *Config) { c.Max = 2 }, obs: obs(0, 2, 1, full(0), full(1)), wantRepl: 2}, + {name: "at MaxNumberOfRunners: no out", obs: func() Observation { + o := obs(0, 2, 1, full(0), full(1)) + o.MaxRunners = 2 + return o + }(), wantRepl: 2}, + {name: "MaxNumberOfRunners above SCALER_MAX: SCALER_MAX wins", cfg: func(c *Config) { c.Max = 2 }, obs: func() Observation { + o := obs(0, 2, 1, full(0), full(1)) + o.MaxRunners = 5 + return o + }(), wantRepl: 2}, + {name: "below SCALER_MIN: out", cfg: func(c *Config) { c.Min = 2 }, obs: obs(0, 1, 0, idle(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "zero replicas: up to the minimum", obs: obs(0, 0, 0), wantScale: ScaleOut, wantRepl: 1}, + {name: "reap Job on next ordinal: out waits", obs: func() Observation { + o := obs(0, 1, 1, full(0)) + o.ReapJobs = map[int]JobState{1: JobRunning} + return o + }(), wantRepl: 1}, + {name: "reap Job on another ordinal: out", obs: func() Observation { + o := obs(0, 1, 1, full(0)) + o.ReapJobs = map[int]JobState{2: JobRunning} + return o + }(), wantScale: ScaleOut, wantRepl: 2}, + {name: "at SCALER_MIN, idle long: no in", obs: obs(time.Hour, 1, 0, idle(0)), state: State{IdleSince: map[int]time.Time{0: t0}}, wantRepl: 1}, + {name: "top idle long enough: in", obs: obs(60*time.Second, 2, 0, idle(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantScale: ScaleIn, wantRepl: 1}, + {name: "top idle not long enough: nothing", obs: obs(59*time.Second, 2, 0, idle(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "top busy: nothing", obs: obs(time.Hour, 2, 0, idle(0), full(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "top unknown to SG: not idle", obs: obs(time.Hour, 2, 0, idle(0)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "queued: no in", obs: obs(time.Hour, 2, 1, full(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "headroom without top too low: no in", cfg: func(c *Config) { c.HeadroomPct = 20 }, + obs: obs(time.Hour, 2, 0, full(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "above max (MaxNumberOfRunners lowered), queued: idle top still parks", obs: func() Observation { + o := obs(time.Hour, 3, 1, full(0), full(1), idle(2)) + o.MaxRunners = 2 + return o + }(), state: State{IdleSince: map[int]time.Time{2: t0}}, wantScale: ScaleIn, wantRepl: 2}, + {name: "settling: no decision", obs: obs(0, 2, 1, full(0)), wantRepl: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := cfg() + if tt.cfg != nil { + tt.cfg(&c) + } + d := Decide(c, tt.obs, tt.state) + if d.Scale != tt.wantScale || d.Replicas != tt.wantRepl { + t.Errorf("got scale %v replicas %d (%s), want %v %d", d.Scale, d.Replicas, d.Why, tt.wantScale, tt.wantRepl) + } + if tt.wantEvent != "" && !hasEvent(d, tt.wantEvent) { + t.Errorf("missing event %s in %+v", tt.wantEvent, d.Events) + } + }) + } +} + +func TestScaleOutResetsTimers(t *testing.T) { + st := State{IdleSince: map[int]time.Time{0: t0}, StuckSince: t0} + d := Decide(cfg(), obs(10*time.Second, 1, 1, full(0)), st) + if d.Scale != ScaleOut { + t.Fatalf("want scale out, got %v", d.Scale) + } + if len(d.State.IdleSince) != 0 || !d.State.StuckSince.IsZero() { + t.Errorf("timers not reset: %+v", d.State) + } + if len(st.IdleSince) != 1 { + t.Error("Decide modified the previous state") + } +} + +// step runs one loop and carries the state and replicas forward. +type sim struct { + t *testing.T + cfg Config + state State + repl int +} + +func (s *sim) loop(at time.Duration, queued int, runners ...Runner) Decision { + s.t.Helper() + d := Decide(s.cfg, obs(at, s.repl, queued, runners...), s.state) + s.state, s.repl = d.State, d.Replicas + return d +} + +// A workflow bigger than any runner pod: the idle runner never takes it. +func TestOversizedWorkflow(t *testing.T) { + s := &sim{t: t, cfg: cfg(), repl: 2} + // Two runners awake, both idle, one oversized workflow queued. + if d := s.loop(0, 1, idle(0), idle(1)); d.Scale != ScaleNone || d.Stuck { + t.Fatalf("t=0: want nothing, got %v stuck=%v", d.Scale, d.Stuck) + } + if d := s.loop(179*time.Second, 1, idle(0), idle(1)); d.Stuck { + t.Fatal("stuck before 180s") + } + d := s.loop(180*time.Second, 1, idle(0), idle(1)) + if !d.Stuck || !hasEvent(d, "StuckQueue") { + t.Fatalf("want stuck + StuckQueue at 180s, got stuck=%v events=%+v", d.Stuck, d.Events) + } + if !strings.Contains(d.Events[0].Message, "2048/4096") { + t.Errorf("stuck message lacks the runner size: %s", d.Events[0].Message) + } + // Stuck: the queue is ignored, so the extra runner parks once idle long enough. + if d := s.loop(239*time.Second, 1, idle(0), idle(1)); d.Scale != ScaleNone { + t.Fatalf("parked before idle timeout: %v", d.Scale) + } + if d := s.loop(240*time.Second, 1, idle(0), idle(1)); d.Scale != ScaleIn || d.Replicas != 1 { + t.Fatalf("want park to 1 while stuck, got %v %d", d.Scale, d.Replicas) + } + // Never below the minimum, never out for the stuck workflow. + for at := 255 * time.Second; at < time.Hour; at += 15 * time.Second { + if d := s.loop(at, 1, idle(0)); d.Scale != ScaleNone { + t.Fatalf("t=%s: want nothing, got %v", at, d.Scale) + } + } + // Queue drained (cancelled): stuck clears. + if d := s.loop(time.Hour, 0, idle(0)); d.Stuck || !d.State.StuckSince.IsZero() { + t.Error("stuck not cleared once the queue is empty") + } +} + +// A queued workflow with the runner busy wakes exactly one more runner, and +// no second one while the new runner waits idle for SG's retry. +func TestNoOvershoot(t *testing.T) { + s := &sim{t: t, cfg: cfg(), repl: 1} + if d := s.loop(0, 1, full(0)); d.Scale != ScaleOut || d.Replicas != 2 { + t.Fatalf("want +1, got %v %d", d.Scale, d.Replicas) + } + // Runner 1 starting: not in SG yet, then registered but not connected. + if d := s.loop(15*time.Second, 1, full(0)); d.Scale != ScaleNone || d.Settling != 1 { + t.Fatalf("want settling, got %v settling=%d", d.Scale, d.Settling) + } + if d := s.loop(30*time.Second, 1, full(0), unconnected(1, StatusActive)); d.Scale != ScaleNone || d.Settling != 1 { + t.Fatalf("want settling, got %v settling=%d", d.Scale, d.Settling) + } + // Connected and idle: SG's retry will place the workflow there. + for at := 45 * time.Second; at < 150*time.Second; at += 15 * time.Second { + if d := s.loop(at, 1, full(0), idle(1)); d.Scale != ScaleNone { + t.Fatalf("t=%s: overshoot: %v to %d", at, d.Scale, d.Replicas) + } + } + // Placed. + if d := s.loop(150*time.Second, 0, full(0), full(1)); d.Scale != ScaleNone || d.Replicas != 2 { + t.Fatalf("want 2 runners, got %v %d", d.Scale, d.Replicas) + } +} + +// Two workflows that each fill a runner: headroom 0 ends with 2 runners, +// headroom 20 with a warm third. +func TestHeadroom(t *testing.T) { + for _, tc := range []struct { + headroom int + want int + }{{0, 2}, {20, 3}} { + c := cfg() + c.HeadroomPct = tc.headroom + s := &sim{t: t, cfg: c, repl: 1} + s.loop(0, 2, idle(0)) // runner 0 idle: SG places one + s.loop(15*time.Second, 1, full(0)) // one left queued: +1 + s.loop(30*time.Second, 1, full(0)) // runner 1 settling + s.loop(45*time.Second, 1, full(0), idle(1)) // idle: SG places the second + s.loop(60*time.Second, 0, full(0), full(1)) // both full + var runners = []Runner{full(0), full(1)} + if s.repl == 3 { + runners = append(runners, idle(2)) + s.loop(75*time.Second, 0, full(0), full(1)) // runner 2 settling + } + for at := 90 * time.Second; at < time.Hour; at += 15 * time.Second { + s.loop(at, 0, runners...) + } + if s.repl != tc.want { + t.Errorf("headroom %d: %d runners, want %d", tc.headroom, s.repl, tc.want) + } + } +} + +// A runner that never connects: waited for, then left out, warned about and +// drained; then it connects and the scaler undoes its own drain. +func TestSettleTimeout(t *testing.T) { + s := &sim{t: t, cfg: cfg(), repl: 2} + broken := unconnected(1, StatusActive) + d := s.loop(0, 1, full(0), broken) + if d.Scale != ScaleNone || d.Settling != 1 || len(d.StatusChanges) != 0 { + t.Fatalf("want settling, got %+v", d) + } + if d := s.loop(299*time.Second, 1, full(0), broken); d.Scale != ScaleNone || d.Settling != 1 { + t.Fatalf("decided before the settle timeout: %v", d.Scale) + } + d = s.loop(300*time.Second, 1, full(0), broken) + if d.Settling != 0 || !hasEvent(d, "RunnerNotConnected") { + t.Fatalf("want timed out + warning, got settling=%d events=%+v", d.Settling, d.Events) + } + want := []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusDraining, Reason: "DrainedUnconnectedRunner"}} + if !reflect.DeepEqual(d.StatusChanges, want) { + t.Fatalf("status changes = %+v, want %+v", d.StatusChanges, want) + } + if d.Scale != ScaleOut || d.Replicas != 3 { + t.Fatalf("broken runner should not block scaling: %v %d", d.Scale, d.Replicas) + } + // Already drained: no second drain, still warned. + drained := unconnected(1, StatusDraining) + d = Decide(s.cfg, obs(400*time.Second, 2, 0, full(0), drained), d.State) + if len(d.StatusChanges) != 0 || !hasEvent(d, "RunnerNotConnected") { + t.Fatalf("want warning only, got %+v", d) + } +} + +func TestParkedRunnerDrains(t *testing.T) { + mark := func(o Observation, n int) Observation { + c := o.Claims[n] + c.DrainedByScaler = true + o.Claims[n] = c + return o + } + withClaim := func(o Observation, n int) Observation { + o.Claims[n] = Claim{Name: claimName(n), ParkedAt: t0, Stamped: true} + return o + } + tests := []struct { + name string + obs Observation + want []StatusChange + wantClear []string + }{ + {name: "parked, ACTIVE, gone: drain", + obs: withClaim(obs(0, 1, 0, full(0), unconnected(1, StatusActive)), 1), + want: []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusDraining, Reason: "DrainedParkedRunner"}}}, + {name: "parked, ACTIVE, still connected (preStop running): leave it", + obs: withClaim(obs(0, 1, 0, full(0), idle(1)), 1)}, + {name: "parked, DRAINING: leave it", + obs: withClaim(obs(0, 1, 0, full(0), unconnected(1, StatusDraining)), 1)}, + {name: "parked, ACTIVE, no PVC (reaped): leave it", + obs: obs(0, 1, 0, full(0), unconnected(1, StatusActive))}, + {name: "parked, marked but still ACTIVE (drain failed): drain again", + obs: mark(withClaim(obs(0, 1, 0, full(0), unconnected(1, StatusActive)), 1), 1), + want: []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusDraining, Reason: "DrainedParkedRunner"}}}, + {name: "woke, DRAINING, marked: reactivate", + obs: mark(obs(0, 2, 0, full(0), func() Runner { r := full(1); r.Status = StatusDraining; return r }()), 1), + want: []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusActive, Reason: "Reactivated"}}}, + {name: "woke, DRAINING, not marked (drained by someone else): leave it", + obs: obs(0, 2, 0, full(0), func() Runner { r := full(1); r.Status = StatusDraining; return r }())}, + {name: "woke, DRAINING, marked, not connected yet: wait", + obs: mark(obs(0, 2, 0, full(0), unconnected(1, StatusDraining)), 1)}, + {name: "woke, ACTIVE, marked: clear the stale mark", + obs: mark(obs(0, 2, 0, full(0), full(1)), 1), wantClear: []string{claimName(1)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := Decide(cfg(), tt.obs, State{}) + if !reflect.DeepEqual(d.StatusChanges, tt.want) { + t.Errorf("status changes = %+v, want %+v", d.StatusChanges, tt.want) + } + if !reflect.DeepEqual(d.ClearDrainMarks, tt.wantClear) { + t.Errorf("clear marks = %v, want %v", d.ClearDrainMarks, tt.wantClear) + } + }) + } +} + +// The runner being parked this loop is not reactivated. +func TestNoReactivateWhileParking(t *testing.T) { + r := idle(1) + r.Status = StatusDraining + o := obs(time.Hour, 2, 0, idle(0), r) + o.Claims[1] = Claim{Name: claimName(1), DrainedByScaler: true} + d := Decide(cfg(), o, State{IdleSince: map[int]time.Time{1: t0}}) + if d.Scale != ScaleIn || len(d.StatusChanges) != 0 { + t.Errorf("got %v changes=%+v", d.Scale, d.StatusChanges) + } +} + +func TestIdleTimerTracksTopOnly(t *testing.T) { + d := Decide(cfg(), obs(0, 3, 0, idle(0), idle(1), full(2)), State{IdleSince: map[int]time.Time{1: t0.Add(-time.Hour)}}) + if len(d.State.IdleSince) != 0 { + t.Errorf("stale idle timers kept: %v", d.State.IdleSince) + } + d = Decide(cfg(), obs(0, 3, 0, idle(0), idle(1), idle(2)), State{}) + if !d.State.IdleSince[2].Equal(t0) || len(d.State.IdleSince) != 1 { + t.Errorf("idle timer for top not started: %v", d.State.IdleSince) + } +} + +func TestPlanReap(t *testing.T) { + ttl := 12 * time.Hour + now := t0.Add(ttl) + cl := map[int]Claim{ + 0: {Name: "s-0", Stamped: true, ParkedAt: t0}, // awake: clear + 1: {Name: "s-1"}, // awake: nothing + 2: {Name: "s-2"}, // parked: stamp + 3: {Name: "s-3", Stamped: true}, // unreadable stamp: restamp + 4: {Name: "s-4", Stamped: true, ParkedAt: now.Add(-time.Hour)}, // within TTL + 5: {Name: "s-5", Stamped: true, ParkedAt: t0}, // past TTL, no Job: start + 6: {Name: "s-6", Stamped: true, ParkedAt: t0}, // Job running + 7: {Name: "s-7", Stamped: true, ParkedAt: t0}, // Job done: cleanup + 8: {Name: "s-8", Stamped: true, ParkedAt: t0}, // Job failed: retry + } + jobs := map[int]JobState{6: JobRunning, 7: JobSucceeded, 8: JobFailed} + got := PlanReap(cl, jobs, 2, now, ttl) + want := []ReapAction{ + {Kind: ReapClearStamp, Ordinal: 0, Claim: "s-0"}, + {Kind: ReapStamp, Ordinal: 2, Claim: "s-2"}, + {Kind: ReapStamp, Ordinal: 3, Claim: "s-3"}, + {Kind: ReapStartJob, Ordinal: 5, Claim: "s-5", ParkedFor: ttl}, + {Kind: ReapCleanup, Ordinal: 7, Claim: "s-7", ParkedFor: ttl}, + {Kind: ReapRetry, Ordinal: 8, Claim: "s-8", ParkedFor: ttl}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("PlanReap =\n%+v\nwant\n%+v", got, want) + } +} + +// Reaping uses the replicas after this loop's scale in. +func TestDecideStampsJustParked(t *testing.T) { + o := obs(time.Hour, 2, 0, idle(0), idle(1)) + d := Decide(cfg(), o, State{IdleSince: map[int]time.Time{1: t0}}) + want := []ReapAction{{Kind: ReapStamp, Ordinal: 1, Claim: claimName(1)}} + if d.Scale != ScaleIn || !reflect.DeepEqual(d.Reap, want) { + t.Errorf("got %v reap=%+v", d.Scale, d.Reap) + } +} + +func TestStateRoundTrip(t *testing.T) { + s := State{ + IdleSince: map[int]time.Time{2: t0}, + StuckSince: t0.Add(time.Minute), + UnconnectedSince: map[int]time.Time{0: t0, 3: t0.Add(time.Second)}, + } + ann := map[string]string{} + for k, v := range EncodeState(s) { + if v != nil { + ann[k] = *v + } + } + if ann[AnnotationIdleSince] != `{"2":1700000000}` { + t.Errorf("idle annotation = %q", ann[AnnotationIdleSince]) + } + if got := DecodeState(ann); !reflect.DeepEqual(got, s) { + t.Errorf("round trip = %+v, want %+v", got, s) + } + empty := EncodeState(State{}) + for _, k := range StateAnnotations { + if v, ok := empty[k]; !ok || v != nil { + t.Errorf("empty state should remove %s", k) + } + } + bad := DecodeState(map[string]string{AnnotationIdleSince: "{", AnnotationStuckSince: "x", + AnnotationUnconnectedSince: `{"a":1,"-1":2,"4":0,"5":7}`}) + if len(bad.IdleSince) != 0 || !bad.StuckSince.IsZero() || len(bad.UnconnectedSince) != 1 { + t.Errorf("bad annotations decoded to %+v", bad) + } +} diff --git a/k8s/scaler/internal/decide/state.go b/k8s/scaler/internal/decide/state.go new file mode 100644 index 0000000..1f795ae --- /dev/null +++ b/k8s/scaler/internal/decide/state.go @@ -0,0 +1,77 @@ +package decide + +import ( + "encoding/json" + "strconv" + "time" +) + +// Annotations on the StatefulSet that keep the scaler's timers across +// restarts and leader changes. Values are Unix epochs; per-ordinal timers +// are JSON objects keyed by ordinal. +const ( + AnnotationIdleSince = "sg-runner.stackguardian.io/scaler-idle-since" + AnnotationStuckSince = "sg-runner.stackguardian.io/scaler-stuck-since" + AnnotationUnconnectedSince = "sg-runner.stackguardian.io/scaler-unconnected-since" +) + +// StateAnnotations lists every annotation the state is stored in. +var StateAnnotations = []string{AnnotationIdleSince, AnnotationStuckSince, AnnotationUnconnectedSince} + +// DecodeState reads the timers from StatefulSet annotations. Unreadable +// values are dropped: a timer that restarts is the safe failure. +func DecodeState(annotations map[string]string) State { + s := State{IdleSince: decodeTimers(annotations[AnnotationIdleSince]), + UnconnectedSince: decodeTimers(annotations[AnnotationUnconnectedSince])} + if v, err := strconv.ParseInt(annotations[AnnotationStuckSince], 10, 64); err == nil && v > 0 { + s.StuckSince = time.Unix(v, 0) + } + return s +} + +// EncodeState renders the timers as annotation values; an empty timer maps +// to nil (remove the annotation). +func EncodeState(s State) map[string]*string { + out := map[string]*string{ + AnnotationIdleSince: encodeTimers(s.IdleSince), + AnnotationUnconnectedSince: encodeTimers(s.UnconnectedSince), + AnnotationStuckSince: nil, + } + if !s.StuckSince.IsZero() { + v := strconv.FormatInt(s.StuckSince.Unix(), 10) + out[AnnotationStuckSince] = &v + } + return out +} + +func decodeTimers(raw string) map[int]time.Time { + out := map[int]time.Time{} + if raw == "" { + return out + } + var m map[string]int64 + if json.Unmarshal([]byte(raw), &m) != nil { + return out + } + for k, v := range m { + n, err := strconv.Atoi(k) + if err != nil || n < 0 || v <= 0 { + continue + } + out[n] = time.Unix(v, 0) + } + return out +} + +func encodeTimers(t map[int]time.Time) *string { + if len(t) == 0 { + return nil + } + m := make(map[string]int64, len(t)) + for n, at := range t { + m[strconv.Itoa(n)] = at.Unix() + } + b, _ := json.Marshal(m) // map keys are sorted: stable output + v := string(b) + return &v +} diff --git a/k8s/scaler/internal/sg/sg.go b/k8s/scaler/internal/sg/sg.go new file mode 100644 index 0000000..6fe76ee --- /dev/null +++ b/k8s/scaler/internal/sg/sg.go @@ -0,0 +1,208 @@ +// Package sg is a minimal client for the StackGuardian runner group API, +// authenticated with the runner group's node token. +// +// The runner group response carries secrets (the group's runner token), so +// responses are decoded into structs holding only the fields the scaler +// needs, and neither bodies nor decoder messages end up in errors. +package sg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// DefaultBaseAPI is used when the credentials don't name one. +const DefaultBaseAPI = "https://api.app.stackguardian.io/api/v1" + +// maxBody caps what is read from a response. +const maxBody = 16 << 20 + +// Credentials identify the runner group. +type Credentials struct { + Token string + Organization string + RunnerGroup string + BaseAPI string +} + +// LoadCredentials reads the credentials Secret mounted as one file per key. +func LoadCredentials(dir string) (Credentials, error) { + read := func(key string, required bool) (string, error) { + b, err := os.ReadFile(filepath.Join(dir, key)) + if err != nil { + if !required && errors.Is(err, os.ErrNotExist) { + return "", nil + } + return "", fmt.Errorf("read credential %s: %w", key, err) + } + v := strings.TrimSpace(string(b)) + if required && v == "" { + return "", fmt.Errorf("credential %s is empty", key) + } + return v, nil + } + var c Credentials + var err error + if c.Token, err = read("SG_NODE_TOKEN", true); err != nil { + return c, err + } + if c.Organization, err = read("SG_ORGANIZATION", true); err != nil { + return c, err + } + if c.RunnerGroup, err = read("SG_RUNNER_GROUP", true); err != nil { + return c, err + } + if c.BaseAPI, err = read("SG_BASE_API", false); err != nil { + return c, err + } + if c.BaseAPI == "" { + c.BaseAPI = DefaultBaseAPI + } + return c, nil +} + +// Client calls the runner group API. +type Client struct { + creds Credentials + http *http.Client +} + +// New returns a client; a nil http.Client gets a 10s timeout. +func New(creds Credentials, hc *http.Client) *Client { + if hc == nil { + hc = &http.Client{Timeout: 10 * time.Second} + } + creds.BaseAPI = strings.TrimRight(creds.BaseAPI, "/") + return &Client{creds: creds, http: hc} +} + +// Resource is a registered or remaining resource of a runner. +type Resource struct { + Name string `json:"name"` + IntegerValue float64 `json:"integerValue"` +} + +// Instance is a runner (ECS container instance) of the group. +type Instance struct { + RunnerID string `json:"runnerId"` + Status string `json:"status"` + AgentConnected bool `json:"agentConnected"` + RunningTasksCount int `json:"runningTasksCount"` + PendingTasksCount int `json:"pendingTasksCount"` + RegisteredResources []Resource `json:"registeredResources"` + RemainingResources []Resource `json:"remainingResources"` + InstanceDetails []struct { + ComputerName string `json:"ComputerName"` + } `json:"instanceDetails"` +} + +// ComputerName is the runner's host name: the pod's FQDN for k8s runners. +func (i Instance) ComputerName() string { + if len(i.InstanceDetails) == 0 { + return "" + } + return i.InstanceDetails[0].ComputerName +} + +// ResourceValue returns the named resource's value from a list, 0 if absent. +func ResourceValue(list []Resource, name string) float64 { + for _, r := range list { + if r.Name == name { + return r.IntegerValue + } + } + return 0 +} + +// RunnerGroup is the part of the runner group the scaler uses. +type RunnerGroup struct { + QueuedWorkflowsCount int `json:"QueuedWorkflowsCount"` + MaxNumberOfRunners *int `json:"MaxNumberOfRunners"` + ContainerInstances []Instance `json:"ContainerInstances"` +} + +// APIError is a non-2xx answer. It never holds the response body. +type APIError struct { + Method string + Path string + StatusCode int +} + +func (e *APIError) Error() string { + return fmt.Sprintf("SG API %s %s: HTTP %d", e.Method, e.Path, e.StatusCode) +} + +// RunnerGroup fetches the runner group with its active workflow counts. +func (c *Client) RunnerGroup(ctx context.Context) (*RunnerGroup, error) { + body, err := c.do(ctx, http.MethodGet, "/?getActiveWorkflows=true", nil) + if err != nil { + return nil, err + } + var resp struct { + Msg *RunnerGroup `json:"msg"` + } + if err := json.Unmarshal(body, &resp); err != nil { + // Decoder messages can quote input: report only where it failed. + var se *json.SyntaxError + if errors.As(err, &se) { + return nil, fmt.Errorf("SG API GET runner group: invalid JSON at offset %d", se.Offset) + } + return nil, errors.New("SG API GET runner group: unexpected response shape") + } + if resp.Msg == nil { + return nil, errors.New("SG API GET runner group: response has no msg") + } + return resp.Msg, nil +} + +// SetRunnerStatus sets a runner ACTIVE or DRAINING through SG, which changes +// the ECS container instance state on its side. +func (c *Client) SetRunnerStatus(ctx context.Context, runnerID, status string) error { + payload, err := json.Marshal(map[string]string{"Status": status, "RunnerId": runnerID}) + if err != nil { + return err + } + _, err = c.do(ctx, http.MethodPost, "/runner_status/", payload) + return err +} + +func (c *Client) do(ctx context.Context, method, path string, payload []byte) ([]byte, error) { + u := c.creds.BaseAPI + "/orgs/" + url.PathEscape(c.creds.Organization) + + "/runnergroups/" + url.PathEscape(c.creds.RunnerGroup) + path + var body io.Reader + if payload != nil { + body = bytes.NewReader(payload) + } + req, err := http.NewRequestWithContext(ctx, method, u, body) + if err != nil { + return nil, fmt.Errorf("SG API %s %s: %w", method, path, err) + } + req.Header.Set("Authorization", "apikey "+c.creds.Token) + req.Header.Set("Accept", "application/json") + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("SG API %s %s: %w", method, path, err) + } + defer resp.Body.Close() + b, err := io.ReadAll(io.LimitReader(resp.Body, maxBody)) + if err != nil { + return nil, fmt.Errorf("SG API %s %s: read response: %w", method, path, err) + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, &APIError{Method: method, Path: path, StatusCode: resp.StatusCode} + } + return b, nil +} diff --git a/k8s/scaler/internal/sg/sg_test.go b/k8s/scaler/internal/sg/sg_test.go new file mode 100644 index 0000000..4350968 --- /dev/null +++ b/k8s/scaler/internal/sg/sg_test.go @@ -0,0 +1,181 @@ +package sg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +const fakeRunnerToken = "sgrt_SUPERSECRET_runner_token_123" + +const groupJSON = `{"msg": { + "ResourceName": "rg", + "RunnerToken": "` + fakeRunnerToken + `", + "QueuedWorkflowsCount": 2, + "MaxNumberOfRunners": 4, + "ContainerInstances": [ + {"runnerId": "r-0", "status": "ACTIVE", "agentConnected": true, + "runningTasksCount": 1, "pendingTasksCount": 1, + "registeredResources": [{"name": "CPU", "integerValue": 2048}, {"name": "MEMORY", "integerValue": 4096}], + "remainingResources": [{"name": "CPU", "integerValue": 1024}, {"name": "MEMORY", "integerValue": 3584}], + "instanceDetails": [{"ComputerName": "sg-runner-0.sg-runner.sg-runner.svc.cluster.local"}]}, + {"runnerId": "vm-1", "status": "DRAINING", "agentConnected": false, + "instanceDetails": [{"ComputerName": "ip-10-0-0-1"}]} + ]}}` + +func fakeSG(t *testing.T, handler http.HandlerFunc) (*Client, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return New(Credentials{Token: "node-token", Organization: "org", RunnerGroup: "rg", BaseAPI: srv.URL + "/api/v1/"}, nil), srv +} + +func TestRunnerGroup(t *testing.T) { + c, _ := fakeSG(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/orgs/org/runnergroups/rg/" || + r.URL.Query().Get("getActiveWorkflows") != "true" { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + } + if got := r.Header.Get("Authorization"); got != "apikey node-token" { + t.Errorf("Authorization = %q", got) + } + io.WriteString(w, groupJSON) + }) + g, err := c.RunnerGroup(context.Background()) + if err != nil { + t.Fatal(err) + } + if g.QueuedWorkflowsCount != 2 || g.MaxNumberOfRunners == nil || *g.MaxNumberOfRunners != 4 || len(g.ContainerInstances) != 2 { + t.Fatalf("decoded %+v", g) + } + i := g.ContainerInstances[0] + if i.RunnerID != "r-0" || !i.AgentConnected || i.RunningTasksCount+i.PendingTasksCount != 2 || + ResourceValue(i.RegisteredResources, "CPU") != 2048 || ResourceValue(i.RemainingResources, "MEMORY") != 3584 || + !strings.HasPrefix(i.ComputerName(), "sg-runner-0.") { + t.Errorf("instance %+v", i) + } + if g.ContainerInstances[1].ComputerName() != "ip-10-0-0-1" || ResourceValue(nil, "CPU") != 0 { + t.Error("second instance") + } +} + +func TestRunnerGroupNullMax(t *testing.T) { + c, _ := fakeSG(t, func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"msg": {"MaxNumberOfRunners": null, "ContainerInstances": null}}`) + }) + g, err := c.RunnerGroup(context.Background()) + if err != nil || g.MaxNumberOfRunners != nil || g.QueuedWorkflowsCount != 0 { + t.Fatalf("got %+v, %v", g, err) + } +} + +func TestSetRunnerStatus(t *testing.T) { + var got map[string]string + c, _ := fakeSG(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/orgs/org/runnergroups/rg/runner_status/" { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Error("missing content type") + } + json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(http.StatusCreated) + }) + if err := c.SetRunnerStatus(context.Background(), "r-1", "DRAINING"); err != nil { + t.Fatal(err) + } + if got["Status"] != "DRAINING" || got["RunnerId"] != "r-1" { + t.Errorf("payload %v", got) + } +} + +// Responses carry the runner token: no error may echo a body, whatever +// goes wrong. +func TestErrorsNeverLeakBodies(t *testing.T) { + cases := map[string]http.HandlerFunc{ + "http 500": func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + io.WriteString(w, groupJSON) + }, + "http 403": func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(403) + io.WriteString(w, `{"msg":"`+fakeRunnerToken+`"}`) + }, + "truncated json": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, groupJSON[:len(groupJSON)-40]) + }, + "wrong type": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"msg": {"QueuedWorkflowsCount": "`+fakeRunnerToken+`"}}`) + }, + "syntax error after secret": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"msg": {"RunnerToken": "`+fakeRunnerToken+`" x}}`) + }, + "no msg": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"RunnerToken": "`+fakeRunnerToken+`"}`) + }, + } + for name, h := range cases { + t.Run(name, func(t *testing.T) { + var logs bytes.Buffer + log := slog.New(slog.NewJSONHandler(&logs, nil)) + c, _ := fakeSG(t, h) + _, err := c.RunnerGroup(context.Background()) + if err == nil { + t.Fatal("want an error") + } + log.Error("loop failed", "err", err) + serr := c.SetRunnerStatus(context.Background(), "r", "ACTIVE") + log.Error("status", "err", serr) + for _, s := range []string{err.Error(), logs.String()} { + if strings.Contains(s, fakeRunnerToken) || strings.Contains(s, "node-token") { + t.Errorf("secret leaked: %s", s) + } + } + }) + } + var apiErr *APIError + c, _ := fakeSG(t, cases["http 403"]) + if _, err := c.RunnerGroup(context.Background()); !errors.As(err, &apiErr) || apiErr.StatusCode != 403 { + t.Errorf("want APIError 403, got %v", err) + } +} + +func TestUnreachable(t *testing.T) { + c, srv := fakeSG(t, func(w http.ResponseWriter, r *http.Request) {}) + srv.Close() + if _, err := c.RunnerGroup(context.Background()); err == nil || strings.Contains(err.Error(), "node-token") { + t.Errorf("got %v", err) + } +} + +func TestLoadCredentials(t *testing.T) { + dir := t.TempDir() + write := func(k, v string) { os.WriteFile(filepath.Join(dir, k), []byte(v), 0o600) } + if _, err := LoadCredentials(dir); err == nil { + t.Error("want error without files") + } + write("SG_NODE_TOKEN", "tok\n") + write("SG_ORGANIZATION", "org\n") + write("SG_RUNNER_GROUP", "rg") + c, err := LoadCredentials(dir) + if err != nil || c != (Credentials{Token: "tok", Organization: "org", RunnerGroup: "rg", BaseAPI: DefaultBaseAPI}) { + t.Fatalf("got %+v, %v", c, err) + } + write("SG_BASE_API", "https://example.test/api/v1\n") + if c, _ := LoadCredentials(dir); c.BaseAPI != "https://example.test/api/v1" { + t.Errorf("base = %q", c.BaseAPI) + } + write("SG_RUNNER_GROUP", " \n") + if _, err := LoadCredentials(dir); err == nil { + t.Error("want error for an empty runner group") + } +} From 234e82a7791cb8d031b6c93cd8c94b7c4caa0eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:54:06 +0200 Subject: [PATCH 14/43] feat: run the go scaler loop with events, metrics, leader election and persisted timers --- k8s/scaler/cmd/sg-runner-scaler/main.go | 193 +++++++ k8s/scaler/go.mod | 58 ++- k8s/scaler/go.sum | 174 +++++++ k8s/scaler/internal/config/config.go | 110 ++++ k8s/scaler/internal/config/config_test.go | 57 +++ k8s/scaler/internal/controller/controller.go | 452 +++++++++++++++++ .../internal/controller/controller_test.go | 473 ++++++++++++++++++ k8s/scaler/internal/kube/kube.go | 240 +++++++++ k8s/scaler/internal/kube/kube_test.go | 156 ++++++ k8s/scaler/internal/metrics/metrics.go | 89 ++++ 10 files changed, 2001 insertions(+), 1 deletion(-) create mode 100644 k8s/scaler/cmd/sg-runner-scaler/main.go create mode 100644 k8s/scaler/go.sum create mode 100644 k8s/scaler/internal/config/config.go create mode 100644 k8s/scaler/internal/config/config_test.go create mode 100644 k8s/scaler/internal/controller/controller.go create mode 100644 k8s/scaler/internal/controller/controller_test.go create mode 100644 k8s/scaler/internal/kube/kube.go create mode 100644 k8s/scaler/internal/kube/kube_test.go create mode 100644 k8s/scaler/internal/metrics/metrics.go diff --git a/k8s/scaler/cmd/sg-runner-scaler/main.go b/k8s/scaler/cmd/sg-runner-scaler/main.go new file mode 100644 index 0000000..7697fde --- /dev/null +++ b/k8s/scaler/cmd/sg-runner-scaler/main.go @@ -0,0 +1,193 @@ +// sg-runner-scaler sizes the sg-runner StatefulSet from the runner group's +// state in StackGuardian, parks idle runners and deregisters runners parked +// for too long. See k8s/README.md, "Autoscaling". +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "log/slog" + "net/http" + "os" + "os/signal" + "sync/atomic" + "syscall" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/config" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/controller" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + slog.SetDefault(log) + klog.SetSlogLogger(log.With("source", "client-go")) + if err := run(log); err != nil { + log.Error("exiting", "err", err) + os.Exit(1) + } +} + +func run(log *slog.Logger) error { + cfg, err := config.Load(nil) + if err != nil { + return err + } + if cfg.MinRaised { + log.Warn("SCALER_MIN raised to 1: SG pauses workflows without an ACTIVE runner", "given", cfg.MinGiven) + } + creds, err := sg.LoadCredentials(cfg.CredentialsDir) + if err != nil { + return err + } + + restCfg, err := rest.InClusterConfig() + if err != nil { + // Out of cluster (development): the usual kubeconfig. + restCfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return err + } + } + restCfg.Timeout = 15 * time.Second + restCfg.UserAgent = "sg-runner-scaler" + client, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return err + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + m := metrics.New() + broadcaster := record.NewBroadcaster(record.WithContext(ctx)) + broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(cfg.Namespace)}) + defer broadcaster.Shutdown() + recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "sg-runner-scaler"}) + + ctrl := &controller.Controller{ + Config: cfg.Rules, + Interval: cfg.Interval, + SG: sg.New(creds, nil), + Kube: kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret), + Events: recorder, + Metrics: m, + Log: log, + } + + var leading atomic.Bool + srv := serve(cfg, m, ctrl, &leading, log) + defer func() { + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutdown) + }() + + lead := func(ctx context.Context) { + leading.Store(true) + m.Leader.Set(1) + _ = ctrl.Run(ctx) + } + if !cfg.LeaderElect { + lead(ctx) + return nil + } + + id, err := identity() + if err != nil { + return err + } + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{Name: cfg.StatefulSet + "-scaler", Namespace: cfg.Namespace}, + Client: client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{Identity: id}, + } + log.Info("waiting for the leader lease", "lease", lock.LeaseMeta.Name, "identity", id) + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + Name: lock.LeaseMeta.Name, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + log.Info("leading", "identity", id) + lead(ctx) + }, + OnStoppedLeading: func() { + leading.Store(false) + m.Leader.Set(0) + log.Info("stopped leading", "identity", id) + }, + OnNewLeader: func(current string) { + if current != id { + log.Info("following", "leader", current) + } + }, + }, + }) + if ctx.Err() == nil { + // Lost the lease without being asked to stop: restart clean. + return errors.New("lost the leader lease") + } + return nil +} + +func identity() (string, error) { + host, err := os.Hostname() + if err != nil { + return "", err + } + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + return "", err + } + return host + "_" + hex.EncodeToString(b), nil +} + +// serve exposes /metrics, /healthz and /readyz. Not leading is healthy and +// ready (a standby); a leader whose loop stopped running is not healthy. +func serve(cfg config.Config, m *metrics.Metrics, ctrl *controller.Controller, leading *atomic.Bool, log *slog.Logger) *http.Server { + stale := 2*controller.DefaultMaxBackoff + cfg.Interval + time.Minute + mux := http.NewServeMux() + mux.Handle("/metrics", m.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if leading.Load() { + if last := ctrl.LastLoop(); !last.IsZero() && time.Since(last) > stale { + http.Error(w, "scaling loop stalled", http.StatusServiceUnavailable) + return + } + } + _, _ = w.Write([]byte("ok\n")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok\n")) + }) + srv := &http.Server{Addr: cfg.MetricsAddr, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error("metrics server", "err", err) + os.Exit(1) + } + }() + return srv +} diff --git a/k8s/scaler/go.mod b/k8s/scaler/go.mod index 4a4d10b..2fa0dab 100644 --- a/k8s/scaler/go.mod +++ b/k8s/scaler/go.mod @@ -1,3 +1,59 @@ module github.com/StackGuardian/sg-runner/k8s/scaler -go 1.24 +go 1.24.0 + +require ( + github.com/prometheus/client_golang v1.23.2 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/k8s/scaler/go.sum b/k8s/scaler/go.sum new file mode 100644 index 0000000..7b25e0e --- /dev/null +++ b/k8s/scaler/go.sum @@ -0,0 +1,174 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/k8s/scaler/internal/config/config.go b/k8s/scaler/internal/config/config.go new file mode 100644 index 0000000..f8d19ea --- /dev/null +++ b/k8s/scaler/internal/config/config.go @@ -0,0 +1,110 @@ +// Package config reads the scaler's settings from the environment, with the +// names and defaults scaler.sh used. +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" +) + +const serviceAccountNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + +// Config is everything the scaler is told from outside. +type Config struct { + Rules decide.Config + Interval time.Duration + StatefulSet string + Image string + CredentialsSecret string + CredentialsDir string + Namespace string + MetricsAddr string + LeaderElect bool + // MinRaised reports a SCALER_MIN below 1 (MinGiven) raised to 1. + MinRaised bool + MinGiven int +} + +// Load reads the environment through getenv (os.Getenv when nil). +func Load(getenv func(string) string) (Config, error) { + if getenv == nil { + getenv = os.Getenv + } + var errs []string + str := func(key, def string) string { + if v := strings.TrimSpace(getenv(key)); v != "" { + return v + } + return def + } + num := func(key string, def, lowest int) int { + raw := str(key, "") + if raw == "" { + return def + } + v, err := strconv.Atoi(raw) + if err != nil || v < lowest { + errs = append(errs, fmt.Sprintf("%s=%q: want an integer >= %d", key, raw, lowest)) + return def + } + return v + } + secs := func(key string, def, lowest int) time.Duration { + return time.Duration(num(key, def, lowest)) * time.Second + } + + c := Config{ + Interval: secs("SCALER_INTERVAL", 15, 1), + StatefulSet: str("SCALER_STATEFULSET", "sg-runner"), + Image: str("SCALER_IMAGE", "sg-runner-k8s:dev"), + CredentialsSecret: str("SCALER_CREDENTIALS_SECRET", "sg-runner-credentials"), + CredentialsDir: str("SG_CREDENTIALS_DIR", "/etc/sg-runner/credentials"), + MetricsAddr: str("SCALER_METRICS_ADDR", ":9090"), + Rules: decide.Config{ + Min: num("SCALER_MIN", 1, -1<<31), + Max: num("SCALER_MAX", 10, 1), + HeadroomPct: num("SCALER_HEADROOM_PCT", 0, 0), + IdleAfter: secs("SCALER_IDLE_SECONDS", 60, 0), + ParkTTL: secs("SCALER_PARK_TTL", 43200, 0), + StuckAfter: secs("SCALER_STUCK_SECONDS", 180, 0), + SettleTimeout: secs("SCALER_SETTLE_TIMEOUT", 300, 0), + }, + } + if c.Rules.HeadroomPct > 100 { + errs = append(errs, fmt.Sprintf("SCALER_HEADROOM_PCT=%d: want 0-100", c.Rules.HeadroomPct)) + } + // SG pauses workflows without an ACTIVE, connected runner. + if c.Rules.Min < 1 { + c.MinRaised, c.MinGiven = true, c.Rules.Min + c.Rules.Min = 1 + } + if c.Rules.Min > c.Rules.Max { + errs = append(errs, fmt.Sprintf("SCALER_MIN=%d is above SCALER_MAX=%d", c.Rules.Min, c.Rules.Max)) + } + switch v := strings.ToLower(str("SCALER_LEADER_ELECT", "true")); v { + case "true", "1", "yes": + c.LeaderElect = true + case "false", "0", "no": + default: + errs = append(errs, fmt.Sprintf("SCALER_LEADER_ELECT=%q: want true or false", v)) + } + + c.Namespace = str("POD_NAMESPACE", "") + if c.Namespace == "" { + if b, err := os.ReadFile(serviceAccountNamespace); err == nil { + c.Namespace = strings.TrimSpace(string(b)) + } + } + if c.Namespace == "" { + errs = append(errs, "no namespace: set POD_NAMESPACE") + } + if len(errs) > 0 { + return c, fmt.Errorf("invalid configuration: %s", strings.Join(errs, "; ")) + } + return c, nil +} diff --git a/k8s/scaler/internal/config/config_test.go b/k8s/scaler/internal/config/config_test.go new file mode 100644 index 0000000..4f71251 --- /dev/null +++ b/k8s/scaler/internal/config/config_test.go @@ -0,0 +1,57 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +func env(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestDefaults(t *testing.T) { + c, err := Load(env(map[string]string{"POD_NAMESPACE": "sg-runner"})) + if err != nil { + t.Fatal(err) + } + r := c.Rules + if c.Interval != 15*time.Second || r.Min != 1 || r.Max != 10 || r.HeadroomPct != 0 || + r.IdleAfter != 60*time.Second || r.ParkTTL != 43200*time.Second || r.StuckAfter != 180*time.Second || + r.SettleTimeout != 300*time.Second || c.StatefulSet != "sg-runner" || c.Image != "sg-runner-k8s:dev" || + c.CredentialsSecret != "sg-runner-credentials" || c.CredentialsDir != "/etc/sg-runner/credentials" || + c.MetricsAddr != ":9090" || !c.LeaderElect || c.Namespace != "sg-runner" || c.MinRaised { + t.Errorf("defaults %+v", c) + } +} + +func TestOverrides(t *testing.T) { + c, err := Load(env(map[string]string{ + "POD_NAMESPACE": "ns", "SCALER_INTERVAL": "5", "SCALER_MIN": "0", "SCALER_MAX": "3", + "SCALER_HEADROOM_PCT": "20", "SCALER_LEADER_ELECT": "false", "SG_CREDENTIALS_DIR": "/c", + "SCALER_STATEFULSET": "r", "SCALER_IMAGE": "i:1", + })) + if err != nil { + t.Fatal(err) + } + if c.Interval != 5*time.Second || c.Rules.Min != 1 || !c.MinRaised || c.MinGiven != 0 || c.Rules.Max != 3 || + c.Rules.HeadroomPct != 20 || c.LeaderElect || c.CredentialsDir != "/c" || c.StatefulSet != "r" || c.Image != "i:1" { + t.Errorf("overrides %+v", c) + } +} + +func TestInvalid(t *testing.T) { + for _, m := range []map[string]string{ + {"SCALER_MAX": "x"}, + {"SCALER_MAX": "0"}, + {"SCALER_INTERVAL": "0"}, + {"SCALER_HEADROOM_PCT": "101"}, + {"SCALER_MIN": "5", "SCALER_MAX": "2"}, + {"SCALER_LEADER_ELECT": "maybe"}, + } { + m["POD_NAMESPACE"] = "ns" + if _, err := Load(env(m)); err == nil || !strings.Contains(err.Error(), "invalid configuration") { + t.Errorf("%v: want an error, got %v", m, err) + } + } +} diff --git a/k8s/scaler/internal/controller/controller.go b/k8s/scaler/internal/controller/controller.go new file mode 100644 index 0000000..dd40b44 --- /dev/null +++ b/k8s/scaler/internal/controller/controller.go @@ -0,0 +1,452 @@ +// Package controller runs the scaling loop: observe SG and the cluster, +// decide, act, report. +package controller + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "regexp" + "strconv" + "sync/atomic" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +// SG is the part of the SG API the loop uses. +type SG interface { + RunnerGroup(ctx context.Context) (*sg.RunnerGroup, error) + SetRunnerStatus(ctx context.Context, runnerID, status string) error +} + +// Defaults for the loop's back-off and event rate limits. +const ( + DefaultMaxBackoff = 5 * time.Minute + DefaultFailureEvents = 3 + DefaultWarnEvery = 10 * time.Minute +) + +// Controller is the scaling loop for one runner StatefulSet. +type Controller struct { + Config decide.Config + Interval time.Duration + SG SG + Kube *kube.Kube + Events record.EventRecorder + Metrics *metrics.Metrics + Log *slog.Logger + Now func() time.Time + + // MaxBackoff caps the delay after failed loops. + MaxBackoff time.Duration + // FailureEvents is how many failed loops in a row raise a Warning. + FailureEvents int + // WarnEvery rate-limits repeats of the same Warning event. + WarnEvery time.Duration + + state *decide.State + persisted map[string]*string + failures int + warned map[string]time.Time + // conditions are the keys of the warnings decide raised last loop. + conditions map[string]bool + // lastLoop is the Unix nano time of the last loop attempt. + lastLoop atomic.Int64 + ordinal *regexp.Regexp +} + +func (c *Controller) init() { + if c.Now == nil { + c.Now = time.Now + } + if c.Log == nil { + c.Log = slog.Default() + } + if c.MaxBackoff == 0 { + c.MaxBackoff = DefaultMaxBackoff + } + if c.FailureEvents == 0 { + c.FailureEvents = DefaultFailureEvents + } + if c.WarnEvery == 0 { + c.WarnEvery = DefaultWarnEvery + } + if c.warned == nil { + c.warned = map[string]time.Time{} + } + if c.ordinal == nil { + c.ordinal = regexp.MustCompile(`^` + regexp.QuoteMeta(c.Kube.StatefulSet) + `-(\d+)\.`) + } +} + +// LastLoop is when the last loop started; zero before the first. +func (c *Controller) LastLoop() time.Time { + ns := c.lastLoop.Load() + if ns == 0 { + return time.Time{} + } + return time.Unix(0, ns) +} + +// Run loops until ctx is done. +func (c *Controller) Run(ctx context.Context) error { + c.init() + c.Log.Info("scaling", "statefulset", c.Kube.StatefulSet, "namespace", c.Kube.Namespace, + "min", c.Config.Min, "max", c.Config.Max, "headroom_pct", c.Config.HeadroomPct, + "idle", c.Config.IdleAfter, "park_ttl", c.Config.ParkTTL, "stuck_after", c.Config.StuckAfter, + "settle_timeout", c.Config.SettleTimeout, "interval", c.Interval) + // Timers are re-read from the StatefulSet when (re)starting, e.g. after + // taking over the lease. + c.state = nil + for { + _ = c.Once(ctx) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.Delay()): + } + } +} + +// Delay is the wait before the next loop: the interval, doubled for every +// failed loop in a row, up to MaxBackoff. +func (c *Controller) Delay() time.Duration { + d := c.Interval + for i := 0; i < c.failures && d < c.MaxBackoff; i++ { + d *= 2 + } + return min(d, max(c.MaxBackoff, c.Interval)) +} + +// loopError marks which API failed a loop. +type loopError struct { + reason string + err error +} + +func (e *loopError) Error() string { return e.err.Error() } +func (e *loopError) Unwrap() error { return e.err } + +// Once runs one loop. On an SG or Kubernetes API error it makes no +// decision and returns the error; the next loop waits longer. +func (c *Controller) Once(ctx context.Context) error { + c.init() + start := c.Now() + c.lastLoop.Store(start.UnixNano()) + defer func() { c.Metrics.LoopDuration.Observe(c.Now().Sub(start).Seconds()) }() + + sts, err := c.loop(ctx, start) + if err == nil { + if c.failures > 0 { + c.Log.Info("recovered", "after_failures", c.failures) + } + c.failures = 0 + c.Metrics.ConsecutiveFailures.Set(0) + delete(c.warned, "SGUnreachable") + delete(c.warned, "KubernetesAPIError") + return nil + } + c.failures++ + c.Metrics.ConsecutiveFailures.Set(float64(c.failures)) + reason := "KubernetesAPIError" + var le *loopError + if errors.As(err, &le) { + reason = le.reason + } + c.Log.Error("loop failed, skipping decisions", "reason", reason, "err", err, + "consecutive_failures", c.failures, "next_in", c.Delay()) + if c.failures >= c.FailureEvents && sts != nil { + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: reason, Key: reason, + Message: fmt.Sprintf("%d loops failed in a row, not scaling: %v", c.failures, err)}) + } + return err +} + +func (c *Controller) sgErr(err error) error { + c.Metrics.SGErrors.Inc() + return &loopError{reason: "SGUnreachable", err: err} +} + +func (c *Controller) kubeErr(what string, err error) error { + c.Metrics.KubeErrors.Inc() + return &loopError{reason: "KubernetesAPIError", err: fmt.Errorf("%s: %w", what, err)} +} + +// loop returns the StatefulSet when it got that far, for events. +func (c *Controller) loop(ctx context.Context, now time.Time) (*appsv1.StatefulSet, error) { + sts, err := c.Kube.GetStatefulSet(ctx) + if err != nil { + return nil, c.kubeErr("get statefulset", err) + } + group, err := c.SG.RunnerGroup(ctx) + if err != nil { + return sts, c.sgErr(err) + } + claims, err := c.Kube.Claims(ctx) + if err != nil { + return sts, c.kubeErr("list pvcs", err) + } + jobs, err := c.Kube.ReapJobs(ctx) + if err != nil { + return sts, c.kubeErr("list reap jobs", err) + } + if c.state == nil { + s := decide.DecodeState(sts.Annotations) + c.state = &s + c.persisted = decide.EncodeState(s) + } + + obs := decide.Observation{ + Now: now, Replicas: kube.Replicas(sts), Queued: group.QueuedWorkflowsCount, + Runners: c.runners(group), Claims: claims, ReapJobs: jobs, + } + if group.MaxNumberOfRunners != nil { + obs.MaxRunners = *group.MaxNumberOfRunners + } + d := decide.Decide(c.Config, obs, *c.state) + + c.Log.Info("loop", "replicas", obs.Replicas, "max", d.Max, "queued", obs.Queued, + "reserved_pct", d.ReservedPct, "free_pct", d.FreePct, "settling", d.Settling, + "unconnected", len(d.Unconnected), "stuck", d.Stuck, "scale", d.Scale.String(), "why", d.Why) + + active := map[string]bool{} + for _, e := range d.Events { + active[e.Key] = true + c.Log.Warn(e.Message, "reason", e.Reason) + c.warn(sts, e) + } + // A condition that cleared and comes back is reported again right away. + for k := range c.conditions { + if !active[k] { + delete(c.warned, k) + } + } + c.conditions = active + + if d.Scale != decide.ScaleNone { + if err := c.Kube.Scale(ctx, d.Replicas); err != nil { + return sts, c.kubeErr("scale statefulset", err) + } + c.Metrics.ScaleEvents.WithLabelValues(d.Scale.String()).Inc() + reason, msg := "ScaledOut", fmt.Sprintf("replicas %d -> %d: %s", obs.Replicas, d.Replicas, d.Why) + if d.Scale == decide.ScaleIn { + reason = "ScaledIn" + msg = fmt.Sprintf("parking runner %d (replicas %d -> %d): %s", d.Replicas, obs.Replicas, d.Replicas, d.Why) + } else if _, parked := claims[obs.Replicas]; parked { + msg += fmt.Sprintf("; waking parked runner %d", obs.Replicas) + } else { + msg += fmt.Sprintf("; registering new runner %d", obs.Replicas) + } + c.Log.Info(msg, "reason", reason) + c.Events.Event(sts, corev1.EventTypeNormal, reason, msg) + } + c.state = &d.State + c.persistState(ctx) + + for _, sc := range d.StatusChanges { + c.changeStatus(ctx, sts, sc, now) + } + for _, name := range d.ClearDrainMarks { + if err := c.Kube.AnnotateClaim(ctx, name, map[string]*string{kube.AnnotationDrainedByScaler: nil}); err != nil { + c.Metrics.KubeErrors.Inc() + c.Log.Error("clear drain mark", "pvc", name, "err", err) + } + } + c.reap(ctx, sts, d.Reap, now) + + parked := 0 + for n := range claims { + if n >= d.Replicas { + parked++ + } + } + m := c.Metrics + m.Queued.Set(float64(obs.Queued)) + m.ReservedPct.Set(float64(d.ReservedPct)) + m.FreePct.Set(float64(d.FreePct)) + m.Replicas.Set(float64(d.Replicas)) + m.Awake.Set(float64(obs.Replicas)) + m.Parked.Set(float64(parked)) + m.Max.Set(float64(d.Max)) + m.Unconnected.Set(float64(len(d.Unconnected))) + m.Settling.Set(float64(d.Settling)) + m.Stuck.Set(b2f(d.Stuck)) + return sts, nil +} + +func b2f(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// runners maps SG runners to StatefulSet ordinals through their host name +// (the pod's FQDN); others (VM runners of the same group) are left out. +func (c *Controller) runners(g *sg.RunnerGroup) []decide.Runner { + var out []decide.Runner + for _, i := range g.ContainerInstances { + m := c.ordinal.FindStringSubmatch(i.ComputerName()) + if m == nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + out = append(out, decide.Runner{ + Ordinal: n, ID: i.RunnerID, Status: i.Status, Connected: i.AgentConnected, + Busy: i.RunningTasksCount + i.PendingTasksCount, + CPU: sg.ResourceValue(i.RegisteredResources, "CPU"), + CPUFree: sg.ResourceValue(i.RemainingResources, "CPU"), + Mem: sg.ResourceValue(i.RegisteredResources, "MEMORY"), + MemFree: sg.ResourceValue(i.RemainingResources, "MEMORY"), + }) + } + return out +} + +// persistState writes changed timers to the StatefulSet. A failure is +// retried next loop; the timers stay in memory meanwhile. +func (c *Controller) persistState(ctx context.Context) { + want := decide.EncodeState(*c.state) + if maps.EqualFunc(want, c.persisted, func(a, b *string) bool { + return (a == nil && b == nil) || (a != nil && b != nil && *a == *b) + }) { + return + } + if err := c.Kube.AnnotateStatefulSet(ctx, want); err != nil { + c.Metrics.KubeErrors.Inc() + c.Log.Error("persist timers on the statefulset", "err", err) + return + } + c.persisted = want +} + +// changeStatus drains or reactivates a runner through SG. A drain is marked +// on the runner's PVC first, so a drain is never left without its mark +// (which would keep the runner DRAINING for good); a reactivation clears +// the mark after SG accepted it. +func (c *Controller) changeStatus(ctx context.Context, sts *appsv1.StatefulSet, sc decide.StatusChange, now time.Time) { + log := c.Log.With("runner", sc.Ordinal, "status", sc.Status, "reason", sc.Reason) + fail := func(what string, err error) { + c.Metrics.StatusChanges.WithLabelValues(sc.Reason, "failed").Inc() + log.Error(what, "err", err) + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: "StatusChangeFailed", + Key: fmt.Sprintf("StatusChangeFailed/%d/%s", sc.Ordinal, sc.Status), + Message: fmt.Sprintf("could not set runner %d %s (%s): %v", sc.Ordinal, sc.Status, sc.Reason, err)}) + } + if sc.Status == decide.StatusDraining { + v := strconv.FormatInt(now.Unix(), 10) + if err := c.Kube.AnnotateClaim(ctx, sc.Claim, map[string]*string{kube.AnnotationDrainedByScaler: &v}); err != nil { + c.Metrics.KubeErrors.Inc() + fail("mark drain on pvc", err) + return + } + } + if err := c.SG.SetRunnerStatus(ctx, sc.RunnerID, sc.Status); err != nil { + c.Metrics.SGErrors.Inc() + fail("set runner status", err) + return + } + if sc.Status == decide.StatusActive { + if err := c.Kube.AnnotateClaim(ctx, sc.Claim, map[string]*string{kube.AnnotationDrainedByScaler: nil}); err != nil { + // Retried next loop: an ACTIVE runner's mark is cleared as stale. + c.Metrics.KubeErrors.Inc() + log.Error("clear drain mark", "err", err) + } + } + c.Metrics.StatusChanges.WithLabelValues(sc.Reason, "ok").Inc() + var msg string + switch sc.Reason { + case "DrainedParkedRunner": + msg = fmt.Sprintf("runner %d is parked but SG had it ACTIVE (its pod went away without draining): set DRAINING", sc.Ordinal) + case "DrainedUnconnectedRunner": + msg = fmt.Sprintf("runner %d is awake but not connected past the settle timeout: set DRAINING until it connects", sc.Ordinal) + default: + msg = fmt.Sprintf("runner %d awake and connected again: set ACTIVE (undoing the scaler's drain)", sc.Ordinal) + } + log.Info(msg) + c.Events.Event(sts, corev1.EventTypeNormal, sc.Reason, msg) +} + +func (c *Controller) reap(ctx context.Context, sts *appsv1.StatefulSet, actions []decide.ReapAction, now time.Time) { + for _, a := range actions { + log := c.Log.With("runner", a.Ordinal, "pvc", a.Claim) + kerr := func(what string, err error) { + c.Metrics.KubeErrors.Inc() + log.Error(what, "err", err) + } + switch a.Kind { + case decide.ReapStamp: + v := strconv.FormatInt(now.Unix(), 10) + if err := c.Kube.AnnotateClaim(ctx, a.Claim, map[string]*string{kube.AnnotationParkedAt: &v}); err != nil { + kerr("stamp parked-at", err) + continue + } + log.Info("runner parked") + c.Events.Eventf(sts, corev1.EventTypeNormal, "Parked", "runner %d parked (pod gone, registration kept on %s)", a.Ordinal, a.Claim) + case decide.ReapClearStamp: + if err := c.Kube.AnnotateClaim(ctx, a.Claim, map[string]*string{kube.AnnotationParkedAt: nil}); err != nil { + kerr("clear parked-at", err) + continue + } + log.Info("runner awake") + c.Events.Eventf(sts, corev1.EventTypeNormal, "Woke", "runner %d awake again", a.Ordinal) + case decide.ReapStartJob: + if err := c.Kube.CreateReapJob(ctx, a.Ordinal); err != nil { + kerr("create reap job", err) + continue + } + c.Metrics.Reaps.WithLabelValues("started").Inc() + log.Info("deregistering parked runner", "parked_for", a.ParkedFor.Round(time.Second)) + c.Events.Eventf(sts, corev1.EventTypeNormal, "Reaping", + "runner %d parked for %s: deregistering it (job/%s)", a.Ordinal, a.ParkedFor.Round(time.Second), c.Kube.JobName(a.Ordinal)) + case decide.ReapCleanup: + if err := c.Kube.DeleteClaim(ctx, a.Claim); err != nil { + kerr("delete state pvc", err) + continue + } + if err := c.Kube.DeleteClaim(ctx, c.Kube.DockerClaimName(a.Ordinal)); err != nil { + kerr("delete docker pvc", err) + continue + } + if err := c.Kube.DeleteReapJob(ctx, a.Ordinal); err != nil { + kerr("delete reap job", err) + } + c.Metrics.Reaps.WithLabelValues("succeeded").Inc() + log.Info("runner deregistered, PVCs deleted") + c.Events.Eventf(sts, corev1.EventTypeNormal, "Reaped", "runner %d deregistered from SG and its PVCs deleted", a.Ordinal) + case decide.ReapRetry: + if err := c.Kube.DeleteReapJob(ctx, a.Ordinal); err != nil { + kerr("delete failed reap job", err) + continue + } + c.Metrics.Reaps.WithLabelValues("failed").Inc() + log.Warn("deregistration failed, retrying next loop") + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: "ReapFailed", + Key: fmt.Sprintf("ReapFailed/%d", a.Ordinal), + Message: fmt.Sprintf("deregistering parked runner %d failed; retrying (identity kept on %s)", a.Ordinal, a.Claim)}) + } + } +} + +// warn emits a Warning event unless the same one went out within WarnEvery. +func (c *Controller) warn(sts *appsv1.StatefulSet, e decide.Event) { + now := c.Now() + if last, ok := c.warned[e.Key]; ok && now.Sub(last) < c.WarnEvery { + return + } + c.warned[e.Key] = now + c.Events.Event(sts, e.Type, e.Reason, e.Message) +} diff --git a/k8s/scaler/internal/controller/controller_test.go b/k8s/scaler/internal/controller/controller_test.go new file mode 100644 index 0000000..50627fd --- /dev/null +++ b/k8s/scaler/internal/controller/controller_test.go @@ -0,0 +1,473 @@ +package controller + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +const ns = "sg" + +var t0 = time.Unix(1_700_000_000, 0) + +type fakeSG struct { + group *sg.RunnerGroup + err error + statuses []string +} + +func (f *fakeSG) RunnerGroup(context.Context) (*sg.RunnerGroup, error) { + if f.err != nil { + return nil, f.err + } + return f.group, nil +} + +func (f *fakeSG) SetRunnerStatus(_ context.Context, id, status string) error { + f.statuses = append(f.statuses, id+"="+status) + return nil +} + +func inst(n int, status string, connected bool, busy int) sg.Instance { + i := sg.Instance{ + RunnerID: fmt.Sprintf("r-%d", n), Status: status, AgentConnected: connected, RunningTasksCount: busy, + RegisteredResources: []sg.Resource{{Name: "CPU", IntegerValue: 2048}, {Name: "MEMORY", IntegerValue: 4096}}, + RemainingResources: []sg.Resource{{Name: "CPU", IntegerValue: 2048}, {Name: "MEMORY", IntegerValue: 4096}}, + } + if busy > 0 { + i.RemainingResources = []sg.Resource{{Name: "CPU", IntegerValue: 0}, {Name: "MEMORY", IntegerValue: 0}} + } + i.InstanceDetails = append(i.InstanceDetails, struct { + ComputerName string `json:"ComputerName"` + }{fmt.Sprintf("sg-runner-%d.sg-runner.sg.svc.cluster.local", n)}) + return i +} + +func statefulSet(replicas int32) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-runner", Namespace: ns}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(replicas)}, + } +} + +func claim(name string, ann map[string]string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Annotations: ann}} +} + +type env struct { + t *testing.T + ctrl *Controller + sg *fakeSG + client *fake.Clientset + events *record.FakeRecorder + now time.Time + logs *bytes.Buffer +} + +func newEnv(t *testing.T, s SG, objs ...runtime.Object) *env { + t.Helper() + e := &env{t: t, client: fake.NewClientset(objs...), events: record.NewFakeRecorder(100), now: t0, logs: &bytes.Buffer{}} + if f, ok := s.(*fakeSG); ok { + e.sg = f + } + e.ctrl = e.newController(s) + return e +} + +func (e *env) newController(s SG) *Controller { + return &Controller{ + Config: decide.Config{Min: 1, Max: 10, IdleAfter: time.Minute, StuckAfter: 3 * time.Minute, + SettleTimeout: 5 * time.Minute, ParkTTL: 12 * time.Hour}, + Interval: 15 * time.Second, + SG: s, + Kube: kube.New(e.client, ns, "sg-runner", "img:1", "creds"), + Events: e.events, + Metrics: metrics.New(), + Log: slog.New(slog.NewTextHandler(e.logs, nil)), + Now: func() time.Time { return e.now }, + } +} + +func (e *env) once() error { + e.t.Helper() + return e.ctrl.Once(context.Background()) +} + +func (e *env) sts() *appsv1.StatefulSet { + s, err := e.client.AppsV1().StatefulSets(ns).Get(context.Background(), "sg-runner", metav1.GetOptions{}) + if err != nil { + e.t.Fatal(err) + } + return s +} + +func (e *env) claim(name string) (*corev1.PersistentVolumeClaim, error) { + return e.client.CoreV1().PersistentVolumeClaims(ns).Get(context.Background(), name, metav1.GetOptions{}) +} + +func (e *env) drainEvents() []string { + var out []string + for { + select { + case ev := <-e.events.Events: + out = append(out, ev) + default: + return out + } + } +} + +func hasEvent(events []string, prefix string) bool { + for _, ev := range events { + if strings.HasPrefix(ev, prefix) { + return true + } + } + return false +} + +func TestScaleOut(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{QueuedWorkflowsCount: 1, ContainerInstances: []sg.Instance{ + inst(0, "ACTIVE", true, 1), + {RunnerID: "vm", Status: "ACTIVE", AgentConnected: true}, // VM runner of the same group: ignored + }}} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil)) + if err := e.once(); err != nil { + t.Fatal(err) + } + if r := kube.Replicas(e.sts()); r != 2 { + t.Fatalf("replicas = %d, want 2", r) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal ScaledOut replicas 1 -> 2") || !strings.Contains(strings.Join(ev, "|"), "registering new runner 1") { + t.Errorf("events %v", ev) + } + if v := testutil.ToFloat64(e.ctrl.Metrics.ScaleEvents.WithLabelValues("out")); v != 1 { + t.Errorf("scale_events_total{out} = %v", v) + } + if v := testutil.ToFloat64(e.ctrl.Metrics.Queued); v != 1 { + t.Errorf("queued = %v", v) + } +} + +// Timers live on the StatefulSet: a new scaler (restart, new leader) keeps +// counting where the old one stopped. +func TestTimersSurviveRestart(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 0), inst(1, "ACTIVE", true, 0)}}} + e := newEnv(t, f, statefulSet(2), claim("state-sg-runner-0", nil), claim("state-sg-runner-1", nil)) + if err := e.once(); err != nil { + t.Fatal(err) + } + if got := e.sts().Annotations[decide.AnnotationIdleSince]; got != fmt.Sprintf(`{"1":%d}`, t0.Unix()) { + t.Fatalf("idle-since annotation = %q", got) + } + // Restarted scaler, 60s later. + e.ctrl = e.newController(f) + e.now = t0.Add(time.Minute) + if err := e.once(); err != nil { + t.Fatal(err) + } + s := e.sts() + if kube.Replicas(s) != 1 { + t.Fatalf("replicas = %d, want 1 (parked with the persisted timer)", kube.Replicas(s)) + } + if _, ok := s.Annotations[decide.AnnotationIdleSince]; ok { + t.Errorf("idle-since should be cleared after parking: %v", s.Annotations) + } + // The parked runner's PVC gets stamped. + p, _ := e.claim("state-sg-runner-1") + if p.Annotations[kube.AnnotationParkedAt] != fmt.Sprint(e.now.Unix()) { + t.Errorf("parked-at = %v", p.Annotations) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal ScaledIn") || !hasEvent(ev, "Normal Parked") { + t.Errorf("events %v", ev) + } +} + +// A runner parked without its preStop drain is drained and marked; on +// wake the scaler reactivates it and clears the mark. Unmarked drains stick. +func TestDrainParkedAndReactivate(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{ + inst(0, "ACTIVE", true, 1), inst(1, "ACTIVE", false, 0), inst(2, "DRAINING", false, 0), + }}} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil), claim("state-sg-runner-1", nil), claim("state-sg-runner-2", nil)) + if err := e.once(); err != nil { + t.Fatal(err) + } + if strings.Join(f.statuses, ",") != "r-1=DRAINING" { + t.Fatalf("status calls = %v", f.statuses) + } + p, _ := e.claim("state-sg-runner-1") + if p.Annotations[kube.AnnotationDrainedByScaler] != fmt.Sprint(t0.Unix()) { + t.Fatalf("drain mark = %v", p.Annotations) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal DrainedParkedRunner") { + t.Errorf("events %v", ev) + } + + // Woken (someone scales to 3), both connected again, DRAINING. + f.statuses = nil + if err := e.ctrl.Kube.Scale(context.Background(), 3); err != nil { + t.Fatal(err) + } + f.group.ContainerInstances = []sg.Instance{inst(0, "ACTIVE", true, 1), inst(1, "DRAINING", true, 0), inst(2, "DRAINING", true, 0)} + e.now = t0.Add(time.Minute) + if err := e.once(); err != nil { + t.Fatal(err) + } + if strings.Join(f.statuses, ",") != "r-1=ACTIVE" { + t.Fatalf("status calls = %v (runner 2 was drained by someone else)", f.statuses) + } + p, _ = e.claim("state-sg-runner-1") + if _, ok := p.Annotations[kube.AnnotationDrainedByScaler]; ok { + t.Errorf("drain mark not cleared: %v", p.Annotations) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal Reactivated") { + t.Errorf("events %v", ev) + } + if v := testutil.ToFloat64(e.ctrl.Metrics.StatusChanges.WithLabelValues("Reactivated", "ok")); v != 1 { + t.Errorf("status changes metric = %v", v) + } +} + +func TestReap(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 1), inst(1, "DRAINING", false, 0)}}} + parkedAt := map[string]string{kube.AnnotationParkedAt: fmt.Sprint(t0.Unix())} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil), + claim("state-sg-runner-1", parkedAt), claim("docker-sg-runner-1", nil)) + ctx := context.Background() + jobs := e.client.BatchV1().Jobs(ns) + + // Within the TTL: nothing. + e.now = t0.Add(11 * time.Hour) + if err := e.once(); err != nil { + t.Fatal(err) + } + if l, _ := jobs.List(ctx, metav1.ListOptions{}); len(l.Items) != 0 { + t.Fatal("job started before the TTL") + } + + // Past it: a reap Job, which blocks waking that ordinal. + e.now = t0.Add(12 * time.Hour) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, err := jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if j.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != "state-sg-runner-1" { + t.Error("job mounts the wrong PVC") + } + f.group.QueuedWorkflowsCount = 1 + if err := e.once(); err != nil { + t.Fatal(err) + } + if kube.Replicas(e.sts()) != 1 { + t.Fatal("woke a runner being deregistered") + } + f.group.QueuedWorkflowsCount = 0 + + // Failed: deleted for a retry, warned. + j.Status.Failed = 1 + if _, err := jobs.UpdateStatus(ctx, j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.drainEvents() + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}); err == nil { + t.Fatal("failed job not deleted") + } + if ev := e.drainEvents(); !hasEvent(ev, "Warning ReapFailed") { + t.Errorf("events %v", ev) + } + + // Retried, succeeds: PVCs and Job deleted. + if err := e.once(); err != nil { + t.Fatal(err) + } + j, _ = jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}) + j.Status.Succeeded = 1 + if _, err := jobs.UpdateStatus(ctx, j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := e.once(); err != nil { + t.Fatal(err) + } + for _, name := range []string{"state-sg-runner-1", "docker-sg-runner-1"} { + if _, err := e.claim(name); err == nil { + t.Errorf("%s not deleted", name) + } + } + if _, err := jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}); err == nil { + t.Error("job not deleted") + } + if _, err := e.claim("state-sg-runner-0"); err != nil { + t.Error("awake runner's PVC deleted") + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal Reaped") { + t.Errorf("events %v", ev) + } + m := e.ctrl.Metrics.Reaps + if testutil.ToFloat64(m.WithLabelValues("started")) != 2 || testutil.ToFloat64(m.WithLabelValues("failed")) != 1 || + testutil.ToFloat64(m.WithLabelValues("succeeded")) != 1 { + t.Error("reap metrics") + } +} + +func TestBackoffAndFailureEvents(t *testing.T) { + f := &fakeSG{err: errors.New("SG API GET /: HTTP 503")} + e := newEnv(t, f, statefulSet(2)) + var delays []time.Duration + for i := 0; i < 7; i++ { + if err := e.once(); err == nil { + t.Fatal("want an error") + } + delays = append(delays, e.ctrl.Delay()) + } + want := []time.Duration{30, 60, 120, 240, 300, 300, 300} + for i := range want { + if delays[i] != want[i]*time.Second { + t.Fatalf("delays = %v", delays) + } + } + if kube.Replicas(e.sts()) != 2 { + t.Error("scaled while SG was unreachable") + } + ev := e.drainEvents() + if len(ev) != 1 || !hasEvent(ev, "Warning SGUnreachable") { + t.Errorf("want one rate-limited SGUnreachable warning, got %v", ev) + } + if testutil.ToFloat64(e.ctrl.Metrics.SGErrors) != 7 || testutil.ToFloat64(e.ctrl.Metrics.ConsecutiveFailures) != 7 { + t.Error("error metrics") + } + // Recovered. + f.err = nil + f.group = &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 1), inst(1, "ACTIVE", true, 1)}} + if err := e.once(); err != nil { + t.Fatal(err) + } + if e.ctrl.Delay() != 15*time.Second || testutil.ToFloat64(e.ctrl.Metrics.ConsecutiveFailures) != 0 { + t.Error("back-off not reset") + } +} + +func TestKubeErrorSkipsDecisions(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{QueuedWorkflowsCount: 1, ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 1)}}} + e := newEnv(t, f) // no StatefulSet + if err := e.once(); err == nil { + t.Fatal("want an error") + } + if testutil.ToFloat64(e.ctrl.Metrics.KubeErrors) != 1 { + t.Error("kube error not counted") + } +} + +// Identical warnings go out once per WarnEvery while the condition lasts, +// and again as soon as it comes back after clearing. +func TestWarningRateLimit(t *testing.T) { + stuck := &sg.RunnerGroup{QueuedWorkflowsCount: 1, ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 0)}} + f := &fakeSG{group: stuck} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil)) + count := func() int { + n := 0 + for _, ev := range e.drainEvents() { + if strings.HasPrefix(ev, "Warning StuckQueue") { + n++ + } + } + return n + } + for at := time.Duration(0); at <= 12*time.Minute; at += 15 * time.Second { + e.now = t0.Add(at) + if err := e.once(); err != nil { + t.Fatal(err) + } + } + // Stuck from 3m: warned at 3m and 13m would be next; 3m..12m is one. + if n := count(); n != 1 { + t.Fatalf("StuckQueue warnings = %d, want 1", n) + } + if testutil.ToFloat64(e.ctrl.Metrics.Stuck) != 1 { + t.Error("stuck metric") + } + f.group = &sg.RunnerGroup{ContainerInstances: stuck.ContainerInstances} + e.now = e.now.Add(15 * time.Second) + e.once() + f.group = stuck + for at := time.Duration(0); at <= 3*time.Minute; at += 15 * time.Second { + e.now = e.now.Add(15 * time.Second) + e.once() + } + if n := count(); n != 1 { + t.Fatalf("StuckQueue warnings after the condition came back = %d, want 1", n) + } +} + +// End to end with the real SG client: the runner group's token must not +// reach logs, events or errors. +func TestNoSecretsInLogs(t *testing.T) { + const secret = "sgrt_TOPSECRET_runner_token" + body := `{"msg": {"RunnerToken": "` + secret + `", "QueuedWorkflowsCount": 1, "ContainerInstances": [ + {"runnerId": "r-0", "status": "ACTIVE", "agentConnected": true, "runningTasksCount": 1, + "registeredResources": [{"name": "CPU", "integerValue": 2048}, {"name": "MEMORY", "integerValue": 4096}], + "remainingResources": [{"name": "CPU", "integerValue": 0}, {"name": "MEMORY", "integerValue": 0}], + "instanceDetails": [{"ComputerName": "sg-runner-0.sg-runner.sg.svc"}]}]}}` + fail := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fail { + w.WriteHeader(500) + } + io.WriteString(w, body) + })) + defer srv.Close() + client := sg.New(sg.Credentials{Token: "node-tok", Organization: "o", RunnerGroup: "g", BaseAPI: srv.URL}, nil) + e := newEnv(t, client, statefulSet(1), claim("state-sg-runner-0", nil)) + e.ctrl.FailureEvents = 1 + if err := e.once(); err != nil { + t.Fatal(err) + } + fail = true + err := e.once() + if err == nil { + t.Fatal("want an error") + } + all := e.logs.String() + strings.Join(e.drainEvents(), "\n") + err.Error() + if strings.Contains(all, secret) || strings.Contains(all, "node-tok") { + t.Errorf("secret leaked:\n%s", all) + } + if !strings.Contains(all, "HTTP 500") { + t.Errorf("failure not reported:\n%s", all) + } +} + +func TestDelayCap(t *testing.T) { + c := &Controller{Interval: 10 * time.Minute, MaxBackoff: 5 * time.Minute} + c.failures = 3 + if c.Delay() != 10*time.Minute { + t.Errorf("an interval above the cap is kept: %v", c.Delay()) + } +} diff --git a/k8s/scaler/internal/kube/kube.go b/k8s/scaler/internal/kube/kube.go new file mode 100644 index 0000000..8794ada --- /dev/null +++ b/k8s/scaler/internal/kube/kube.go @@ -0,0 +1,240 @@ +// Package kube is the scaler's Kubernetes side: the runner StatefulSet, its +// state PVCs and the reap Jobs. +package kube + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" +) + +// Annotations on a runner's state PVC. +const ( + // AnnotationParkedAt is when the runner was first seen parked (epoch). + AnnotationParkedAt = "sg-runner.stackguardian.io/parked-at" + // AnnotationDrainedByScaler marks a drain the scaler did (epoch), so it + // undoes only its own drains on wake. + AnnotationDrainedByScaler = "sg-runner.stackguardian.io/drained-by-scaler" +) + +// ReapJobTTL deletes finished reap Jobs the scaler didn't get to. +const ReapJobTTL int32 = 24 * 3600 + +// Kube talks to the API server for one runner StatefulSet. +type Kube struct { + Client kubernetes.Interface + Namespace string + StatefulSet string + // Image is the runner image reap Jobs run. + Image string + // CredentialsSecret is the runner group's SG credentials Secret. + CredentialsSecret string + + claimRE *regexp.Regexp + jobRE *regexp.Regexp +} + +// New returns the adapter for one StatefulSet. +func New(c kubernetes.Interface, namespace, statefulSet, image, secret string) *Kube { + q := regexp.QuoteMeta(statefulSet) + return &Kube{ + Client: c, Namespace: namespace, StatefulSet: statefulSet, Image: image, CredentialsSecret: secret, + claimRE: regexp.MustCompile(`^state-` + q + `-(\d+)$`), + jobRE: regexp.MustCompile(`^` + q + `-reap-(\d+)$`), + } +} + +// ClaimName is the state PVC of an ordinal. +func (k *Kube) ClaimName(n int) string { return fmt.Sprintf("state-%s-%d", k.StatefulSet, n) } + +// DockerClaimName is the Docker cache PVC of an ordinal. +func (k *Kube) DockerClaimName(n int) string { return fmt.Sprintf("docker-%s-%d", k.StatefulSet, n) } + +// JobName is the reap Job of an ordinal. +func (k *Kube) JobName(n int) string { return fmt.Sprintf("%s-reap-%d", k.StatefulSet, n) } + +// GetStatefulSet reads the runner StatefulSet. +func (k *Kube) GetStatefulSet(ctx context.Context) (*appsv1.StatefulSet, error) { + return k.Client.AppsV1().StatefulSets(k.Namespace).Get(ctx, k.StatefulSet, metav1.GetOptions{}) +} + +// Replicas is a StatefulSet's spec.replicas (1 when unset, as the API defaults). +func Replicas(s *appsv1.StatefulSet) int { + if s.Spec.Replicas == nil { + return 1 + } + return int(*s.Spec.Replicas) +} + +// Scale sets replicas through the scale subresource. +func (k *Kube) Scale(ctx context.Context, replicas int) error { + body := fmt.Sprintf(`{"spec":{"replicas":%d}}`, replicas) + _, err := k.Client.AppsV1().StatefulSets(k.Namespace).Patch(ctx, k.StatefulSet, + types.MergePatchType, []byte(body), metav1.PatchOptions{}, "scale") + return err +} + +func annotationsPatch(a map[string]*string) ([]byte, error) { + return json.Marshal(map[string]any{"metadata": map[string]any{"annotations": a}}) +} + +// AnnotateStatefulSet merges annotations into the StatefulSet's metadata; +// nil values remove them. +func (k *Kube) AnnotateStatefulSet(ctx context.Context, a map[string]*string) error { + body, err := annotationsPatch(a) + if err != nil { + return err + } + _, err = k.Client.AppsV1().StatefulSets(k.Namespace).Patch(ctx, k.StatefulSet, + types.MergePatchType, body, metav1.PatchOptions{}) + return err +} + +// AnnotateClaim merges annotations into a PVC; nil values remove them. +func (k *Kube) AnnotateClaim(ctx context.Context, name string, a map[string]*string) error { + body, err := annotationsPatch(a) + if err != nil { + return err + } + _, err = k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).Patch(ctx, name, + types.MergePatchType, body, metav1.PatchOptions{}) + return err +} + +// Claims lists the state PVCs by ordinal. +func (k *Kube) Claims(ctx context.Context) (map[int]decide.Claim, error) { + list, err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := map[int]decide.Claim{} + for _, p := range list.Items { + m := k.claimRE.FindStringSubmatch(p.Name) + if m == nil || p.DeletionTimestamp != nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + c := decide.Claim{Name: p.Name} + if v, ok := p.Annotations[AnnotationParkedAt]; ok { + c.Stamped = true + if at, err := strconv.ParseInt(v, 10, 64); err == nil && at > 0 { + c.ParkedAt = time.Unix(at, 0) + } + } + _, c.DrainedByScaler = p.Annotations[AnnotationDrainedByScaler] + out[n] = c + } + return out, nil +} + +// DeleteClaim deletes a PVC; a missing one is fine. +func (k *Kube) DeleteClaim(ctx context.Context, name string) error { + err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + return err +} + +// ReapJobs lists the reap Jobs by ordinal. +func (k *Kube) ReapJobs(ctx context.Context) (map[int]decide.JobState, error) { + list, err := k.Client.BatchV1().Jobs(k.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=" + k.StatefulSet + "-reaper", + }) + if err != nil { + return nil, err + } + out := map[int]decide.JobState{} + for _, j := range list.Items { + m := k.jobRE.FindStringSubmatch(j.Name) + if m == nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + switch { + case j.Status.Succeeded > 0: + out[n] = decide.JobSucceeded + case j.Status.Failed > 0: + out[n] = decide.JobFailed + default: + out[n] = decide.JobRunning + } + } + return out, nil +} + +// ReapJob is the Job deregistering a parked runner from its state PVC +// (`runner.sh reap`). +func (k *Kube) ReapJob(n int) *batchv1.Job { + labels := map[string]string{"app": k.StatefulSet + "-reaper"} + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: k.JobName(n), Namespace: k.Namespace, Labels: labels}, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + TTLSecondsAfterFinished: ptr.To(ReapJobTTL), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: ptr.To(false), + Containers: []corev1.Container{{ + Name: "reap", Image: k.Image, ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"reap"}, + VolumeMounts: []corev1.VolumeMount{ + {Name: "state", MountPath: "/etc/ecs", SubPath: "ecs-config"}, + {Name: "state", MountPath: "/var/lib/ecs/data", SubPath: "ecs-data"}, + {Name: "state", MountPath: "/var/lib/amazon/ssm", SubPath: "ssm"}, + {Name: "sg-credentials", MountPath: "/etc/sg-runner/credentials", ReadOnly: true}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "state", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: k.ClaimName(n)}}}, + {Name: "sg-credentials", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: k.CredentialsSecret, DefaultMode: ptr.To[int32](0o400)}}}, + }, + }, + }, + }, + } +} + +// CreateReapJob starts the reap Job of an ordinal; an existing one is fine. +func (k *Kube) CreateReapJob(ctx context.Context, n int) error { + _, err := k.Client.BatchV1().Jobs(k.Namespace).Create(ctx, k.ReapJob(n), metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +// DeleteReapJob deletes the reap Job of an ordinal and its pod. +func (k *Kube) DeleteReapJob(ctx context.Context, n int) error { + err := k.Client.BatchV1().Jobs(k.Namespace).Delete(ctx, k.JobName(n), metav1.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }) + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/k8s/scaler/internal/kube/kube_test.go b/k8s/scaler/internal/kube/kube_test.go new file mode 100644 index 0000000..ff44724 --- /dev/null +++ b/k8s/scaler/internal/kube/kube_test.go @@ -0,0 +1,156 @@ +package kube + +import ( + "context" + "reflect" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" +) + +const ns = "sg" + +func pvc(name string, ann map[string]string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Annotations: ann}} +} + +func newKube(objs ...runtime.Object) *Kube { + return New(fake.NewClientset(objs...), ns, "sg-runner", "img:1", "creds") +} + +func TestClaims(t *testing.T) { + k := newKube( + pvc("state-sg-runner-0", nil), + pvc("state-sg-runner-1", map[string]string{AnnotationParkedAt: "1700000000", AnnotationDrainedByScaler: "1700000001"}), + pvc("state-sg-runner-2", map[string]string{AnnotationParkedAt: "garbage"}), + pvc("docker-sg-runner-1", nil), + pvc("state-sg-runner-other-3", nil), + pvc("state-sg-runner-x", nil), + ) + got, err := k.Claims(context.Background()) + if err != nil { + t.Fatal(err) + } + want := map[int]decide.Claim{ + 0: {Name: "state-sg-runner-0"}, + 1: {Name: "state-sg-runner-1", Stamped: true, ParkedAt: time.Unix(1700000000, 0), DrainedByScaler: true}, + 2: {Name: "state-sg-runner-2", Stamped: true}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Claims =\n%+v\nwant\n%+v", got, want) + } +} + +func TestAnnotateClaim(t *testing.T) { + k := newKube(pvc("state-sg-runner-1", map[string]string{"keep": "me", AnnotationParkedAt: "1"})) + ctx := context.Background() + v := "42" + if err := k.AnnotateClaim(ctx, "state-sg-runner-1", map[string]*string{AnnotationDrainedByScaler: &v, AnnotationParkedAt: nil}); err != nil { + t.Fatal(err) + } + p, _ := k.Client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "state-sg-runner-1", metav1.GetOptions{}) + if want := map[string]string{"keep": "me", AnnotationDrainedByScaler: "42"}; !reflect.DeepEqual(p.Annotations, want) { + t.Errorf("annotations = %v, want %v", p.Annotations, want) + } +} + +func TestScaleAndAnnotateStatefulSet(t *testing.T) { + k := newKube(&appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-runner", Namespace: ns, Annotations: map[string]string{"helm": "x"}}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To[int32](1)}, + }) + ctx := context.Background() + if err := k.Scale(ctx, 3); err != nil { + t.Fatal(err) + } + v := `{"2":1}` + if err := k.AnnotateStatefulSet(ctx, map[string]*string{decide.AnnotationIdleSince: &v, decide.AnnotationStuckSince: nil}); err != nil { + t.Fatal(err) + } + s, err := k.GetStatefulSet(ctx) + if err != nil { + t.Fatal(err) + } + if Replicas(s) != 3 || s.Annotations["helm"] != "x" || s.Annotations[decide.AnnotationIdleSince] != v { + t.Errorf("statefulset = replicas %d annotations %v", Replicas(s), s.Annotations) + } + if Replicas(&appsv1.StatefulSet{}) != 1 { + t.Error("unset replicas should default to 1") + } +} + +func TestReapJobSpec(t *testing.T) { + k := New(nil, ns, "sg-runner", "img:1", "creds") + j := k.ReapJob(4) + if j.Name != "sg-runner-reap-4" || j.Labels["app"] != "sg-runner-reaper" { + t.Errorf("metadata %+v", j.ObjectMeta) + } + if *j.Spec.BackoffLimit != 0 || *j.Spec.TTLSecondsAfterFinished != ReapJobTTL { + t.Error("backoffLimit/ttl") + } + ps := j.Spec.Template.Spec + if ps.RestartPolicy != corev1.RestartPolicyNever || *ps.AutomountServiceAccountToken { + t.Error("restart policy / token automount") + } + c := ps.Containers[0] + if c.Image != "img:1" || !reflect.DeepEqual(c.Args, []string{"reap"}) || c.ImagePullPolicy != corev1.PullIfNotPresent { + t.Errorf("container %+v", c) + } + wantMounts := []corev1.VolumeMount{ + {Name: "state", MountPath: "/etc/ecs", SubPath: "ecs-config"}, + {Name: "state", MountPath: "/var/lib/ecs/data", SubPath: "ecs-data"}, + {Name: "state", MountPath: "/var/lib/amazon/ssm", SubPath: "ssm"}, + {Name: "sg-credentials", MountPath: "/etc/sg-runner/credentials", ReadOnly: true}, + } + if !reflect.DeepEqual(c.VolumeMounts, wantMounts) { + t.Errorf("mounts %+v", c.VolumeMounts) + } + if ps.Volumes[0].PersistentVolumeClaim.ClaimName != "state-sg-runner-4" || + ps.Volumes[1].Secret.SecretName != "creds" || *ps.Volumes[1].Secret.DefaultMode != 0o400 { + t.Errorf("volumes %+v", ps.Volumes) + } +} + +func TestReapJobLifecycle(t *testing.T) { + k := newKube( + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-5", Namespace: ns, Labels: map[string]string{"app": "sg-runner-reaper"}}, + Status: batchv1.JobStatus{Succeeded: 1}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-6", Namespace: ns, Labels: map[string]string{"app": "sg-runner-reaper"}}, + Status: batchv1.JobStatus{Failed: 1}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: ns, Labels: map[string]string{"app": "sg-runner-reaper"}}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-9", Namespace: ns}}, + ) + ctx := context.Background() + if err := k.CreateReapJob(ctx, 3); err != nil { + t.Fatal(err) + } + if err := k.CreateReapJob(ctx, 3); err != nil { + t.Fatalf("existing job should be fine: %v", err) + } + got, err := k.ReapJobs(ctx) + if err != nil { + t.Fatal(err) + } + want := map[int]decide.JobState{3: decide.JobRunning, 5: decide.JobSucceeded, 6: decide.JobFailed} + if !reflect.DeepEqual(got, want) { + t.Errorf("ReapJobs = %v, want %v", got, want) + } + if err := k.DeleteReapJob(ctx, 5); err != nil { + t.Fatal(err) + } + if err := k.DeleteReapJob(ctx, 5); err != nil { + t.Fatalf("missing job should be fine: %v", err) + } + if err := k.DeleteClaim(ctx, "nope"); err != nil { + t.Fatalf("missing pvc should be fine: %v", err) + } +} diff --git a/k8s/scaler/internal/metrics/metrics.go b/k8s/scaler/internal/metrics/metrics.go new file mode 100644 index 0000000..73330de --- /dev/null +++ b/k8s/scaler/internal/metrics/metrics.go @@ -0,0 +1,89 @@ +// Package metrics defines the scaler's Prometheus metrics. +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ns = "sg_runner_scaler" + +// Metrics are the scaler's gauges and counters, on their own registry. +type Metrics struct { + Registry *prometheus.Registry + + Queued prometheus.Gauge + ReservedPct prometheus.Gauge + FreePct prometheus.Gauge + Replicas prometheus.Gauge + Awake prometheus.Gauge + Parked prometheus.Gauge + Max prometheus.Gauge + Stuck prometheus.Gauge + Unconnected prometheus.Gauge + Settling prometheus.Gauge + Leader prometheus.Gauge + ConsecutiveFailures prometheus.Gauge + + ScaleEvents *prometheus.CounterVec + Reaps *prometheus.CounterVec + StatusChanges *prometheus.CounterVec + SGErrors prometheus.Counter + KubeErrors prometheus.Counter + LoopDuration prometheus.Histogram +} + +// New registers the metrics on a fresh registry. +func New() *Metrics { + g := func(name, help string) prometheus.Gauge { + return prometheus.NewGauge(prometheus.GaugeOpts{Namespace: ns, Name: name, Help: help}) + } + m := &Metrics{ + Registry: prometheus.NewRegistry(), + Queued: g("queued_workflows", "Workflows queued in the runner group."), + ReservedPct: g("reserved_percent", "Share of awake ACTIVE, connected runners' CPU or memory reserved (higher of the two)."), + FreePct: g("free_percent", "100 minus reserved_percent."), + Replicas: g("replicas", "Replicas of the runner StatefulSet after this loop."), + Awake: g("awake_runners", "Runners with a pod (ordinal below replicas) at the start of the loop."), + Parked: g("parked_runners", "Runners parked: a state PVC but no pod."), + Max: g("max_runners", "Effective maximum: the lower of SCALER_MAX and MaxNumberOfRunners."), + Stuck: g("stuck", "1 while queued workflows fit no runner (stuck queue)."), + Unconnected: g("unconnected_runners", "Awake runners not connected to SG."), + Settling: g("settling_runners", "Awake runners not connected yet, within the settle timeout."), + Leader: g("leader", "1 while this replica holds the leader lease."), + ConsecutiveFailures: g("consecutive_failures", "Loops failed in a row (SG or Kubernetes API errors)."), + ScaleEvents: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "scale_events_total", + Help: "Replica changes by direction (out, in)."}, []string{"direction"}), + Reaps: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "reaps_total", + Help: "Reap steps by result (started, succeeded, failed)."}, []string{"result"}), + StatusChanges: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "runner_status_changes_total", + Help: "Runner status changes through SG by reason and result."}, []string{"reason", "result"}), + SGErrors: prometheus.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "sg_api_errors_total", + Help: "Failed SG API calls."}), + KubeErrors: prometheus.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "kube_api_errors_total", + Help: "Failed Kubernetes API calls."}), + LoopDuration: prometheus.NewHistogram(prometheus.HistogramOpts{Namespace: ns, Name: "loop_duration_seconds", + Help: "Duration of a scaling loop.", Buckets: prometheus.ExponentialBuckets(0.05, 2, 10)}), + } + m.Registry.MustRegister( + collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + m.Queued, m.ReservedPct, m.FreePct, m.Replicas, m.Awake, m.Parked, m.Max, m.Stuck, m.Unconnected, + m.Settling, m.Leader, m.ConsecutiveFailures, + m.ScaleEvents, m.Reaps, m.StatusChanges, m.SGErrors, m.KubeErrors, m.LoopDuration, + ) + for _, d := range []string{"out", "in"} { + m.ScaleEvents.WithLabelValues(d) + } + for _, r := range []string{"started", "succeeded", "failed"} { + m.Reaps.WithLabelValues(r) + } + return m +} + +// Handler serves the registry. +func (m *Metrics) Handler() http.Handler { + return promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{Registry: m.Registry}) +} From 2f644f77b169e21c076a0c73de90be40b74267e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 20:55:47 +0200 Subject: [PATCH 15/43] feat: build the go scaler image --- k8s/Makefile | 13 +++++++++++++ k8s/scaler/.dockerignore | 4 ++++ k8s/scaler/Dockerfile | 22 ++++++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 k8s/scaler/.dockerignore create mode 100644 k8s/scaler/Dockerfile diff --git a/k8s/Makefile b/k8s/Makefile index 97f5932..958ab94 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -19,6 +19,19 @@ help: ## List the available targets build: ## Build the runner image (OrbStack k8s uses local images directly) docker build -f Dockerfile -t $(IMAGE) .. +# The scaler (Go, k8s/scaler). SCALER_PLATFORMS builds a multi-arch image instead +# (buildx, e.g. SCALER_PLATFORMS=linux/amd64,linux/arm64 with --push in SCALER_BUILDX_FLAGS). +SCALER_BUILD_IMAGE ?= sg-runner-scaler:dev +SCALER_PLATFORMS ?= +SCALER_BUILDX_FLAGS ?= +.PHONY: build-scaler +build-scaler: ## Build the scaler image (Go) +ifeq ($(SCALER_PLATFORMS),) + docker build -t $(SCALER_BUILD_IMAGE) scaler +else + docker buildx build --platform $(SCALER_PLATFORMS) $(SCALER_BUILDX_FLAGS) -t $(SCALER_BUILD_IMAGE) scaler +endif + secret: ## Create the SG credentials Secret from .env @test -f .env || { echo "Copy .env.example to .env and fill it in first."; exit 1; } kubectl --context $(CONTEXT) apply -f manifests/namespace.yaml diff --git a/k8s/scaler/.dockerignore b/k8s/scaler/.dockerignore new file mode 100644 index 0000000..dc9070a --- /dev/null +++ b/k8s/scaler/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +.dockerignore +*.test +*.out diff --git a/k8s/scaler/Dockerfile b/k8s/scaler/Dockerfile new file mode 100644 index 0000000..507451b --- /dev/null +++ b/k8s/scaler/Dockerfile @@ -0,0 +1,22 @@ +# sg-runner-scaler: sizes the runner StatefulSet (see k8s/README.md, +# "Autoscaling"). Build from k8s/scaler: +# docker build -t sg-runner-scaler:dev . +# docker buildx build --platform linux/amd64,linux/arm64 -t sg-runner-scaler:dev . + +ARG GO_VERSION=1.27 + +# Cross-compiles on the build host's platform: no emulation needed. +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS build +ARG TARGETOS TARGETARCH +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -ldflags="-s -w" -o /out/sg-runner-scaler ./cmd/sg-runner-scaler + +FROM gcr.io/distroless/static:nonroot +COPY --from=build /out/sg-runner-scaler /sg-runner-scaler +USER 65532:65532 +EXPOSE 9090 +ENTRYPOINT ["/sg-runner-scaler"] From 3ee215ab28d4e397f67c76c2679c14027c155c27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:02:46 +0200 Subject: [PATCH 16/43] fix: count a runner reactivated this loop as active --- k8s/scaler/internal/controller/controller.go | 7 +-- k8s/scaler/internal/decide/decide.go | 47 +++++++++++++------- k8s/scaler/internal/decide/decide_test.go | 21 +++++++++ 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/k8s/scaler/internal/controller/controller.go b/k8s/scaler/internal/controller/controller.go index dd40b44..bcb5a21 100644 --- a/k8s/scaler/internal/controller/controller.go +++ b/k8s/scaler/internal/controller/controller.go @@ -103,8 +103,9 @@ func (c *Controller) Run(ctx context.Context) error { c.init() c.Log.Info("scaling", "statefulset", c.Kube.StatefulSet, "namespace", c.Kube.Namespace, "min", c.Config.Min, "max", c.Config.Max, "headroom_pct", c.Config.HeadroomPct, - "idle", c.Config.IdleAfter, "park_ttl", c.Config.ParkTTL, "stuck_after", c.Config.StuckAfter, - "settle_timeout", c.Config.SettleTimeout, "interval", c.Interval) + "idle", c.Config.IdleAfter.String(), "park_ttl", c.Config.ParkTTL.String(), + "stuck_after", c.Config.StuckAfter.String(), "settle_timeout", c.Config.SettleTimeout.String(), + "interval", c.Interval.String()) // Timers are re-read from the StatefulSet when (re)starting, e.g. after // taking over the lease. c.state = nil @@ -164,7 +165,7 @@ func (c *Controller) Once(ctx context.Context) error { reason = le.reason } c.Log.Error("loop failed, skipping decisions", "reason", reason, "err", err, - "consecutive_failures", c.failures, "next_in", c.Delay()) + "consecutive_failures", c.failures, "next_in", c.Delay().String()) if c.failures >= c.FailureEvents && sts != nil { c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: reason, Key: reason, Message: fmt.Sprintf("%d loops failed in a row, not scaling: %v", c.failures, err)}) diff --git a/k8s/scaler/internal/decide/decide.go b/k8s/scaler/internal/decide/decide.go index 4666af7..ac88e9a 100644 --- a/k8s/scaler/internal/decide/decide.go +++ b/k8s/scaler/internal/decide/decide.go @@ -243,6 +243,33 @@ func Decide(cfg Config, obs Observation, prev State) Decision { if obs.MaxRunners > 0 && obs.MaxRunners < d.Max { d.Max = obs.MaxRunners } + + // Undo the scaler's own drains once the runner is awake and connected + // again; drains by anyone else (no mark) stick. Decide as if they were + // ACTIVE already, or the runner reactivated this loop looks busy and + // another one gets woken for nothing. + var reactivate []StatusChange + runners := make([]Runner, len(obs.Runners)) + copy(runners, obs.Runners) + for i, r := range runners { + claim, ok := obs.Claims[r.Ordinal] + if !ok || !claim.DrainedByScaler || r.Ordinal >= replicas || !r.Connected || r.ID == "" { + continue + } + switch r.Status { + case StatusDraining: + reactivate = append(reactivate, StatusChange{ + Ordinal: r.Ordinal, RunnerID: r.ID, Claim: claim.Name, Status: StatusActive, + Reason: "Reactivated", + }) + runners[i].Status = StatusActive + case StatusActive: + // Active again (by someone, or a drain that failed): the mark is stale. + d.ClearDrainMarks = append(d.ClearDrainMarks, claim.Name) + } + } + obs.Runners = runners + d.ReservedPct = ReservedPct(obs.Runners, replicas) d.FreePct = 100 - d.ReservedPct d.Awake = replicas @@ -400,22 +427,10 @@ func Decide(cfg Config, obs Observation, prev State) Decision { }) } - // Undo the scaler's own drains once the runner is awake and connected - // again; drains by anyone else (no mark) stick. - for _, r := range obs.Runners { - claim, ok := obs.Claims[r.Ordinal] - if !ok || !claim.DrainedByScaler || r.Ordinal >= d.Replicas || !r.Connected || r.ID == "" { - continue - } - switch r.Status { - case StatusDraining: - d.StatusChanges = append(d.StatusChanges, StatusChange{ - Ordinal: r.Ordinal, RunnerID: r.ID, Claim: claim.Name, Status: StatusActive, - Reason: "Reactivated", - }) - case StatusActive: - // Active again (by someone, or a drain that failed): the mark is stale. - d.ClearDrainMarks = append(d.ClearDrainMarks, claim.Name) + // Not the runner parked this loop: its pod is on its way down. + for _, sc := range reactivate { + if sc.Ordinal < d.Replicas { + d.StatusChanges = append(d.StatusChanges, sc) } } diff --git a/k8s/scaler/internal/decide/decide_test.go b/k8s/scaler/internal/decide/decide_test.go index 8661d18..b7306ee 100644 --- a/k8s/scaler/internal/decide/decide_test.go +++ b/k8s/scaler/internal/decide/decide_test.go @@ -466,3 +466,24 @@ func TestStateRoundTrip(t *testing.T) { t.Errorf("bad annotations decoded to %+v", bad) } } + +// A woken runner the scaler reactivates this loop counts as ACTIVE: an +// idle one takes the queued workflow, so no other runner is woken. +func TestReactivatedRunnerCountsAsActive(t *testing.T) { + r := idle(1) + r.Status = StatusDraining + o := obs(0, 2, 1, full(0), r) + o.Claims[1] = Claim{Name: claimName(1), DrainedByScaler: true} + d := Decide(cfg(), o, State{}) + if d.Scale != ScaleNone || len(d.StatusChanges) != 1 || d.StatusChanges[0].Status != StatusActive { + t.Errorf("got %v (%s) changes=%+v", d.Scale, d.Why, d.StatusChanges) + } + if o.Runners[1].Status != StatusDraining { + t.Error("Decide modified the observation") + } + // Drained by someone else: it stays DRAINING and doesn't count. + o.Claims[1] = Claim{Name: claimName(1)} + if d := Decide(cfg(), o, State{}); d.Scale != ScaleOut { + t.Errorf("want scale out past a runner drained on purpose, got %v", d.Scale) + } +} From 5e06a9d57c438b190623db64c3ed93378b204748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:05:57 +0200 Subject: [PATCH 17/43] feat: replace scaler.sh with the go scaler --- k8s/Dockerfile | 1 - k8s/README.md | 29 ++- k8s/manifests/scaler.yaml | 54 ++++- k8s/scaler.sh | 337 --------------------------- k8s/scaler/internal/config/config.go | 2 +- 5 files changed, 70 insertions(+), 353 deletions(-) delete mode 100755 k8s/scaler.sh diff --git a/k8s/Dockerfile b/k8s/Dockerfile index a856a47..8a7733a 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -37,6 +37,5 @@ COPY --from=ecs-agent /images /images COPY main.sh /opt/sg-runner/main.sh COPY k8s/runner.sh /opt/sg-runner/runner.sh -COPY k8s/scaler.sh /opt/sg-runner/scaler.sh ENTRYPOINT ["/opt/sg-runner/runner.sh"] diff --git a/k8s/README.md b/k8s/README.md index 0190540..02e7e51 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -92,12 +92,13 @@ itself: a customer's runner has no ECS permissions. Deregistering is explicit: `make deregister` wakes every parked runner (each state PVC) and deregisters them all; `make nuke` does that and deletes -everything. Reaping runners parked for too long is up to the scaler (planned). +everything. Runners parked longer than `SCALER_PARK_TTL` are reaped by the scaler. ## Autoscaling -`manifests/scaler.yaml` runs `scaler.sh`, a small loop that owns the runner -StatefulSet's `replicas`. Every `SCALER_INTERVAL` (15s) it reads the runner +`manifests/scaler.yaml` runs the scaler ([`scaler/`](scaler), Go; image +built with `make build-scaler`), a loop that owns the runner StatefulSet's +`replicas`. Every `SCALER_INTERVAL` (15s) it reads the runner group from SG with the node token (`?getActiveWorkflows=true`) and: | Rule | When | Action | @@ -114,8 +115,26 @@ Replicas stay within `SCALER_MIN` (≥ 1: SG pauses workflows without an While an awake runner hasn't connected yet, the scaler waits instead of deciding again, for up to `SCALER_SETTLE_TIMEOUT` (300s); a runner that still isn't connected by then (failed registration, crash loop) is left out of the -decision and reported every loop, so one broken pod can't freeze scaling. Its Role covers only this StatefulSet's scale, PVCs and reap -Jobs; runner pods have no Kubernetes API access. +decision, reported and set `DRAINING`, so one broken pod can't freeze scaling. + +A parked runner that SG still shows `ACTIVE` (its pod went away without the +`preStop` drain: node loss, force delete) is set `DRAINING` too. Drains the +scaler does are marked on the runner's state PVC +(`sg-runner.stackguardian.io/drained-by-scaler`), and the scaler sets the +runner `ACTIVE` again once it's awake and connected. Drains from anyone else +(SG UI, an autoscaler) leave no mark and stick. + +Two replicas share a leader Lease (`-scaler`; +`SCALER_LEADER_ELECT=false` turns it off). The idle, stuck and settle timers +are kept as annotations on the StatefulSet, so a restart or a new leader +carries on where the last one stopped. On SG or Kubernetes API errors a loop +decides nothing and the next one waits longer (doubling, up to 5 minutes). +Decisions and warnings (`StuckQueue`, `RunnerNotConnected`, `SGUnreachable`, +`ReapFailed`, ...) are Events on the StatefulSet; Prometheus metrics +(`sg_runner_scaler_*`), `/healthz` and `/readyz` are served on port 9090. + +Its Role covers only this StatefulSet, its scale and annotations, PVCs, reap +Jobs, Events and its Lease; runner pods have no Kubernetes API access. ## Known gaps diff --git a/k8s/manifests/scaler.yaml b/k8s/manifests/scaler.yaml index 91deea2..03b84c6 100644 --- a/k8s/manifests/scaler.yaml +++ b/k8s/manifests/scaler.yaml @@ -1,4 +1,5 @@ -# Scaler for the runner StatefulSet (see scaler.sh). It owns spec.replicas. +# Scaler for the runner StatefulSet (k8s/scaler, Go). It owns spec.replicas. +# Two replicas share a leader Lease: one scales, the other stands by. apiVersion: v1 kind: ServiceAccount metadata: @@ -11,22 +12,38 @@ metadata: name: sg-runner-scaler namespace: sg-runner rules: + # Timers (idle/stuck/unconnected) kept as annotations, across restarts. - apiGroups: ["apps"] resources: ["statefulsets"] resourceNames: ["sg-runner"] - verbs: ["get"] + verbs: ["get", "patch"] - apiGroups: ["apps"] resources: ["statefulsets/scale"] resourceNames: ["sg-runner"] verbs: ["get", "patch"] - # Parked-at stamps, and deleting a reaped runner's PVCs. + # Parked-at and drained-by-scaler marks, and deleting a reaped runner's PVCs. - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["list", "get", "patch", "delete"] # Reap Jobs deregister long-parked runners from their PVC. - apiGroups: ["batch"] resources: ["jobs"] - verbs: ["create", "get", "delete"] + verbs: ["create", "get", "list", "delete"] + # Events on the StatefulSet (scaled, parked, reaped, warnings). + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] + # Leader election: Lease -scaler. + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + resourceNames: ["sg-runner-scaler"] + verbs: ["get", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -48,9 +65,7 @@ metadata: name: sg-runner-scaler namespace: sg-runner spec: - replicas: 1 - strategy: - type: Recreate + replicas: 2 selector: matchLabels: app: sg-runner-scaler @@ -60,12 +75,21 @@ spec: app: sg-runner-scaler spec: serviceAccountName: sg-runner-scaler + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + # Makes the 0400 credentials readable by the non-root user. + fsGroup: 65532 + seccompProfile: { type: RuntimeDefault } containers: - name: scaler - image: sg-runner-k8s:dev + image: sg-runner-scaler:dev imagePullPolicy: IfNotPresent - command: ["/opt/sg-runner/scaler.sh"] env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: { fieldPath: metadata.namespace } - { name: SCALER_MIN, value: "1" } - { name: SCALER_MAX, value: "10" } # % of awake runners' capacity to keep free (warm spare); 0 = off. @@ -74,6 +98,18 @@ spec: - { name: SCALER_PARK_TTL, value: "43200" } - { name: SCALER_IMAGE, value: sg-runner-k8s:dev } - { name: SCALER_CREDENTIALS_SECRET, value: sg-runner-credentials } + ports: + - { name: metrics, containerPort: 9090 } + livenessProbe: + httpGet: { path: /healthz, port: metrics } + periodSeconds: 20 + readinessProbe: + httpGet: { path: /readyz, port: metrics } + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } resources: requests: { cpu: 10m, memory: 32Mi } limits: { memory: 128Mi } diff --git a/k8s/scaler.sh b/k8s/scaler.sh deleted file mode 100755 index a6a6d54..0000000 --- a/k8s/scaler.sh +++ /dev/null @@ -1,337 +0,0 @@ -#!/bin/bash -# -# sg-runner scaler (prototype): sizes the runner StatefulSet from the runner -# group's state in SG, and deregisters runners parked for too long. -# -# Every SCALER_INTERVAL seconds: -# out workflows queued and no awake runner fully idle → one more runner -# (wakes a parked one if its PVC is there, registers a new one -# otherwise). With a runner fully idle, SG's next retry places queued -# work there if it fits; another runner of the same size can't help. -# Optional headroom: with SCALER_HEADROOM_PCT > 0, also scale out when -# less than that share of the awake runners' CPU or memory is free, -# so the next workflow finds a warm runner instead of queueing. -# in nothing queued and the highest awake runner idle for -# SCALER_IDLE_SECONDS, and the rest keeps the headroom → park it (StatefulSets always remove the -# highest ordinal, so that's the only one worth checking) -# reap runner parked longer than SCALER_PARK_TTL → deregister it from its -# PVC (a Job running `runner.sh reap`), then delete its PVCs -# stuck workflows queued while an awake runner sat fully idle for -# SCALER_STUCK_SECONDS → what's queued fits no runner (bigger than a -# runner pod), so more runners of the same size won't help: ignore -# the queue (no scale out for it, park as if empty) and warn -# -# Replicas stay within [SCALER_MIN, max], where max is the lower of SCALER_MAX -# and the runner group's MaxNumberOfRunners. SCALER_MIN >= 1: SG pauses -# workflows when no runner is ACTIVE and connected. -# -# All SG calls use the runner group's node token. - -set -o pipefail - -# shellcheck source-path=SCRIPTDIR source=runner.sh -source /opt/sg-runner/runner.sh - -SCALER_INTERVAL="${SCALER_INTERVAL:-15}" -SCALER_MIN="${SCALER_MIN:-1}" -SCALER_MAX="${SCALER_MAX:-10}" -# Share of awake capacity to keep free; 0 = off (scale out on the queue only). -SCALER_HEADROOM_PCT="${SCALER_HEADROOM_PCT:-0}" -SCALER_IDLE_SECONDS="${SCALER_IDLE_SECONDS:-60}" -SCALER_PARK_TTL="${SCALER_PARK_TTL:-43200}" -# Longer than SG's retry delay for queued workflows. -SCALER_STUCK_SECONDS="${SCALER_STUCK_SECONDS:-180}" -# How long to wait for an awake runner to connect before deciding without it. -SCALER_SETTLE_TIMEOUT="${SCALER_SETTLE_TIMEOUT:-300}" -SCALER_STATEFULSET="${SCALER_STATEFULSET:-sg-runner}" -SCALER_IMAGE="${SCALER_IMAGE:-sg-runner-k8s:dev}" -# The runner group's SG credentials, mounted by reap Jobs. -SCALER_CREDENTIALS_SECRET="${SCALER_CREDENTIALS_SECRET:-sg-runner-credentials}" - -readonly SA_DIR="/var/run/secrets/kubernetes.io/serviceaccount" -readonly PARKED_AT_ANNOTATION="sg-runner.stackguardian.io/parked-at" - -# Ordinal of the highest awake runner -> epoch it was first seen idle. -declare -A IDLE_SINCE=() -# Since when workflows have been queued while an awake runner was fully idle. -STUCK_SINCE="" -# Ordinal of an awake runner not connected to SG -> epoch first seen so. -declare -A UNCONNECTED_SINCE=() - -log() { - printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" -} - -kube() { #{{{ - # Usage: kube [body] - # Kubernetes API call with the scaler's service account. PATCH bodies are - # merge patches, POST bodies plain JSON. - local method="$1" path="$2" body="${3:-}" content_type="application/json" - [[ "$method" == PATCH ]] && content_type="application/merge-patch+json" - local args=(-fsS --max-time 10 -X "$method" - --cacert "$SA_DIR/ca.crt" - -H "Authorization: Bearer $(<"$SA_DIR/token")") - [[ -n "$body" ]] && args+=(-H "Content-Type: $content_type" -d "$body") - curl "${args[@]}" "https://kubernetes.default.svc/${path#/}" -} -#}}}: kube - -sg_runner_group() { #{{{ - # Prints the runner group as SG reports it, with active workflow counts. - # It holds secrets (the group's runner token): only extract fields from it. - # Subshell: api_call exits on an empty response, the loop must not. - ( - api_call GET \ - "${SG_BASE_API}/orgs/${SG_ORGANIZATION}/runnergroups/${SG_RUNNER_GROUP}/?getActiveWorkflows=true" \ - >/dev/null 2>&1 && - printf '%s' "$response" - ) -} -#}}}: sg_runner_group - -summarize_group() { #{{{ - # stdin: runner group JSON. Prints what scaling needs, with each SG runner - # mapped to its StatefulSet ordinal through its hostname (the pod's FQDN). - jq -c --arg sts "$SCALER_STATEFULSET" ' - def res($list; $name): [($list // [])[] | select(.name == $name) | .integerValue][0] // 0; - .msg | { - queued: (.QueuedWorkflowsCount // 0), - max_runners: .MaxNumberOfRunners, - runners: [(.ContainerInstances // [])[] | { - ordinal: (try (.instanceDetails[0].ComputerName - | capture("^" + $sts + "-(?[0-9]+)\\.").n | tonumber) catch null), - status, - connected: (.agentConnected == true), - busy: ((.runningTasksCount // 0) + (.pendingTasksCount // 0)), - cpu: res(.registeredResources; "CPU"), - cpu_free: res(.remainingResources; "CPU"), - mem: res(.registeredResources; "MEMORY"), - mem_free: res(.remainingResources; "MEMORY") - } | select(.ordinal != null)] - }' -} -#}}}: summarize_group - -reserved_pct() { #{{{ - # Usage: reserved_pct

- # Percent of CPU or memory (whichever is higher) reserved on runners with an - # ordinal below the given one that are ACTIVE and connected. 100 if none. - jq -r --argjson below "$2" ' - [.runners[] | select(.ordinal < $below and .status == "ACTIVE" and .connected)] as $r - | if ($r | length) == 0 then 100 else - ([$r[].cpu] | add) as $cpu | ([$r[].mem] | add) as $mem - | [(($cpu - ([$r[].cpu_free] | add)) * 100 / ([$cpu, 1] | max)), - (($mem - ([$r[].mem_free] | add)) * 100 / ([$mem, 1] | max))] - | max | floor - end' <<<"$1" -} -#}}}: reserved_pct - -set_replicas() { #{{{ - kube PATCH "apis/apps/v1/namespaces/${NS}/statefulsets/${SCALER_STATEFULSET}/scale" \ - "{\"spec\":{\"replicas\":$1}}" >/dev/null -} -#}}}: set_replicas - -annotate_pvc() { #{{{ - # Usage: annotate_pvc - local value="null" - [[ "$2" != null ]] && value="\"$2\"" - kube PATCH "api/v1/namespaces/${NS}/persistentvolumeclaims/$1" \ - "{\"metadata\":{\"annotations\":{\"${PARKED_AT_ANNOTATION}\":${value}}}}" >/dev/null -} -#}}}: annotate_pvc - -reap_job() { #{{{ - # Usage: reap_job ; prints the Job manifest. - local n="$1" - jq -n --arg n "$n" --arg sts "$SCALER_STATEFULSET" --arg image "$SCALER_IMAGE" \ - --arg secret "$SCALER_CREDENTIALS_SECRET" '{ - apiVersion: "batch/v1", kind: "Job", - metadata: {name: "\($sts)-reap-\($n)", labels: {app: "\($sts)-reaper"}}, - spec: {backoffLimit: 0, template: {spec: { - restartPolicy: "Never", - automountServiceAccountToken: false, - containers: [{ - name: "reap", image: $image, imagePullPolicy: "IfNotPresent", args: ["reap"], - volumeMounts: [ - {name: "state", mountPath: "/etc/ecs", subPath: "ecs-config"}, - {name: "state", mountPath: "/var/lib/ecs/data", subPath: "ecs-data"}, - {name: "state", mountPath: "/var/lib/amazon/ssm", subPath: "ssm"}, - {name: "sg-credentials", mountPath: "/etc/sg-runner/credentials", readOnly: true} - ] - }], - volumes: [ - {name: "state", persistentVolumeClaim: {claimName: "state-\($sts)-\($n)"}}, - {name: "sg-credentials", secret: {secretName: $secret, defaultMode: 256}} - ] - }}} - }' -} -#}}}: reap_job - -reap_parked() { #{{{ - # Usage: reap_parked - # Stamps newly parked runners and deregisters those parked past the TTL. - local replicas="$1" now pvcs pvc n parked_at job job_state - now="$(date +%s)" - pvcs="$(kube GET "api/v1/namespaces/${NS}/persistentvolumeclaims")" || return - - while IFS=$'\t' read -r pvc parked_at; do - n="${pvc##*-}" - [[ "$n" =~ ^[0-9]+$ ]] || continue - - if ((n < replicas)); then - # Awake again: clear a stale stamp. - [[ -n "$parked_at" ]] && annotate_pvc "$pvc" null - continue - fi - if [[ -z "$parked_at" ]]; then - annotate_pvc "$pvc" "$now" - log "runner $n parked" - continue - fi - ((now - parked_at >= SCALER_PARK_TTL)) || continue - - job="${SCALER_STATEFULSET}-reap-${n}" - job_state="$(kube GET "apis/batch/v1/namespaces/${NS}/jobs/${job}" 2>/dev/null | - jq -r 'if .status.succeeded then "done" elif .status.failed then "failed" else "running" end')" - case "$job_state" in - "") - log "runner $n parked for $((now - parked_at))s: deregistering" - kube POST "apis/batch/v1/namespaces/${NS}/jobs" "$(reap_job "$n")" >/dev/null - ;; - done) - log "runner $n deregistered: deleting its PVCs" - kube DELETE "api/v1/namespaces/${NS}/persistentvolumeclaims/${pvc}" >/dev/null - kube DELETE "api/v1/namespaces/${NS}/persistentvolumeclaims/docker-${SCALER_STATEFULSET}-${n}" >/dev/null - kube DELETE "apis/batch/v1/namespaces/${NS}/jobs/${job}?propagationPolicy=Background" >/dev/null - ;; - failed) - # Keep the identity; retry on a later loop. - log "runner $n: deregistration failed, retrying (see job/${job})" - kube DELETE "apis/batch/v1/namespaces/${NS}/jobs/${job}?propagationPolicy=Background" >/dev/null - ;; - esac - done < <(jq -r --arg prefix "state-${SCALER_STATEFULSET}-" --arg a "$PARKED_AT_ANNOTATION" ' - .items[] | select(.metadata.name | startswith($prefix)) - | [.metadata.name, (.metadata.annotations[$a] // "")] | @tsv' <<<"$pvcs") -} -#}}}: reap_parked - -reaping() { #{{{ - # Usage: reaping ; true while a reap Job holds that runner's PVC. - kube GET "apis/batch/v1/namespaces/${NS}/jobs/${SCALER_STATEFULSET}-reap-$1" >/dev/null 2>&1 -} -#}}}: reaping - -scale() { #{{{ - local group summary replicas queued max top top_runner pct free free_without_top now settling - group="$(sg_runner_group)" || { - log "can't read the runner group from SG" - return - } - summary="$(summarize_group <<<"$group")" || return - replicas="$(kube GET "apis/apps/v1/namespaces/${NS}/statefulsets/${SCALER_STATEFULSET}" | - jq -r '.spec.replicas')" || return - - queued="$(jq -r '.queued' <<<"$summary")" - max="$(jq -r --argjson max "$SCALER_MAX" '[.max_runners // $max, $max] | min' <<<"$summary")" - pct="$(reserved_pct "$summary" "$replicas")" - free=$((100 - pct)) - # A runner that is awake but not yet connected: let it settle before - # deciding again, or one queued workflow wakes several runners. One that - # never connects (failed registration, crash loop) mustn't freeze scaling: - # past SCALER_SETTLE_TIMEOUT it's left out and reported. - local unconnected n - unconnected="$(jq -r --argjson r "$replicas" ' - [range(0; $r)] - [.runners[] | select(.connected) | .ordinal] | .[]' <<<"$summary")" - now="$(date +%s)" - for n in "${!UNCONNECTED_SINCE[@]}"; do - grep -qx "$n" <<<"$unconnected" || unset "UNCONNECTED_SINCE[$n]" - done - settling=0 - for n in $unconnected; do - : "${UNCONNECTED_SINCE[$n]:=$now}" - if ((now - UNCONNECTED_SINCE[$n] < SCALER_SETTLE_TIMEOUT)); then - settling=$((settling + 1)) - else - log "WARNING: runner $n awake but not connected to SG for $((now - UNCONNECTED_SINCE[$n]))s;" \ - "scaling without it (check pod/${SCALER_STATEFULSET}-$n)" - fi - done - - # Queued while a runner sits fully idle: once that lasts past SG's retry - # delay, the queued workflows don't fit on a runner of this size. - local stuck=false idle_runner - idle_runner="$(jq -r --argjson r "$replicas" '[.runners[] | select(.ordinal < $r - and .status == "ACTIVE" and .connected and .busy == 0 and .cpu_free == .cpu - and .mem_free == .mem)] | first | if . then "\(.cpu)/\(.mem)" else "" end' <<<"$summary")" - if ((queued > 0)) && [[ -n "$idle_runner" ]]; then - : "${STUCK_SINCE:=$now}" - if ((now - STUCK_SINCE >= SCALER_STUCK_SECONDS)); then - stuck=true - log "WARNING: $queued workflow(s) queued for $((now - STUCK_SINCE))s while a runner" \ - "(CPU/MiB $idle_runner) sits idle: they need more than one runner pod offers." \ - "Ignoring them; raise the dockerd container's limits to run them." - fi - else - STUCK_SINCE="" - fi - - log "replicas=$replicas max=$max queued=$queued reserved=${pct}% free=${free}% settling=$settling stuck=$stuck" - - if ((settling > 0)); then - : - elif ((replicas < max)) && { { ((queued > 0)) && [[ -z "$idle_runner" ]]; } || - ((free < SCALER_HEADROOM_PCT)); }; then - if reaping "$replicas"; then - log "scale out waits: runner $replicas is being deregistered" - else - log "scale out: $replicas -> $((replicas + 1))" - set_replicas $((replicas + 1)) - IDLE_SINCE=() - # The new runner starts idle: don't mistake it for a stuck queue. - STUCK_SINCE="" - fi - elif ((replicas > SCALER_MIN)) && { ((queued == 0)) || [[ "$stuck" == true ]]; }; then - top=$((replicas - 1)) - top_runner="$(jq -c --argjson n "$top" '.runners[] | select(.ordinal == $n)' <<<"$summary")" - if [[ "$(jq -r '.busy' <<<"$top_runner")" != 0 ]]; then - unset "IDLE_SINCE[$top]" - else - : "${IDLE_SINCE[$top]:=$now}" - # Don't park into a scale out: the rest must keep the headroom. - free_without_top=$((100 - $(reserved_pct "$summary" "$top"))) - if ((now - IDLE_SINCE[$top] >= SCALER_IDLE_SECONDS && free_without_top >= SCALER_HEADROOM_PCT)); then - log "scale in: runner $top idle for $((now - IDLE_SINCE[$top]))s, parking ($replicas -> $top)" - set_replicas "$top" - unset "IDLE_SINCE[$top]" - fi - fi - fi - - reap_parked "$(kube GET "apis/apps/v1/namespaces/${NS}/statefulsets/${SCALER_STATEFULSET}" | - jq -r '.spec.replicas')" -} -#}}}: scale - -main() { - if ((SCALER_MIN < 1)); then - log "SCALER_MIN=$SCALER_MIN raised to 1: SG pauses workflows without an ACTIVE runner" - SCALER_MIN=1 - fi - NS="$(<"$SA_DIR/namespace")" - load_sg_credentials_from_dir - log "scaling statefulset/${SCALER_STATEFULSET}: min=$SCALER_MIN max=$SCALER_MAX" \ - "headroom=${SCALER_HEADROOM_PCT}% idle=${SCALER_IDLE_SECONDS}s park-ttl=${SCALER_PARK_TTL}s" \ - "stuck-after=${SCALER_STUCK_SECONDS}s settle-timeout=${SCALER_SETTLE_TIMEOUT}s" - while :; do - scale - sleep "$SCALER_INTERVAL" - done -} - -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main -fi diff --git a/k8s/scaler/internal/config/config.go b/k8s/scaler/internal/config/config.go index f8d19ea..7bc8d82 100644 --- a/k8s/scaler/internal/config/config.go +++ b/k8s/scaler/internal/config/config.go @@ -1,5 +1,5 @@ // Package config reads the scaler's settings from the environment, with the -// names and defaults scaler.sh used. +// names and defaults the original bash scaler used. package config import ( From 4be94a63c6247f50da72215e7d9064ec47b3d2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:07:32 +0200 Subject: [PATCH 18/43] fix: let the scaler list reap jobs --- k8s/chart/sg-runner/templates/scaler-rbac.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/k8s/chart/sg-runner/templates/scaler-rbac.yaml b/k8s/chart/sg-runner/templates/scaler-rbac.yaml index f32d3c4..ba76dac 100644 --- a/k8s/chart/sg-runner/templates/scaler-rbac.yaml +++ b/k8s/chart/sg-runner/templates/scaler-rbac.yaml @@ -31,10 +31,11 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["list", "get", "patch", "delete"] - # Reap Jobs deregister long-parked runners from their PVC. + # Reap Jobs deregister long-parked runners from their PVC; the scaler lists + # them to find reaps in progress. - apiGroups: ["batch"] resources: ["jobs"] - verbs: ["create", "get", "delete"] + verbs: ["create", "get", "list", "delete"] {{- if .Values.scaler.leaderElection.enabled }} # Leader election: one active scaler per StatefulSet. create can't be # restricted by name (the object doesn't exist yet), get/update can. From 862e108afab29126509ec65240a7d731c560b1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:08:06 +0200 Subject: [PATCH 19/43] feat: configure reap job labels, pull secrets and scheduling --- k8s/scaler/cmd/sg-runner-scaler/main.go | 4 +- k8s/scaler/internal/config/config.go | 63 +++++++++++++++++++++++ k8s/scaler/internal/config/config_test.go | 32 ++++++++++++ k8s/scaler/internal/kube/kube.go | 25 ++++++++- k8s/scaler/internal/kube/kube_test.go | 24 +++++++++ 5 files changed, 146 insertions(+), 2 deletions(-) diff --git a/k8s/scaler/cmd/sg-runner-scaler/main.go b/k8s/scaler/cmd/sg-runner-scaler/main.go index 7697fde..9716fb9 100644 --- a/k8s/scaler/cmd/sg-runner-scaler/main.go +++ b/k8s/scaler/cmd/sg-runner-scaler/main.go @@ -83,11 +83,13 @@ func run(log *slog.Logger) error { defer broadcaster.Shutdown() recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "sg-runner-scaler"}) + k := kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret) + k.ReapPod = cfg.ReapPod ctrl := &controller.Controller{ Config: cfg.Rules, Interval: cfg.Interval, SG: sg.New(creds, nil), - Kube: kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret), + Kube: k, Events: recorder, Metrics: m, Log: log, diff --git a/k8s/scaler/internal/config/config.go b/k8s/scaler/internal/config/config.go index 7bc8d82..6395852 100644 --- a/k8s/scaler/internal/config/config.go +++ b/k8s/scaler/internal/config/config.go @@ -3,13 +3,19 @@ package config import ( + "bytes" + "encoding/json" "fmt" "os" "strconv" "strings" "time" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" ) const serviceAccountNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" @@ -25,6 +31,8 @@ type Config struct { Namespace string MetricsAddr string LeaderElect bool + // ReapPod is extra pod settings for reap Jobs. + ReapPod kube.ReapPodOptions // MinRaised reports a SCALER_MIN below 1 (MinGiven) raised to 1. MinRaised bool MinGiven int @@ -94,6 +102,8 @@ func Load(getenv func(string) string) (Config, error) { errs = append(errs, fmt.Sprintf("SCALER_LEADER_ELECT=%q: want true or false", v)) } + c.ReapPod, errs = reapPod(str, errs) + c.Namespace = str("POD_NAMESPACE", "") if c.Namespace == "" { if b, err := os.ReadFile(serviceAccountNamespace); err == nil { @@ -108,3 +118,56 @@ func Load(getenv func(string) string) (Config, error) { } return c, nil } + +// strictJSON decodes raw into v, rejecting unknown fields and trailing data. +func strictJSON(raw string, v any) error { + dec := json.NewDecoder(bytes.NewReader([]byte(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if dec.More() { + return fmt.Errorf("trailing data") + } + return nil +} + +// reapPod reads the reap Job pod settings the Helm chart passes. +func reapPod(str func(string, string) string, errs []string) (kube.ReapPodOptions, []string) { + var o kube.ReapPodOptions + if raw := str("SCALER_REAP_POD_LABELS", ""); raw != "" { + if err := strictJSON(raw, &o.Labels); err != nil { + errs = append(errs, fmt.Sprintf("SCALER_REAP_POD_LABELS: want a JSON object of strings: %v", err)) + } + for k, v := range o.Labels { + for _, msg := range append(validation.IsQualifiedName(k), validation.IsValidLabelValue(v)...) { + errs = append(errs, fmt.Sprintf("SCALER_REAP_POD_LABELS: label %q=%q: %s", k, v, msg)) + } + } + } + if raw := str("SCALER_REAP_IMAGE_PULL_SECRETS", ""); raw != "" { + for _, name := range strings.Split(raw, ",") { + name = strings.TrimSpace(name) + if name == "" { + continue + } + for _, msg := range validation.IsDNS1123Subdomain(name) { + errs = append(errs, fmt.Sprintf("SCALER_REAP_IMAGE_PULL_SECRETS: %q: %s", name, msg)) + } + o.ImagePullSecrets = append(o.ImagePullSecrets, name) + } + } + if raw := str("SCALER_REAP_NODE_SELECTOR", ""); raw != "" { + if err := strictJSON(raw, &o.NodeSelector); err != nil { + errs = append(errs, fmt.Sprintf("SCALER_REAP_NODE_SELECTOR: want a JSON object of strings: %v", err)) + } + } + if raw := str("SCALER_REAP_TOLERATIONS", ""); raw != "" { + var t []corev1.Toleration + if err := strictJSON(raw, &t); err != nil { + errs = append(errs, fmt.Sprintf("SCALER_REAP_TOLERATIONS: want a JSON array of tolerations: %v", err)) + } + o.Tolerations = t + } + return o, errs +} diff --git a/k8s/scaler/internal/config/config_test.go b/k8s/scaler/internal/config/config_test.go index 4f71251..df551b7 100644 --- a/k8s/scaler/internal/config/config_test.go +++ b/k8s/scaler/internal/config/config_test.go @@ -55,3 +55,35 @@ func TestInvalid(t *testing.T) { } } } + +func TestReapPod(t *testing.T) { + c, err := Load(env(map[string]string{ + "POD_NAMESPACE": "ns", + "SCALER_REAP_POD_LABELS": `{"app.kubernetes.io/part-of":"sg-runner","np":"egress"}`, + "SCALER_REAP_IMAGE_PULL_SECRETS": "regcred, other ,", + "SCALER_REAP_NODE_SELECTOR": `{"kubernetes.io/os":"linux"}`, + "SCALER_REAP_TOLERATIONS": `[{"key":"dedicated","operator":"Equal","value":"runners","effect":"NoSchedule"}]`, + })) + if err != nil { + t.Fatal(err) + } + o := c.ReapPod + if o.Labels["np"] != "egress" || len(o.Labels) != 2 || strings.Join(o.ImagePullSecrets, ",") != "regcred,other" || + o.NodeSelector["kubernetes.io/os"] != "linux" || len(o.Tolerations) != 1 || o.Tolerations[0].Value != "runners" { + t.Errorf("reap pod %+v", o) + } + for k, v := range map[string]string{ + "SCALER_REAP_POD_LABELS": `["a"]`, + "SCALER_REAP_IMAGE_PULL_SECRETS": "Bad_Name", + "SCALER_REAP_NODE_SELECTOR": `{"a":1}`, + "SCALER_REAP_TOLERATIONS": `[{"keyy":"x"}]`, + } { + _, err := Load(env(map[string]string{"POD_NAMESPACE": "ns", k: v})) + if err == nil || !strings.Contains(err.Error(), k) { + t.Errorf("%s=%s: want an error naming it, got %v", k, v, err) + } + } + if _, err := Load(env(map[string]string{"POD_NAMESPACE": "ns", "SCALER_REAP_POD_LABELS": `{"bad key!":"x"}`})); err == nil { + t.Error("want an invalid label key rejected") + } +} diff --git a/k8s/scaler/internal/kube/kube.go b/k8s/scaler/internal/kube/kube.go index 8794ada..01783bc 100644 --- a/k8s/scaler/internal/kube/kube.go +++ b/k8s/scaler/internal/kube/kube.go @@ -34,6 +34,16 @@ const ( // ReapJobTTL deletes finished reap Jobs the scaler didn't get to. const ReapJobTTL int32 = 24 * 3600 +// ReapPodOptions are extra settings for reap Jobs, so a chart's +// NetworkPolicy, pull secrets and scheduling cover them. +type ReapPodOptions struct { + // Labels go on the Job and its pod, next to app: -reaper (which wins). + Labels map[string]string + ImagePullSecrets []string + NodeSelector map[string]string + Tolerations []corev1.Toleration +} + // Kube talks to the API server for one runner StatefulSet. type Kube struct { Client kubernetes.Interface @@ -43,6 +53,8 @@ type Kube struct { Image string // CredentialsSecret is the runner group's SG credentials Secret. CredentialsSecret string + // ReapPod customizes reap Jobs. + ReapPod ReapPodOptions claimRE *regexp.Regexp jobRE *regexp.Regexp @@ -186,7 +198,15 @@ func (k *Kube) ReapJobs(ctx context.Context) (map[int]decide.JobState, error) { // ReapJob is the Job deregistering a parked runner from its state PVC // (`runner.sh reap`). func (k *Kube) ReapJob(n int) *batchv1.Job { - labels := map[string]string{"app": k.StatefulSet + "-reaper"} + labels := map[string]string{} + for key, v := range k.ReapPod.Labels { + labels[key] = v + } + labels["app"] = k.StatefulSet + "-reaper" + var pullSecrets []corev1.LocalObjectReference + for _, name := range k.ReapPod.ImagePullSecrets { + pullSecrets = append(pullSecrets, corev1.LocalObjectReference{Name: name}) + } return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: k.JobName(n), Namespace: k.Namespace, Labels: labels}, Spec: batchv1.JobSpec{ @@ -197,6 +217,9 @@ func (k *Kube) ReapJob(n int) *batchv1.Job { Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: ptr.To(false), + ImagePullSecrets: pullSecrets, + NodeSelector: k.ReapPod.NodeSelector, + Tolerations: k.ReapPod.Tolerations, Containers: []corev1.Container{{ Name: "reap", Image: k.Image, ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{"reap"}, diff --git a/k8s/scaler/internal/kube/kube_test.go b/k8s/scaler/internal/kube/kube_test.go index ff44724..2061f92 100644 --- a/k8s/scaler/internal/kube/kube_test.go +++ b/k8s/scaler/internal/kube/kube_test.go @@ -154,3 +154,27 @@ func TestReapJobLifecycle(t *testing.T) { t.Fatalf("missing pvc should be fine: %v", err) } } + +func TestReapJobPodOptions(t *testing.T) { + k := New(nil, ns, "sg-runner", "img:1", "creds") + k.ReapPod = ReapPodOptions{ + Labels: map[string]string{"np": "egress", "app": "hijack"}, + ImagePullSecrets: []string{"regcred"}, + NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpExists}}, + } + j := k.ReapJob(1) + ps := j.Spec.Template.Spec + for _, l := range []map[string]string{j.Labels, j.Spec.Template.Labels} { + if l["np"] != "egress" || l["app"] != "sg-runner-reaper" { + t.Errorf("labels %v", l) + } + } + if len(ps.ImagePullSecrets) != 1 || ps.ImagePullSecrets[0].Name != "regcred" || + ps.NodeSelector["kubernetes.io/os"] != "linux" || len(ps.Tolerations) != 1 { + t.Errorf("pod spec %+v", ps) + } + if len(k.ReapPod.Labels) != 2 || k.ReapPod.Labels["app"] != "hijack" { + t.Error("ReapJob modified the options") + } +} From 6ee0f1e9b52457a3bc9c7d29e0c096dcf43afa86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:08:42 +0200 Subject: [PATCH 20/43] feat: test reap job settings with the chart's values --- k8s/README.md | 1 + k8s/scaler/internal/config/config_test.go | 18 ++++++++++++ k8s/scaler/internal/kube/kube_test.go | 34 +++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/k8s/README.md b/k8s/README.md index 02e7e51..023ff4b 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -135,6 +135,7 @@ Decisions and warnings (`StuckQueue`, `RunnerNotConnected`, `SGUnreachable`, Its Role covers only this StatefulSet, its scale and annotations, PVCs, reap Jobs, Events and its Lease; runner pods have no Kubernetes API access. +Reap Job pods take extra labels, pull secrets, node selector and tolerations from `SCALER_REAP_POD_LABELS`, `SCALER_REAP_IMAGE_PULL_SECRETS`, `SCALER_REAP_NODE_SELECTOR` and `SCALER_REAP_TOLERATIONS`. ## Known gaps diff --git a/k8s/scaler/internal/config/config_test.go b/k8s/scaler/internal/config/config_test.go index df551b7..0bf0cc7 100644 --- a/k8s/scaler/internal/config/config_test.go +++ b/k8s/scaler/internal/config/config_test.go @@ -87,3 +87,21 @@ func TestReapPod(t *testing.T) { t.Error("want an invalid label key rejected") } } + +// The Helm chart renders unset values as "", {} and []. +func TestReapPodChartDefaults(t *testing.T) { + c, err := Load(env(map[string]string{ + "POD_NAMESPACE": "ns", + "SCALER_REAP_POD_LABELS": `{}`, + "SCALER_REAP_IMAGE_PULL_SECRETS": "", + "SCALER_REAP_NODE_SELECTOR": `{}`, + "SCALER_REAP_TOLERATIONS": `[]`, + })) + if err != nil { + t.Fatal(err) + } + o := c.ReapPod + if len(o.Labels) != 0 || len(o.ImagePullSecrets) != 0 || len(o.NodeSelector) != 0 || len(o.Tolerations) != 0 { + t.Errorf("want no reap pod settings, got %+v", o) + } +} diff --git a/k8s/scaler/internal/kube/kube_test.go b/k8s/scaler/internal/kube/kube_test.go index 2061f92..9ee8eed 100644 --- a/k8s/scaler/internal/kube/kube_test.go +++ b/k8s/scaler/internal/kube/kube_test.go @@ -178,3 +178,37 @@ func TestReapJobPodOptions(t *testing.T) { t.Error("ReapJob modified the options") } } + +// The Job as the API server stores it, with the chart's labels. +func TestCreateReapJobWithChartOptions(t *testing.T) { + k := newKube() + k.ReapPod = ReapPodOptions{ + Labels: map[string]string{"app.kubernetes.io/name": "sg-runner", "app.kubernetes.io/instance": "x", + "app.kubernetes.io/component": "reaper"}, + ImagePullSecrets: []string{"regcred"}, + NodeSelector: map[string]string{"kubernetes.io/arch": "arm64"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpEqual, Value: "runners", Effect: corev1.TaintEffectNoSchedule}}, + } + ctx := context.Background() + if err := k.CreateReapJob(ctx, 2); err != nil { + t.Fatal(err) + } + j, err := k.Client.BatchV1().Jobs(ns).Get(ctx, "sg-runner-reap-2", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"app": "sg-runner-reaper", "app.kubernetes.io/name": "sg-runner", + "app.kubernetes.io/instance": "x", "app.kubernetes.io/component": "reaper"} + if !reflect.DeepEqual(j.Labels, want) || !reflect.DeepEqual(j.Spec.Template.Labels, want) { + t.Errorf("labels job=%v pod=%v", j.Labels, j.Spec.Template.Labels) + } + ps := j.Spec.Template.Spec + if !reflect.DeepEqual(ps.ImagePullSecrets, []corev1.LocalObjectReference{{Name: "regcred"}}) || + ps.NodeSelector["kubernetes.io/arch"] != "arm64" || !reflect.DeepEqual(ps.Tolerations, k.ReapPod.Tolerations) { + t.Errorf("pod spec %+v", ps) + } + // Still found by the scaler's label selector. + if jobs, err := k.ReapJobs(ctx); err != nil || jobs[2] != decide.JobRunning { + t.Errorf("ReapJobs = %v, %v", jobs, err) + } +} From fcd4b4257d2f094644cc64fa64a4912781949019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:10:12 +0200 Subject: [PATCH 21/43] docs: drop the last scaler.sh mention from the chart --- k8s/chart/sg-runner/templates/credentials-secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/chart/sg-runner/templates/credentials-secret.yaml b/k8s/chart/sg-runner/templates/credentials-secret.yaml index c7a2ab3..cb4c79e 100644 --- a/k8s/chart/sg-runner/templates/credentials-secret.yaml +++ b/k8s/chart/sg-runner/templates/credentials-secret.yaml @@ -1,7 +1,7 @@ {{- $name := include "sg-runner.credentialsSecretName" . }} {{- if .Values.credentials.create }} {{- $c := .Values.credentials }} -# SG credentials, one file per key when mounted: runner.sh and scaler.sh read +# SG credentials, one file per key when mounted: runner.sh and the scaler read # these names. apiVersion: v1 kind: Secret From ee49eee91c8ec2dd1c5ac6ed820b8eeb6b1fbc91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:11:39 +0200 Subject: [PATCH 22/43] docs: describe the chart, go scaler and runner sizes --- README.md | 35 ++++++++++++------- docs/TODO.md | 50 +++++++++++---------------- k8s/README.md | 95 ++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 119 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 26b3479..3fdb8e1 100644 --- a/README.md +++ b/README.md @@ -192,29 +192,38 @@ To revert you can just run: > **Prototype only.** Not supported for customer use. -[`k8s/`](k8s/README.md) runs a Private Runner as a Kubernetes pod: each +[`k8s/`](k8s/README.md) runs Private Runners as Kubernetes pods, installed with +a Helm chart ([`k8s/chart/sg-runner`](k8s/chart/sg-runner/README.md)). Each StatefulSet replica is one registered runner, with nested Docker, the SSM agent and the ECS agent as containers, and `main.sh` sourced for registration and -network setup. Tested end to end on OrbStack Kubernetes against QA. See -[`k8s/README.md`](k8s/README.md) for usage. +network setup. A scaler ([`k8s/scaler`](k8s/scaler), Go) follows the runner +group's queue in SG: + +- **Parks** idle runners: the pod stops but the runner stays registered, and + wakes again in about 8s (a fresh registration takes about 20s). +- **Wakes or adds** runners when workflows queue, between a minimum of 1 and + the runner group's `MaxNumberOfRunners`; optional headroom keeps a warm spare. +- **Deregisters** runners parked longer than a TTL (12h by default). + +Customers pick the runner size at install (`runner.size`: `small`, `medium`, +`large` or `custom`). Tested end to end on OrbStack Kubernetes against QA with +the earlier shell scaler; see [`k8s/README.md`](k8s/README.md) for usage, +measurements and known gaps. ### Known gaps - **Privileged pod.** Nested Docker needs it, or an equivalent set of capabilities; clusters enforcing restricted Pod Security reject either. -- **No workflow network isolation.** Workflow containers can reach cluster DNS - and the Kubernetes API. Needs an egress NetworkPolicy (see - [`docs/TODO.md`](docs/TODO.md)). -- **Workspaces in the container layer.** `/var/tmp/workspace` isn't on a volume - yet (see [`docs/TODO.md`](docs/TODO.md)). +- **Network isolation is opt-in.** The chart's NetworkPolicy keeps workflow + containers away from cluster addresses and the metadata endpoint, but needs + an enforcing CNI and the cluster's CIDRs. - **No task CPU/memory limits.** The agent can't create task cgroups from its container; the pod's resources are the only cap. - **No proxy support.** `--http-proxy` isn't wired up. -- **Parked runners stay registered.** Scaling down parks runners (stopped, - registration kept, woken in seconds); nothing deregisters them automatically - yet. Long parking periods are untested. -- **SG credentials in the pod.** Mounted into the `ecs-agent` container to set - the runner's status. +- **Unverified:** waking a runner after a long park (hours or days), and the Go + scaler against real SG. +- **SG credentials in the pod.** Mounted into the `ecs-agent` container and the + scaler to set runner status and read the queue. - **`ecs-init` is bypassed.** Agent defaults are copied into `k8s/runner.sh` and may drift with new ECS agent releases. diff --git a/docs/TODO.md b/docs/TODO.md index 8a93d65..357911b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -2,37 +2,25 @@ ## Kubernetes prototype (`k8s/`) -### Workspace volume for the nested Docker daemon - -The run controller and every workflow step bind-mount `/var/tmp/workspace` from -the Docker daemon they run on. In the pod that daemon is the `dockerd` -container, and the path isn't backed by a volume, so workspaces land in the -container's writable layer: they count against the node's container storage -and are lost when the container restarts. - -- Mount a volume at `/var/tmp/workspace` in the `dockerd` container: an - `emptyDir` matches VM behavior (scratch space), a PVC only if runs must - survive a pod restart. -- Size it with `sizeLimit` / storage requests so a large run can't fill the node. - -### NetworkPolicy for workflow isolation - -Workflow containers can resolve cluster DNS and reach the Kubernetes API -(verified on OrbStack: `kubernetes.default.svc` answered with `401`). The -`DOCKER-USER` DROP rules from `main.sh` don't help: they target `sg-net`, and -the run controller and step containers run on the default bridge. - -Nested traffic leaves through the pod IP, so an egress NetworkPolicy on the -runner pod covers workflow containers and the agents alike: - -- Allow DNS to the cluster resolver only if workflows need it; otherwise - resolve via an external resolver. -- Deny the pod, service and node CIDRs (Kubernetes API included) and the - cloud metadata endpoint (`169.254.169.254`). -- Allow everything else (SG API, AWS endpoints, VCS, registries, workflow - targets). -- Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...). Check what - OrbStack's CNI does before testing there. +### Verify the NetworkPolicy on an enforcing CNI + +The chart ships an optional egress NetworkPolicy (`networkPolicy.enabled`) for +runner and reap pods: DNS to kube-dns, everything outside `clusterCIDRs`, no +`169.254.169.254`. Nested workflow traffic leaves through the pod IP, so it is +covered. Not yet tested on a CNI that enforces NetworkPolicy (Calico, Cilium, +the VPC CNI's policy mode): + +- Confirm a workflow container can't reach the Kubernetes API, other pods or + the metadata endpoint, and can still reach SG, AWS, VCS and registries. +- Decide whether workflows may resolve cluster names at all; if not, block + cluster DNS and give `dockerd` an explicit `--dns`. + +### Long parking and hard kills against real SG + +- Park a runner for 12–24h, wake it, and run a workflow on it. +- Force-delete a runner pod during scale-down and check that the scaler drains + it in SG and reactivates it on wake. +- Check whether SG resumes workflows paused while no runner was `ACTIVE`. ### Drop privileged mode diff --git a/k8s/README.md b/k8s/README.md index ce3259e..8836ae1 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -2,8 +2,16 @@ > **Status: prototype.** Not supported, not documented for customers. -Runs a Private Runner as a pod. Each StatefulSet replica is one registered -runner: it stands in for the VM that `main.sh register` normally prepares. +Runs Private Runners as pods. Each StatefulSet replica is one registered +runner: it stands in for the VM that `main.sh register` normally prepares. A +scaler parks idle runners (pod stopped, registration kept, woken again in +seconds) and wakes or adds runners when workflows queue. + +| Component | Where | What | +|---|---|---| +| Runner image | [`Dockerfile`](Dockerfile), [`runner.sh`](runner.sh) | nested `dockerd`, SSM agent, ECS agent; `runner.sh` sources `main.sh` | +| Scaler image | [`scaler/`](scaler) (Go) | sizes the StatefulSet from SG's queue, parks, wakes and reaps runners | +| Helm chart | [`chart/sg-runner`](chart/sg-runner/README.md) | runner StatefulSet, scaler, credentials, optional NetworkPolicy and PDB | ## How it maps to a VM @@ -17,16 +25,17 @@ runner: it stands in for the VM that `main.sh register` normally prepares. | `/root/.aws/credentials` (SSM → agent) | in-memory `emptyDir` shared by both agents | | `configure_local_network` (iptables/sysctl) | same function, run in the pod's network namespace | -All containers use one image (`Dockerfile`) and `runner.sh`, which sources -`main.sh` for the API, config and network helpers. +All runner containers use one image (`Dockerfile`) and `runner.sh`, which +sources `main.sh` for the API, config and network helpers. ## Usage (OrbStack Kubernetes) ```sh cd k8s cp .env.example .env # fill in a throwaway runner group's credentials -make build # build sg-runner-k8s:dev -make secret # namespace + credentials Secret +make build # runner image sg-runner-k8s:dev +make build-scaler # scaler image sg-runner-scaler:dev +make secret # namespace (privileged PSA) + credentials Secret make up # helm upgrade --install chart/sg-runner; first start registers the runner make logs # follow all containers make status # pod state + ECS agent metadata @@ -41,6 +50,35 @@ make nuke # deregister, then uninstall, delete PVCs, namespace `make nuke` deregisters first and stops if that fails: deleting the PVC without deregistering would leave a dead runner in the runner group. +Outside OrbStack, install the chart directly; the namespace must allow +privileged pods (the chart doesn't create it): + +```sh +kubectl create namespace sg-runner +kubectl label namespace sg-runner pod-security.kubernetes.io/enforce=privileged +helm upgrade --install sg-runner k8s/chart/sg-runner -n sg-runner \ + --set credentials.existingSecret=sg-runner-credentials \ + --set runner.size=medium +``` + +### Choosing the runner size + +Every workflow reserves 0.5 vCPU / 512 MiB for its run controller plus its own +resource setting (default 0.5 vCPU / 1 GiB), so a default workflow takes +1 vCPU / 1.5 GiB. `runner.size` picks the `dockerd` container's limits, which +is what the runner advertises: + +| `runner.size` | Limits | Default workflows at once | +|---|---|---| +| `small` (default) | 2 vCPU / 4Gi | 2 | +| `medium` | 4 vCPU / 8Gi | 4 | +| `large` | 8 vCPU / 16Gi | 8 | +| `custom` | `runner.resources` (limits required) | | + +The runner must be larger than the largest workflow (its setting plus +0.5 vCPU / 512 MiB), or that workflow stays queued; the scaler then warns +(`StuckQueue`) instead of waking every runner. + Image versions are pinned as build args in `Dockerfile` (`ECS_AGENT_VERSION`, `SSM_AGENT_VERSION`, `DOCKER_VERSION`). `make build-multiarch` builds `linux/amd64` and `linux/arm64` on a dedicated buildx builder and keeps only @@ -161,7 +199,22 @@ Decisions and warnings (`StuckQueue`, `RunnerNotConnected`, `SGUnreachable`, Its Role covers only this StatefulSet, its scale and annotations, PVCs, reap Jobs, Events and its Lease; runner pods have no Kubernetes API access. -Reap Job pods take extra labels, pull secrets, node selector and tolerations from `SCALER_REAP_POD_LABELS`, `SCALER_REAP_IMAGE_PULL_SECRETS`, `SCALER_REAP_NODE_SELECTOR` and `SCALER_REAP_TOLERATIONS`. +Reap Job pods take extra labels, pull secrets, node selector and tolerations +from `SCALER_REAP_POD_LABELS`, `SCALER_REAP_IMAGE_PULL_SECRETS`, +`SCALER_REAP_NODE_SELECTOR` and `SCALER_REAP_TOLERATIONS`; the chart sets them +from the runner's values. + +The scaler reads the SG credentials once, at start: restart it +(`kubectl rollout restart deploy/-sg-runner-scaler`) after rotating +the node token. + +## Development + +```sh +make lint test # repo root: shellcheck + bats (main.sh, k8s/runner.sh) +(cd k8s/scaler && go vet ./... && go test ./...) +helm lint --strict k8s/chart/sg-runner --set credentials.existingSecret=x +``` ## Known gaps @@ -169,11 +222,11 @@ Reap Job pods take extra labels, pull secrets, node selector and tolerations fro Non-privileged works with `SYS_ADMIN`, `NET_ADMIN`, `SYS_RESOURCE`, Unconfined seccomp and read-write remounts of `/sys/fs/cgroup` and `/proc/sys` (verified on OrbStack), but that's no real isolation gain. See [`docs/TODO.md`](../docs/TODO.md). -- **Network isolation.** The VM rule (`DOCKER-USER` DROP for - `169.254.169.254,10.0.0.0/24`) doesn't fit a cluster: job containers can - resolve cluster DNS and reach the Kubernetes API (verified). Needs a - NetworkPolicy on the runner pod and a CNI that enforces it - ([`docs/TODO.md`](../docs/TODO.md)). +- **Network isolation is opt-in.** Workflow containers can resolve cluster DNS + and reach the Kubernetes API (verified). The chart's NetworkPolicy + (`networkPolicy.enabled`, with the cluster's CIDRs) blocks cluster ranges and + the metadata endpoint for runner and reap pods; it needs a CNI that enforces + NetworkPolicy and isn't tested on one yet. - **Workspaces are scratch.** `/var/tmp/workspace` is an `emptyDir` on `dockerd` (`runner.workspace.sizeLimit`, default 20Gi): a run doesn't survive a pod restart, and a workspace past the limit gets the pod evicted. @@ -187,17 +240,25 @@ Reap Job pods take extra labels, pull secrets, node selector and tolerations fro workflow. - **Restarting a single runner pauses new workflows** for the ~10s it takes: SG pauses workflows submitted while no runner is `ACTIVE`, and a StatefulSet - stops the old pod before starting the new one. Use `SCALER_MIN=2` where that - matters. + stops the old pod before starting the new one. `podDisruptionBudget.enabled` + keeps node drains from evicting the last runner; for rollouts use + `scaler.min=2` where that matters. Unverified: whether SG resumes such + workflows on its own once a runner reconnects. - **Unverified: long parking.** Waking was tested after seconds. Whether SSM and ECS reconnect after hours or days parked isn't tested yet. -- **Unverified: hard kills.** A pod killed without its `preStop` hook (node - crash) parks as `ACTIVE` but disconnected; how ECS treats that isn't tested. +- **Hard kills.** A pod killed without its `preStop` hook (node loss, force + delete) parks as `ACTIVE` but disconnected; the scaler sets it `DRAINING` and + reactivates it on wake. Unverified against real SG/ECS (tested with a fake + SG). If someone also drains such a runner from the SG UI while the scaler's + mark is on it, the scaler reactivates it on wake. - **SG credentials in the pod.** The credentials Secret is mounted - (read-only) into `ecs-agent` for the status changes. + (read-only) into `ecs-agent` for the status changes, and into the scaler. - **Pods end in `Error`.** The containers exit non-zero on SIGTERM after a clean drain; cosmetic. - **SSM Run Command** lands in the `ssm-agent` container (it has the Docker CLI and socket), not on a host. Unverified whether SG relies on it. - `ecs-init` is bypassed: agent defaults it sets are copied in `runner.sh` and may drift with new agent releases. +- **Go scaler not yet run against real SG.** Its rules were validated on QA in + the shell version it replaces, and the Go version against a fake SG; the + chart + Go scaler end to end on QA is still to do. From 59b8fca0d8a00e75871c5f5abd47b15d55af19ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:18:37 +0200 Subject: [PATCH 23/43] feat: add k8s CI for the scaler, chart, images and a kind smoke install --- .github/scripts/chart-check.sh | 134 ++++++++++++++++++ .github/scripts/kind-smoke.sh | 201 +++++++++++++++++++++++++++ .github/workflows/k8s.yml | 242 +++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 10 ++ 4 files changed, 587 insertions(+) create mode 100755 .github/scripts/chart-check.sh create mode 100755 .github/scripts/kind-smoke.sh create mode 100644 .github/workflows/k8s.yml diff --git a/.github/scripts/chart-check.sh b/.github/scripts/chart-check.sh new file mode 100755 index 0000000..2daa798 --- /dev/null +++ b/.github/scripts/chart-check.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Lints the sg-runner chart, renders the key value scenarios, validates every +# rendered manifest with kubeconform and checks that the schema and the +# templates reject bad values. Needs helm (v3 or v4) and kubeconform on PATH; +# never talks to a cluster. +# +# .github/scripts/chart-check.sh [out-dir] +# +# KUBE_VERSION: Kubernetes version the manifests are validated against. +set -euo pipefail + +CHART=k8s/chart/sg-runner +OUT=${1:-${RUNNER_TEMP:-/tmp}/chart-check} +KUBE_VERSION=${KUBE_VERSION:-1.37.0} +# Release name = chart name, as the k8s/Makefile installs it. +RELEASE=sg-runner +# Every scenario needs credentials; the Secret doesn't have to exist to render. +CREDS=(--set credentials.existingSecret=sg-runner-credentials) + +rm -rf "$OUT" && mkdir -p "$OUT" +failed=0 + +render() { # + local name=$1 + shift + echo "::group::render $name" + if helm template "$RELEASE" "$CHART" --namespace sg-runner "$@" >"$OUT/$name.yaml"; then + echo "rendered $(grep -c '^kind:' "$OUT/$name.yaml") objects" + else + echo "::error title=helm template::scenario '$name' failed to render" + failed=1 + fi + echo "::endgroup::" +} + +# Rendering must fail, and with the expected message (so an unrelated error +# doesn't count as the rejection). +reject() { # + local name=$1 want=$2 + shift 2 + if helm template "$RELEASE" "$CHART" "$@" >/dev/null 2>"$OUT/$name.err"; then + echo "::error title=helm template::scenario '$name' rendered, expected a rejection" + failed=1 + elif ! grep -Eq "$want" "$OUT/$name.err"; then + echo "::error title=helm template::scenario '$name' failed for another reason:" + cat "$OUT/$name.err" + failed=1 + else + echo "rejected as expected: $name ($(head -c 200 "$OUT/$name.err" | tr '\n' ' '))" + fi +} + +# Must contain the given kind (guards against a scenario silently rendering +# nothing new). +expect_kind() { # + grep -q "^kind: $2\$" "$OUT/$1.yaml" \ + || { echo "::error title=helm template::scenario '$1' has no $2"; failed=1; } +} +expect_no_kind() { # + ! grep -q "^kind: $2\$" "$OUT/$1.yaml" \ + || { echo "::error title=helm template::scenario '$1' unexpectedly has a $2"; failed=1; } +} + +echo "::group::helm lint --strict" +helm version +helm lint --strict "$CHART" "${CREDS[@]}" || failed=1 +echo "::endgroup::" + +render defaults "${CREDS[@]}" +expect_kind defaults StatefulSet +expect_kind defaults Deployment +expect_kind defaults Role + +render credentials-create \ + --set credentials.create=true \ + --set credentials.nodeToken=ci-dummy-token \ + --set credentials.organization=ci-org \ + --set credentials.runnerGroup=ci-group \ + --set credentials.apiUrl=http://127.0.0.1:9 +expect_kind credentials-create Secret + +render scaler-disabled "${CREDS[@]}" --set scaler.enabled=false --set runner.replicas=2 +expect_no_kind scaler-disabled Deployment +expect_no_kind scaler-disabled Role + +render network-policy "${CREDS[@]}" \ + --set networkPolicy.enabled=true \ + --set 'networkPolicy.clusterCIDRs={10.0.0.0/8,172.16.0.0/12}' +expect_kind network-policy NetworkPolicy + +render size-custom "${CREDS[@]}" \ + --set runner.size=custom \ + --set runner.resources.requests.cpu=3 \ + --set runner.resources.requests.memory=6Gi \ + --set runner.resources.limits.cpu=3 \ + --set runner.resources.limits.memory=6Gi + +render pdb "${CREDS[@]}" --set podDisruptionBudget.enabled=true --set scaler.min=2 +expect_kind pdb PodDisruptionBudget + +# --api-versions makes the ServiceMonitor render, as on a cluster with the +# Prometheus Operator CRDs. +render metrics "${CREDS[@]}" \ + --set scaler.metrics.enabled=true \ + --set scaler.metrics.serviceMonitor.enabled=true \ + --api-versions monitoring.coreos.com/v1/ServiceMonitor +expect_kind metrics Service +expect_kind metrics ServiceMonitor + +echo "::group::kubeconform (Kubernetes $KUBE_VERSION)" +# ServiceMonitor is validated against the community CRDs catalog. +kubeconform -strict -summary -output text \ + -kubernetes-version "$KUBE_VERSION" \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + "$OUT"/*.yaml || failed=1 +echo "::endgroup::" + +echo "::group::rejections" +reject size-huge 'runner[./]size' "${CREDS[@]}" --set runner.size=huge +reject scaler-min-0 'scaler[./]min' "${CREDS[@]}" --set scaler.min=0 +reject no-credentials 'credentials' +reject both-credentials 'not both' "${CREDS[@]}" --set credentials.create=true +reject custom-without-limits 'limits' "${CREDS[@]}" --set runner.size=custom +reject network-policy-no-cidrs 'clusterCIDRs' "${CREDS[@]}" --set networkPolicy.enabled=true +reject replicas-without-lease 'leaderElection' "${CREDS[@]}" \ + --set scaler.replicas=2 --set scaler.leaderElection.enabled=false +echo "::endgroup::" + +if [[ $failed -ne 0 ]]; then + echo "chart check failed" + exit 1 +fi +echo "chart check passed" diff --git a/.github/scripts/kind-smoke.sh b/.github/scripts/kind-smoke.sh new file mode 100755 index 0000000..b8d2afd --- /dev/null +++ b/.github/scripts/kind-smoke.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Smoke-installs the sg-runner chart on a throwaway kind cluster with locally +# built images and checks what can be checked without StackGuardian: the +# scaler starts and is ready, finds SG unreachable and leaks no credentials, +# the runner's register init container fails as expected, and the scaler's +# RBAC is as narrow as intended. +# +# SG_BASE_API in the dummy Secret is http://127.0.0.1:9 (discard port on the +# pod's own loopback): nothing ever reaches StackGuardian, and registration +# fails before any AWS call. +# +# .github/scripts/kind-smoke.sh run # load images, install, assert +# .github/scripts/kind-smoke.sh diag DIR # dump cluster state into DIR +# +# The images must exist locally (RUNNER_IMAGE, SCALER_IMAGE); the cluster +# must exist already (kind create cluster --name "$CLUSTER"). +set -euo pipefail + +CLUSTER=${CLUSTER:-sg-runner-ci} +CONTEXT=${KUBE_CONTEXT:-kind-$CLUSTER} +RUNNER_IMAGE=${RUNNER_IMAGE:-sg-runner-k8s:ci} +SCALER_IMAGE=${SCALER_IMAGE:-sg-runner-scaler:ci} +NS=sg-runner +RELEASE=sg-runner +STS=sg-runner +SCALER=sg-runner-scaler +SECRET=sg-runner-credentials +CHART=k8s/chart/sg-runner +TIMEOUT=${TIMEOUT:-240} +# Distinctive, so finding it in any log means a credential leaked. +TOKEN=ci-dummy-node-token-6f1d3c9a +SA=system:serviceaccount:$NS:$SCALER + +# Only ever a kind cluster: never a real or shared context. +[[ $CONTEXT == kind-* ]] || { echo "refusing to use context '$CONTEXT' (not kind-*)"; exit 2; } + +k() { kubectl --context "$CONTEXT" -n "$NS" "$@"; } + +step() { echo "::group::$*"; } +endstep() { echo "::endgroup::"; } +fail() { echo "::error title=kind smoke::$*"; exit 1; } + +# Polls a command until it succeeds or TIMEOUT seconds pass. +until_ok() { # + local what=$1 deadline=$((SECONDS + TIMEOUT)) + shift + until "$@"; do + ((SECONDS < deadline)) || fail "timed out after ${TIMEOUT}s waiting for: $what" + sleep 5 + done +} + +split_image() { # -> "repo tag" + echo "${1%:*} ${1##*:}" +} + +cmd_run() { + step "load images into kind/$CLUSTER" + kind load docker-image --name "$CLUSTER" "$RUNNER_IMAGE" "$SCALER_IMAGE" + endstep + + step "namespace and dummy credentials" + kubectl --context "$CONTEXT" create namespace "$NS" --dry-run=client -o yaml \ + | kubectl --context "$CONTEXT" apply -f - + kubectl --context "$CONTEXT" label namespace "$NS" --overwrite \ + pod-security.kubernetes.io/enforce=privileged + k create secret generic "$SECRET" \ + --from-literal=SG_NODE_TOKEN="$TOKEN" \ + --from-literal=SG_ORGANIZATION=ci-org \ + --from-literal=SG_RUNNER_GROUP=ci-group \ + --from-literal=SG_BASE_API=http://127.0.0.1:9 \ + --dry-run=client -o yaml | k apply -f - + endstep + + step "helm install" + local runner_repo runner_tag scaler_repo scaler_tag + read -r runner_repo runner_tag <<<"$(split_image "$RUNNER_IMAGE")" + read -r scaler_repo scaler_tag <<<"$(split_image "$SCALER_IMAGE")" + # Small volumes: kind's local-path provisioner ignores sizes anyway. + helm --kube-context "$CONTEXT" -n "$NS" upgrade --install "$RELEASE" "$CHART" \ + --set credentials.existingSecret="$SECRET" \ + --set image.repository="$runner_repo" --set image.tag="$runner_tag" \ + --set image.pullPolicy=Never \ + --set scaler.image.repository="$scaler_repo" --set scaler.image.tag="$scaler_tag" \ + --set scaler.image.pullPolicy=Never \ + --set scaler.interval=5 \ + --set runner.storage.state.size=64Mi --set runner.storage.docker.size=256Mi \ + --set runner.terminationGracePeriodSeconds=5 + endstep + + step "scaler Deployment becomes Available" + k wait --for=condition=Available "deployment/$SCALER" --timeout="${TIMEOUT}s" + endstep + + step "scaler started and finds SG unreachable" + # The scaler logs JSON; a failed loop against the unreachable API carries + # reason SGUnreachable. + until_ok "scaler log: loop failed with SGUnreachable" scaler_saw_sg_unreachable + k logs "deployment/$SCALER" --tail=20 + endstep + + step "runner StatefulSet exists, register init container fails" + k get "statefulset/$STS" + until_ok "register init container of $STS-0 to exit non-zero" register_failed + local logs code + code=$(register_exit_code) + logs=$(k logs "$STS-0" -c register --previous 2>/dev/null || k logs "$STS-0" -c register) + # Deduplicated: a failing API call can repeat the same error many times. + awk '!seen[$0]++' <<<"$logs" | tail -n 20 + grep -Eq "Could not fetch data from API|API call failed" <<<"$logs" \ + || fail "register exited $code, but not on the SG API call" + echo "register exited $code on the SG API call, as expected" + if ((code >= 128)); then + # err() reports through the API, whose failure calls err() again: with SG + # unreachable that recurses until bash crashes instead of exiting 1. + echo "::warning title=kind smoke::register crashed (exit $code, $(grep -c 'API call failed' <<<"$logs") failed API calls) instead of exiting cleanly" + fi + endstep + + step "no credentials in logs or events" + local all + all=$( + k logs "deployment/$SCALER" --all-containers 2>&1 + k logs "$STS-0" -c register --previous 2>&1 || true + k logs "$STS-0" -c register 2>&1 || true + k get events -o yaml 2>&1 + ) + if grep -qF "$TOKEN" <<<"$all"; then + fail "the dummy SG_NODE_TOKEN shows up in logs or events" + fi + echo "dummy token not found in scaler logs, register logs or events" + endstep + + step "scaler RBAC" + can "patch statefulsets.apps/$STS --subresource=scale" yes + can "get statefulsets.apps/$STS --subresource=scale" yes + can "patch statefulsets.apps/$STS" yes + can "create jobs.batch" yes + can "patch statefulsets.apps/other --subresource=scale" no + can "delete statefulsets.apps/$STS" no + can "delete pods" no + can "create pods" no + can "get secrets" no + can "create pods/exec" no + endstep + + echo "kind smoke passed" +} + +scaler_saw_sg_unreachable() { + k logs "deployment/$SCALER" 2>/dev/null \ + | grep '"loop failed, skipping decisions"' | grep -q '"reason":"SGUnreachable"' +} + +# Exit code of the register init container's last finished run, if any. +register_exit_code() { + local status + status=$(k get pod "$STS-0" -o jsonpath='{.status.initContainerStatuses[?(@.name=="register")]}' 2>/dev/null) \ + || return 0 + jq -r '(.state.terminated // .lastState.terminated // {}).exitCode // empty' <<<"$status" +} + +register_failed() { + local code + code=$(register_exit_code) + [[ -n $code && $code != 0 ]] +} + +can() { # + local got + # shellcheck disable=SC2086 # $1 is a verb plus resource plus flags + got=$(k auth can-i $1 --as="$SA" 2>/dev/null || true) + if [[ $got != "$2" ]]; then + fail "scaler can-i '$1': got '$got', want '$2'" + fi + echo "can-i $1: $got" +} + +cmd_diag() { + local dir=${1:?usage: kind-smoke.sh diag DIR} + mkdir -p "$dir" + k get all,pvc,secret,lease,role,rolebinding,serviceaccount -o wide >"$dir/get.txt" 2>&1 || true + k describe statefulset,deployment,pods,pvc >"$dir/describe.txt" 2>&1 || true + k get events --sort-by=.lastTimestamp >"$dir/events.txt" 2>&1 || true + k logs "deployment/$SCALER" --all-containers --prefix >"$dir/scaler.log" 2>&1 || true + k logs "$STS-0" --all-containers --prefix >"$dir/runner-0.log" 2>&1 || true + k logs "$STS-0" -c register --previous >"$dir/runner-0-register-previous.log" 2>&1 || true + helm --kube-context "$CONTEXT" -n "$NS" get all "$RELEASE" >"$dir/helm-release.txt" 2>&1 || true + kubectl --context "$CONTEXT" get nodes -o wide >"$dir/nodes.txt" 2>&1 || true + # Only dummy values in this cluster, but keep secret data out regardless. + sed -i.bak "s/$TOKEN//g" "$dir"/*.txt "$dir"/*.log && rm -f "$dir"/*.bak +} + +case ${1:-} in + run) cmd_run ;; + diag) cmd_diag "${2:-}" ;; + *) + echo "usage: $0 run | diag DIR" + exit 2 + ;; +esac diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml new file mode 100644 index 0000000..5ccb81a --- /dev/null +++ b/.github/workflows/k8s.yml @@ -0,0 +1,242 @@ +name: k8s + +# Kubernetes runner: the Go scaler, the Helm chart, both images and a kind +# smoke install. Never pushes an image and never contacts StackGuardian. +# +# Path-filtered so PRs that only touch the VM runner skip the slow image and +# kind jobs. main.sh is in the filter because the runner image bundles it. +# If any of these jobs become required checks, note that a skipped workflow +# leaves them pending; run it with workflow_dispatch in that case. +on: + pull_request: + branches: [main] + paths: + - k8s/** + - main.sh + - .github/workflows/k8s.yml + - .github/scripts/** + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Matches the kind node image below; kubeconform validates against it. + KUBE_VERSION: "1.37.0" + KIND_VERSION: v0.33.0 + KIND_NODE_IMAGE: kindest/node:v1.37.0@sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5 + KUBECONFORM_VERSION: v0.8.0 + KUBECONFORM_SHA256: 9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883 + TRIVY_VERSION: v0.75.0 + +jobs: + scaler: + name: scaler (go) + runs-on: ubuntu-latest + defaults: + run: + working-directory: k8s/scaler + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: k8s/scaler/go.mod + cache-dependency-path: k8s/scaler/go.sum + - name: gofmt + run: | + unformatted=$(gofmt -l .) + if [[ -n $unformatted ]]; then + echo "::error title=gofmt::not gofmt-formatted:" + echo "$unformatted" + exit 1 + fi + - name: go vet + run: go vet ./... + - name: go test -race + run: | + go test -race -covermode=atomic -coverprofile=coverage.out ./... + { + echo '### Scaler coverage' + echo '```' + go tool cover -func=coverage.out | tail -n 1 + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + chart: + name: chart (helm ${{ matrix.helm }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + helm: [v3.22.0, v4.3.0] + steps: + - uses: actions/checkout@v7 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: ${{ matrix.helm }} + - name: Install kubeconform + run: | + curl -fsSL -o "$RUNNER_TEMP/kubeconform.tar.gz" \ + "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" + echo "${KUBECONFORM_SHA256} $RUNNER_TEMP/kubeconform.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/kubeconform.tar.gz" -C "$RUNNER_TEMP" kubeconform + echo "$RUNNER_TEMP" >>"$GITHUB_PATH" + # Lint, render the value scenarios, kubeconform them, check rejections. + - name: Chart check + run: .github/scripts/chart-check.sh "$RUNNER_TEMP/rendered" + - name: Upload rendered manifests + if: failure() + uses: actions/upload-artifact@v7 + with: + name: rendered-helm-${{ matrix.helm }} + path: ${{ runner.temp }}/rendered + retention-days: 7 + + images: + name: image (${{ matrix.image }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - image: runner + context: . + file: k8s/Dockerfile + tag: sg-runner-k8s:ci + - image: scaler + context: k8s/scaler + file: k8s/scaler/Dockerfile + tag: sg-runner-scaler:ci + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + # Both platforms, kept as build cache only: proves arm64 builds too. + - name: Build linux/amd64 + linux/arm64 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + platforms: linux/amd64,linux/arm64 + push: false + outputs: type=cacheonly + provenance: false + cache-from: type=gha,scope=${{ matrix.image }} + cache-to: type=gha,mode=max,scope=${{ matrix.image }} + # amd64 again from the cache, loaded into the local daemon for Trivy. + - name: Load linux/amd64 for scanning + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + platforms: linux/amd64 + push: false + load: true + provenance: false + tags: ${{ matrix.tag }} + cache-from: type=gha,scope=${{ matrix.image }} + # Report HIGH and CRITICAL for both images, never failing here. + - name: Trivy report + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ matrix.tag }} + version: ${{ env.TRIVY_VERSION }} + severity: HIGH,CRITICAL + format: table + output: trivy-${{ matrix.image }}.txt + exit-code: "0" + - name: Trivy summary + if: always() + run: | + [[ -f trivy-${{ matrix.image }}.txt ]] || exit 0 + cat trivy-${{ matrix.image }}.txt + { + echo '### Trivy: ${{ matrix.tag }} (HIGH, CRITICAL)' + echo '
report' + echo + echo '```' + head -c 900000 trivy-${{ matrix.image }}.txt + echo '```' + echo '
' + } >>"$GITHUB_STEP_SUMMARY" + # Gate: fail on CRITICAL vulnerabilities that have a fix, scaler only. + # The scaler is our own Go binary on distroless, so a fixable CRITICAL + # is actionable here (bump a module, rebuild on a newer Go patch). + # The runner image is not gated: it bundles upstream binaries (static + # Docker/containerd, the SSM agent, the ECS agent) built with Go + # releases Trivy marks as "fixed" upstream, which we can only clear by + # waiting for upstream releases. Gating it would keep CI red on things + # this repo can't fix; it is reported above instead. + - name: Trivy gate (CRITICAL, fixable) + if: matrix.image == 'scaler' + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ matrix.tag }} + version: ${{ env.TRIVY_VERSION }} + skip-setup-trivy: true + severity: CRITICAL + ignore-unfixed: true + format: table + exit-code: "1" + + kind-smoke: + name: kind smoke install + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + # amd64 only (the CI runner's arch); reuses the images job's cache when + # it's there. + - name: Build runner image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: k8s/Dockerfile + platforms: linux/amd64 + load: true + provenance: false + tags: sg-runner-k8s:ci + cache-from: type=gha,scope=runner + - name: Build scaler image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: k8s/scaler + platforms: linux/amd64 + load: true + provenance: false + tags: sg-runner-scaler:ci + cache-from: type=gha,scope=scaler + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - name: Create kind cluster + uses: helm/kind-action@1544676c07bb3570fcb70166beaa07c4528fd4aa # v1.15.1 + with: + version: ${{ env.KIND_VERSION }} + node_image: ${{ env.KIND_NODE_IMAGE }} + cluster_name: sg-runner-ci + wait: 120s + # Dummy credentials, SG_BASE_API=http://127.0.0.1:9: never reaches SG. + - name: Install and check + env: + CLUSTER: sg-runner-ci + RUNNER_IMAGE: sg-runner-k8s:ci + SCALER_IMAGE: sg-runner-scaler:ci + run: .github/scripts/kind-smoke.sh run + - name: Collect diagnostics + if: failure() + env: + CLUSTER: sg-runner-ci + run: .github/scripts/kind-smoke.sh diag "$RUNNER_TEMP/kind-diag" + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: kind-smoke-diagnostics + path: ${{ runner.temp }}/kind-diag + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29f1e44..bb11008 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,18 @@ name: test +# Shell: main.sh and k8s/runner.sh (shellcheck), and the bats unit + smoke +# tiers, which include the k8s/runner.sh tests (test/unit/k8s_*.bats). on: pull_request: branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: lint: From 1292675f4f0e860279e4f31b4472f2c45330817f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:20:42 +0200 Subject: [PATCH 24/43] fix: stop error reports from retrying themselves when the api is down --- main.sh | 12 +++++++- test/unit/update_runner_group.bats | 49 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 test/unit/update_runner_group.bats diff --git a/main.sh b/main.sh index 5a1e1e6..ee30a1e 100755 --- a/main.sh +++ b/main.sh @@ -328,6 +328,12 @@ EOF #}}}: api_call update_runner_group() { #{{{ + # Best-effort report, sent at most once per error: api_call reports its own + # failures through err(), which would call back in here and retry until the + # stack overflows (one authenticated request per round) whenever the API is + # unreachable. The guard is dynamically scoped, so nested calls see it. + [[ -n "${REPORTING_RUNNER_GROUP_ERROR:-}" ]] && return 0 + local REPORTING_RUNNER_GROUP_ERROR=1 local err_msg ip_addr payload url url="${SG_BASE_API}/orgs/${ORGANIZATION_ID}/runnergroups/${RUNNER_GROUP_ID}/" @@ -344,11 +350,15 @@ update_runner_group() { #{{{ --arg command "${0} $*" \ '{RunnerRegistrationErrors: {($ip): {RunnerId: $runner_id, error: $error, timestamp: $timestamp, command: $command}}}') - if api_call "PATCH" "$url" "$payload"; then + # Subshell: api_call exits on an empty response, and a failed report must + # not end a caller that only meant to log a non-fatal error. + if (api_call "PATCH" "$url" "$payload"); then debug "updated runner group with error msg" else debug "failed to update runner group with error msg" fi + # debug() returns non-zero when --debug is off; a report never fails err(). + return 0 } #}}}: update_runner_group diff --git a/test/unit/update_runner_group.bats b/test/unit/update_runner_group.bats new file mode 100644 index 0000000..30d6405 --- /dev/null +++ b/test/unit/update_runner_group.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# +# Unit tests for update_runner_group: error reports are best effort and sent +# once, even when the API is unreachable (api_call reports its own failure +# through err(), which calls update_runner_group again). + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" + export SG_NODE_TOKEN="test-token" + export ORGANIZATION_ID="demo-org" + export RUNNER_GROUP_ID="demo-group" +} + +curl_calls() { + grep -c '^curl ' "${MOCK_CALL_LOG}" || true +} + +@test "err with an unreachable API reports once and returns" { + export MOCK_CURL_OUTPUT="" + load_main + + run err "Registration failed" "boom" + assert_success + assert_output --partial "Registration failed" + assert_equal "$(curl_calls)" "1" +} + +@test "die with an unreachable API exits 1 after a single report" { + export MOCK_CURL_OUTPUT="" + load_main + + run die "Could not fetch data from API." "000" + assert_failure 1 + assert_equal "$(curl_calls)" "1" +} + +@test "err reports to the runner group endpoint when the API answers" { + export MOCK_CURL_OUTPUT="$(cat "${BATS_TEST_DIRNAME}/../fixtures/api/200_ok.http")" + load_main + + run err "Registration failed" "boom" + assert_success + assert_equal "$(curl_calls)" "1" + # The JSON payload spans lines in the call log, so check method and URL apart. + run grep -E '^curl .*-X PATCH' "${MOCK_CALL_LOG}" + assert_success + run grep -F '/orgs/demo-org/runnergroups/demo-group/' "${MOCK_CALL_LOG}" + assert_success +} From f19cb70b633c694f1c200016ca077e303caa4af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:22:00 +0200 Subject: [PATCH 25/43] fix: pick up debian security updates in the runner image --- k8s/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/Dockerfile b/k8s/Dockerfile index e4e70c3..4ec4032 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -15,7 +15,9 @@ ARG DOCKER_VERSION=27.5.1 # SSM agent: .../SSMAgent/latest/VERSION names the current release. ARG SSM_AGENT_VERSION=3.3.5390.0 +# upgrade: pick up Debian security fixes newer than the base image. RUN apt-get update \ + && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ bash ca-certificates curl jq iproute2 iptables procps xz-utils \ && rm -rf /var/lib/apt/lists/* From 6ba8ef6ea94ddf89ded174dae7e40c4d3f197160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Tue, 13 Oct 2026 21:23:56 +0200 Subject: [PATCH 26/43] feat: make the kind smoke strict about register failing cleanly --- .github/scripts/kind-smoke.sh | 59 ++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/.github/scripts/kind-smoke.sh b/.github/scripts/kind-smoke.sh index b8d2afd..fa1b338 100755 --- a/.github/scripts/kind-smoke.sh +++ b/.github/scripts/kind-smoke.sh @@ -27,6 +27,8 @@ SCALER=sg-runner-scaler SECRET=sg-runner-credentials CHART=k8s/chart/sg-runner TIMEOUT=${TIMEOUT:-240} +# Failed SG API calls one register run may make (see the register step). +MAX_API_ATTEMPTS=3 # Distinctive, so finding it in any log means a credential leaked. TOKEN=ci-dummy-node-token-6f1d3c9a SA=system:serviceaccount:$NS:$SCALER @@ -101,20 +103,22 @@ cmd_run() { step "runner StatefulSet exists, register init container fails" k get "statefulset/$STS" - until_ok "register init container of $STS-0 to exit non-zero" register_failed - local logs code - code=$(register_exit_code) - logs=$(k logs "$STS-0" -c register --previous 2>/dev/null || k logs "$STS-0" -c register) - # Deduplicated: a failing API call can repeat the same error many times. - awk '!seen[$0]++' <<<"$logs" | tail -n 20 - grep -Eq "Could not fetch data from API|API call failed" <<<"$logs" \ - || fail "register exited $code, but not on the SG API call" - echo "register exited $code on the SG API call, as expected" - if ((code >= 128)); then - # err() reports through the API, whose failure calls err() again: with SG - # unreachable that recurses until bash crashes instead of exiting 1. - echo "::warning title=kind smoke::register crashed (exit $code, $(grep -c 'API call failed' <<<"$logs") failed API calls) instead of exiting cleanly" - fi + until_ok "a finished run of the register init container of $STS-0" register_finished + local logs=$REGISTER_LOG code=$REGISTER_CODE attempts + echo "$logs" | tail -n 20 + # Exactly 1: die() on the failed SG call. 0 would mean it registered + # against nothing; >=128 means it crashed (e.g. error reporting recursing + # into itself while the API is down). + [[ $code == 1 ]] || fail "register exited $code, want 1" + grep -q "API call failed" <<<"$logs" \ + || fail "register exited 1, but not on the SG API call" + grep -qF "$TOKEN" <<<"$logs" && fail "register printed the dummy SG_NODE_TOKEN" + # Each failed API call logs one "API call failed": the registration call + # plus at most one best-effort error report. More means it is retrying. + attempts=$(grep -c "API call failed" <<<"$logs") + ((attempts <= MAX_API_ATTEMPTS)) \ + || fail "register made $attempts failed API calls in one run, want <= $MAX_API_ATTEMPTS" + echo "register exited 1 after $attempts failed API calls, as expected" endstep step "no credentials in logs or events" @@ -152,18 +156,23 @@ scaler_saw_sg_unreachable() { | grep '"loop failed, skipping decisions"' | grep -q '"reason":"SGUnreachable"' } -# Exit code of the register init container's last finished run, if any. -register_exit_code() { - local status +# Sets REGISTER_CODE and REGISTER_LOG from the register init container's last +# finished run: the current container once it has terminated, else the +# previous one (it restarts in a loop). Fails until a run has finished and its +# log is retrievable. +register_finished() { + local status prev=() status=$(k get pod "$STS-0" -o jsonpath='{.status.initContainerStatuses[?(@.name=="register")]}' 2>/dev/null) \ - || return 0 - jq -r '(.state.terminated // .lastState.terminated // {}).exitCode // empty' <<<"$status" -} - -register_failed() { - local code - code=$(register_exit_code) - [[ -n $code && $code != 0 ]] + || return 1 + REGISTER_CODE=$(jq -r '.state.terminated.exitCode // empty' <<<"$status") + if [[ -z $REGISTER_CODE ]]; then + REGISTER_CODE=$(jq -r '.lastState.terminated.exitCode // empty' <<<"$status") + prev=(--previous) + fi + [[ -n $REGISTER_CODE ]] || return 1 + REGISTER_LOG=$(k logs "$STS-0" -c register "${prev[@]}" 2>/dev/null) || return 1 + # Gone between the two calls (restarted again): try the next round. + [[ -n $REGISTER_LOG && $REGISTER_LOG != "unable to retrieve container logs"* ]] } can() { # From 9203eab64a26195debdc3e70ba60dfa5071d1102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 14 Oct 2026 16:59:09 +0200 Subject: [PATCH 27/43] feat: scaler teardown mode, autoscaler nudge and config from a configmap --- k8s/scaler/cmd/sg-runner-scaler/main.go | 72 ++++- k8s/scaler/internal/config/config.go | 17 ++ k8s/scaler/internal/config/config_test.go | 6 +- k8s/scaler/internal/controller/controller.go | 159 ++++++++++ .../internal/controller/controller_test.go | 2 +- k8s/scaler/internal/controller/nudge_test.go | 263 ++++++++++++++++ k8s/scaler/internal/kube/kube.go | 85 +++++- k8s/scaler/internal/kube/kube_test.go | 15 +- k8s/scaler/internal/kube/nudge.go | 288 ++++++++++++++++++ k8s/scaler/internal/kube/nudge_test.go | 130 ++++++++ k8s/scaler/internal/metrics/metrics.go | 8 +- k8s/scaler/internal/sg/sg.go | 3 +- k8s/scaler/internal/teardown/teardown.go | 213 +++++++++++++ k8s/scaler/internal/teardown/teardown_test.go | 202 ++++++++++++ 14 files changed, 1432 insertions(+), 31 deletions(-) create mode 100644 k8s/scaler/internal/controller/nudge_test.go create mode 100644 k8s/scaler/internal/kube/nudge.go create mode 100644 k8s/scaler/internal/kube/nudge_test.go create mode 100644 k8s/scaler/internal/teardown/teardown.go create mode 100644 k8s/scaler/internal/teardown/teardown_test.go diff --git a/k8s/scaler/cmd/sg-runner-scaler/main.go b/k8s/scaler/cmd/sg-runner-scaler/main.go index 9716fb9..27b07fb 100644 --- a/k8s/scaler/cmd/sg-runner-scaler/main.go +++ b/k8s/scaler/cmd/sg-runner-scaler/main.go @@ -1,6 +1,10 @@ // sg-runner-scaler sizes the sg-runner StatefulSet from the runner group's // state in StackGuardian, parks idle runners and deregisters runners parked // for too long. See k8s/README.md, "Autoscaling". +// +// `sg-runner-scaler teardown` instead deregisters every runner and deletes +// their PVCs: the Helm chart's pre-delete hook, so `helm uninstall` leaves +// nothing behind. package main import ( @@ -33,12 +37,17 @@ import ( "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/teardown" ) func main() { log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) slog.SetDefault(log) klog.SetSlogLogger(log.With("source", "client-go")) + run := run + if len(os.Args) > 1 && os.Args[1] == "teardown" { + run = runTeardown + } if err := run(log); err != nil { log.Error("exiting", "err", err) os.Exit(1) @@ -58,18 +67,7 @@ func run(log *slog.Logger) error { return err } - restCfg, err := rest.InClusterConfig() - if err != nil { - // Out of cluster (development): the usual kubeconfig. - restCfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}).ClientConfig() - if err != nil { - return err - } - } - restCfg.Timeout = 15 * time.Second - restCfg.UserAgent = "sg-runner-scaler" - client, err := kubernetes.NewForConfig(restCfg) + client, err := kubeClient() if err != nil { return err } @@ -83,8 +81,9 @@ func run(log *slog.Logger) error { defer broadcaster.Shutdown() recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "sg-runner-scaler"}) - k := kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret) + k := kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret, cfg.ConfigMap) k.ReapPod = cfg.ReapPod + k.NudgeImage = cfg.NudgeImage ctrl := &controller.Controller{ Config: cfg.Rules, Interval: cfg.Interval, @@ -93,7 +92,9 @@ func run(log *slog.Logger) error { Events: recorder, Metrics: m, Log: log, + Nudge: cfg.Nudge, } + ctrl.NudgeCooldown = cfg.NudgeCooldown var leading atomic.Bool srv := serve(cfg, m, ctrl, &leading, log) @@ -154,6 +155,51 @@ func run(log *slog.Logger) error { return nil } +// runTeardown is the pre-delete hook: deregister every runner, delete PVCs. +// It runs until done or until the Job's activeDeadlineSeconds stops it. +func runTeardown(log *slog.Logger) error { + cfg, err := config.Load(nil) + if err != nil { + return err + } + client, err := kubeClient() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + k := kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret, cfg.ConfigMap) + k.ReapPod = cfg.ReapPod + td := &teardown.Teardown{ + Kube: k, + ScalerDeployment: os.Getenv("TEARDOWN_SCALER_DEPLOYMENT"), + ScalerSelector: os.Getenv("TEARDOWN_SCALER_SELECTOR"), + RunnerSelector: os.Getenv("TEARDOWN_RUNNER_SELECTOR"), + Log: log, + } + if td.RunnerSelector == "" || (td.ScalerDeployment != "" && td.ScalerSelector == "") { + return errors.New("teardown needs TEARDOWN_RUNNER_SELECTOR, and TEARDOWN_SCALER_SELECTOR with TEARDOWN_SCALER_DEPLOYMENT") + } + return td.Run(ctx) +} + +// kubeClient connects in cluster, or with the usual kubeconfig out of +// cluster (development). +func kubeClient() (kubernetes.Interface, error) { + restCfg, err := rest.InClusterConfig() + if err != nil { + restCfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, err + } + } + restCfg.Timeout = 15 * time.Second + restCfg.UserAgent = "sg-runner-scaler" + return kubernetes.NewForConfig(restCfg) +} + func identity() (string, error) { host, err := os.Hostname() if err != nil { diff --git a/k8s/scaler/internal/config/config.go b/k8s/scaler/internal/config/config.go index 6395852..5b44ef7 100644 --- a/k8s/scaler/internal/config/config.go +++ b/k8s/scaler/internal/config/config.go @@ -27,10 +27,16 @@ type Config struct { StatefulSet string Image string CredentialsSecret string + ConfigMap string CredentialsDir string Namespace string MetricsAddr string LeaderElect bool + // Nudge starts placeholder pods when the cluster autoscaler won't add a + // node for a runner pod; NudgeImage is their image. + Nudge bool + NudgeImage string + NudgeCooldown time.Duration // ReapPod is extra pod settings for reap Jobs. ReapPod kube.ReapPodOptions // MinRaised reports a SCALER_MIN below 1 (MinGiven) raised to 1. @@ -71,6 +77,7 @@ func Load(getenv func(string) string) (Config, error) { StatefulSet: str("SCALER_STATEFULSET", "sg-runner"), Image: str("SCALER_IMAGE", "sg-runner-k8s:dev"), CredentialsSecret: str("SCALER_CREDENTIALS_SECRET", "sg-runner-credentials"), + ConfigMap: str("SCALER_CONFIGMAP", "sg-runner-config"), CredentialsDir: str("SG_CREDENTIALS_DIR", "/etc/sg-runner/credentials"), MetricsAddr: str("SCALER_METRICS_ADDR", ":9090"), Rules: decide.Config{ @@ -102,6 +109,16 @@ func Load(getenv func(string) string) (Config, error) { errs = append(errs, fmt.Sprintf("SCALER_LEADER_ELECT=%q: want true or false", v)) } + switch v := strings.ToLower(str("SCALER_NUDGE", "true")); v { + case "true", "1", "yes": + c.Nudge = true + case "false", "0", "no": + default: + errs = append(errs, fmt.Sprintf("SCALER_NUDGE=%q: want true or false", v)) + } + c.NudgeImage = str("SCALER_NUDGE_IMAGE", kube.DefaultNudgeImage) + c.NudgeCooldown = secs("SCALER_NUDGE_COOLDOWN", 180, 30) + c.ReapPod, errs = reapPod(str, errs) c.Namespace = str("POD_NAMESPACE", "") diff --git a/k8s/scaler/internal/config/config_test.go b/k8s/scaler/internal/config/config_test.go index 0bf0cc7..56dcf82 100644 --- a/k8s/scaler/internal/config/config_test.go +++ b/k8s/scaler/internal/config/config_test.go @@ -19,8 +19,10 @@ func TestDefaults(t *testing.T) { if c.Interval != 15*time.Second || r.Min != 1 || r.Max != 10 || r.HeadroomPct != 0 || r.IdleAfter != 60*time.Second || r.ParkTTL != 43200*time.Second || r.StuckAfter != 180*time.Second || r.SettleTimeout != 300*time.Second || c.StatefulSet != "sg-runner" || c.Image != "sg-runner-k8s:dev" || - c.CredentialsSecret != "sg-runner-credentials" || c.CredentialsDir != "/etc/sg-runner/credentials" || - c.MetricsAddr != ":9090" || !c.LeaderElect || c.Namespace != "sg-runner" || c.MinRaised { + c.CredentialsSecret != "sg-runner-credentials" || c.ConfigMap != "sg-runner-config" || + c.CredentialsDir != "/etc/sg-runner/credentials" || + c.MetricsAddr != ":9090" || !c.LeaderElect || c.Namespace != "sg-runner" || c.MinRaised || + !c.Nudge || c.NudgeImage != "registry.k8s.io/pause:3.10" || c.NudgeCooldown != 180*time.Second { t.Errorf("defaults %+v", c) } } diff --git a/k8s/scaler/internal/controller/controller.go b/k8s/scaler/internal/controller/controller.go index bcb5a21..f70836d 100644 --- a/k8s/scaler/internal/controller/controller.go +++ b/k8s/scaler/internal/controller/controller.go @@ -15,6 +15,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" @@ -53,6 +54,12 @@ type Controller struct { FailureEvents int // WarnEvery rate-limits repeats of the same Warning event. WarnEvery time.Duration + // Nudge starts placeholder pods for runner pods the cluster autoscaler + // won't add a node for (see kube.NudgeJob). + Nudge bool + // NudgeCooldown spaces placeholders for the same runner + // (DefaultNudgeCooldown when zero). + NudgeCooldown time.Duration state *decide.State persisted map[string]*string @@ -63,8 +70,26 @@ type Controller struct { // lastLoop is the Unix nano time of the last loop attempt. lastLoop atomic.Int64 ordinal *regexp.Regexp + // nudges are each unschedulable runner's placeholder attempts. + nudges map[int]nudgeAttempt } +// nudgeAttempt is when a runner's last placeholder started, and how many it +// had since it became unschedulable. +type nudgeAttempt struct { + at time.Time + attempts int +} + +// DefaultNudgeCooldown spaces placeholders for the same runner. It must +// outlast a node's boot: a runner still Pending sooner is just waiting for +// the node already requested. +const DefaultNudgeCooldown = 3 * time.Minute + +// NudgeWarnAfter is how many placeholders a runner may need before the +// scaler reports it as blocked. +const NudgeWarnAfter = 2 + func (c *Controller) init() { if c.Now == nil { c.Now = time.Now @@ -84,6 +109,12 @@ func (c *Controller) init() { if c.warned == nil { c.warned = map[string]time.Time{} } + if c.nudges == nil { + c.nudges = map[int]nudgeAttempt{} + } + if c.NudgeCooldown == 0 { + c.NudgeCooldown = DefaultNudgeCooldown + } if c.ordinal == nil { c.ordinal = regexp.MustCompile(`^` + regexp.QuoteMeta(c.Kube.StatefulSet) + `-(\d+)\.`) } @@ -264,6 +295,7 @@ func (c *Controller) loop(ctx context.Context, now time.Time) (*appsv1.StatefulS } } c.reap(ctx, sts, d.Reap, now) + c.nudge(ctx, sts, d.Replicas, now) parked := 0 for n := range claims { @@ -442,6 +474,133 @@ func (c *Controller) reap(ctx context.Context, sts *appsv1.StatefulSet, actions } } +// nudge gets nodes for awake runners whose pods the cluster autoscaler +// refused to add a node for: a placeholder without volumes asks for one, and +// is deleted as soon as the autoscaler accepts (TriggeredScaleUp), so the +// runner takes the node the moment it joins. A runner still unschedulable a +// cooldown later gets another placeholder; repeated attempts are reported. +// Best effort: errors are logged and retried next loop. +func (c *Controller) nudge(ctx context.Context, sts *appsv1.StatefulSet, replicas int, now time.Time) { + if !c.Nudge { + return + } + kerr := func(what string, err error) { + c.Metrics.KubeErrors.Inc() + c.Log.Error(what, "err", err) + } + sel, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector) + if err != nil { + c.Log.Error("runner selector", "err", err) + return + } + pods, err := c.Kube.RunnerPods(ctx, sel.String()) + if err != nil { + kerr("list runner pods", err) + return + } + jobs, err := c.Kube.NudgeJobs(ctx) + if err != nil { + kerr("list placeholder jobs", err) + return + } + blocked := func(n int, why string) { + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: "ScaleUpBlocked", + Key: fmt.Sprintf("ScaleUpBlocked/%d", n), + Message: fmt.Sprintf("runner %d can't get a node: %s; retrying in %s", n, why, c.NudgeCooldown)}) + } + + for n, j := range jobs { + p := pods[n] + job := c.Kube.NudgeJobName(n) + log := c.Log.With("runner", n, "job", job) + remove := func() bool { + if err := c.Kube.DeleteNudgeJob(ctx, n); err != nil { + kerr("delete placeholder job", err) + return false + } + return true + } + if n >= replicas || p == nil || !kube.Unschedulable(p) { + // Placed, parked or gone: nothing left to ask for. + if remove() { + log.Info("placeholder no longer needed, deleted") + } + continue + } + reason, msg, err := c.Kube.NudgeOutcome(ctx, n, j.Created) + if err != nil { + kerr("list placeholder events", err) + continue + } + switch { + case reason == kube.EventScaleUpTriggered || j.State == kube.NudgeRunning: + // A node is coming (or there): free it for the runner now. + if !remove() { + continue + } + c.Metrics.Nudges.WithLabelValues("node_requested").Inc() + log.Info("cluster autoscaler is adding a node; placeholder deleted so the runner takes it") + c.Events.Eventf(sts, corev1.EventTypeNormal, "NodeRequested", + "runner %d: the cluster autoscaler is adding a node for job/%s; deleted the placeholder so the runner takes the node", n, job) + case reason == kube.EventScaleUpRefused: + if !remove() { + continue + } + c.Metrics.Nudges.WithLabelValues("refused").Inc() + log.Warn("cluster autoscaler refused the placeholder too", "message", msg) + blocked(n, "the cluster autoscaler won't add a node even without volumes ("+msg+")") + case j.State == kube.NudgeFailed: + if !remove() { + continue + } + c.Metrics.Nudges.WithLabelValues("refused").Inc() + log.Warn("no answer from the cluster autoscaler within the placeholder's deadline") + blocked(n, fmt.Sprintf("no node for its placeholder within %s", time.Duration(kube.NudgeDeadline)*time.Second)) + } + } + + for n, p := range pods { + if n >= replicas || !kube.Unschedulable(p) { + delete(c.nudges, n) + continue + } + if _, ok := jobs[n]; ok { + continue + } + last := c.nudges[n] + if last.attempts > 0 && now.Sub(last.at) < c.NudgeCooldown { + continue + } + refused, err := c.Kube.ScaleUpRefused(ctx, p) + if err != nil { + kerr("list autoscaler events", err) + continue + } + if !refused { + continue + } + if last.attempts >= NudgeWarnAfter { + blocked(n, fmt.Sprintf("still unschedulable after %d node requests (the new node went to another pod, or never came)", last.attempts)) + } + if err := c.Kube.CreateNudgeJob(ctx, n, p); err != nil { + kerr("create placeholder job", err) + continue + } + c.nudges[n] = nudgeAttempt{at: now, attempts: last.attempts + 1} + c.Metrics.Nudges.WithLabelValues("started").Inc() + c.Log.Info("cluster autoscaler won't add a node for the runner pod; asking with a placeholder", + "runner", n, "job", c.Kube.NudgeJobName(n), "attempt", last.attempts+1) + c.Events.Eventf(sts, corev1.EventTypeNormal, "NudgingAutoscaler", + "runner %d can't be scheduled and the cluster autoscaler won't add a node for it (its volumes): job/%s asks for one", + n, c.Kube.NudgeJobName(n)) + } + for n := range c.nudges { + if _, ok := pods[n]; !ok { + delete(c.nudges, n) + } + } +} + // warn emits a Warning event unless the same one went out within WarnEvery. func (c *Controller) warn(sts *appsv1.StatefulSet, e decide.Event) { now := c.Now() diff --git a/k8s/scaler/internal/controller/controller_test.go b/k8s/scaler/internal/controller/controller_test.go index 50627fd..4ebd83b 100644 --- a/k8s/scaler/internal/controller/controller_test.go +++ b/k8s/scaler/internal/controller/controller_test.go @@ -102,7 +102,7 @@ func (e *env) newController(s SG) *Controller { SettleTimeout: 5 * time.Minute, ParkTTL: 12 * time.Hour}, Interval: 15 * time.Second, SG: s, - Kube: kube.New(e.client, ns, "sg-runner", "img:1", "creds"), + Kube: kube.New(e.client, ns, "sg-runner", "img:1", "creds", "cfg"), Events: e.events, Metrics: metrics.New(), Log: slog.New(slog.NewTextHandler(e.logs, nil)), diff --git a/k8s/scaler/internal/controller/nudge_test.go b/k8s/scaler/internal/controller/nudge_test.go new file mode 100644 index 0000000..9b3b5ba --- /dev/null +++ b/k8s/scaler/internal/controller/nudge_test.go @@ -0,0 +1,263 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +var runnerLabels = map[string]string{"app.kubernetes.io/component": "runner"} + +func runnerPod(n string, node string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-" + n, Namespace: ns, UID: types.UID("uid-" + n), Labels: runnerLabels}, + Spec: corev1.PodSpec{ + NodeName: node, + Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{MatchLabels: runnerLabels}}}}}, + Containers: []corev1.Container{{Name: "dockerd", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("6500m"), corev1.ResourceMemory: resource.MustParse("11Gi")}}}}, + Volumes: []corev1.Volume{{Name: "state"}}, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + if node == "" { + p.Status = corev1.PodStatus{Phase: corev1.PodPending, Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable}}} + } + return p +} + +func refusedEvent(pod string) *corev1.Event { + return &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: pod + ".1", Namespace: ns}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: pod, Namespace: ns}, + Reason: "NotTriggerScaleUp", Source: corev1.EventSource{Component: "cluster-autoscaler"}, + } +} + +// nudgeEnv: runner 0 runs and is busy, runner 1 (awake, new) can't be +// scheduled. +func nudgeEnv(t *testing.T, objs ...runtime.Object) *env { + sts := statefulSet(2) + sts.Spec.Selector = &metav1.LabelSelector{MatchLabels: runnerLabels} + objs = append(objs, sts, runnerPod("0", "node-a"), runnerPod("1", ""), + claim("state-sg-runner-0", nil), claim("state-sg-runner-1", nil)) + e := newEnv(t, &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, decide.StatusActive, true, 1)}}}, objs...) + e.ctrl.Nudge = true + return e +} + +func (e *env) job(name string) (*batchv1.Job, error) { + return e.client.BatchV1().Jobs(ns).Get(context.Background(), name, metav1.GetOptions{}) +} + +func TestNudgeStartsPlaceholderWhenAutoscalerRefuses(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, err := e.job("sg-runner-nudge-1") + if err != nil { + t.Fatalf("no placeholder job: %v\n%s", err, e.logs) + } + ps := j.Spec.Template.Spec + req := ps.Containers[0].Resources.Requests + if req.Cpu().String() != "6500m" || req.Memory().String() != "11Gi" { + t.Errorf("placeholder requests %v, want the runner's", req) + } + if len(ps.Volumes) != 0 { + t.Errorf("placeholder has volumes %v", ps.Volumes) + } + if ps.Affinity == nil || ps.Affinity.PodAntiAffinity == nil { + t.Errorf("placeholder lost the runner's anti-affinity") + } + if j.Labels["app.kubernetes.io/component"] == "runner" { + t.Errorf("placeholder carries runner labels: %v", j.Labels) + } + if !hasEvent(e.drainEvents(), "Normal NudgingAutoscaler runner 1") { + t.Error("no NudgingAutoscaler event") + } + if _, err := e.job("sg-runner-nudge-0"); err == nil { + t.Error("placeholder for the scheduled runner 0") + } +} + +func TestNudgeWaitsForTheAutoscalersVerdict(t *testing.T) { + // No NotTriggerScaleUp: the autoscaler may still add a node itself. + e := nudgeEnv(t) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("placeholder started without the autoscaler refusing") + } +} + +// autoscalerSays records the cluster autoscaler's answer on the runner's +// current placeholder pod. +func (e *env) autoscalerSays(n, reason string) { + e.t.Helper() + ev := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "ph-" + n + "-" + reason + e.now.String(), Namespace: ns}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: "sg-runner-nudge-" + n + "-xyz", Namespace: ns}, + Reason: reason, Message: "max node group size reached", LastTimestamp: metav1.NewTime(e.now), + } + if _, err := e.client.CoreV1().Events(ns).Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { + e.t.Fatal(err) + } +} + +func TestNudgeReleasesThePlaceholderOnceTheAutoscalerAccepts(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + e.autoscalerSays("1", "TriggeredScaleUp") + e.drainEvents() + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Fatal("placeholder kept after the autoscaler accepted") + } + if !hasEvent(e.drainEvents(), "Normal NodeRequested runner 1") { + t.Error("no NodeRequested event") + } + if v := testutil.ToFloat64(e.ctrl.Metrics.Nudges.WithLabelValues("node_requested")); v != 1 { + t.Errorf("node_requested = %v, want 1", v) + } + // The node is booting and the runner still Pending: no new placeholder + // within the cooldown. + e.now = e.now.Add(time.Minute) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("new placeholder within the cooldown") + } + // Past it, still unschedulable: ask again, and that's the attempt + // after which it's reported. + e.now = e.now.Add(e.ctrl.NudgeCooldown) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err != nil { + t.Fatalf("no placeholder after the cooldown: %v", err) + } + if hasEvent(e.drainEvents(), "Warning ScaleUpBlocked") { + t.Error("blocked after only one earlier attempt") + } + _ = e.client.BatchV1().Jobs(ns).Delete(context.Background(), "sg-runner-nudge-1", metav1.DeleteOptions{}) + e.now = e.now.Add(e.ctrl.NudgeCooldown) + if err := e.once(); err != nil { + t.Fatal(err) + } + if !hasEvent(e.drainEvents(), "Warning ScaleUpBlocked runner 1 can't get a node: still unschedulable after 2") { + t.Error("no ScaleUpBlocked after repeated attempts") + } +} + +func TestNudgeHandsOverARunningPlaceholder(t *testing.T) { + // The placeholder found room without a scale-up (a node freed up). + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, _ := e.job("sg-runner-nudge-1") + j.Status.Ready = ptr.To[int32](1) + if _, err := e.client.BatchV1().Jobs(ns).UpdateStatus(context.Background(), j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("running placeholder kept") + } +} + +func TestNudgeReportsAPlaceholderTheAutoscalerRefuses(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + e.autoscalerSays("1", "NotTriggerScaleUp") + e.drainEvents() + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("refused placeholder kept") + } + if !hasEvent(e.drainEvents(), "Warning ScaleUpBlocked runner 1 can't get a node: the cluster autoscaler won't add a node even without volumes (max node group size reached)") { + t.Error("no ScaleUpBlocked warning with the autoscaler's reason") + } +} + +func TestNudgeDeletesPlaceholderOnceTheRunnerIsPlaced(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.client.CoreV1().Pods(ns).Update(context.Background(), runnerPod("1", "node-b"), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("placeholder kept after the runner was placed") + } +} + +func TestNudgeReportsAPlaceholderPastItsDeadline(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, _ := e.job("sg-runner-nudge-1") + j.Status.Failed = 1 + if _, err := e.client.BatchV1().Jobs(ns).UpdateStatus(context.Background(), j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.drainEvents() + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("failed placeholder kept") + } + if !hasEvent(e.drainEvents(), "Warning ScaleUpBlocked runner 1") { + t.Error("no ScaleUpBlocked warning") + } +} + +func TestNudgeOff(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + e.ctrl.Nudge = false + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("placeholder with nudging off") + } +} diff --git a/k8s/scaler/internal/kube/kube.go b/k8s/scaler/internal/kube/kube.go index 01783bc..797bdfc 100644 --- a/k8s/scaler/internal/kube/kube.go +++ b/k8s/scaler/internal/kube/kube.go @@ -51,22 +51,31 @@ type Kube struct { StatefulSet string // Image is the runner image reap Jobs run. Image string - // CredentialsSecret is the runner group's SG credentials Secret. + // CredentialsSecret holds the runner group's node token (key + // SG_NODE_TOKEN). CredentialsSecret string - // ReapPod customizes reap Jobs. + // ConfigMap holds the runner group (SG_ORGANIZATION, SG_RUNNER_GROUP, + // SG_BASE_API). + ConfigMap string + // ReapPod customizes reap Jobs (and placeholder Jobs' pull secrets). ReapPod ReapPodOptions + // NudgeImage is the placeholder Jobs' image (DefaultNudgeImage when empty). + NudgeImage string - claimRE *regexp.Regexp - jobRE *regexp.Regexp + claimRE *regexp.Regexp + dockerRE *regexp.Regexp + jobRE *regexp.Regexp } // New returns the adapter for one StatefulSet. -func New(c kubernetes.Interface, namespace, statefulSet, image, secret string) *Kube { +func New(c kubernetes.Interface, namespace, statefulSet, image, secret, configMap string) *Kube { q := regexp.QuoteMeta(statefulSet) return &Kube{ - Client: c, Namespace: namespace, StatefulSet: statefulSet, Image: image, CredentialsSecret: secret, - claimRE: regexp.MustCompile(`^state-` + q + `-(\d+)$`), - jobRE: regexp.MustCompile(`^` + q + `-reap-(\d+)$`), + Client: c, Namespace: namespace, StatefulSet: statefulSet, Image: image, + CredentialsSecret: secret, ConfigMap: configMap, + claimRE: regexp.MustCompile(`^state-` + q + `-(\d+)$`), + dockerRE: regexp.MustCompile(`^docker-` + q + `-(\d+)$`), + jobRE: regexp.MustCompile(`^` + q + `-reap-(\d+)$`), } } @@ -156,6 +165,51 @@ func (k *Kube) Claims(ctx context.Context) (map[int]decide.Claim, error) { return out, nil } +// DockerClaims lists the ordinals that have a Docker cache PVC. +func (k *Kube) DockerClaims(ctx context.Context) ([]int, error) { + list, err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + var out []int + for _, p := range list.Items { + m := k.dockerRE.FindStringSubmatch(p.Name) + if m == nil || p.DeletionTimestamp != nil { + continue + } + if n, err := strconv.Atoi(m[1]); err == nil { + out = append(out, n) + } + } + return out, nil +} + +// ScaleDeployment sets a Deployment's replicas through the scale subresource. +func (k *Kube) ScaleDeployment(ctx context.Context, name string, replicas int) error { + body := fmt.Sprintf(`{"spec":{"replicas":%d}}`, replicas) + _, err := k.Client.AppsV1().Deployments(k.Namespace).Patch(ctx, name, + types.MergePatchType, []byte(body), metav1.PatchOptions{}, "scale") + return err +} + +// CountPods counts the pods matching a label selector. +func (k *Kube) CountPods(ctx context.Context, selector string) (int, error) { + list, err := k.Client.CoreV1().Pods(k.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return 0, err + } + return len(list.Items), nil +} + +// ReapJobExists tells whether the reap Job of an ordinal still exists. +func (k *Kube) ReapJobExists(ctx context.Context, n int) (bool, error) { + _, err := k.Client.BatchV1().Jobs(k.Namespace).Get(ctx, k.JobName(n), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil, err +} + // DeleteClaim deletes a PVC; a missing one is fine. func (k *Kube) DeleteClaim(ctx context.Context, name string) error { err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).Delete(ctx, name, metav1.DeleteOptions{}) @@ -233,8 +287,21 @@ func (k *Kube) ReapJob(n int) *batchv1.Job { Volumes: []corev1.Volume{ {Name: "state", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: k.ClaimName(n)}}}, + // The node token and the runner group as files in one + // directory, as runner.sh reads them. {Name: "sg-credentials", VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: k.CredentialsSecret, DefaultMode: ptr.To[int32](0o400)}}}, + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr.To[int32](0o400), + Sources: []corev1.VolumeProjection{ + {Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: k.CredentialsSecret}, + Items: []corev1.KeyToPath{{Key: "SG_NODE_TOKEN", Path: "SG_NODE_TOKEN"}}, + }}, + {ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: k.ConfigMap}, + }}, + }, + }}}, }, }, }, diff --git a/k8s/scaler/internal/kube/kube_test.go b/k8s/scaler/internal/kube/kube_test.go index 9ee8eed..d6da133 100644 --- a/k8s/scaler/internal/kube/kube_test.go +++ b/k8s/scaler/internal/kube/kube_test.go @@ -24,7 +24,7 @@ func pvc(name string, ann map[string]string) *corev1.PersistentVolumeClaim { } func newKube(objs ...runtime.Object) *Kube { - return New(fake.NewClientset(objs...), ns, "sg-runner", "img:1", "creds") + return New(fake.NewClientset(objs...), ns, "sg-runner", "img:1", "creds", "cfg") } func TestClaims(t *testing.T) { @@ -89,7 +89,7 @@ func TestScaleAndAnnotateStatefulSet(t *testing.T) { } func TestReapJobSpec(t *testing.T) { - k := New(nil, ns, "sg-runner", "img:1", "creds") + k := New(nil, ns, "sg-runner", "img:1", "creds", "cfg") j := k.ReapJob(4) if j.Name != "sg-runner-reap-4" || j.Labels["app"] != "sg-runner-reaper" { t.Errorf("metadata %+v", j.ObjectMeta) @@ -115,9 +115,16 @@ func TestReapJobSpec(t *testing.T) { t.Errorf("mounts %+v", c.VolumeMounts) } if ps.Volumes[0].PersistentVolumeClaim.ClaimName != "state-sg-runner-4" || - ps.Volumes[1].Secret.SecretName != "creds" || *ps.Volumes[1].Secret.DefaultMode != 0o400 { + *ps.Volumes[1].Projected.DefaultMode != 0o400 { t.Errorf("volumes %+v", ps.Volumes) } + // Only the token from the Secret, the rest from the ConfigMap. + src := ps.Volumes[1].Projected.Sources + wantToken := []corev1.KeyToPath{{Key: "SG_NODE_TOKEN", Path: "SG_NODE_TOKEN"}} + if len(src) != 2 || src[0].Secret.Name != "creds" || !reflect.DeepEqual(src[0].Secret.Items, wantToken) || + src[1].ConfigMap.Name != "cfg" || src[1].ConfigMap.Items != nil { + t.Errorf("credential sources %+v", src) + } } func TestReapJobLifecycle(t *testing.T) { @@ -156,7 +163,7 @@ func TestReapJobLifecycle(t *testing.T) { } func TestReapJobPodOptions(t *testing.T) { - k := New(nil, ns, "sg-runner", "img:1", "creds") + k := New(nil, ns, "sg-runner", "img:1", "creds", "cfg") k.ReapPod = ReapPodOptions{ Labels: map[string]string{"np": "egress", "app": "hijack"}, ImagePullSecrets: []string{"regcred"}, diff --git a/k8s/scaler/internal/kube/nudge.go b/k8s/scaler/internal/kube/nudge.go new file mode 100644 index 0000000..7f30d9c --- /dev/null +++ b/k8s/scaler/internal/kube/nudge.go @@ -0,0 +1,288 @@ +package kube + +// Nudging the cluster autoscaler. Some autoscalers (GKE's, at least) never +// add a node for a pod with a persistent disk: the template node they +// simulate lacks the disk driver's zone label, so a runner with its PVCs +// stays Pending ("NotTriggerScaleUp ... didn't match Pod's node +// affinity/selector") while the same pod without volumes would scale up. A +// nudge is a placeholder Job with the runner pod's requests and scheduling +// rules but no volumes: the autoscaler adds a node for it, and once it runs, +// deleting it leaves that node to the runner. + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/utils/ptr" +) + +// DefaultNudgeImage is the placeholder's image: it only has to sleep. +const DefaultNudgeImage = "registry.k8s.io/pause:3.10" + +// NudgeDeadline bounds a placeholder that never gets a node (e.g. the node +// pool's maximum is reached). +const NudgeDeadline int64 = 15 * 60 + +// NudgeState is a placeholder Job's progress. +type NudgeState int + +// Placeholder progress. +const ( + // NudgeWaiting: created, its pod not running yet (a node is coming). + NudgeWaiting NudgeState = iota + // NudgeRunning: its pod runs, so the node is there. + NudgeRunning + // NudgeFailed: past its deadline without a node. + NudgeFailed +) + +func (k *Kube) nudgeApp() string { return k.StatefulSet + "-nudge" } + +// NudgeJobName is the placeholder Job of an ordinal. +func (k *Kube) NudgeJobName(n int) string { return fmt.Sprintf("%s-nudge-%d", k.StatefulSet, n) } + +// RunnerPods lists the runner StatefulSet's pods by ordinal. +func (k *Kube) RunnerPods(ctx context.Context, selector string) (map[int]*corev1.Pod, error) { + list, err := k.Client.CoreV1().Pods(k.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`^` + regexp.QuoteMeta(k.StatefulSet) + `-(\d+)$`) + out := map[int]*corev1.Pod{} + for i := range list.Items { + p := &list.Items[i] + m := re.FindStringSubmatch(p.Name) + if m == nil { + continue + } + if n, err := strconv.Atoi(m[1]); err == nil { + out[n] = p + } + } + return out, nil +} + +// Unschedulable reports a pod the scheduler can't place on any node. +func Unschedulable(p *corev1.Pod) bool { + if p.Spec.NodeName != "" || p.DeletionTimestamp != nil || p.Status.Phase != corev1.PodPending { + return false + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse && c.Reason == corev1.PodReasonUnschedulable { + return true + } + } + return false +} + +// Cluster autoscaler event reasons on a pending pod. +const ( + // EventScaleUpRefused: no node group can host the pod. + EventScaleUpRefused = "NotTriggerScaleUp" + // EventScaleUpTriggered: a node group is being resized for it. + EventScaleUpTriggered = "TriggeredScaleUp" +) + +// podEvents lists a reason's events on pods; the field selector is checked +// again here (fake clients ignore it). +func (k *Kube) podEvents(ctx context.Context, reason string, match func(corev1.ObjectReference) bool) ([]corev1.Event, error) { + sel := fields.Set{"involvedObject.kind": "Pod", "reason": reason}.AsSelector().String() + list, err := k.Client.CoreV1().Events(k.Namespace).List(ctx, metav1.ListOptions{FieldSelector: sel}) + if err != nil { + return nil, err + } + var out []corev1.Event + for _, e := range list.Items { + if e.InvolvedObject.Kind == "Pod" && e.Reason == reason && match(e.InvolvedObject) { + out = append(out, e) + } + } + return out, nil +} + +// ScaleUpRefused reports whether the cluster autoscaler said it won't add a +// node for this pod (event NotTriggerScaleUp). +func (k *Kube) ScaleUpRefused(ctx context.Context, p *corev1.Pod) (bool, error) { + ev, err := k.podEvents(ctx, EventScaleUpRefused, func(o corev1.ObjectReference) bool { + return o.Name == p.Name && (o.UID == "" || o.UID == p.UID) + }) + return len(ev) > 0, err +} + +// NudgeOutcome is what the cluster autoscaler said about an ordinal's +// current placeholder pod (events since the Job's creation; earlier ones +// belong to earlier placeholders): the event reason (EventScaleUpTriggered or +// EventScaleUpRefused) and its message, or empty while it hasn't decided. +func (k *Kube) NudgeOutcome(ctx context.Context, n int, since time.Time) (reason, message string, err error) { + prefix := k.NudgeJobName(n) + "-" + since = since.Truncate(time.Second) + isPlaceholder := func(o corev1.ObjectReference) bool { return strings.HasPrefix(o.Name, prefix) } + // A triggered scale-up wins: the node is coming even if the autoscaler + // also refused once (e.g. while another scale-up was in flight). + for _, r := range []string{EventScaleUpTriggered, EventScaleUpRefused} { + ev, err := k.podEvents(ctx, r, isPlaceholder) + if err != nil { + return "", "", err + } + for _, e := range ev { + if !eventTime(e).Before(since) { + return r, e.Message, nil + } + } + } + return "", "", nil +} + +// eventTime is when an event last happened. +func eventTime(e corev1.Event) time.Time { + switch { + case !e.LastTimestamp.IsZero(): + return e.LastTimestamp.Time + case !e.EventTime.IsZero(): + return e.EventTime.Time + default: + return e.FirstTimestamp.Time + } +} + +// Nudge is a placeholder Job: its progress and when it was created. +type Nudge struct { + State NudgeState + Created time.Time +} + +// NudgeJobs lists the placeholder Jobs by ordinal. +func (k *Kube) NudgeJobs(ctx context.Context) (map[int]Nudge, error) { + list, err := k.Client.BatchV1().Jobs(k.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "app=" + k.nudgeApp()}) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`^` + regexp.QuoteMeta(k.StatefulSet) + `-nudge-(\d+)$`) + out := map[int]Nudge{} + for _, j := range list.Items { + m := re.FindStringSubmatch(j.Name) + if m == nil || j.DeletionTimestamp != nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + nj := Nudge{State: NudgeWaiting, Created: j.CreationTimestamp.Time} + switch { + case j.Status.Failed > 0: + nj.State = NudgeFailed + case j.Status.Ready != nil && *j.Status.Ready > 0: + nj.State = NudgeRunning + } + out[n] = nj + } + return out, nil +} + +// PodRequests is what the scheduler reserves for a pod: the containers' +// requests summed, or an init container's if larger. +func PodRequests(p *corev1.Pod) corev1.ResourceList { + out := corev1.ResourceList{} + for _, c := range p.Spec.Containers { + for name, q := range c.Resources.Requests { + sum := out[name] + sum.Add(q) + out[name] = sum + } + } + for _, c := range p.Spec.InitContainers { + for name, q := range c.Resources.Requests { + if cur, ok := out[name]; !ok || q.Cmp(cur) > 0 { + out[name] = q.DeepCopy() + } + } + } + return out +} + +// NudgeJob is the placeholder for a runner pod: its requests, node +// selection, tolerations and affinity (so it asks for the same kind of node), +// no volumes. +func (k *Kube) NudgeJob(n int, runner *corev1.Pod) *batchv1.Job { + labels := map[string]string{"app": k.nudgeApp()} + var pullSecrets []corev1.LocalObjectReference + for _, name := range k.ReapPod.ImagePullSecrets { + pullSecrets = append(pullSecrets, corev1.LocalObjectReference{Name: name}) + } + image := k.NudgeImage + if image == "" { + image = DefaultNudgeImage + } + requests := PodRequests(runner) + // Nothing to reserve otherwise, and the pause container needs no more. + if len(requests) == 0 { + requests = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1m")} + } + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: k.NudgeJobName(n), Namespace: k.Namespace, Labels: labels}, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + ActiveDeadlineSeconds: ptr.To(NudgeDeadline), + TTLSecondsAfterFinished: ptr.To[int32](600), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: ptr.To(false), + TerminationGracePeriodSeconds: ptr.To[int64](0), + ImagePullSecrets: pullSecrets, + NodeSelector: runner.Spec.NodeSelector, + Tolerations: runner.Spec.Tolerations, + Affinity: runner.Spec.Affinity, + PriorityClassName: runner.Spec.PriorityClassName, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65535), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: "placeholder", Image: image, ImagePullPolicy: corev1.PullIfNotPresent, + Resources: corev1.ResourceRequirements{Requests: requests}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }, + }, + }, + } +} + +// CreateNudgeJob starts the placeholder for a runner pod; an existing one is +// fine. +func (k *Kube) CreateNudgeJob(ctx context.Context, n int, runner *corev1.Pod) error { + _, err := k.Client.BatchV1().Jobs(k.Namespace).Create(ctx, k.NudgeJob(n, runner), metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +// DeleteNudgeJob deletes a placeholder Job and its pod; a missing one is fine. +func (k *Kube) DeleteNudgeJob(ctx context.Context, n int) error { + err := k.Client.BatchV1().Jobs(k.Namespace).Delete(ctx, k.NudgeJobName(n), metav1.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }) + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/k8s/scaler/internal/kube/nudge_test.go b/k8s/scaler/internal/kube/nudge_test.go new file mode 100644 index 0000000..9cf1dca --- /dev/null +++ b/k8s/scaler/internal/kube/nudge_test.go @@ -0,0 +1,130 @@ +package kube + +import ( + "context" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestPodRequestsSumsContainersAndTakesTheLargerInit(t *testing.T) { + q := resource.MustParse + p := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: q("8"), corev1.ResourceMemory: q("1Gi")}}}}, + Containers: []corev1.Container{ + {Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: q("6500m"), corev1.ResourceMemory: q("11Gi")}}}, + {Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceMemory: q("32Mi")}}}, + {}, + }, + }} + got := PodRequests(p) + if got.Cpu().String() != "8" || got.Memory().String() != "11296Mi" { + t.Errorf("PodRequests = cpu %s memory %s, want 8 and 11296Mi", got.Cpu(), got.Memory()) + } +} + +func TestUnschedulable(t *testing.T) { + cond := func(reason string) []corev1.PodCondition { + return []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: reason}} + } + cases := map[string]struct { + pod corev1.Pod + want bool + }{ + "unschedulable": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: cond(corev1.PodReasonUnschedulable)}}, true}, + "not tried yet": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, false}, + "other reason": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: cond("SchedulingGated")}}, false}, + "bound": {corev1.Pod{Spec: corev1.PodSpec{NodeName: "n"}, Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: cond(corev1.PodReasonUnschedulable)}}, false}, + "running": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodRunning}}, false}, + } + for name, c := range cases { + if got := Unschedulable(&c.pod); got != c.want { + t.Errorf("%s: Unschedulable = %v, want %v", name, got, c.want) + } + } +} + +func TestNudgeJobs(t *testing.T) { + job := func(name string, st batchv1.JobStatus) *batchv1.Job { + return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"app": "sg-runner-nudge"}}, Status: st} + } + k := newKube( + job("sg-runner-nudge-1", batchv1.JobStatus{}), + job("sg-runner-nudge-2", batchv1.JobStatus{Ready: ptr.To[int32](1)}), + job("sg-runner-nudge-3", batchv1.JobStatus{Failed: 1}), + ) + got, err := k.NudgeJobs(context.Background()) + if err != nil { + t.Fatal(err) + } + want := map[int]NudgeState{1: NudgeWaiting, 2: NudgeRunning, 3: NudgeFailed} + for n, s := range want { + if got[n].State != s { + t.Errorf("ordinal %d: %v, want %v", n, got[n].State, s) + } + } + if len(got) != len(want) { + t.Errorf("NudgeJobs = %v", got) + } +} + +func TestNudgeJobIsAPlainPlaceholder(t *testing.T) { + k := newKube() + k.ReapPod.ImagePullSecrets = []string{"regcred"} + runner := &corev1.Pod{Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"pool": "runners"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpExists}}, + Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2")}}}}, + Volumes: []corev1.Volume{{Name: "state"}}, + }} + j := k.NudgeJob(4, runner) + ps := j.Spec.Template.Spec + if j.Name != "sg-runner-nudge-4" || j.Labels["app"] != "sg-runner-nudge" { + t.Errorf("name/labels %s %v", j.Name, j.Labels) + } + if ps.Containers[0].Image != DefaultNudgeImage || len(ps.Volumes) != 0 || *ps.AutomountServiceAccountToken { + t.Errorf("not a plain placeholder: %+v", ps) + } + if ps.NodeSelector["pool"] != "runners" || len(ps.Tolerations) != 1 || ps.ImagePullSecrets[0].Name != "regcred" { + t.Errorf("scheduling not copied: %+v", ps) + } + if *j.Spec.ActiveDeadlineSeconds != NudgeDeadline || *j.Spec.BackoffLimit != 0 { + t.Errorf("deadline/backoff %+v", j.Spec) + } +} + +func TestNudgeOutcomeReadsOnlyTheCurrentPlaceholder(t *testing.T) { + created := time.Unix(1_700_000_000, 0) + ev := func(name, pod, reason string, at time.Time) *corev1.Event { + return &corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: pod}, Reason: reason, Message: reason + " msg", + LastTimestamp: metav1.NewTime(at)} + } + k := newKube( + // An earlier placeholder for runner 2, and another runner's. + ev("e1", "sg-runner-nudge-2-old", EventScaleUpTriggered, created.Add(-5*time.Minute)), + ev("e2", "sg-runner-nudge-3-abc", EventScaleUpTriggered, created.Add(time.Minute)), + ev("e3", "sg-runner-2", EventScaleUpRefused, created.Add(time.Minute)), + ) + ctx := context.Background() + if r, _, err := k.NudgeOutcome(ctx, 2, created); err != nil || r != "" { + t.Fatalf("outcome before any answer = %q, %v; want none", r, err) + } + + _, _ = k.Client.CoreV1().Events(ns).Create(ctx, ev("e4", "sg-runner-nudge-2-new", EventScaleUpRefused, created.Add(time.Second)), metav1.CreateOptions{}) + if r, msg, _ := k.NudgeOutcome(ctx, 2, created); r != EventScaleUpRefused || msg != "NotTriggerScaleUp msg" { + t.Errorf("outcome = %q %q, want the refusal", r, msg) + } + // Triggered wins over an earlier refusal of the same placeholder. + _, _ = k.Client.CoreV1().Events(ns).Create(ctx, ev("e5", "sg-runner-nudge-2-new", EventScaleUpTriggered, created.Add(2*time.Second)), metav1.CreateOptions{}) + if r, _, _ := k.NudgeOutcome(ctx, 2, created); r != EventScaleUpTriggered { + t.Errorf("outcome = %q, want %q", r, EventScaleUpTriggered) + } +} diff --git a/k8s/scaler/internal/metrics/metrics.go b/k8s/scaler/internal/metrics/metrics.go index 73330de..94c9e07 100644 --- a/k8s/scaler/internal/metrics/metrics.go +++ b/k8s/scaler/internal/metrics/metrics.go @@ -30,6 +30,7 @@ type Metrics struct { ScaleEvents *prometheus.CounterVec Reaps *prometheus.CounterVec + Nudges *prometheus.CounterVec StatusChanges *prometheus.CounterVec SGErrors prometheus.Counter KubeErrors prometheus.Counter @@ -59,6 +60,8 @@ func New() *Metrics { Help: "Replica changes by direction (out, in)."}, []string{"direction"}), Reaps: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "reaps_total", Help: "Reap steps by result (started, succeeded, failed)."}, []string{"result"}), + Nudges: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "autoscaler_nudges_total", + Help: "Placeholder pods asking the cluster autoscaler for a node, by result (started, node_requested, refused)."}, []string{"result"}), StatusChanges: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "runner_status_changes_total", Help: "Runner status changes through SG by reason and result."}, []string{"reason", "result"}), SGErrors: prometheus.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "sg_api_errors_total", @@ -72,7 +75,7 @@ func New() *Metrics { collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), m.Queued, m.ReservedPct, m.FreePct, m.Replicas, m.Awake, m.Parked, m.Max, m.Stuck, m.Unconnected, m.Settling, m.Leader, m.ConsecutiveFailures, - m.ScaleEvents, m.Reaps, m.StatusChanges, m.SGErrors, m.KubeErrors, m.LoopDuration, + m.ScaleEvents, m.Reaps, m.Nudges, m.StatusChanges, m.SGErrors, m.KubeErrors, m.LoopDuration, ) for _, d := range []string{"out", "in"} { m.ScaleEvents.WithLabelValues(d) @@ -80,6 +83,9 @@ func New() *Metrics { for _, r := range []string{"started", "succeeded", "failed"} { m.Reaps.WithLabelValues(r) } + for _, r := range []string{"started", "node_requested", "refused"} { + m.Nudges.WithLabelValues(r) + } return m } diff --git a/k8s/scaler/internal/sg/sg.go b/k8s/scaler/internal/sg/sg.go index 6fe76ee..42196a5 100644 --- a/k8s/scaler/internal/sg/sg.go +++ b/k8s/scaler/internal/sg/sg.go @@ -35,7 +35,8 @@ type Credentials struct { BaseAPI string } -// LoadCredentials reads the credentials Secret mounted as one file per key. +// LoadCredentials reads the node token (Secret) and the runner group +// (ConfigMap), mounted together as one file per key. func LoadCredentials(dir string) (Credentials, error) { read := func(key string, required bool) (string, error) { b, err := os.ReadFile(filepath.Join(dir, key)) diff --git a/k8s/scaler/internal/teardown/teardown.go b/k8s/scaler/internal/teardown/teardown.go new file mode 100644 index 0000000..822b817 --- /dev/null +++ b/k8s/scaler/internal/teardown/teardown.go @@ -0,0 +1,213 @@ +// Package teardown removes every runner of a StatefulSet from StackGuardian +// before Helm deletes the release (the chart's pre-delete hook), so +// `helm uninstall` leaves no registered runners and no PVCs behind. +package teardown + +import ( + "context" + "fmt" + "log/slog" + "slices" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" +) + +// Teardown parks, deregisters and deletes every runner of one StatefulSet. +type Teardown struct { + Kube *kube.Kube + // ScalerDeployment is stopped first, so it can't wake, park or reap + // runners meanwhile. Empty = no scaler. + ScalerDeployment string + // ScalerSelector and RunnerSelector select the scaler's and the runners' + // pods, to wait for them to be gone. + ScalerSelector string + RunnerSelector string + // Poll is how often to check progress (5s when zero). + Poll time.Duration + Log *slog.Logger +} + +// Run stops the scaler, parks every runner (each one's preStop drain waits +// for its running workflows), deregisters each runner from its state PVC +// with a reap Job, and deletes the PVCs of every runner it deregistered. A +// runner that fails to deregister keeps its PVCs, and Run returns an error, +// so Helm stops before anything is orphaned. +func (t *Teardown) Run(ctx context.Context) error { + if t.Poll == 0 { + t.Poll = 5 * time.Second + } + if t.ScalerDeployment != "" { + t.Log.Info("stopping the scaler", "deployment", t.ScalerDeployment) + if err := t.Kube.ScaleDeployment(ctx, t.ScalerDeployment, 0); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("stop the scaler: %w", err) + } + if err := t.waitNoPods(ctx, t.ScalerSelector, "scaler"); err != nil { + return err + } + } + + t.Log.Info("parking every runner", "statefulset", t.Kube.StatefulSet) + if err := t.Kube.Scale(ctx, 0); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("park the runners: %w", err) + } + if err := t.waitNoPods(ctx, t.RunnerSelector, "runner"); err != nil { + return err + } + + claims, err := t.Kube.Claims(ctx) + if err != nil { + return fmt.Errorf("list runner PVCs: %w", err) + } + ordinals := make([]int, 0, len(claims)) + for n := range claims { + ordinals = append(ordinals, n) + } + slices.Sort(ordinals) + t.Log.Info("deregistering runners", "ordinals", ordinals) + + // A leftover reap Job (a failed reap by the scaler) would read as this + // run's result: start every reap afresh. + for _, n := range ordinals { + if err := t.deleteReapJob(ctx, n); err != nil { + return err + } + } + for _, n := range ordinals { + if err := t.Kube.CreateReapJob(ctx, n); err != nil { + return fmt.Errorf("start reap Job for runner %d: %w", n, err) + } + } + states, err := t.waitReaps(ctx, ordinals) + if err != nil { + return err + } + + var failed []int + for _, n := range ordinals { + if states[n] != decide.JobSucceeded { + failed = append(failed, n) + t.Log.Error("could not deregister runner; keeping its PVCs", "ordinal", n, "job", t.Kube.JobName(n)) + continue + } + for _, claim := range []string{t.Kube.ClaimName(n), t.Kube.DockerClaimName(n)} { + if err := t.Kube.DeleteClaim(ctx, claim); err != nil { + return fmt.Errorf("delete PVC %s: %w", claim, err) + } + } + if err := t.Kube.DeleteReapJob(ctx, n); err != nil { + return fmt.Errorf("delete reap Job %s: %w", t.Kube.JobName(n), err) + } + t.Log.Info("runner deregistered, PVCs deleted", "ordinal", n) + } + + // Placeholders asking the cluster autoscaler for nodes (the scaler is + // stopped, nothing needs them). + nudges, err := t.Kube.NudgeJobs(ctx) + if err != nil { + return fmt.Errorf("list placeholder Jobs: %w", err) + } + for n := range nudges { + if err := t.Kube.DeleteNudgeJob(ctx, n); err != nil { + return fmt.Errorf("delete placeholder Job %s: %w", t.Kube.NudgeJobName(n), err) + } + } + + // Docker caches whose runner is gone already (reaped earlier, or its state + // PVC deleted by hand). + docker, err := t.Kube.DockerClaims(ctx) + if err != nil { + return fmt.Errorf("list Docker cache PVCs: %w", err) + } + for _, n := range docker { + if _, ok := claims[n]; ok { + continue + } + if err := t.Kube.DeleteClaim(ctx, t.Kube.DockerClaimName(n)); err != nil { + return fmt.Errorf("delete PVC %s: %w", t.Kube.DockerClaimName(n), err) + } + } + + if len(failed) > 0 { + return fmt.Errorf("could not deregister runners %v: see the logs of their %s-reap- Jobs; "+ + "their PVCs are kept so nothing is orphaned", failed, t.Kube.StatefulSet) + } + t.Log.Info("teardown complete", "runners", len(ordinals)) + return nil +} + +// waitNoPods waits until no pod matches selector. Runners can take as long as +// their running workflows (their preStop drain waits for them). +func (t *Teardown) waitNoPods(ctx context.Context, selector, what string) error { + var last time.Time + for { + n, err := t.Kube.CountPods(ctx, selector) + if err != nil { + return fmt.Errorf("list %s pods: %w", what, err) + } + if n == 0 { + return nil + } + if time.Since(last) >= time.Minute { + t.Log.Info("waiting for pods to stop (runners wait for their running workflows)", "component", what, "pods", n) + last = time.Now() + } + if err := t.sleep(ctx); err != nil { + return err + } + } +} + +// deleteReapJob deletes the reap Job of an ordinal and waits until it's gone. +func (t *Teardown) deleteReapJob(ctx context.Context, n int) error { + if err := t.Kube.DeleteReapJob(ctx, n); err != nil { + return fmt.Errorf("delete old reap Job %s: %w", t.Kube.JobName(n), err) + } + for { + exists, err := t.Kube.ReapJobExists(ctx, n) + if err != nil { + return fmt.Errorf("check old reap Job %s: %w", t.Kube.JobName(n), err) + } + if !exists { + return nil + } + if err := t.sleep(ctx); err != nil { + return err + } + } +} + +// waitReaps waits until every ordinal's reap Job has finished. +func (t *Teardown) waitReaps(ctx context.Context, ordinals []int) (map[int]decide.JobState, error) { + for { + jobs, err := t.Kube.ReapJobs(ctx) + if err != nil { + return nil, fmt.Errorf("list reap Jobs: %w", err) + } + done := true + for _, n := range ordinals { + if s, ok := jobs[n]; !ok || s == decide.JobRunning { + done = false + break + } + } + if done { + return jobs, nil + } + if err := t.sleep(ctx); err != nil { + return nil, err + } + } +} + +func (t *Teardown) sleep(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(t.Poll): + return nil + } +} diff --git a/k8s/scaler/internal/teardown/teardown_test.go b/k8s/scaler/internal/teardown/teardown_test.go new file mode 100644 index 0000000..ed997bf --- /dev/null +++ b/k8s/scaler/internal/teardown/teardown_test.go @@ -0,0 +1,202 @@ +package teardown + +import ( + "bytes" + "context" + "log/slog" + "slices" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" +) + +const ns = "sg" + +func pod(name, component string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, + Labels: map[string]string{"app.kubernetes.io/component": component}}} +} + +func pvc(name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} +} + +// cluster stands in for the controllers and the kubelet: it removes pods +// once their workload is scaled to 0 and finishes reap Jobs, failing those +// of the ordinals in fail. +type cluster struct { + t *testing.T + k *kube.Kube + fail map[string]bool + logs *bytes.Buffer +} + +func newCluster(t *testing.T, fail []string, objs ...runtime.Object) *cluster { + c := &cluster{t: t, k: kube.New(fake.NewClientset(objs...), ns, "sg-runner", "runner:1", "creds", "cfg"), + fail: map[string]bool{}, logs: &bytes.Buffer{}} + for _, name := range fail { + c.fail[name] = true + } + return c +} + +func (c *cluster) step(ctx context.Context) { + cs := c.k.Client + if d, err := cs.AppsV1().Deployments(ns).Get(ctx, "sg-runner-scaler", metav1.GetOptions{}); err == nil && *d.Spec.Replicas == 0 { + c.deletePods(ctx, "scaler") + } + if s, err := c.k.GetStatefulSet(ctx); err == nil && *s.Spec.Replicas == 0 { + c.deletePods(ctx, "runner") + } + jobs, _ := cs.BatchV1().Jobs(ns).List(ctx, metav1.ListOptions{}) + for _, j := range jobs.Items { + if j.Status.Succeeded+j.Status.Failed > 0 { + continue + } + if c.fail[j.Name] { + j.Status.Failed = 1 + } else { + j.Status.Succeeded = 1 + } + if _, err := cs.BatchV1().Jobs(ns).UpdateStatus(ctx, &j, metav1.UpdateOptions{}); err != nil { + c.t.Error(err) + } + } +} + +func (c *cluster) deletePods(ctx context.Context, component string) { + pods, _ := c.k.Client.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/component=" + component}) + for _, p := range pods.Items { + _ = c.k.Client.CoreV1().Pods(ns).Delete(ctx, p.Name, metav1.DeleteOptions{}) + } +} + +func (c *cluster) run() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + go func() { + for ctx.Err() == nil { + c.step(ctx) + time.Sleep(time.Millisecond) + } + }() + td := &Teardown{ + Kube: c.k, + ScalerDeployment: "sg-runner-scaler", + ScalerSelector: "app.kubernetes.io/component=scaler", + RunnerSelector: "app.kubernetes.io/component=runner", + Poll: time.Millisecond, + Log: slog.New(slog.NewJSONHandler(c.logs, nil)), + } + return td.Run(ctx) +} + +func (c *cluster) claims() []string { + list, err := c.k.Client.CoreV1().PersistentVolumeClaims(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + c.t.Fatal(err) + } + var names []string + for _, p := range list.Items { + names = append(names, p.Name) + } + slices.Sort(names) + return names +} + +func workloads() []runtime.Object { + return []runtime.Object{ + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-scaler", Namespace: ns}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)}}, + &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner", Namespace: ns}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To[int32](2)}}, + pod("sg-runner-scaler-abc", "scaler"), + pod("sg-runner-0", "runner"), + pod("sg-runner-1", "runner"), + } +} + +func TestTeardownDeregistersEveryRunnerAndDeletesPVCs(t *testing.T) { + objs := append(workloads(), + // Awake runners 0 and 1, parked runner 3, an orphaned Docker cache 5, + // and a PVC of another release that must stay. + pvc("state-sg-runner-0"), pvc("docker-sg-runner-0"), + pvc("state-sg-runner-1"), pvc("docker-sg-runner-1"), + pvc("state-sg-runner-3"), pvc("docker-sg-runner-3"), + pvc("docker-sg-runner-5"), + pvc("state-other-0"), + // A failed reap the scaler left behind: must not count as this run's. + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-3", Namespace: ns, + Labels: map[string]string{"app": "sg-runner-reaper"}}, Status: batchv1.JobStatus{Failed: 1}}, + ) + c := newCluster(t, nil, objs...) + if err := c.run(); err != nil { + t.Fatalf("Run: %v\n%s", err, c.logs) + } + if got, want := c.claims(), []string{"state-other-0"}; !slices.Equal(got, want) { + t.Errorf("PVCs left = %v, want %v", got, want) + } + jobs, _ := c.k.Client.BatchV1().Jobs(ns).List(context.Background(), metav1.ListOptions{}) + if len(jobs.Items) != 0 { + t.Errorf("%d reap Jobs left, want 0", len(jobs.Items)) + } + if !strings.Contains(c.logs.String(), `"teardown complete"`) { + t.Errorf("no completion log:\n%s", c.logs) + } +} + +func TestTeardownKeepsPVCsOfRunnersItCouldNotDeregister(t *testing.T) { + objs := append(workloads(), + pvc("state-sg-runner-0"), pvc("docker-sg-runner-0"), + pvc("state-sg-runner-1"), pvc("docker-sg-runner-1"), + ) + c := newCluster(t, []string{"sg-runner-reap-1"}, objs...) + err := c.run() + if err == nil || !strings.Contains(err.Error(), "could not deregister runners [1]") { + t.Fatalf("Run error = %v, want one naming runner 1", err) + } + if got, want := c.claims(), []string{"docker-sg-runner-1", "state-sg-runner-1"}; !slices.Equal(got, want) { + t.Errorf("PVCs left = %v, want %v", got, want) + } + // The failed Job stays for its logs. + if _, err := c.k.Client.BatchV1().Jobs(ns).Get(context.Background(), "sg-runner-reap-1", metav1.GetOptions{}); err != nil { + t.Errorf("failed reap Job deleted: %v", err) + } +} + +func TestTeardownWithoutScalerOrRunners(t *testing.T) { + // Scaler disabled and the StatefulSet already gone: nothing to do, no error. + c := newCluster(t, nil, pvc("docker-sg-runner-2")) + ctx := context.Background() + td := &Teardown{Kube: c.k, RunnerSelector: "app.kubernetes.io/component=runner", Poll: time.Millisecond, + Log: slog.New(slog.NewJSONHandler(c.logs, nil))} + if err := td.Run(ctx); err != nil { + t.Fatalf("Run: %v\n%s", err, c.logs) + } + if got := c.claims(); len(got) != 0 { + t.Errorf("PVCs left = %v, want none", got) + } +} + +func TestTeardownStopsOnCancel(t *testing.T) { + // A runner that never stops (a workflow still running) blocks until the + // context ends: the Job's deadline, not a hang. + c := newCluster(t, nil, pod("sg-runner-0", "runner")) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + td := &Teardown{Kube: c.k, RunnerSelector: "app.kubernetes.io/component=runner", Poll: time.Millisecond, + Log: slog.New(slog.NewJSONHandler(c.logs, nil))} + if err := td.Run(ctx); err == nil { + t.Fatal("Run returned nil with a runner pod that never stops") + } +} From ff1f476e0959a12c82b7447185bb792c89651d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 14 Oct 2026 16:59:10 +0200 Subject: [PATCH 28/43] feat: workflow logs sidecar and reap without a registration --- k8s/runner.sh | 118 +++++++++++++++++++++++++++++++++++++- test/unit/k8s_runner.bats | 74 ++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 3 deletions(-) diff --git a/k8s/runner.sh b/k8s/runner.sh index e8f6b3e..d72a467 100755 --- a/k8s/runner.sh +++ b/k8s/runner.sh @@ -14,6 +14,7 @@ # wait-drained dockerd/ssm-agent preStop: outlive the drain # deregister run via `kubectl exec -i`; reads SG credentials from stdin # reap scaler Job: deregister a runner parked too long, from its PVC +# workflow-logs opt-in sidecar: workflow step output, tagged with its run # # main.sh is sourced for the API, config and network helpers; its # BASH_SOURCE guard keeps main() from running. @@ -57,6 +58,14 @@ SG_PRUNE_UNTIL="${SG_PRUNE_UNTIL:-24h}" SG_PRUNE_IMAGES_ABOVE_PCT="${SG_PRUNE_IMAGES_ABOVE_PCT:-70}" DOCKER_DATA_ROOT="${DOCKER_DATA_ROOT:-/var/lib/docker}" +# workflow-logs sidecar: "text" lines or "json" objects, one per output line; +# [SG_DEBUG] lines only with SG_WORKFLOW_LOGS_DEBUG=true. +SG_WORKFLOW_LOGS_FORMAT="${SG_WORKFLOW_LOGS_FORMAT:-text}" +SG_WORKFLOW_LOGS_DEBUG="${SG_WORKFLOW_LOGS_DEBUG:-false}" +# Containers being followed (one directory each), so a container listed at +# start and also seen starting is followed once. +FOLLOWED_DIR="${FOLLOWED_DIR:-/tmp/sg-workflow-logs}" + # What the agent sizing reads; overridable so tests can feed fake files. PROC_STATUS_FILE="${PROC_STATUS_FILE:-/proc/self/status}" MEMINFO_FILE="${MEMINFO_FILE:-/proc/meminfo}" @@ -374,9 +383,16 @@ cmd_deregister() { #{{{ #}}}: cmd_deregister cmd_reap() { #{{{ - # One-off Job the scaler starts for a runner parked longer than its TTL: the - # pod is gone, so deregister using the identity on the mounted state PVC. + # One-off Job the scaler (parked longer than its TTL) or the uninstall hook + # starts: the pod is gone, so deregister using the identity on the mounted + # state PVC. load_sg_credentials_from_dir + # Never registered (registration failed, or already deregistered and + # wiped): nothing in SG to remove, and the PVC can go. + if [[ ! -s "$ECS_CONFIG_DIR/ecs.config" ]]; then + info "Not registered:" "nothing to deregister" + return 0 + fi deregister_runner } #}}}: cmd_reap @@ -496,6 +512,101 @@ cmd_wait_drained() { #{{{ } #}}}: cmd_wait_drained +container_log_tags() { #{{{ + # Usage: container_log_tags + # Prints "run workflow" for a step (user) container, "-" where unknown, or + # fails for a run controller (admin) container, an ECS task: it relays its + # steps' output, so following it too would print every step line twice. + # Steps mount their run's workspace, .../wfs//wfruns//... + # No env is read: it holds the run's credentials. + local task_arn sources run="-" wf="-" + IFS='|' read -r task_arn sources <<<"$1" + [[ -z "$task_arn" ]] || return 1 + [[ "$sources" =~ /wfruns/([^/[:space:]]+) ]] && run="${BASH_REMATCH[1]}" + [[ "$sources" =~ /wfs/([^/[:space:]]+)/wfruns/ ]] && wf="${BASH_REMATCH[1]}" + echo "$run $wf" +} +#}}}: container_log_tags + +format_log_lines() { #{{{ + # Usage: format_log_lines + # Reads `docker logs -t` lines ("