diff --git a/.github/scripts/chart-check.sh b/.github/scripts/chart-check.sh new file mode 100755 index 0000000..c159ba2 --- /dev/null +++ b/.github/scripts/chart-check.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# Lints the sg-runner chart, renders the key value scenarios, validates every +# rendered manifest with kubeconform and checks that the schema and the +# templates reject bad values. Needs helm (v3 or v4) and kubeconform on PATH; +# never talks to a cluster. +# +# .github/scripts/chart-check.sh [out-dir] +# +# KUBE_VERSION: Kubernetes version the manifests are validated against. +set -euo pipefail + +CHART=k8s/chart/sg-runner +OUT=${1:-${RUNNER_TEMP:-/tmp}/chart-check} +KUBE_VERSION=${KUBE_VERSION:-1.37.0} +# Release name = chart name, as the k8s/Makefile installs it. +RELEASE=sg-runner +# Every scenario needs credentials; the Secret doesn't have to exist to render. +CREDS=(--set credentials.existingSecret=sg-runner-credentials + --set sg.organization=ci-org --set sg.runnerGroup=ci-group) + +rm -rf "$OUT" && mkdir -p "$OUT" +failed=0 + +render() { # + local name=$1 + shift + echo "::group::render $name" + if helm template "$RELEASE" "$CHART" --namespace sg-runner "$@" >"$OUT/$name.yaml"; then + echo "rendered $(grep -c '^kind:' "$OUT/$name.yaml") objects" + else + echo "::error title=helm template::scenario '$name' failed to render" + failed=1 + fi + echo "::endgroup::" +} + +# Rendering must fail, and with the expected message (so an unrelated error +# doesn't count as the rejection). +reject() { # + local name=$1 want=$2 + shift 2 + if helm template "$RELEASE" "$CHART" "$@" >/dev/null 2>"$OUT/$name.err"; then + echo "::error title=helm template::scenario '$name' rendered, expected a rejection" + failed=1 + elif ! grep -Eq "$want" "$OUT/$name.err"; then + echo "::error title=helm template::scenario '$name' failed for another reason:" + cat "$OUT/$name.err" + failed=1 + else + echo "rejected as expected: $name ($(head -c 200 "$OUT/$name.err" | tr '\n' ' '))" + fi +} + +# Must contain the given kind (guards against a scenario silently rendering +# nothing new). +expect_kind() { # + grep -q "^kind: $2\$" "$OUT/$1.yaml" \ + || { echo "::error title=helm template::scenario '$1' has no $2"; failed=1; } +} +expect_no_kind() { # + ! grep -q "^kind: $2\$" "$OUT/$1.yaml" \ + || { echo "::error title=helm template::scenario '$1' unexpectedly has a $2"; failed=1; } +} +expect_text() { # + grep -qF -- "$2" "$OUT/$1.yaml" \ + || { echo "::error title=helm template::scenario '$1' lacks: $2"; failed=1; } +} +expect_count() { # + local got + got=$(grep -cF -- "$2" "$OUT/$1.yaml" || true) + [[ $got == "$3" ]] \ + || { echo "::error title=helm template::scenario '$1' has $got x '$2', want $3"; failed=1; } +} + +echo "::group::helm lint --strict" +helm version +helm lint --strict "$CHART" "${CREDS[@]}" || failed=1 +echo "::endgroup::" + +render defaults "${CREDS[@]}" +expect_kind defaults StatefulSet +expect_kind defaults Deployment +expect_kind defaults Role +# The uninstall hook (pre-delete Job). +expect_kind defaults Job +# The register init container sees what ecs-agent will advertise, to +# re-register a runner whose size changed. +expect_count defaults 'name: SG_TASK_CPU_LIMIT' 2 +expect_count defaults 'name: SG_TASK_MEMORY_LIMIT_MIB' 2 +expect_text defaults '{ name: SG_REREGISTER_ON_RESIZE, value: "true" }' + +render credentials-create \ + --set credentials.create=true \ + --set credentials.nodeToken=ci-dummy-token \ + --set sg.organization=ci-org \ + --set sg.runnerGroup=ci-group \ + --set sg.apiUrl=http://127.0.0.1:9 +expect_kind credentials-create Secret +expect_kind credentials-create ConfigMap + +render scaler-disabled "${CREDS[@]}" --set scaler.enabled=false --set runner.replicas=2 \ + --set uninstall.deregister=false +expect_no_kind scaler-disabled Deployment +# Only the runner's Role (the steps broker), not the scaler's. +[[ $(grep -c '^kind: Role$' "$OUT/scaler-disabled.yaml") == 1 ]] \ + || { echo "::error title=helm template::scenario 'scaler-disabled' should have only the runner's Role"; failed=1; } +expect_no_kind scaler-disabled Job + +# Without the scaler, the uninstall hook still cleans up, with its own Role. +render uninstall-without-scaler "${CREDS[@]}" --set scaler.enabled=false +expect_no_kind uninstall-without-scaler Deployment +expect_kind uninstall-without-scaler Job +expect_kind uninstall-without-scaler Role + +render network-policy "${CREDS[@]}" \ + --set networkPolicy.enabled=true \ + --set 'networkPolicy.clusterCIDRs={10.0.0.0/8,172.16.0.0/12}' +expect_kind network-policy NetworkPolicy + +# The runner pod's init containers (register, then the steps sidecar). +init_containers() { # + awk '/^kind: StatefulSet$/ {sts = 1} sts && /^ initContainers:$/ {on = 1; next} + on && /^ containers:$/ {exit} on' "$OUT/$1.yaml" +} + +# Defaults: size small, the broker a native sidecar guarding dockerd, the +# runner's RBAC, a workspace PVC per pod, one runner per node. +expect_kind defaults ServiceAccount +expect_kind defaults RoleBinding +expect_text defaults 'volumeClaimTemplate:' +expect_text defaults 'topologyKey: kubernetes.io/hostname' +expect_count defaults '{ name: SG_TASK_CPU_LIMIT, value: "2" }' 2 +expect_count defaults '{ name: SG_TASK_MEMORY_LIMIT_MIB, value: "4096" }' 2 +expect_count defaults '{ name: SG_DOCKERD_OVERHEAD_MIB, value: "0" }' 2 +expect_text defaults 'verbs: ["create", "get", "list", "watch", "patch", "delete"]' +# Published images by default, tagged with the chart's appVersion ("dev" in +# git; the release workflow sets the version). +expect_text defaults 'image: ghcr.io/stackguardian/sg-runner-k8s:dev' +expect_text defaults 'image: ghcr.io/stackguardian/sg-runner-scaler:dev' +expect_text defaults '{ name: BROKER_IMAGE, value: "ghcr.io/stackguardian/sg-runner-scaler:dev" }' +expect_text defaults '{ name: SCALER_IMAGE, value: "ghcr.io/stackguardian/sg-runner-k8s:dev" }' +expect_text defaults '{ name: BROKER_IMAGE_PULL_POLICY, value: "IfNotPresent" }' +expect_text defaults '{ name: FINISHED_TTL, value: "1h" }' +expect_text defaults '{ name: GUARD_SOCKET, value: /run/sg-guard/sg-steps-guard.sock }' +expect_text defaults '{ name: docker-plugins, mountPath: /run/docker/plugins }' +expect_text defaults '{ name: docker-plugins, mountPath: /run/sg-guard }' +# The sidecar outlives the drain by itself: no drain marker, no preStop. +expect_count defaults 'DRAINED_MARKER' 0 +expect_count defaults "command: ['/sg-runner-steps', 'wait-drained']" 0 +expect_count defaults 'timeoutSeconds: 6' 2 +if ! init_containers defaults | grep -q -- '- name: steps' \ + || ! init_containers defaults | grep -q 'restartPolicy: Always' \ + || ! init_containers defaults | grep -qF "command: ['/sg-runner-steps', 'healthcheck']"; then + echo "::error title=helm template::scenario 'defaults': the steps broker isn't a native sidecar with a healthcheck" + failed=1 +fi +if grep -q 'workflow-logs' "$OUT/defaults.yaml"; then + echo "::error title=helm template::scenario 'defaults' still has workflow-logs" + failed=1 +fi +if [[ $(grep -c 'mountPath: /var/run/secrets/kubernetes.io/serviceaccount' "$OUT/defaults.yaml") -ne 1 ]]; then + echo "::error title=helm template::scenario 'defaults' must mount the API token in exactly one container" + failed=1 +fi + +# Presets set what the runner advertises and the controllers' room. +render size-large "${CREDS[@]}" --set runner.size=large +expect_count size-large '{ name: SG_TASK_CPU_LIMIT, value: "8" }' 2 +expect_count size-large '{ name: SG_TASK_MEMORY_LIMIT_MIB, value: "16384" }' 2 +expect_text size-large 'memory: 8Gi' + +render size-custom "${CREDS[@]}" \ + --set runner.size=custom \ + --set runner.advertise.cpu=3 --set runner.advertise.memory=3072Mi \ + --set runner.controllerResources.requests.cpu=250m \ + --set runner.controllerResources.requests.memory=512Mi \ + --set runner.controllerResources.limits.cpu=1 \ + --set runner.controllerResources.limits.memory=1536Mi +expect_count size-custom '{ name: SG_TASK_CPU_LIMIT, value: "3" }' 2 +expect_count size-custom '{ name: SG_TASK_MEMORY_LIMIT_MIB, value: "3072" }' 2 +expect_text size-custom 'memory: 1536Mi' + +# Every knob at once; the NetworkPolicy covers step pods. +render everything "${CREDS[@]}" --set runner.size=medium \ + --set networkPolicy.enabled=true --set 'networkPolicy.clusterCIDRs={10.0.0.0/8}' \ + --set 'runner.ecsAgent.extraEnv[0].name=ECS_IMAGE_PULL_BEHAVIOR' \ + --set 'runner.ecsAgent.extraEnv[0].value=prefer-cached' \ + --set 'runner.dockerd.extraEnv[0].name=SG_PRUNE_INTERVAL_SECONDS' \ + --set-string 'runner.dockerd.extraEnv[0].value=0' \ + --set runner.steps.step.imagePullPolicy=IfNotPresent \ + --set 'imagePullSecrets[0].name=regcred' --set 'imagePullSecrets[1].name=other' \ + --set runner.reregisterOnResize=false \ + --set 'runner.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey=zone' +expect_kind everything NetworkPolicy +expect_text everything 'values: [runner, reaper, step]' +expect_count everything '{ name: SG_TASK_CPU_LIMIT, value: "4" }' 2 +expect_text everything '{ name: BROKER_IMAGE_PULL_SECRETS, value: "regcred,other" }' +expect_text everything '{ name: STEP_IMAGE_PULL_POLICY, value: IfNotPresent }' +expect_text everything 'name: SG_PRUNE_INTERVAL_SECONDS' +expect_text everything 'name: ECS_IMAGE_PULL_BEHAVIOR' +expect_text everything '{ name: SG_REREGISTER_ON_RESIZE, value: "false" }' +expect_text everything 'topologyKey: zone' +expect_text everything 'topologyKey: kubernetes.io/hostname' + +render pdb "${CREDS[@]}" --set podDisruptionBudget.enabled=true --set scaler.min=2 +expect_kind pdb PodDisruptionBudget + +# --api-versions makes the ServiceMonitor render, as on a cluster with the +# Prometheus Operator CRDs. +render metrics "${CREDS[@]}" \ + --set scaler.metrics.enabled=true \ + --set scaler.metrics.serviceMonitor.enabled=true \ + --api-versions monitoring.coreos.com/v1/ServiceMonitor +expect_kind metrics Service +expect_kind metrics ServiceMonitor + +# imagePullSecrets as plain names and {name:} objects, mixed: every pod gets +# {name:} refs, the scaler and the broker the names. +render pull-secrets-mixed "${CREDS[@]}" \ + --set-json 'imagePullSecrets=["plain",{"name":"obj"}]' +expect_count pull-secrets-mixed '- name: plain' 3 +expect_count pull-secrets-mixed '- name: obj' 3 +expect_text pull-secrets-mixed '{ name: BROKER_IMAGE_PULL_SECRETS, value: "plain,obj" }' +expect_count pull-secrets-mixed 'value: "plain,obj"' 3 + +echo "::group::kubeconform (Kubernetes $KUBE_VERSION)" +# ServiceMonitor is validated against the community CRDs catalog. +kubeconform -strict -summary -output text \ + -kubernetes-version "$KUBE_VERSION" \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \ + "$OUT"/*.yaml || failed=1 +echo "::endgroup::" + +echo "::group::rejections" +reject size-huge 'runner[./]size' "${CREDS[@]}" --set runner.size=huge +reject scaler-min-0 'scaler[./]min' "${CREDS[@]}" --set scaler.min=0 +reject no-credentials 'credentials' --set sg.organization=o --set sg.runnerGroup=g +reject both-credentials 'not both' "${CREDS[@]}" --set credentials.create=true +reject no-runner-group 'sg.organization and sg.runnerGroup' \ + --set credentials.existingSecret=x --set sg.organization=o +reject token-only-secret 'additional properties|additionalProperties|organization' \ + --set credentials.existingSecret=x --set credentials.organization=o \ + --set sg.organization=o --set sg.runnerGroup=g +reject network-policy-no-cidrs 'clusterCIDRs' "${CREDS[@]}" --set networkPolicy.enabled=true +reject custom-without-advertise 'advertise' "${CREDS[@]}" --set runner.size=custom \ + --set runner.controllerResources.limits.cpu=1 --set runner.controllerResources.limits.memory=1Gi +reject custom-without-controllers 'controllerResources' "${CREDS[@]}" --set runner.size=custom \ + --set runner.advertise.cpu=2 --set runner.advertise.memory=4Gi +reject fractional-cpu 'advertise[./]cpu' "${CREDS[@]}" --set runner.size=custom \ + --set runner.advertise.cpu=1.5 --set runner.advertise.memory=4Gi \ + --set runner.controllerResources.limits.cpu=1 --set runner.controllerResources.limits.memory=1Gi +reject memory-bytes 'advertise[./]memory' "${CREDS[@]}" --set runner.size=custom \ + --set runner.advertise.cpu=2 --set runner.advertise.memory=6442450944 \ + --set runner.controllerResources.limits.cpu=1 --set runner.controllerResources.limits.memory=1Gi +reject bad-timeout 'startTimeout' "${CREDS[@]}" --set runner.steps.startTimeout=30 +reject bad-pull-policy 'imagePullPolicy' "${CREDS[@]}" --set runner.steps.step.imagePullPolicy=Sometimes +reject old-native-steps 'nativeSteps|additional' "${CREDS[@]}" --set runner.nativeSteps.enabled=true +reject pull-secret-empty-name 'imagePullSecrets' "${CREDS[@]}" --set-json 'imagePullSecrets=[{"name":""}]' +reject replicas-without-lease 'leaderElection' "${CREDS[@]}" \ + --set scaler.replicas=2 --set scaler.leaderElection.enabled=false +echo "::endgroup::" + +if [[ $failed -ne 0 ]]; then + echo "chart check failed" + exit 1 +fi +echo "chart check passed" diff --git a/.github/scripts/kind-smoke.sh b/.github/scripts/kind-smoke.sh new file mode 100755 index 0000000..d0bdcf5 --- /dev/null +++ b/.github/scripts/kind-smoke.sh @@ -0,0 +1,623 @@ +#!/usr/bin/env bash +# Smoke-installs the sg-runner chart on a throwaway kind cluster with locally +# built images and checks what can be checked without StackGuardian: the +# scaler starts and is ready, finds SG unreachable and leaks no credentials, +# the runner's register init container fails as expected, the scaler's +# RBAC is as narrow as intended, the runner pod's steps broker is a native +# sidecar and dockerd is guarded by it, the broker runs steps as pods with +# the chart's runner RBAC, and `helm uninstall` leaves no PVCs behind. +# +# sg.apiUrl is http://127.0.0.1:9 (discard port on the pod's own loopback): +# nothing ever reaches StackGuardian, and registration fails before any AWS +# call. +# +# .github/scripts/kind-smoke.sh run # load images, install, assert +# .github/scripts/kind-smoke.sh diag DIR # dump cluster state into DIR +# +# The images must exist locally (RUNNER_IMAGE, SCALER_IMAGE); the cluster +# must exist already (kind create cluster --name "$CLUSTER"). +# +# CONTROLLER_REPO (optional, local runs): an sg-run-controller checkout whose +# unit/test_step_broker_live.py then runs against the broker too. +set -euo pipefail + +CLUSTER=${CLUSTER:-sg-runner-ci} +CONTEXT=${KUBE_CONTEXT:-kind-$CLUSTER} +RUNNER_IMAGE=${RUNNER_IMAGE:-sg-runner-k8s:ci} +SCALER_IMAGE=${SCALER_IMAGE:-sg-runner-scaler:ci} +NS=sg-runner +RELEASE=sg-runner +STS=sg-runner +SCALER=sg-runner-scaler +SECRET=sg-runner-credentials +CHART=k8s/chart/sg-runner +TIMEOUT=${TIMEOUT:-240} +# Failed SG API calls one register run may make (see the register step). +MAX_API_ATTEMPTS=3 +# Distinctive, so finding it in any log means a credential leaked. +TOKEN=ci-dummy-node-token-6f1d3c9a +SA=system:serviceaccount:$NS:$SCALER +# Native steps: the chart's runner ServiceAccount, the broker's test pod. +RUNNER_SA=system:serviceaccount:$NS:$STS +STEPS_POD=steps-smoke +STEPS_SOCK=/var/tmp/workspace/.sg-runner/steps.sock +# A run's workspace as a run controller would lay it out. +STEPS_RUN=/var/tmp/workspace/orgs/ci/wfruns/r1 +STEP_IMAGE=alpine:3.20 +# Finished step pods are kept this long in the test pod (the broker's GC +# sweeps every minute). +STEPS_FINISHED_TTL=90s +# A registry host only the node's containerd knows (a hosts.toml mirror to +# Docker Hub): the node pulls it, the broker can't read its config, so the +# step runs unwrapped. (An image only `kind load`ed doesn't do: the kubelet +# re-pulls images it has no pull record for.) +MIRROR_HOST=ci-local.invalid +UNWRAPPED_STEP_IMAGE=$MIRROR_HOST/library/$STEP_IMAGE +OUTCOME_LABEL=sg-runner.stackguardian.io/outcome +WRAPPED_ANNOTATION=sg-runner.stackguardian.io/wrapped +CONTROLLER_REPO=${CONTROLLER_REPO:-} + +# Only ever a kind cluster: never a real or shared context. +[[ $CONTEXT == kind-* ]] || { echo "refusing to use context '$CONTEXT' (not kind-*)"; exit 2; } + +k() { kubectl --context "$CONTEXT" -n "$NS" "$@"; } + +step() { echo "::group::$*"; } +endstep() { echo "::endgroup::"; } +fail() { echo "::error title=kind smoke::$*"; exit 1; } + +# Polls a command until it succeeds or TIMEOUT seconds pass. +until_ok() { # + local what=$1 deadline=$((SECONDS + TIMEOUT)) + shift + until "$@"; do + ((SECONDS < deadline)) || fail "timed out after ${TIMEOUT}s waiting for: $what" + sleep 5 + done +} + +split_image() { # -> "repo tag" + echo "${1%:*} ${1##*:}" +} + +cmd_run() { + step "load images into kind/$CLUSTER" + kind load docker-image --name "$CLUSTER" "$RUNNER_IMAGE" "$SCALER_IMAGE" + endstep + + step "namespace and dummy credentials" + kubectl --context "$CONTEXT" create namespace "$NS" --dry-run=client -o yaml \ + | kubectl --context "$CONTEXT" apply -f - + kubectl --context "$CONTEXT" label namespace "$NS" --overwrite \ + pod-security.kubernetes.io/enforce=privileged + # Only the token is secret; the runner group goes in as chart values. + k create secret generic "$SECRET" \ + --from-literal=SG_NODE_TOKEN="$TOKEN" \ + --dry-run=client -o yaml | k apply -f - + endstep + + step "helm install" + local runner_repo runner_tag scaler_repo scaler_tag + read -r runner_repo runner_tag <<<"$(split_image "$RUNNER_IMAGE")" + read -r scaler_repo scaler_tag <<<"$(split_image "$SCALER_IMAGE")" + # Small volumes: kind's local-path provisioner ignores sizes anyway. + helm --kube-context "$CONTEXT" -n "$NS" upgrade --install "$RELEASE" "$CHART" \ + --set credentials.existingSecret="$SECRET" \ + --set sg.organization=ci-org --set sg.runnerGroup=ci-group \ + --set sg.apiUrl=http://127.0.0.1:9 \ + --set image.repository="$runner_repo" --set image.tag="$runner_tag" \ + --set image.pullPolicy=Never \ + --set scaler.image.repository="$scaler_repo" --set scaler.image.tag="$scaler_tag" \ + --set scaler.image.pullPolicy=Never \ + --set scaler.interval=5 \ + --set runner.storage.state.size=64Mi --set runner.storage.docker.size=256Mi \ + --set runner.terminationGracePeriodSeconds=5 + endstep + + step "scaler Deployment becomes Available" + k wait --for=condition=Available "deployment/$SCALER" --timeout="${TIMEOUT}s" + endstep + + step "scaler started and finds SG unreachable" + # The scaler logs JSON; a failed loop against the unreachable API carries + # reason SGUnreachable. + until_ok "scaler log: loop failed with SGUnreachable" scaler_saw_sg_unreachable + k logs "deployment/$SCALER" --tail=20 + endstep + + step "runner pod: steps broker sidecar, guarded dockerd" + # The register init container fails here, so neither ever runs: check the + # spec and the image instead. + local policy + policy=$(k get "statefulset/$STS" \ + -o jsonpath='{.spec.template.spec.initContainers[?(@.name=="steps")].restartPolicy}') + [[ $policy == Always ]] || fail "the steps container isn't a native sidecar (restartPolicy '$policy')" + echo "steps: init container with restartPolicy Always" + docker run --rm --entrypoint grep "$RUNNER_IMAGE" -q -- '--authorization-plugin=' /opt/sg-runner/runner.sh \ + || fail "the runner image's dockerd doesn't use the authorization plugin" + echo "runner image: dockerd starts with --authorization-plugin" + endstep + + step "runner StatefulSet exists, register init container fails" + k get "statefulset/$STS" + until_ok "a finished run of the register init container of $STS-0" register_finished + local logs=$REGISTER_LOG code=$REGISTER_CODE attempts + echo "$logs" | tail -n 20 + # Exactly 1: die() on the failed SG call. 0 would mean it registered + # against nothing; >=128 means it crashed (e.g. error reporting recursing + # into itself while the API is down). + [[ $code == 1 ]] || fail "register exited $code, want 1" + grep -q "API call failed" <<<"$logs" \ + || fail "register exited 1, but not on the SG API call" + grep -qF "$TOKEN" <<<"$logs" && fail "register printed the dummy SG_NODE_TOKEN" + # Each failed API call logs one "API call failed": the registration call + # plus at most one best-effort error report. More means it is retrying. + attempts=$(grep -c "API call failed" <<<"$logs") + ((attempts <= MAX_API_ATTEMPTS)) \ + || fail "register made $attempts failed API calls in one run, want <= $MAX_API_ATTEMPTS" + echo "register exited 1 after $attempts failed API calls, as expected" + endstep + + step "no credentials in logs or events" + local all + all=$( + k logs "deployment/$SCALER" --all-containers 2>&1 + k logs "$STS-0" -c register --previous 2>&1 || true + k logs "$STS-0" -c register 2>&1 || true + k get events -o yaml 2>&1 + ) + if grep -qF "$TOKEN" <<<"$all"; then + fail "the dummy SG_NODE_TOKEN shows up in logs or events" + fi + echo "dummy token not found in scaler logs, register logs or events" + endstep + + step "scaler RBAC" + can "patch statefulsets.apps/$STS --subresource=scale" yes + can "get statefulsets.apps/$STS --subresource=scale" yes + can "patch statefulsets.apps/$STS" yes + can "create jobs.batch" yes + # scaler.nudge: read-only on runner pods and autoscaler events. + can "list pods" yes + can "list events" yes + can "patch statefulsets.apps/other --subresource=scale" no + can "delete statefulsets.apps/$STS" no + can "delete pods" no + can "create pods" no + can "get secrets" no + can "create pods/exec" no + endstep + + native_steps + + step "helm uninstall deregisters and deletes every runner's PVCs" + # The pre-delete hook parks the runner, runs a reap Job against its state + # PVC (never registered here: nothing to deregister), then deletes the PVCs. + k get pvc + helm --kube-context "$CONTEXT" -n "$NS" uninstall "$RELEASE" --timeout "${TIMEOUT}s" + # The workspace PVC (generic ephemeral) goes with its pod, through the + # garbage collector: give it a moment. + until_ok "nothing left after helm uninstall" nothing_left + echo "nothing left after helm uninstall" + endstep + + echo "kind smoke passed" +} + +scaler_saw_sg_unreachable() { + k logs "deployment/$SCALER" 2>/dev/null \ + | grep '"loop failed, skipping decisions"' | grep -q '"reason":"SGUnreachable"' +} + +# Sets REGISTER_CODE and REGISTER_LOG from the register init container's last +# finished run: the current container once it has terminated, else the +# previous one (it restarts in a loop). Fails until a run has finished and its +# log is retrievable. +register_finished() { + local status prev=() + status=$(k get pod "$STS-0" -o jsonpath='{.status.initContainerStatuses[?(@.name=="register")]}' 2>/dev/null) \ + || return 1 + REGISTER_CODE=$(jq -r '.state.terminated.exitCode // empty' <<<"$status") + if [[ -z $REGISTER_CODE ]]; then + REGISTER_CODE=$(jq -r '.lastState.terminated.exitCode // empty' <<<"$status") + prev=(--previous) + fi + [[ -n $REGISTER_CODE ]] || return 1 + REGISTER_LOG=$(k logs "$STS-0" -c register "${prev[@]}" 2>/dev/null) || return 1 + # Gone between the two calls (restarted again): try the next round. + [[ -n $REGISTER_LOG && $REGISTER_LOG != "unable to retrieve container logs"* ]] +} + +can() { # [service account] + local got as=${3:-$SA} + # shellcheck disable=SC2086 # $1 is a verb plus resource plus flags + got=$(k auth can-i $1 --as="$as" 2>/dev/null || true) + if [[ $got != "$2" ]]; then + fail "${as##*:} can-i '$1': got '$got', want '$2'" + fi + echo "can-i $1: $got" +} + +# Native steps. The runner pod never gets past register without SG, so the +# broker runs in a test pod of its own with the chart's runner ServiceAccount +# (which also checks the chart's RBAC), next to a client container laid out +# like the runner's dockerd: the workspace PVC at /var/tmp/workspace, the +# broker's socket below it. +native_steps() { + step "native steps: broker test pod" + local node + for node in $(kind get nodes --name "$CLUSTER"); do + docker exec -i "$node" sh -c "mkdir -p /etc/containerd/certs.d/$MIRROR_HOST \ + && cat >/etc/containerd/certs.d/$MIRROR_HOST/hosts.toml" </mnt/sg_workspace/out.txt; echo done"]') + out=$(api GET "/v1/steps/$id/logs?follow=1" | cut -d' ' -f2-) + [[ $out == $'env=bar\ndone' ]] || fail "step logs: $out" + echo "logs: $(tr '\n' ' ' <<<"$out")" + expect_state "$id" exited 0 + [[ $(kc cat "$STEPS_RUN/shared-workspace/out.txt") == hi ]] \ + || fail "the step's workspace write isn't visible to the client" + echo "workspace write visible to the client" + + expect_error InvalidMount POST /v1/steps "$(step_json '["true"]' '"mounts":[{"source":"/var/tmp/workspace","target":"/w"}]')" + expect_error InvalidMount POST /v1/steps "$(step_json '["true"]' "\"mounts\":[{\"source\":\"$STEPS_RUN/../r2/x\",\"target\":\"/w\"}]")" + expect_error InvalidMount POST /v1/steps "$(step_json '["true"]' '"mounts":[{"source":"/var/tmp/workspace/orgs/ci/wfruns/r2/x","target":"/w"}]')" + expect_error InvalidEnv POST /v1/steps "$(step_json '["true"]' '"env":{"BAD KEY":"x"}')" + expect_error NotFound GET "/v1/steps/kube-apiserver-$CLUSTER-control-plane" + + id=$(STEP_IMAGE=nonexistent.invalid/nope:1 create_step '["true"]') + expect_state "$id" failed "" 'ErrImagePull|ImagePullBackOff' + id=$(create_step '["true"]' '"memoryMiB":9999999') + expect_state "$id" failed "" Unschedulable + + # TERM trapped: the step exits 143 itself within the grace. + id=$(create_step '["sh","-c","trap \"echo got TERM; exit 143\" TERM; while true; do sleep 1; done"]') + until_ok "step $id running" step_running "$id" + api POST "/v1/steps/$id/stop" '{"graceSeconds":30}' >/dev/null + expect_state "$id" exited 143 + + # Kept for FINISHED_TTL, then deleted by the GC; the state stays. + until_ok "step pod $first deleted after FINISHED_TTL" step_gone "$first" + [[ $(api GET "/v1/steps/$first" | jq -r .exitCode) == 0 ]] || fail "deleted step $first lost its exit code" + echo "step $first deleted after FINISHED_TTL, exit code still served" + endstep + + step "native steps: dockerd guard" + # As dockerd asks it (the authorization plugin protocol): a step container + # (labelled with its run) is refused, a run controller allowed. + [[ $(guard /Plugin.Activate '{}' | jq -c .Implements) == '["authz"]' ]] \ + || fail "the guard doesn't activate as an authz plugin" + out=$(guard /AuthZPlugin.AuthZReq "$(authz_req '{"Image":"alpine","Labels":{"wfrunId":"r1"}}')") + [[ $(jq -r .Allow <<<"$out") == false ]] || fail "the guard allowed a step container: $out" + echo "step container refused: $(jq -r .Msg <<<"$out")" + out=$(guard /AuthZPlugin.AuthZReq "$(authz_req '{"Image":"controller","Labels":{"com.amazonaws.ecs.task-arn":"x"}}')") + [[ $(jq -r .Allow <<<"$out") == true ]] || fail "the guard refused a run controller: $out" + echo "run controller allowed" + endstep + + if [[ -n $CONTROLLER_REPO ]]; then + controller_live_test + fi + + step "native steps: cleanup" + k delete pod "$STEPS_POD" --timeout=60s + k delete pvc "$STEPS_POD-workspace" --timeout=60s + # Step pods and Secrets are owned by the test pod. + until_ok "step pods gone" steps_all_gone + echo "native steps cleaned up" + endstep +} + +steps_pod_manifest() { + cat </dev/null && touch /tmp/ready; sleep infinity & wait'] + readinessProbe: + exec: { command: [test, -e, /tmp/ready] } + periodSeconds: 2 + volumeMounts: + - { name: workspace, mountPath: /var/tmp/workspace } + - { name: socket, mountPath: /var/tmp/workspace/.sg-runner } + # Where dockerd would find the guard. + - { name: guard, mountPath: /run/docker/plugins } + volumes: + - name: workspace + persistentVolumeClaim: { claimName: $STEPS_POD-workspace } + - name: socket + emptyDir: {} + - name: guard + emptyDir: {} + # As the chart's steps container gets it. + - name: api-access + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: { path: token, expirationSeconds: 3607 } + - configMap: + name: kube-root-ca.crt + items: [{ key: ca.crt, path: ca.crt }] + - downwardAPI: + items: + - path: namespace + fieldRef: { apiVersion: v1, fieldPath: metadata.namespace } +EOF +} + +# Calls the guard's plugin API; prints the body. +guard() { # + kc curl -sS --unix-socket /run/docker/plugins/sg-steps-guard.sock -X POST \ + -H 'Content-Type: application/json' -d "$2" "http://guard$1" +} + +authz_req() { # + jq -nc --arg b "$(printf '%s' "$1" | base64 | tr -d '\n')" \ + '{User: "", RequestMethod: "POST", RequestUri: "/v1.47/containers/create", RequestBody: $b}' +} + +# In the client container. +kc() { k exec "$STEPS_POD" -c client -- "$@"; } + +# Calls the broker; prints the body, fails on a curl error (not on a 4xx). +api() { # [json body] + local args=(curl -sS --unix-socket "$STEPS_SOCK" -X "$1" "http://broker$2") + [[ -n ${3:-} ]] && args+=(-H 'Content-Type: application/json' -d "$3") + kc "${args[@]}" +} + +step_json() { # [extra fields] + echo "{\"name\":\"ci-r1-step\",\"org\":\"/orgs/ci\",\"run\":\"/wfruns/r1\",\"stepIdx\":0, + \"image\":\"$STEP_IMAGE\",\"args\":$1,\"env\":{\"FOO\":\"bar\"}, + \"mounts\":[{\"source\":\"$STEPS_RUN/shared-workspace\",\"target\":\"/mnt/sg_workspace\",\"readOnly\":false}], + \"cpuShares\":64,\"memoryMiB\":64,\"timeoutSeconds\":300${2:+,$2}}" +} + +create_step() { # [extra fields] -> id + local out id + out=$(api POST /v1/steps "$(step_json "$@")") + id=$(jq -r '.id // empty' <<<"$out") + [[ -n $id ]] || fail "create step: $out" + echo "$id" +} + +# Waits for a terminal state and checks it. +expect_state() { # [reason regex] + local id=$1 want=$2 code=$3 reason=${4:-} st + until_ok "step $id terminal" step_terminal "$id" + st=$STEP_STATE + [[ $(jq -r .state <<<"$st") == "$want" ]] || fail "step $id: want $want, got $st" + if [[ -n $code ]]; then + [[ $(jq -r .exitCode <<<"$st") == "$code" ]] || fail "step $id: want exit code $code, got $st" + fi + if [[ -n $reason ]]; then + [[ $(jq -r .reason <<<"$st") =~ ^($reason)$ ]] || fail "step $id: want reason $reason, got $st" + fi + echo "step $id: $(jq -c '{state, exitCode, reason}' <<<"$st")" +} + +step_terminal() { + STEP_STATE=$(api GET "/v1/steps/$1") || return 1 + [[ $(jq -r .state <<<"$STEP_STATE") =~ ^(exited|failed)$ ]] +} + +step_running() { + [[ $(api GET "/v1/steps/$1" | jq -r .state) == running ]] +} + +step_gone() { + [[ -z $(k get "pod/$1" "secret/$1-env" -o name --ignore-not-found 2>/dev/null) ]] +} + +secret_gone() { + [[ -z $(k get "secret/$1-env" -o name --ignore-not-found 2>/dev/null) ]] +} + +# A finished step pod as the broker leaves it. Polled as a whole: the broker +# labels the pod once its container has terminated, which can be a moment +# before the kubelet updates the pod's phase. +expect_pod() { # + local got="" want="$2 $3 $4 $5" deadline=$((SECONDS + TIMEOUT)) + until got=$(pod_outcome "$1") && [[ $got == "$want" ]]; do + ((SECONDS < deadline)) \ + || fail "step pod $1: want '$want' (phase reason outcome wrapped), last seen '$got' after ${TIMEOUT}s" + sleep 2 + done + echo "step pod $1: $got" +} + +pod_outcome() { # -> "phase reason outcome wrapped" + k get "pod/$1" -o json 2>/dev/null | jq -r --arg l "$OUTCOME_LABEL" --arg a "$WRAPPED_ANNOTATION" ' + [.status.phase, + ((.status.containerStatuses // [])[] | select(.name == "step") | .state.terminated.reason // ""), + .metadata.labels[$l] // "", .metadata.annotations[$a] // ""] | join(" ")' +} + +steps_all_gone() { + [[ -z $(k get pods,secrets -l app.kubernetes.io/component=step -o name 2>/dev/null) ]] +} + +nothing_left() { + LEFT=$(k get pvc,statefulset,deployment,job -o name 2>&1) + [[ -z $LEFT ]] || { echo "left: $LEFT"; return 1; } +} + +expect_error() { # [json body] + local want=$1 out + shift + out=$(api "$@") + [[ $(jq -r .reason <<<"$out" 2>/dev/null) == "$want" ]] || fail "$1 $2: want $want, got $out" + echo "$1 $2: $want" +} + +# The run controller's live broker test (unit/test_step_broker_live.py) from a +# local sg-run-controller checkout, in the client container. +controller_live_test() { + step "native steps: run controller live test" + local test=$CONTROLLER_REPO/unit/test_step_broker_live.py + [[ -f $test ]] || fail "CONTROLLER_REPO is set, but $test doesn't exist" + kc mkdir -p /tmp/controller/unit + tar -C "$CONTROLLER_REPO" -cf - src/run_controller unit/test_step_broker_live.py \ + | k exec -i "$STEPS_POD" -c client -- tar -C /tmp/controller -xf - + kc pip install -q --root-user-action=ignore --disable-pip-version-check docker==6.0.1 requests==2.31.0 'urllib3<2' pytest + kc sh -c "cd /tmp/controller && STEP_BROKER_SOCKET=$STEPS_SOCK STEP_BROKER_WORKSPACES_DIR=/var/tmp/workspace \ + STEP_BROKER_IMAGE=$STEP_IMAGE \ + PYTHONPATH=src:src/run_controller python -m pytest -q unit/test_step_broker_live.py" + endstep +} + +cmd_diag() { + local dir=${1:?usage: kind-smoke.sh diag DIR} + mkdir -p "$dir" + k get all,pvc,secret,lease,role,rolebinding,serviceaccount -o wide >"$dir/get.txt" 2>&1 || true + k describe statefulset,deployment,pods,pvc >"$dir/describe.txt" 2>&1 || true + k get events --sort-by=.lastTimestamp >"$dir/events.txt" 2>&1 || true + k logs "deployment/$SCALER" --all-containers --prefix >"$dir/scaler.log" 2>&1 || true + k logs "$STS-0" --all-containers --prefix >"$dir/runner-0.log" 2>&1 || true + k logs "$STS-0" -c register --previous >"$dir/runner-0-register-previous.log" 2>&1 || true + # Native steps: the broker test pod and its step pods. + k logs "$STEPS_POD" --all-containers --prefix >"$dir/steps-smoke.log" 2>&1 || true + k get pods,secrets -l app.kubernetes.io/component=step -o wide >"$dir/step-pods.txt" 2>&1 || true + k describe pods -l app.kubernetes.io/component=step >"$dir/step-pods-describe.txt" 2>&1 || true + helm --kube-context "$CONTEXT" -n "$NS" get all "$RELEASE" >"$dir/helm-release.txt" 2>&1 || true + kubectl --context "$CONTEXT" get nodes -o wide >"$dir/nodes.txt" 2>&1 || true + # Only dummy values in this cluster, but keep secret data out regardless. + sed -i.bak "s/$TOKEN//g" "$dir"/*.txt "$dir"/*.log && rm -f "$dir"/*.bak +} + +case ${1:-} in + run) cmd_run ;; + diag) cmd_diag "${2:-}" ;; + *) + echo "usage: $0 run | diag DIR" + exit 2 + ;; +esac diff --git a/.github/workflows/k8s-release.yml b/.github/workflows/k8s-release.yml new file mode 100644 index 0000000..134cd46 --- /dev/null +++ b/.github/workflows/k8s-release.yml @@ -0,0 +1,221 @@ +name: k8s-release + +# Publishes the Kubernetes runner (beta, GHCR): the runner and scaler images +# (linux/amd64 + linux/arm64, with build provenance), the Helm chart as an OCI +# artifact, and a GitHub release. +# +# git tag k8s/v0.9.0 && git push origin k8s/v0.9.0 # publish 0.9.0 +# workflow_dispatch (version input) # dry run: builds, pushes nothing +# +# The version (tag minus `k8s/v`) is the image tag, the chart version and the +# chart's appVersion. Nothing published is ever overwritten: an existing image +# tag or chart version fails the run. +on: + push: + tags: ["k8s/v*"] + workflow_dispatch: + inputs: + version: + description: Version to build (dry run, nothing is pushed) + default: 0.0.0-dryrun + required: true + +permissions: + contents: read + +concurrency: + group: k8s-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + version: + name: version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + push: ${{ steps.version.outputs.push }} + registry: ${{ steps.version.outputs.registry }} + steps: + - name: Version from the tag, or the dry run's input + id: version + env: + INPUT_VERSION: ${{ inputs.version }} + OWNER: ${{ github.repository_owner }} + run: | + if [[ $GITHUB_EVENT_NAME == push ]]; then + version=${GITHUB_REF_NAME#k8s/v} + push=true + else + version=$INPUT_VERSION + push=false + fi + # SemVer without build metadata: `+` isn't valid in an image tag. + semver='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$' + if [[ ! $version =~ $semver ]]; then + echo "::error title=version::'$version' isn't a SemVer version (X.Y.Z or X.Y.Z-pre)" + exit 1 + fi + { + echo "version=$version" + echo "push=$push" + # Registry paths must be lowercase. + echo "registry=ghcr.io/${OWNER,,}" + } >>"$GITHUB_OUTPUT" + echo "### k8s release $version (push: $push)" >>"$GITHUB_STEP_SUMMARY" + + # The whole k8s CI: Go, chart, images, kind smoke. + ci: + name: ci + uses: ./.github/workflows/k8s.yml + + images: + name: image (${{ matrix.image }}) + needs: [version, ci] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + strategy: + fail-fast: true + matrix: + include: + - image: runner + name: sg-runner-k8s + context: . + file: k8s/Dockerfile + - image: scaler + name: sg-runner-scaler + context: k8s/scaler + file: k8s/scaler/Dockerfile + env: + VERSION: ${{ needs.version.outputs.version }} + PUSH: ${{ needs.version.outputs.push }} + IMAGE: ${{ needs.version.outputs.registry }}/${{ matrix.name }} + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Log in to GHCR + if: env.PUSH == 'true' + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "$TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + - name: Refuse to overwrite ${{ env.IMAGE }}:${{ env.VERSION }} + if: env.PUSH == 'true' + run: | + if docker manifest inspect "$IMAGE:$VERSION" >/dev/null 2>&1; then + echo "::error title=image exists::$IMAGE:$VERSION is already published" + exit 1 + fi + - name: Build (and push on a tag) + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + platforms: linux/amd64,linux/arm64 + push: ${{ env.PUSH == 'true' }} + outputs: ${{ env.PUSH != 'true' && 'type=cacheonly' || '' }} + tags: ${{ env.IMAGE }}:${{ env.VERSION }} + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.version=${{ env.VERSION }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.licenses=Apache-2.0 + # Provenance comes from the attestation below. + provenance: false + cache-from: type=gha,scope=${{ matrix.image }} + - name: Attest build provenance + if: env.PUSH == 'true' + uses: actions/attest-build-provenance@v4 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + - name: Summary + run: | + if [[ $PUSH == true ]]; then + echo "- \`$IMAGE:$VERSION\` (\`${{ steps.build.outputs.digest }}\`)" >>"$GITHUB_STEP_SUMMARY" + else + echo "- dry run: built \`$IMAGE:$VERSION\` for linux/amd64 + linux/arm64, not pushed" >>"$GITHUB_STEP_SUMMARY" + fi + + chart: + name: chart + needs: [version, images] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + VERSION: ${{ needs.version.outputs.version }} + PUSH: ${{ needs.version.outputs.push }} + CHARTS: oci://${{ needs.version.outputs.registry }}/charts + steps: + - uses: actions/checkout@v7 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - name: Package + run: | + helm package k8s/chart/sg-runner --version "$VERSION" --app-version "$VERSION" -d "$RUNNER_TEMP" + # The packaged chart's defaults point at this release's images. + helm template sg-runner "$RUNNER_TEMP/sg-runner-$VERSION.tgz" \ + --set credentials.existingSecret=x --set sg.organization=o --set sg.runnerGroup=g \ + | grep -E '^\s+image: ' | sort -u + - name: Log in to GHCR + if: env.PUSH == 'true' + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "$TOKEN" | helm registry login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + - name: Refuse to overwrite sg-runner ${{ env.VERSION }} + if: env.PUSH == 'true' + run: | + if helm show chart "$CHARTS/sg-runner" --version "$VERSION" >/dev/null 2>&1; then + echo "::error title=chart exists::$CHARTS/sg-runner $VERSION is already published" + exit 1 + fi + - name: Push + if: env.PUSH == 'true' + run: helm push "$RUNNER_TEMP/sg-runner-$VERSION.tgz" "$CHARTS" + - name: Upload the packaged chart + uses: actions/upload-artifact@v7 + with: + name: sg-runner-chart-${{ env.VERSION }} + path: ${{ runner.temp }}/sg-runner-${{ env.VERSION }}.tgz + retention-days: 7 + + release: + name: GitHub release + needs: [version, chart] + if: needs.version.outputs.push == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + env: + VERSION: ${{ needs.version.outputs.version }} + REGISTRY: ${{ needs.version.outputs.registry }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Create the release + run: | + prerelease=() + [[ $VERSION == *-* ]] && prerelease=(--prerelease) + cat >"$RUNNER_TEMP/notes.md" <>"$GITHUB_PATH" + # Lint, render the value scenarios, kubeconform them, check rejections. + - name: Chart check + run: .github/scripts/chart-check.sh "$RUNNER_TEMP/rendered" + - name: Upload rendered manifests + if: failure() + uses: actions/upload-artifact@v7 + with: + name: rendered-helm-${{ matrix.helm }} + path: ${{ runner.temp }}/rendered + retention-days: 7 + + images: + name: image (${{ matrix.image }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - image: runner + context: . + file: k8s/Dockerfile + tag: sg-runner-k8s:ci + - image: scaler + context: k8s/scaler + file: k8s/scaler/Dockerfile + tag: sg-runner-scaler:ci + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + # Both platforms, kept as build cache only: proves arm64 builds too. + - name: Build linux/amd64 + linux/arm64 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + platforms: linux/amd64,linux/arm64 + push: false + outputs: type=cacheonly + provenance: false + cache-from: type=gha,scope=${{ matrix.image }} + cache-to: type=gha,mode=max,scope=${{ matrix.image }} + # amd64 again from the cache, loaded into the local daemon for Trivy. + - name: Load linux/amd64 for scanning + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + platforms: linux/amd64 + push: false + load: true + provenance: false + tags: ${{ matrix.tag }} + cache-from: type=gha,scope=${{ matrix.image }} + # Report HIGH and CRITICAL for both images, never failing here. + - name: Trivy report + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ matrix.tag }} + version: ${{ env.TRIVY_VERSION }} + severity: HIGH,CRITICAL + format: table + output: trivy-${{ matrix.image }}.txt + exit-code: "0" + - name: Trivy summary + if: always() + run: | + [[ -f trivy-${{ matrix.image }}.txt ]] || exit 0 + cat trivy-${{ matrix.image }}.txt + { + echo '### Trivy: ${{ matrix.tag }} (HIGH, CRITICAL)' + echo '
report' + echo + echo '```' + head -c 900000 trivy-${{ matrix.image }}.txt + echo '```' + echo '
' + } >>"$GITHUB_STEP_SUMMARY" + # Gate: fail on CRITICAL vulnerabilities that have a fix, scaler only. + # The scaler is our own Go binary on distroless, so a fixable CRITICAL + # is actionable here (bump a module, rebuild on a newer Go patch). + # The runner image is not gated: it bundles upstream binaries (static + # Docker/containerd, the SSM agent, the ECS agent) built with Go + # releases Trivy marks as "fixed" upstream, which we can only clear by + # waiting for upstream releases. Gating it would keep CI red on things + # this repo can't fix; it is reported above instead. + - name: Trivy gate (CRITICAL, fixable) + if: matrix.image == 'scaler' + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ matrix.tag }} + version: ${{ env.TRIVY_VERSION }} + skip-setup-trivy: true + severity: CRITICAL + ignore-unfixed: true + format: table + exit-code: "1" + + kind-smoke: + name: kind smoke install + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + # amd64 only (the CI runner's arch); reuses the images job's cache when + # it's there. + - name: Build runner image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: k8s/Dockerfile + platforms: linux/amd64 + load: true + provenance: false + tags: sg-runner-k8s:ci + cache-from: type=gha,scope=runner + - name: Build scaler image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: k8s/scaler + platforms: linux/amd64 + load: true + provenance: false + tags: sg-runner-scaler:ci + cache-from: type=gha,scope=scaler + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.0 + - name: Create kind cluster + uses: helm/kind-action@1544676c07bb3570fcb70166beaa07c4528fd4aa # v1.15.1 + with: + version: ${{ env.KIND_VERSION }} + node_image: ${{ env.KIND_NODE_IMAGE }} + cluster_name: sg-runner-ci + wait: 120s + # Dummy token, sg.apiUrl=http://127.0.0.1:9: never reaches SG. + - name: Install and check + env: + CLUSTER: sg-runner-ci + RUNNER_IMAGE: sg-runner-k8s:ci + SCALER_IMAGE: sg-runner-scaler:ci + run: .github/scripts/kind-smoke.sh run + - name: Collect diagnostics + if: failure() + env: + CLUSTER: sg-runner-ci + run: .github/scripts/kind-smoke.sh diag "$RUNNER_TEMP/kind-diag" + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: kind-smoke-diagnostics + path: ${{ runner.temp }}/kind-diag + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29f1e44..bb11008 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,18 @@ name: test +# Shell: main.sh and k8s/runner.sh (shellcheck), and the bats unit + smoke +# tiers, which include the k8s/runner.sh tests (test/unit/k8s_*.bats). on: pull_request: branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: lint: diff --git a/Makefile b/Makefile index 7e5c748..0c5f9b8 100644 --- a/Makefile +++ b/Makefile @@ -23,11 +23,13 @@ test-unit: ## Run the unit tier (pure functions, no I/O) test-smoke: ## Run the smoke tier (subprocess CLI-contract) $(BATS) --recursive test/smoke -SHELL_SOURCES := main.sh test/mocks/bin/* test/helpers/*.bash +SHELL_SOURCES := main.sh k8s/runner.sh test/mocks/bin/* test/helpers/*.bash lint: ## shellcheck correctness + modern-idiom gate (config in .shellcheckrc) # General correctness (warnings and up); SC2034 is disabled in .shellcheckrc. shellcheck --severity=warning main.sh + # -x: follows runner.sh's source of main.sh. + shellcheck -x --severity=warning k8s/runner.sh shellcheck test/mocks/bin/* shellcheck test/helpers/*.bash # Idiom gate: enforce [[ ]] over [ ] (SC2292) and command substitution over diff --git a/README.md b/README.md index e8c0c6d..d5b5c58 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ - [3.4 Restart](#34-restart) - [System diagnostics](#system-diagnostics) - [Managing `cgroupsv2`](#managing-cgroupsv2) +- [Kubernetes (prototype)](#kubernetes-prototype) - [Troubleshooting](#troubleshooting) - [Development](#development) @@ -187,6 +188,48 @@ To revert you can just run: ./main.sh cgroupsv2 enable ``` +## Kubernetes (prototype) + +> **Prototype only.** Not supported for customer use. Needs a run controller +> release with step broker support, which StackGuardian hasn't shipped yet. + +[`k8s/`](k8s/README.md) runs Private Runners as Kubernetes pods, installed with +a Helm chart ([`k8s/chart/sg-runner`](k8s/chart/sg-runner/README.md), Kubernetes +1.29+). Each StatefulSet replica is one registered runner: a nested Docker +daemon for the run controller, the SSM and ECS agents, and `main.sh` sourced +for registration and network setup. Workflow **steps** run as their own +Kubernetes pods, created by a step broker in the runner pod +([`k8s/docs/native-steps.md`](k8s/docs/native-steps.md)); a guard refuses step +containers in the nested Docker daemon. A scaler +([`k8s/scaler`](k8s/scaler), Go) follows the runner group's queue in SG: + +- **Parks** idle runners: the pod stops but the runner stays registered, and + wakes again in about 8s (a fresh registration takes about 20s). +- **Wakes or adds** runners when workflows queue, between a minimum of 1 and + the runner group's `MaxNumberOfRunners`; optional headroom keeps a warm spare. +- **Deregisters** runners parked longer than a TTL (12h by default). + +The runner size (`runner.size`: `small`, `medium`, `large` or `custom`) sets +the workflow capacity a runner advertises; one runner per node. See +[`k8s/README.md`](k8s/README.md) for usage, measurements and known gaps. + +### Known gaps + +- **Privileged pod.** The nested Docker daemon (run controllers) needs it; + clusters enforcing restricted Pod Security reject it. Workflow steps run as + unprivileged pods. +- **Network isolation is opt-in.** The chart's NetworkPolicy keeps runner and + step pods away from cluster addresses and the metadata endpoint, but needs an + enforcing CNI and the cluster's CIDRs. +- **amd64 nodes** for StackGuardian's step images. +- **No proxy support.** `--http-proxy` isn't wired up. +- **Unverified:** waking a runner after a long park (hours or days), the Go + scaler against real SG, and GKE/EKS specifics (see `k8s/docs/open-items.md`). +- **SG credentials in the pod.** Mounted into the `ecs-agent` container and the + scaler to set runner status and read the queue. +- **`ecs-init` is bypassed.** Agent defaults are copied into `k8s/runner.sh` and + may drift with new ECS agent releases. + ## Troubleshooting - StackGuardian uses AWS SSM to setup connection between SG control plane and runners, you can diagnose SSM client using `ssm-cli get-diagnostics --output table` diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..42f1187 --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,46 @@ +# TODO + +## Kubernetes prototype (`k8s/`) + +### Verify the NetworkPolicy on an enforcing CNI + +The chart ships an optional egress NetworkPolicy (`networkPolicy.enabled`) for +runner, step and reap pods: DNS to kube-dns, everything outside `clusterCIDRs`, +no `169.254.169.254`. Not yet tested on a CNI that enforces NetworkPolicy +(Calico, Cilium, the VPC CNI's policy mode): + +- Confirm a step pod can't reach the Kubernetes API, other pods or the metadata + endpoint, and can still reach SG, AWS, VCS and registries. +- The step broker needs the Kubernetes API: today that means an + `extraEgress` rule for the API endpoint (open to the whole runner pod); + replace it with a dedicated value. +- Decide whether steps may resolve cluster names at all; if not, give step + pods their own `dnsConfig` and block cluster DNS. + +### Long parking and hard kills against real SG + +- Park a runner for 12–24h, wake it, and run a workflow on it. +- Force-delete a runner pod during scale-down and check that the scaler drains + it in SG and reactivates it on wake. +- Check whether SG resumes workflows paused while no runner was `ACTIVE`. + +### Drop privileged mode + +Spike results on OrbStack (k8s 1.36, cri-dockerd, arm64): + +| Option | Result | +|---|---| +| `hostUsers: false` (user namespaces) | Not testable: cri-dockerd doesn't support user namespaces | +| Capabilities only | Works with `SYS_ADMIN`, `NET_ADMIN`, `SYS_RESOURCE`, seccomp/AppArmor `Unconfined` and `mount -o remount,rw` of `/sys/fs/cgroup` and `/proc/sys`. Not a real gain: `CAP_SYS_ADMIN` in the host user namespace with seccomp off is close to privileged | +| Rootless dind | Containers fail to mount `/proc` without `procMount: Unmasked` (needs user namespaces); task-credential DNAT would target the wrong loopback | +| Sysbox | Not tried (needs a node install) | + +Next: test `hostUsers: false` (with `procMount: Unmasked`) on containerd 2.x +or CRI-O, kernel 6.3+, and Sysbox on a cluster where it can be installed. +Both keep root inside the pod unprivileged on the node. + +### Move image and chart distribution to Amazon ECR Public + +Move image and chart distribution to Amazon ECR Public (AWS-native, OIDC +role, public.ecr.aws alias); GHCR is the beta choice +(`.github/workflows/k8s-release.yml`). diff --git a/k8s/.env.example b/k8s/.env.example new file mode 100644 index 0000000..9b394f0 --- /dev/null +++ b/k8s/.env.example @@ -0,0 +1,7 @@ +# Copy to k8s/.env (gitignored) and fill in. Use a throwaway runner group. +# `make secret` puts only SG_NODE_TOKEN in a Secret; `make up` passes the rest +# as chart values (sg.*), which the chart keeps in a ConfigMap. +SG_NODE_TOKEN= +SG_ORGANIZATION= +SG_RUNNER_GROUP= +SG_BASE_API=https://api.app.stackguardian.io/api/v1 diff --git a/k8s/Dockerfile b/k8s/Dockerfile new file mode 100644 index 0000000..4ec4032 --- /dev/null +++ b/k8s/Dockerfile @@ -0,0 +1,49 @@ +# sg-runner Kubernetes prototype image: one image backs every container of +# the pod (see runner.sh). Build from the repo root: +# docker build -f k8s/Dockerfile -t sg-runner-k8s:dev . + +# Pinned for reproducible builds; each exists for linux/amd64 and linux/arm64. +# ECS agent: Docker Hub tags (https://hub.docker.com/r/amazon/amazon-ecs-agent/tags). +ARG ECS_AGENT_VERSION=v1.108.0 + +FROM amazon/amazon-ecs-agent:${ECS_AGENT_VERSION} AS ecs-agent + +FROM debian:bookworm-slim + +ARG TARGETARCH +ARG DOCKER_VERSION=27.5.1 +# SSM agent: .../SSMAgent/latest/VERSION names the current release. +ARG SSM_AGENT_VERSION=3.3.5390.0 + +# upgrade: pick up Debian security fixes newer than the base image. +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + bash ca-certificates curl jq iproute2 iptables procps xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# Docker engine + CLI (static build), run nested in the privileged container. +RUN arch="$([ "$TARGETARCH" = arm64 ] && echo aarch64 || echo x86_64)" \ + && curl -fsSL "https://download.docker.com/linux/static/stable/${arch}/docker-${DOCKER_VERSION}.tgz" \ + | tar -xz -C /usr/local/bin --strip-components=1 + +# SSM agent. Unpacked rather than installed: the postinst expects systemd. +# Staged first: extracting onto / would replace the merged-usr /lib symlink. +RUN curl -fsSL -o /tmp/ssm.deb \ + "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/${SSM_AGENT_VERSION}/debian_${TARGETARCH}/amazon-ssm-agent.deb" \ + && mkdir /tmp/ssm \ + && dpkg-deb -x /tmp/ssm.deb /tmp/ssm \ + && cp /tmp/ssm/usr/bin/* /usr/bin/ \ + && cp -r /tmp/ssm/etc/amazon /etc/ \ + && cp /etc/amazon/ssm/seelog.xml.template /etc/amazon/ssm/seelog.xml \ + && rm -rf /tmp/ssm /tmp/ssm.deb + +# ECS agent, run directly instead of through ecs-init. +COPY --from=ecs-agent /agent /usr/local/bin/ecs-agent +COPY --from=ecs-agent /amazon-ecs-cni-plugins /amazon-ecs-cni-plugins +COPY --from=ecs-agent /images /images + +COPY main.sh /opt/sg-runner/main.sh +COPY k8s/runner.sh /opt/sg-runner/runner.sh + +ENTRYPOINT ["/opt/sg-runner/runner.sh"] diff --git a/k8s/Makefile b/k8s/Makefile new file mode 100644 index 0000000..07805c0 --- /dev/null +++ b/k8s/Makefile @@ -0,0 +1,115 @@ +# sg-runner Kubernetes prototype. Run from k8s/. +# Every kubectl call is pinned to CONTEXT so this never touches a real cluster +# by accident. + +CONTEXT ?= orbstack +IMAGE ?= sg-runner-k8s:dev +NS := sg-runner +# Release name = chart name, so the StatefulSet (and its pods and PVCs) is +# named sg-runner too. +RELEASE := sg-runner +STS := $(RELEASE) +POD := $(STS)-0 +CHART := chart/sg-runner +SECRET := sg-runner-credentials +KUBECTL := kubectl --context $(CONTEXT) -n $(NS) +HELM := helm --kube-context $(CONTEXT) -n $(NS) + +# build-multiarch: builds every platform and keeps only the build cache unless +# PUSH=1, which pushes $(REGISTRY)/$(IMAGE) (REGISTRY required). The default +# docker driver can't build multi-platform without the containerd image store, +# so it runs on its own docker-container builder, created on first use and +# never made the default. +PLATFORMS ?= linux/amd64,linux/arm64 +BUILDER ?= sg-runner-multiarch +PUSH ?= 0 +REGISTRY ?= + +.DEFAULT_GOAL := help + +.PHONY: help build build-multiarch secret up status logs down nuke + +help: ## List the available targets + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "} {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}' + +build: ## Build the runner image (OrbStack k8s uses local images directly) + docker build -f Dockerfile -t $(IMAGE) .. + +build-multiarch: ## Build amd64 + arm64 (PLATFORMS); cache only, or PUSH=1 REGISTRY=... to push +ifeq ($(PUSH),1) + @test -n "$(REGISTRY)" || { echo "PUSH=1 needs REGISTRY=."; exit 1; } +endif + @docker buildx inspect $(BUILDER) >/dev/null 2>&1 \ + || docker buildx create --name $(BUILDER) --driver docker-container +ifeq ($(PUSH),1) + docker buildx build --builder $(BUILDER) -f Dockerfile --platform $(PLATFORMS) \ + -t $(REGISTRY)/$(IMAGE) --push .. +else + docker buildx build --builder $(BUILDER) -f Dockerfile --platform $(PLATFORMS) \ + --output type=cacheonly .. +endif + +# The scaler (Go, k8s/scaler). SCALER_PLATFORMS builds a multi-arch image instead +# (buildx, e.g. SCALER_PLATFORMS=linux/amd64,linux/arm64 with --push in SCALER_BUILDX_FLAGS). +SCALER_BUILD_IMAGE ?= sg-runner-scaler:dev +SCALER_PLATFORMS ?= +SCALER_BUILDX_FLAGS ?= +.PHONY: build-scaler +build-scaler: ## Build the scaler image (Go) +ifeq ($(SCALER_PLATFORMS),) + docker build -t $(SCALER_BUILD_IMAGE) scaler +else + docker buildx build --platform $(SCALER_PLATFORMS) $(SCALER_BUILDX_FLAGS) -t $(SCALER_BUILD_IMAGE) scaler +endif + +# The namespace must allow privileged pods (dockerd); the chart doesn't create it. +# Only the node token goes in the Secret; `up` passes the runner group (not +# secret) as chart values, which the chart puts in a ConfigMap. +secret: ## Create the namespace and the node token Secret from .env + @test -f .env || { echo "Copy .env.example to .env and fill it in first."; exit 1; } + kubectl --context $(CONTEXT) create namespace $(NS) --dry-run=client -o yaml \ + | kubectl --context $(CONTEXT) apply -f - + kubectl --context $(CONTEXT) label namespace $(NS) --overwrite \ + pod-security.kubernetes.io/enforce=privileged + grep '^SG_NODE_TOKEN=' .env \ + | $(KUBECTL) create secret generic $(SECRET) --from-env-file=/dev/stdin \ + --dry-run=client -o yaml | $(KUBECTL) apply -f - + +# IMAGE and SCALER_BUILD_IMAGE are split into repository and tag for the chart: +# the local builds, not the chart's published defaults. HELM_ARGS go last, e.g. +# make up HELM_ARGS="--set runner.size=medium ..." +HELM_ARGS ?= +up: ## Install or upgrade the chart (registers on first start) + @test -f .env || { echo "Copy .env.example to .env and fill it in first."; exit 1; } + . ./.env && $(HELM) upgrade --install $(RELEASE) $(CHART) \ + --set credentials.existingSecret=$(SECRET) \ + --set-string sg.organization="$$SG_ORGANIZATION" \ + --set-string sg.runnerGroup="$$SG_RUNNER_GROUP" \ + --set-string sg.apiUrl="$${SG_BASE_API:-}" \ + --set image.repository=$(firstword $(subst :, ,$(IMAGE))) \ + --set image.tag=$(lastword $(subst :, ,$(IMAGE))) \ + --set scaler.image.repository=$(firstword $(subst :, ,$(SCALER_BUILD_IMAGE))) \ + --set scaler.image.tag=$(lastword $(subst :, ,$(SCALER_BUILD_IMAGE))) \ + $(HELM_ARGS) + +status: ## Show runner pods and state PVCs (a PVC without a pod is a parked runner) + $(KUBECTL) get statefulset,deployment,pods,pvc -o wide + -$(KUBECTL) exec $(POD) -c ecs-agent -- curl -fsS localhost:51678/v1/metadata + +logs: ## Tail all runner containers + $(KUBECTL) logs $(POD) --all-containers --prefix -f --max-log-requests 10 + +# --no-hooks skips the chart's uninstall cleanup: the runners' PVCs aren't +# part of the release, so every runner parks (drain on the way out) and stays +# registered in SG. With the scaler gone nothing reaps them; `make up` wakes +# them again. +down: ## Uninstall the chart, keep PVCs (runners stay registered, parked) + -$(HELM) uninstall $(RELEASE) --no-hooks + +# The chart's pre-delete hook parks every runner, deregisters each from SG +# and deletes their PVCs; it fails (and the release stays) rather than orphan +# a runner. The timeout covers running workflows. +nuke: ## Uninstall: deregister every runner, delete PVCs, then the namespace + $(HELM) uninstall $(RELEASE) --timeout 30m + -kubectl --context $(CONTEXT) delete namespace $(NS) diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..f391a2d --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,304 @@ +# sg-runner on Kubernetes (prototype) + +> **Status: prototype.** Not supported, not documented for customers. + +Runs Private Runners as pods (Kubernetes ≥ 1.29). Each StatefulSet replica is +one registered runner on a node of its own: it stands in for the VM that +`main.sh register` normally prepares. Run controllers run in the pod's nested +Docker; each workflow step runs as a pod of its own on the runner's node, +created by the `steps` broker sidecar ([`docs/steps-api.md`](docs/steps-api.md)). +A scaler parks idle runners (pod stopped, registration kept, woken again in +seconds) and wakes or adds runners when workflows queue. + +| Component | Where | What | +|---|---|---| +| Runner image | [`Dockerfile`](Dockerfile), [`runner.sh`](runner.sh) | nested `dockerd`, SSM agent, ECS agent; `runner.sh` sources `main.sh` | +| Scaler image | [`scaler/`](scaler) (Go) | sizes the StatefulSet from SG's queue, parks, wakes and reaps runners; also ships the steps broker (`/sg-runner-steps`) | +| Helm chart | [`chart/sg-runner`](chart/sg-runner/README.md) | runner StatefulSet, scaler, credentials, optional NetworkPolicy and PDB | + +## How it maps to a VM + +| VM (`main.sh` + `ecs-anywhere-install.sh`) | Pod | +|--------------------------------------------|----------------------------------------------------------| +| `systemd` + `dockerd` | `dockerd` container (privileged, nested Docker, run controllers only; guarded by the broker's authorization plugin) | +| workflow step containers | step pods on the runner's node (`steps` sidecar) | +| `amazon-ssm-agent` service | `ssm-agent` container | +| `ecs-init` → `ecs-agent` container | `ecs-agent` container, env loaded from `ecs.config` | +| `register` (API + SSM activation) | `register` init container, skipped if state exists | +| `/etc/ecs`, `/var/lib/ecs`, `/var/lib/amazon/ssm` | `state` PVC: runner identity survives pod restarts | +| `/root/.aws/credentials` (SSM → agent) | in-memory `emptyDir` shared by both agents | +| `configure_local_network` (iptables/sysctl) | same function, run in the pod's network namespace | + +All runner containers use one image (`Dockerfile`) and `runner.sh`, which +sources `main.sh` for the API, config and network helpers. + +## Usage (OrbStack Kubernetes) + +```sh +cd k8s +cp .env.example .env # fill in a throwaway runner group's credentials +make build # runner image sg-runner-k8s:dev +make build-scaler # scaler image sg-runner-scaler:dev +make secret # namespace (privileged PSA) + credentials Secret +make up # helm upgrade --install chart/sg-runner; first start registers the runner +make logs # follow all containers +make status # pod state + ECS agent metadata + +kubectl -n sg-runner scale sts sg-runner --replicas=N # park / wake runners + +make down # uninstall --no-hooks, keep PVCs (runners stay registered, parked) +make nuke # helm uninstall (deregisters every runner, deletes PVCs), then the namespace +``` + +`helm uninstall` cleans up through the chart's pre-delete hook (see the +[chart README](chart/sg-runner/README.md#uninstall)). If a runner can't be +deregistered, the hook fails and keeps that runner's PVCs, and `make nuke` +stops before deleting the namespace: deleting the PVC without deregistering +would leave a dead runner in the runner group. + +Outside OrbStack, install the published chart (GHCR, beta; its defaults use +the images of the same release). Only the node token is a Secret; the runner +group is plain values. Label the namespace where Pod Security is enforced +(runner pods are privileged): + +```sh +kubectl create namespace sg-runner +kubectl label namespace sg-runner pod-security.kubernetes.io/enforce=privileged +kubectl -n sg-runner create secret generic sg-runner-credentials \ + --from-literal=SG_NODE_TOKEN=... +helm upgrade --install sg-runner oci://ghcr.io/stackguardian/charts/sg-runner --version \ + -n sg-runner \ + --set sg.organization= --set sg.runnerGroup= \ + --set credentials.existingSecret=sg-runner-credentials \ + --set runner.size=medium +``` + +The chart in this repo (`k8s/chart/sg-runner`) installs the same way; its +default images carry the `dev` tag, so point `image.*` and `scaler.image.*` at +images you built (`make up` uses the local `:dev` builds). + +### Choosing the runner size + +Every workflow reserves 0.5 vCPU / 512 MiB for its run controller plus its own +resource setting (default 0.5 vCPU / 1 GiB), so a default workflow takes +1 vCPU / 1.5 GiB. `runner.size` picks what the runner advertises (step pods +and run controllers together; the node needs it free) and the `dockerd` +container's resources, which only hold the run controllers: + +| `runner.size` | Advertised | Default workflows at once | +|---|---|---| +| `small` (default) | 2 vCPU / 4Gi | 2 | +| `medium` | 4 vCPU / 8Gi | 4 | +| `large` | 8 vCPU / 16Gi | 8 | +| `custom` | `runner.advertise` + `runner.controllerResources` (required) | | + +The runner must be larger than the largest workflow (its setting plus +0.5 vCPU / 512 MiB), or that workflow stays queued; the scaler then warns +(`StuckQueue`) instead of waking every runner. + +Image versions are pinned as build args in `Dockerfile` (`ECS_AGENT_VERSION`, +`SSM_AGENT_VERSION`, `DOCKER_VERSION`). `make build-multiarch` builds +`linux/amd64` and `linux/arm64` on a dedicated buildx builder and keeps only +the build cache; `PUSH=1 REGISTRY=` pushes the result. + +## Workflow steps + +Steps are pods (`app.kubernetes.io/component=step`), so `kubectl logs` shows +their output; finished ones stay for `runner.steps.finishedTTL`, labelled with +their outcome. Details in the +[chart README](chart/sg-runner/README.md#how-a-runner-runs-workflows). + +## Docker cleanup + +On a VM, `main.sh prune` runs from cron. In the pod, the `dockerd` container +prunes the nested Docker itself: once at start (a runner may park before a +full interval) and then every `SG_PRUNE_INTERVAL_SECONDS`. + +| Env (on `dockerd`) | Default | | +|---|---|---| +| `SG_PRUNE_INTERVAL_SECONDS` | `14400` (4h, as the VM cron) | `0` turns cleanup off | +| `SG_PRUNE_UNTIL` | `24h` | minimum age of stopped containers, images and build cache it removes | +| `SG_PRUNE_IMAGES_ABOVE_PCT` | `70` | unused images go only once the Docker volume is this full (`0`: every run) | + +Every sweep that can remove a container is age-guarded: an unguarded prune +deletes containers the ECS agent created but hasn't started yet. Volumes are +pruned on their own (`docker volume prune`). Unused images are the wake-up +cache, so they're kept until the volume fills up; Docker filters images by +creation time, not last use, so a prune under pressure removes every unused +image older than `SG_PRUNE_UNTIL`. + +## Resource sizing + +The ECS agent sizes the runner from the node, not the pod: CPU is +`runtime.NumCPU() x 1024` (CPU affinity, not the cgroup quota) and memory is +`MemTotal` from `/proc/meminfo`. Workflows run inside the `dockerd` container, +so its limits are what the runner should advertise. `runner.sh ecs-agent` +reads them through the downward API and: + +- sets `ECS_RESERVED_MEMORY` to node memory minus (limit - 256 MiB for dockerd, + `SG_DOCKERD_OVERHEAD_MIB`). The agent parses it as `uint16`; a reservation + above 65535 MiB would silently become 0, so the agent refuses to start instead. +- pins the agent to `limits.cpu` CPUs (rounded up) with `taskset`. + +Without limits on `dockerd`, the runner advertises the whole node. + +## Parking: stopping and scaling down + +Scaling down doesn't deregister a runner, it **parks** it: the pod stops, the +registration and Docker image cache stay on its PVCs, and scaling back up +wakes the same runner without registering again. + +| Measured on QA | Time to ECS `ACTIVE` + connected | +|---|---| +| Wake a parked runner (`kubectl scale` up) | **8s**, same container instance, image cache kept | +| Fresh runner (new ordinal, full registration) | 20s, plus image pulls on its first workflow | +| Park a runner (`kubectl scale` down, idle) | 14s | + +Every shutdown (scale-down, restart, rollout, eviction) parks the same way. +The `ecs-agent` container's `preStop` hook (`runner.sh drain`): + +1. sets the runner `DRAINING` through SG's `runner_status` API (the same call + `sg-runner-autoscaler` uses), so no new workflows are placed on it, and + leaves a marker on the state PVC; +2. waits for running workflows, up to `terminationGracePeriodSeconds` (1h). + `dockerd` and `ssm-agent` wait for the hook (`wait-drained`), so workflows + keep their Docker daemon and the agent keeps rotating credentials. + +On wake the runner sets itself `ACTIVE` again once the agent has reconnected, +only if the marker is there. A runner that was already `DRAINING` when it +parked, drained on purpose from the SG UI or by an autoscaler, stays drained. + +A parked runner shows in SG as registered but disconnected and `DRAINING`; +it still counts against the runner group's `MaxNumberOfRunners`. Pods are +managed in parallel (`podManagementPolicy: Parallel`), so several runners park +or wake at once. + +Status changes use the runner group's node token, the same credential the +runner registers with; no extra key is needed. The runner never calls ECS +itself: a customer's runner has no ECS permissions. + +Runners are deregistered when they're reaped (parked longer than +`SCALER_PARK_TTL`) and on `helm uninstall`, which deregisters every runner, +awake or parked, from its state PVC and deletes the PVCs. + +## Autoscaling + +The chart's scaler Deployment ([`chart/sg-runner`](chart/sg-runner/README.md)) +runs the scaler ([`scaler/`](scaler), Go; image built with `make build-scaler`), +a loop that owns the runner StatefulSet's `replicas`. Every `SCALER_INTERVAL` +(15s) it reads the runner +group from SG with the node token (`?getActiveWorkflows=true`) and: + +| Rule | When | Action | +|---|---|---| +| Scale out | workflows queued and no awake runner fully idle | +1 runner: wakes a parked one, or registers a new one | +| Headroom (optional) | less than `SCALER_HEADROOM_PCT` of awake runners' CPU or memory free (0 = off, the default) | +1 runner, kept as a warm spare | +| Scale in | nothing queued, highest awake runner idle for `SCALER_IDLE_SECONDS` (60), the rest still keeps the headroom | park it (-1) | +| Stuck queue | workflows queued while an awake runner sits fully idle for `SCALER_STUCK_SECONDS` (180) | warn, ignore the queue (it needs more than a runner pod offers) | +| Reap | runner parked longer than `SCALER_PARK_TTL` (12h) | Job deregisters it from its PVC, then its PVCs are deleted | + +Replicas stay within `SCALER_MIN` (≥ 1: SG pauses workflows without an +`ACTIVE` runner) and the lower of `SCALER_MAX` and the runner group's +`MaxNumberOfRunners`. Runners map to ordinals by hostname (the pod's FQDN). +While an awake runner hasn't connected yet, the scaler waits instead of +deciding again, for up to `SCALER_SETTLE_TIMEOUT` (300s); a runner that still +isn't connected by then (failed registration, crash loop) is left out of the +decision, reported and set `DRAINING`, so one broken pod can't freeze scaling. + +A parked runner that SG still shows `ACTIVE` (its pod went away without the +`preStop` drain: node loss, force delete) is set `DRAINING` too. Drains the +scaler does are marked on the runner's state PVC +(`sg-runner.stackguardian.io/drained-by-scaler`), and the scaler sets the +runner `ACTIVE` again once it's awake and connected. Drains from anyone else +(SG UI, an autoscaler) leave no mark and stick. + +Two replicas share a leader Lease (`-scaler`; +`SCALER_LEADER_ELECT=false` turns it off). The idle, stuck and settle timers +are kept as annotations on the StatefulSet, so a restart or a new leader +carries on where the last one stopped. On SG or Kubernetes API errors a loop +decides nothing and the next one waits longer (doubling, up to 5 minutes). +Decisions and warnings (`StuckQueue`, `RunnerNotConnected`, `SGUnreachable`, +`ReapFailed`, ...) are Events on the StatefulSet; Prometheus metrics +(`sg_runner_scaler_*`), `/healthz` and `/readyz` are served on port 9090. + +Its Role covers only this StatefulSet, its scale and annotations, PVCs, reap +Jobs, Events and its Lease; runner pods have no Kubernetes API access. +Reap Job pods take extra labels, pull secrets, node selector and tolerations +from `SCALER_REAP_POD_LABELS`, `SCALER_REAP_IMAGE_PULL_SECRETS`, +`SCALER_REAP_NODE_SELECTOR` and `SCALER_REAP_TOLERATIONS`; the chart sets them +from the runner's values. + +The scaler reads the SG credentials once, at start: restart it +(`kubectl rollout restart deploy/-sg-runner-scaler`) after rotating +the node token. + +## Development + +```sh +make lint test # repo root: shellcheck + bats (main.sh, k8s/runner.sh) +(cd k8s/scaler && go vet ./... && go test ./...) +helm lint --strict k8s/chart/sg-runner --set credentials.existingSecret=x +``` + +## Releasing + +`.github/workflows/k8s-release.yml` publishes to GHCR: push a tag +`k8s/vX.Y.Z` (SemVer, optionally `-pre`) on `main` after the merge. After the +full k8s CI (`k8s.yml`) passes, it pushes the runner and scaler images +(`ghcr.io/stackguardian/sg-runner-k8s`, `sg-runner-scaler`, linux/amd64 and +arm64, with build provenance) and the chart +(`oci://ghcr.io/stackguardian/charts/sg-runner`), all as version `X.Y.Z` (the +chart's appVersion too), and creates the GitHub release with the install +command. An existing image tag or chart version fails the run instead of +being overwritten. A dry run (`workflow_dispatch` with a `version`) builds +and packages everything and pushes nothing. After the first publish, make +the three GHCR packages public once in the org's package settings. + +## Known gaps + +- **Privileged pod.** Needed for nested Docker; restricted PSA clusters reject it. + Non-privileged works with `SYS_ADMIN`, `NET_ADMIN`, `SYS_RESOURCE`, Unconfined + seccomp and read-write remounts of `/sys/fs/cgroup` and `/proc/sys` (verified on + OrbStack), but that's no real isolation gain. See [`docs/TODO.md`](../docs/TODO.md). +- **Network isolation is opt-in.** Workflow containers can resolve cluster DNS + and reach the Kubernetes API (verified). The chart's NetworkPolicy + (`networkPolicy.enabled`, with the cluster's CIDRs) blocks cluster ranges and + the metadata endpoint for runner, step and reap pods; it needs a CNI that + enforces NetworkPolicy and isn't tested on one yet. The steps broker needs + the Kubernetes API: allow its endpoint in `networkPolicy.extraEgress`. +- **Workspaces are scratch.** `/var/tmp/workspace` is a PVC per runner pod + (`-workspace`, generic ephemeral, `runner.steps.workspace.size`, + default 20Gi), deleted with the pod: a run doesn't survive a pod restart. +- **Task cgroup limits off** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`): the agent + can't create task cgroups from its container. Run controllers share the + `dockerd` container's limits; steps get their own pod requests and limits. +- **No proxy support** yet (`--http-proxy` path not wired). +- **Workflows bigger than a runner pod never run.** A workflow reserves + 0.5 vCPU / 512 MiB plus its own resource setting; if that exceeds the + `dockerd` container's limits it stays queued. The scaler detects it and warns + instead of waking every runner, but the pod size must cover the largest + workflow. +- **Restarting a single runner pauses new workflows** for the ~10s it takes: + SG pauses workflows submitted while no runner is `ACTIVE`, and a StatefulSet + stops the old pod before starting the new one. `podDisruptionBudget.enabled` + keeps node drains from evicting the last runner; for rollouts use + `scaler.min=2` where that matters. Unverified: whether SG resumes such + workflows on its own once a runner reconnects. +- **Unverified: long parking.** Waking was tested after seconds. Whether SSM + and ECS reconnect after hours or days parked isn't tested yet. +- **Hard kills.** A pod killed without its `preStop` hook (node loss, force + delete) parks as `ACTIVE` but disconnected; the scaler sets it `DRAINING` and + reactivates it on wake. Unverified against real SG/ECS (tested with a fake + SG). If someone also drains such a runner from the SG UI while the scaler's + mark is on it, the scaler reactivates it on wake. +- **SG credentials in the pod.** The credentials Secret is mounted + (read-only) into `ecs-agent` for the status changes, and into the scaler. +- **Pods end in `Error`.** The containers exit non-zero on SIGTERM after a + clean drain; cosmetic. +- **SSM Run Command** lands in the `ssm-agent` container (it has the Docker + CLI and socket), not on a host. Unverified whether SG relies on it. +- `ecs-init` is bypassed: agent defaults it sets are copied in `runner.sh` and + may drift with new agent releases. +- **Go scaler not yet run against real SG.** Its rules were validated on QA in + the shell version it replaces, and the Go version against a fake SG; the + chart + Go scaler end to end on QA is still to do. diff --git a/k8s/chart/sg-runner/.helmignore b/k8s/chart/sg-runner/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s/chart/sg-runner/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/chart/sg-runner/Chart.yaml b/k8s/chart/sg-runner/Chart.yaml new file mode 100644 index 0000000..d6884c1 --- /dev/null +++ b/k8s/chart/sg-runner/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: sg-runner +description: StackGuardian Private Runner on Kubernetes (prototype). Each StatefulSet replica is one registered runner. +type: application +# Chart version, bumped on every chart change. +version: 0.9.0 +# The prototype image is built locally (`make build`), there is no released +# tag yet. Used as the image tag when image.tag is empty. +appVersion: "dev" +# The steps broker is a native sidecar (init container with restartPolicy +# Always). +kubeVersion: ">=1.29.0-0" +keywords: + - stackguardian + - runner +home: https://github.com/StackGuardian/sg-runner diff --git a/k8s/chart/sg-runner/README.md b/k8s/chart/sg-runner/README.md new file mode 100644 index 0000000..c408b4d --- /dev/null +++ b/k8s/chart/sg-runner/README.md @@ -0,0 +1,289 @@ +# sg-runner Helm chart (prototype) + +> **Status: prototype.** Not supported, not documented for customers. + +Runs StackGuardian Private Runners on Kubernetes (≥ 1.29). Each replica of the +runner StatefulSet is one registered runner on a node of its own, running each +workflow step as a pod ([How a runner runs workflows](#how-a-runner-runs-workflows)); +a scaler Deployment wakes, parks and reaps runners from the runner group's +queue. Design and known gaps: [`k8s/README.md`](../../README.md). + +## Install + +The chart and its images are published to GHCR (beta): the chart as +`oci://ghcr.io/stackguardian/charts/sg-runner`, the images as +`ghcr.io/stackguardian/sg-runner-k8s` and `ghcr.io/stackguardian/sg-runner-scaler`, +all with the same version (the chart's defaults point at its own release's +images). Versions are on the repository's releases (`k8s/v*` tags). + +Only the runner group's node token is secret. The organization, runner group +and API URL are plain values (`sg.*`); the chart keeps them in a ConfigMap. + +```sh +kubectl create namespace sg-runner +kubectl -n sg-runner create secret generic sg-runner-credentials \ + --from-literal=SG_NODE_TOKEN=... +helm -n sg-runner upgrade --install sg-runner oci://ghcr.io/stackguardian/charts/sg-runner --version \ + --set sg.organization= --set sg.runnerGroup= \ + --set credentials.existingSecret=sg-runner-credentials --set runner.size=medium +``` + +From a checkout, `./k8s/chart/sg-runner` works the same; its images default to +the `dev` tag, so set `image.*` and `scaler.image.*` to images you built +(`make up` does). + +Or let the chart create the Secret too (`credentials.create=true`, +`credentials.nodeToken=...`), with `--create-namespace`: one `helm install`. +The token then also lives in Helm's release record; prefer `existingSecret` +(e.g. filled by External Secrets) outside testing. + +Runner pods are privileged (nested Docker). On clusters that enforce Pod +Security, label the namespace first; the chart doesn't create or label it: + +```sh +kubectl label namespace sg-runner pod-security.kubernetes.io/enforce=privileged +``` + +From `k8s/`, `make secret up` does the same against OrbStack. + +Release `sg-runner` names the StatefulSet `sg-runner`, so pods `sg-runner-N` and +PVCs `state-sg-runner-N` / `docker-sg-runner-N`. Other release names get +`-sg-runner`. Don't rename an existing install: the runners +registered from the old PVCs would be orphaned. + +## Uninstall + +```sh +helm -n sg-runner uninstall sg-runner --timeout 30m +kubectl delete namespace sg-runner # if nothing else lives there +``` + +`helm uninstall` removes the runners from StackGuardian too. A pre-delete hook +Job (`-uninstall`, the scaler image in `teardown` mode): + +1. stops the scaler, +2. parks every runner: each sets itself `DRAINING` and waits for its running + workflows, +3. deregisters every runner, awake or parked, with a reap Job against its + state PVC (`runner.sh reap`), +4. deletes the runners' PVCs, which `volumeClaimTemplates` keep out of the + release. + +Running workflows finish first, so give `--timeout` enough for them (Helm's +default is 5m; the Job itself gives up after `uninstall.activeDeadlineSeconds`, +by default `runner.terminationGracePeriodSeconds` + 15 minutes). If Helm times +out, the Job keeps going; running `helm uninstall` again restarts it, and +runners already deregistered are skipped. + +If a runner can't be deregistered, the hook fails and keeps that runner's +PVCs, so nothing is orphaned; the release stays. Read +`kubectl -n sg-runner logs job/sg-runner-reap-`, fix the cause, and +uninstall again. + +To keep the runners registered instead (parked, PVCs kept, e.g. to reinstall +later): `helm uninstall --no-hooks`, or `uninstall.deregister=false`. + +## How a runner runs workflows + +Each runner pod takes a node of its own. Its run controllers (the ECS tasks) +run in the pod's nested Docker (`dockerd`); each workflow step runs as a pod +of its own on the runner's node. A `steps` sidecar in the runner pod (the +broker, `/sg-runner-steps` in the scaler image) creates the step pods for the +controllers over a unix socket ([`steps-api.md`](../../docs/steps-api.md)). + +- **One runner per node.** A required pod anti-affinity on + `kubernetes.io/hostname` is appended to `runner.affinity`'s own + `podAntiAffinity` terms (everything else in `runner.affinity` is kept). Step + pods are pinned to their runner's node and mount its workspace. +- **The broker is a native sidecar** (an init container with + `restartPolicy: Always`, hence Kubernetes ≥ 1.29): it starts before + `dockerd`, once its health check passes, and stops after every other + container, so it outlives a drain. +- **dockerd is guarded** by the broker: it runs with + `--authorization-plugin=sg-steps-guard`, the plugin socket on a shared + `docker-plugins` volume. The guard refuses step containers in dockerd + (`POST /containers/create` with a `wfrunId` label), so steps can only run as + pods. It fails closed: `dockerd` doesn't start without the socket (it waits + up to 60s, then fails), and while the guard isn't answering dockerd refuses + every API call. So a crash-looping broker blocks the runner. +- **Workspace** is a PVC per pod (`-workspace`, a generic ephemeral + volume created and deleted with the pod, `runner.steps.workspace`), mounted + by the run controllers and the step pods. +- **API access.** The runner pod gets a ServiceAccount (``) with a + Role for pods (create, get, list, watch, patch, delete), `pods/log` (get) + and Secrets (create, delete only: never read back). Pod-level token + automount stays off; only the `steps` container mounts a token. +- **Logs**: steps are pods, so + `kubectl logs -l app.kubernetes.io/component=step` shows them. Finished + step pods stay for `runner.steps.finishedTTL` (1h), labelled with their + outcome; their Secrets (env, registry credentials) are deleted as soon as + they finish: + `kubectl get pods -l app.kubernetes.io/component=step -L sg-runner.stackguardian.io/outcome`. +- **Exit codes 11 and 12** (approval required, completed early) are outcomes, + not failures: the broker runs each step through a small wrapper (an init + container from the broker image copies it in) so those pods end + `Completed`. To wrap a step it reads the image's ENTRYPOINT/CMD from its + registry (the image config only, no layers). If that fails, the step runs + unwrapped, and exits 11/12 then show as `Error` (the run's result is the + same; annotation `sg-runner.stackguardian.io/not-wrapped-reason` says why). + So the `steps` container calls registries over HTTPS, which the + NetworkPolicy's egress allows like any other external destination. + Wrapped step pods pull the broker image with `imagePullSecrets` + (`BROKER_IMAGE_PULL_SECRETS`). +- **NetworkPolicy** covers step pods too (`app.kubernetes.io/component: + step`). It also blocks the Kubernetes API, which the broker needs: allow the + API server's endpoint (`kubectl get endpoints kubernetes`) with + `networkPolicy.extraEgress`. NetworkPolicy can't tell containers apart, so + that opens the API endpoint to the whole runner pod (the run controllers + too, without a token). + +## Runner size + +`runner.size` sets what the runner advertises to SG for workflows (step pods +and run controllers together), and the resources of the `dockerd` container, +which only holds the run controllers. Nothing is kept back for dockerd. + +| Size | Advertised | Controllers (dockerd) requests | limits | +|---|---|---|---| +| `small` | 2 CPU / 4Gi | 500m / 1Gi | 2 / 2Gi | +| `medium` | 4 CPU / 8Gi | 1 / 2Gi | 4 / 4Gi | +| `large` | 8 CPU / 16Gi | 2 / 4Gi | 8 / 8Gi | +| `custom` | `runner.advertise` (required) | `runner.controllerResources` (limits required) | | + +A workflow reserves 0.5 vCPU / 512 MiB for its run controller plus its own +setting (default 0.5 vCPU / 1 GiB): 1 vCPU / 1.5 GiB by default. The runner +must be at least the largest workflow + 0.5 vCPU / 512 MiB, else that workflow +stays queued (the scaler warns). Size the controllers' memory limit at ~512Mi +per concurrent workflow run. + +The node needs the advertised capacity free for step pods, plus the runner +pod's own requests. The agent can't advertise more than the node has, and +node memory minus the advertised memory must stay under 64 GiB (the agent's +reservation is a uint16 of MiB; the runner refuses to start otherwise). + +**Resizing.** ECS keeps a runner's CPU and memory from its first +registration, so a new size alone changes nothing. With +`runner.reregisterOnResize` (default), a runner pod that starts with another +size than its registration (`runner.size` or `runner.advertise` changed) +deregisters and registers again, if nothing runs on it; it gets a new runner +ID in SG. With tasks running or pending it keeps the old size and logs a +warning; restart it once idle. If SG can't be read, the old registration +stays. + +## Values + +| Key | Default | Description | +|---|---|---| +| `nameOverride` / `fullnameOverride` | `""` | Rename resources; `fullnameOverride` also renames the StatefulSet (max 52 chars). | +| `image.repository` / `.tag` / `.pullPolicy` | `ghcr.io/stackguardian/sg-runner-k8s` / `""` / `IfNotPresent` | Runner image; also used by reap Jobs. Empty tag = `appVersion` (the release's version). | +| `imagePullSecrets` | `[]` | For runner, scaler and reap Job pods (and native steps' wrapper). Secret names or `{name: ...}` objects, mixed as you like. | +| `sg.organization` / `sg.runnerGroup` | `""` | Required: the runner group to register into. In the `-config` ConfigMap. | +| `sg.apiUrl` | `""` | StackGuardian API; empty = `https://api.app.stackguardian.io/api/v1`. | +| `credentials.existingSecret` | `""` | Secret with key `SG_NODE_TOKEN` (the runner group's node token); other keys are ignored. | +| `credentials.create` / `nodeToken` | `false` / `""` | Create that Secret from `nodeToken`. Exactly one of this or `existingSecret`. | +| `runner.replicas` | `1` | Only with `scaler.enabled=false`. | +| `runner.size` | `small` | `small`, `medium`, `large` or `custom`: what the runner advertises and the controllers' room ([Runner size](#runner-size)). | +| `runner.advertise.cpu` / `.memory` | empty | `size=custom` only, required: what the runner advertises to SG. Whole CPUs; `Mi` or `Gi`. | +| `runner.controllerResources` | `{}` | `size=custom` only, limits required: dockerd resources (run controllers only); ~512Mi per concurrent run. | +| `runner.storage.state.size` / `.storageClassName` | `1Gi` / `""` | Runner identity PVC. Empty class = cluster default. | +| `runner.storage.docker.size` / `.storageClassName` | `20Gi` / `""` | Docker data root PVC (image cache). | +| `runner.dockerd.securityContext` | `{privileged: true}` | Nested Docker needs privileged for now. | +| `runner.dockerd.extraEnv` | `[]` | Extra env vars for the `dockerd` container (advanced/testing, e.g. `SG_PRUNE_INTERVAL_SECONDS=0` turns off the nested Docker prune loop). | +| `runner.ecsAgent.extraEnv` | `[]` | Extra env vars for the `ecs-agent` container (advanced/testing, e.g. `ECS_IMAGE_PULL_BEHAVIOR=prefer-cached`). | +| `runner.steps.workspace.size` / `.storageClassName` / `.accessModes` | `20Gi` / `""` / `[ReadWriteOnce]` | Workspace PVC per pod (`-workspace`) at `/var/tmp/workspace` on dockerd, mounted by step pods. Empty class = cluster default. | +| `runner.steps.schedulingTimeout` / `startTimeout` / `finishedTTL` | `5m` / `30m` / `1h` | Step pod fails if unschedulable / still pending this long. Finished step pods are kept for `finishedTTL`, then deleted; their Secrets go when they finish. | +| `runner.steps.step.nodeSelector` / `.tolerations` | empty | Step pod scheduling; empty = `runner.nodeSelector` / `runner.tolerations`. | +| `runner.steps.step.imagePullPolicy` | `""` | `STEP_IMAGE_PULL_POLICY`. Empty = automatic (`Always` for `latest` or no tag, else `IfNotPresent`); `IfNotPresent` / `Never` for air-gapped clusters or preloaded images. | +| `runner.steps.broker.image.repository` / `.tag` / `.pullPolicy` | `""` / `""` / `IfNotPresent` | `steps` sidecar image, also the step pods' wrapper init container (`BROKER_IMAGE`, `BROKER_IMAGE_PULL_POLICY`); empty repository = the scaler image. | +| `runner.steps.broker.resources` | 10m / 32Mi, limit 256Mi | | +| `runner.steps.broker.extraEnv` | `[]` | Extra env vars for the broker. | +| `runner.reregisterOnResize` | `true` | `SG_REREGISTER_ON_RESIZE` on the register init container: register an idle runner again when its registered size differs from what it now advertises ([Runner size](#runner-size)). | +| `runner.terminationGracePeriodSeconds` | `3600` | Max wait for running workflows when a runner stops. | +| `runner.podAnnotations` / `podLabels` / `nodeSelector` / `tolerations` / `affinity` | empty | Runner pod scheduling and metadata; `nodeSelector` and `tolerations` also apply to reap Jobs. | +| `scaler.enabled` | `true` | Scaler Deployment + RBAC. When on, the chart leaves the StatefulSet's `replicas` to it. | +| `scaler.image.repository` / `.tag` / `.pullPolicy` | `ghcr.io/stackguardian/sg-runner-scaler` / `""` / `IfNotPresent` | Scaler image (Go, distroless, non-root); also the steps broker's. Empty tag = `appVersion`. | +| `scaler.command` | `[]` | Empty = image entrypoint. | +| `scaler.replicas` | `1` | 2 is safe with leader election; > 1 without it fails rendering. | +| `scaler.leaderElection.enabled` | `true` | `SCALER_LEADER_ELECT`; lease `-scaler`. Also switches the rollout to `RollingUpdate`. | +| `scaler.min` | `1` | `SCALER_MIN`, ≥ 1 (SG pauses workflows without an ACTIVE runner). | +| `scaler.max` | `10` | `SCALER_MAX`, also capped by `MaxNumberOfRunners`. | +| `scaler.headroomPct` | `0` | `SCALER_HEADROOM_PCT`: capacity to keep free, 0 = off. | +| `scaler.idleSeconds` | `60` | `SCALER_IDLE_SECONDS`: idle time before parking. | +| `scaler.parkTTL` | `43200` | `SCALER_PARK_TTL`: parked time before reaping (deregister + delete PVCs). | +| `scaler.stuckSeconds` | `180` | `SCALER_STUCK_SECONDS`: queue + idle runner for this long = workflow too big. | +| `scaler.settleTimeout` | `300` | `SCALER_SETTLE_TIMEOUT`: wait for a runner to connect. | +| `scaler.interval` | `15` | `SCALER_INTERVAL`: seconds between decisions. | +| `scaler.nudge.enabled` / `.image` | `true` / `registry.k8s.io/pause:3.10` | `SCALER_NUDGE`: placeholder pods for runners the cluster autoscaler won't add a node for ([Notes](#notes)). | +| `scaler.nudge.cooldownSeconds` | `180` | `SCALER_NUDGE_COOLDOWN`: wait before asking again for a still-Pending runner; must outlast a node's boot. | +| `scaler.extraEnv` | `[]` | Extra env vars for the scaler. | +| `scaler.metrics.port` | `9090` | `SCALER_METRICS_ADDR=:`, container port `metrics`, always on: `/metrics`, `/healthz` (liveness), `/readyz` (readiness). | +| `scaler.metrics.enabled` | `false` | Adds a metrics Service. | +| `scaler.metrics.serviceMonitor.enabled` / `.interval` / `.labels` | `false` / `30s` / `{}` | ServiceMonitor, only if `monitoring.coreos.com/v1` is served. | +| `scaler.resources` | 10m / 32Mi, limit 128Mi | | +| `scaler.podSecurityContext` | non-root 65532, `fsGroup` 65532, seccomp `RuntimeDefault` | `fsGroup` + file mode `0440` let the non-root scaler read the token and runner group. | +| `scaler.securityContext` | read-only root FS, no privilege escalation, drop `ALL` | | +| `scaler.podAnnotations` / `podLabels` / scheduling | empty | | +| `networkPolicy.enabled` | `false` | Egress isolation for runner pods; needs an enforcing CNI. | +| `networkPolicy.clusterCIDRs` | `[]` | Pod, service, node CIDRs (+ API endpoint). IPv4 ones required when enabled; IPv6 ones open IPv6 egress except them. | +| `networkPolicy.blockMetadata` | `true` | Deny `169.254.169.254/32`. | +| `networkPolicy.blockMetadataIPv6` | `false` | Deny `fd00:ec2::254/128`; only matters with IPv6 egress open. | +| `networkPolicy.allowClusterDNS` / `dnsSelector` | `true` / kube-system `k8s-app: kube-dns` | Allow DNS (53/UDP+TCP) to the cluster resolver. | +| `networkPolicy.extraEgress` | `[]` | Extra egress rules, as-is. | +| `podDisruptionBudget.enabled` / `minAvailable` | `false` / `1` | Keep drains from evicting the last runner; with one awake runner it blocks drains, so pair with `scaler.min: 2`. | +| `uninstall.deregister` | `true` | `helm uninstall` deregisters every runner and deletes their PVCs ([Uninstall](#uninstall)). | +| `uninstall.activeDeadlineSeconds` | empty | Upper bound for that cleanup; empty = `runner.terminationGracePeriodSeconds` + 900. | +| `uninstall.resources` | 10m / 32Mi | The cleanup Job's resources. | + +## Notes + +- **Replicas and the scaler.** With the scaler on, the chart leaves the + StatefulSet's `replicas` out and Helm upgrades keep the live count. Turning + the scaler on for an install that had it off removes the field Helm managed, + which resets the StatefulSet to 1 replica: every other runner parks until the + scaler wakes them for queued work. +- **Cluster autoscaler and runner volumes.** Some cluster autoscalers (GKE's, + at least) never add a node for a pod with a persistent disk: a runner that + needs a new node, new or waking, stays Pending with `NotTriggerScaleUp … + didn't match Pod's node affinity/selector`, while the same pod without + volumes scales up. With `scaler.nudge.enabled` (default), the scaler then + starts a placeholder Job (`-nudge-`, a `pause` container + with the runner's requests, node selector, tolerations and affinity, no + volumes). As soon as the autoscaler accepts it (`TriggeredScaleUp`), the + scaler deletes it, and the runner takes the new node when it joins. A runner + still Pending after `scaler.nudge.cooldownSeconds` gets another placeholder. + Events `NudgingAutoscaler`, `NodeRequested`, `ScaleUpBlocked` (the + autoscaler refused the placeholder too, e.g. the node pool is at its + maximum, or repeated attempts didn't place the runner); metric + `sg_runner_scaler_autoscaler_nudges_total`. It only acts after the + autoscaler's refusal, so autoscalers that scale for runners themselves are + left alone. Needs read access to pods and events in the namespace. +- **Storage values are fixed after install**: `volumeClaimTemplates` are + immutable on a StatefulSet. +- **NetworkPolicy** covers runner pods (run controllers and agents alike: + nested traffic leaves through the pod IP), step pods + (`app.kubernetes.io/component: step`) and reap Job pods, which the scaler + labels `app.kubernetes.io/component: reaper` (`SCALER_REAP_POD_LABELS`). +- **IPv6 gap.** The policy is IPv4: it allows no IPv6 egress unless IPv6 + CIDRs are listed in `clusterCIDRs` (dual stack), which opens `::/0` except + those and, with `blockMetadataIPv6`, the AWS IPv6 metadata endpoint. Other + clouds' IPv6 metadata addresses aren't covered. + +## Migrating from the raw manifests + +The old `k8s/manifests/` used `app: sg-runner` selectors. A StatefulSet's +selector is immutable and Helm won't adopt objects it didn't create, so the +chart can't take over in place. The PVCs carry the runner identities and keep +their names (`state-sg-runner-N`, `docker-sg-runner-N`) as long as the release +is named `sg-runner`, so: + +1. Note the current replica count: `kubectl -n sg-runner get sts sg-runner`. +2. Delete the old scaler, then the StatefulSet (runners drain and park; PVCs + stay): `kubectl -n sg-runner delete deploy/sg-runner-scaler sts/sg-runner`, + and the old RBAC (`sa`, `role`, `rolebinding` named `sg-runner-scaler`). +3. Install the chart as release `sg-runner` in the same namespace with + `credentials.existingSecret=sg-runner-credentials` plus `sg.organization` + and `sg.runnerGroup` (`make up`). Only the Secret's `SG_NODE_TOKEN` key is + used now. +4. The StatefulSet starts at 1 replica; the scaler wakes the other parked + runners as work queues (or reaps them after `parkTTL`). Without the + scaler, set `runner.replicas` to the noted count. diff --git a/k8s/chart/sg-runner/templates/NOTES.txt b/k8s/chart/sg-runner/templates/NOTES.txt new file mode 100644 index 0000000..bdca857 --- /dev/null +++ b/k8s/chart/sg-runner/templates/NOTES.txt @@ -0,0 +1,60 @@ +{{- $sts := include "sg-runner.runnerName" . -}} +{{- $ns := .Release.Namespace -}} +sg-runner {{ .Chart.Version }} installed as StatefulSet {{ $ns }}/{{ $sts }} (size: {{ .Values.runner.size }}). + +Runner group: {{ .Values.sg.organization }}/{{ .Values.sg.runnerGroup }}. Each replica registers one runner on its +first start. + +Runner pods are privileged (nested Docker). Where Pod Security is enforced, the +namespace needs: + + kubectl label namespace {{ $ns }} pod-security.kubernetes.io/enforce=privileged + +Runners: + kubectl -n {{ $ns }} get statefulset,pods -l app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=runner + +Parked runners are state PVCs without a pod (still registered in SG, shown as +DRAINING and disconnected): + kubectl -n {{ $ns }} get pvc | grep '^state-{{ $sts }}-' +{{- if .Values.scaler.enabled }} + +The scaler owns the replica count ({{ .Values.scaler.min }}..{{ .Values.scaler.max }}). Watch its decisions: + kubectl -n {{ $ns }} logs -f deploy/{{ include "sg-runner.scalerName" . }} +{{- if .Values.scaler.leaderElection.enabled }} +With several scaler replicas only the lease holder decides: + kubectl -n {{ $ns }} get lease {{ include "sg-runner.runnerName" . }}-scaler +{{- end }} + +Don't `kubectl scale` the StatefulSet by hand; the scaler overrides it. +{{- else }} + +Scaler disabled: the StatefulSet runs {{ .Values.runner.replicas }} replica(s). Scaling down parks runners, +scaling up wakes them: + kubectl -n {{ $ns }} scale statefulset {{ $sts }} --replicas=N +{{- end }} + +{{- if .Values.uninstall.deregister }} +`helm uninstall` deregisters every runner and deletes their PVCs; it waits for +running workflows, so pass a --timeout that covers them: + helm -n {{ $ns }} uninstall {{ .Release.Name }} --timeout 30m +{{- else }} +uninstall.deregister=false: `helm uninstall` keeps the PVCs, and runners stay +registered in SG as parked. +{{- end }} +{{- $size := include "sg-runner.size" . | fromYaml }} + +One runner per node, advertising {{ $size.cpu }} CPU / {{ $size.memory }} for workflows. +Each workflow step runs as a pod on its runner's node: + kubectl -n {{ $ns }} get pods -l app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=step + kubectl -n {{ $ns }} logs -f +Finished step pods stay for {{ .Values.runner.steps.finishedTTL }}, labelled with their outcome: + kubectl -n {{ $ns }} get pods -l app.kubernetes.io/component=step -L sg-runner.stackguardian.io/outcome +The steps broker in each runner pod creates them: + kubectl -n {{ $ns }} logs -f {{ $sts }}-0 -c steps +{{- if .Values.networkPolicy.enabled }} + +NetworkPolicy {{ $sts }} only takes effect with a CNI that enforces it. +It blocks the Kubernetes API too, which the steps broker needs: allow the API +server's endpoint (`kubectl get endpoints kubernetes`) in +networkPolicy.extraEgress. That opens it to everything in the runner pod. +{{- end }} diff --git a/k8s/chart/sg-runner/templates/_helpers.tpl b/k8s/chart/sg-runner/templates/_helpers.tpl new file mode 100644 index 0000000..19942bb --- /dev/null +++ b/k8s/chart/sg-runner/templates/_helpers.tpl @@ -0,0 +1,295 @@ +{{/* +Chart name. +*/}} +{{- define "sg-runner.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified release name; also the runner StatefulSet's name. Kept to 52 +characters: the StatefulSet controller adds a `controller-revision-hash` label +of - that must fit in 63. +*/}} +{{- define "sg-runner.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 52 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 52 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 52 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Runner StatefulSet (and headless Service) name. Pods are -, PVCs +state-- and docker--; the scaler maps SG runners +to ordinals through that name. +*/}} +{{- define "sg-runner.runnerName" -}} +{{- include "sg-runner.fullname" . }} +{{- end }} + +{{- define "sg-runner.scalerName" -}} +{{- printf "%s-scaler" (include "sg-runner.fullname" .) }} +{{- end }} + +{{- define "sg-runner.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. Pass a dict with "ctx" (root context) and "component". +*/}} +{{- define "sg-runner.labels" -}} +helm.sh/chart: {{ include "sg-runner.chart" .ctx }} +{{ include "sg-runner.selectorLabels" . }} +{{- if .ctx.Chart.AppVersion }} +app.kubernetes.io/version: {{ .ctx.Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .ctx.Release.Service }} +app.kubernetes.io/part-of: sg-runner +{{- end }} + +{{/* +Selector labels: runner and scaler pods share name/instance, the component +tells them apart. +*/}} +{{- define "sg-runner.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sg-runner.name" .ctx }} +app.kubernetes.io/instance: {{ .ctx.Release.Name }} +app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{/* +Labels of reap Job pods: the runner's selector labels with component reaper, +so the NetworkPolicy covers them too. +*/}} +{{- define "sg-runner.reaperLabels" -}} +{{- include "sg-runner.selectorLabels" (dict "ctx" . "component" "reaper") }} +{{- end }} + +{{/* +Env for whatever starts reap Jobs (the scaler, the uninstall hook): the +StatefulSet, and how to run reap Job pods. +*/}} +{{- define "sg-runner.reapEnv" -}} +- name: POD_NAMESPACE + valueFrom: + fieldRef: { fieldPath: metadata.namespace } +- { name: SCALER_STATEFULSET, value: {{ include "sg-runner.runnerName" . | quote }} } +# Reap Jobs run the runner image (not the scaler's) with the node token +# Secret and the runner group ConfigMap. +- { name: SCALER_IMAGE, value: {{ include "sg-runner.image" . | quote }} } +- { name: SCALER_CREDENTIALS_SECRET, value: {{ include "sg-runner.credentialsSecretName" . | quote }} } +- { name: SCALER_CONFIGMAP, value: {{ include "sg-runner.configName" . | quote }} } +# Reap Job pods: labelled like runners (component reaper) so the +# NetworkPolicy covers them, scheduled like runners (they mount a runner's +# state PVC, which may be bound to a node or zone). +- name: SCALER_REAP_POD_LABELS + value: {{ include "sg-runner.reaperLabels" . | fromYaml | toJson | quote }} +- name: SCALER_REAP_IMAGE_PULL_SECRETS + value: {{ include "sg-runner.pullSecretNames" . | quote }} +- name: SCALER_REAP_NODE_SELECTOR + value: {{ default dict .Values.runner.nodeSelector | toJson | quote }} +- name: SCALER_REAP_TOLERATIONS + value: {{ default list .Values.runner.tolerations | toJson | quote }} +{{- end }} + +{{/* +A label map as a selector string: k1=v1,k2=v2 (sorted). +*/}} +{{- define "sg-runner.selectorString" -}} +{{- $pairs := list }} +{{- range $k := keys . | sortAlpha }} +{{- $pairs = append $pairs (printf "%s=%s" $k (get $ $k)) }} +{{- end }} +{{- join "," $pairs }} +{{- end }} + +{{/* +imagePullSecrets names, comma-separated. Entries are Secret names or +{name: ...} objects, mixed as you like. +*/}} +{{- define "sg-runner.pullSecretNames" -}} +{{- $names := list }} +{{- range .Values.imagePullSecrets }} +{{- if kindIs "string" . }} +{{- $names = append $names . }} +{{- else }} +{{- $names = append $names .name }} +{{- end }} +{{- end }} +{{- join "," $names }} +{{- end }} + +{{/* +imagePullSecrets as a pod spec takes them ({name: ...} each), for every pod +the chart renders. +*/}} +{{- define "sg-runner.imagePullSecrets" -}} +{{- $refs := list }} +{{- range splitList "," (include "sg-runner.pullSecretNames" .) }} +{{- with . }} +{{- $refs = append $refs (dict "name" .) }} +{{- end }} +{{- end }} +{{- with $refs }} +{{- toYaml . }} +{{- end }} +{{- end }} + +{{/* +Runner image reference; the scaler also uses it for reap Jobs. +*/}} +{{- define "sg-runner.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag | toString) }} +{{- end }} + +{{/* +Scaler image reference (the steps broker's too, unless set apart). +*/}} +{{- define "sg-runner.scalerImage" -}} +{{- printf "%s:%s" .Values.scaler.image.repository (default .Chart.AppVersion .Values.scaler.image.tag | toString) }} +{{- end }} + +{{/* +Name of the SG credentials Secret. Fails rendering when neither an existing +Secret nor credentials.create is configured, or both are. +*/}} +{{- define "sg-runner.credentialsSecretName" -}} +{{- $c := .Values.credentials }} +{{- if and $c.existingSecret $c.create }} +{{- fail "credentials: set either credentials.existingSecret or credentials.create=true, not both" }} +{{- else if $c.existingSecret }} +{{- $c.existingSecret }} +{{- else if $c.create }} +{{- printf "%s-credentials" (include "sg-runner.fullname" .) }} +{{- else }} +{{- fail "credentials: set credentials.existingSecret (a Secret with key SG_NODE_TOKEN) or credentials.create=true with credentials.nodeToken" }} +{{- end }} +{{- end }} + +{{/* +Name of the ConfigMap with the runner group (SG_ORGANIZATION, SG_RUNNER_GROUP, +SG_BASE_API). Fails rendering without sg.organization and sg.runnerGroup. +*/}} +{{- define "sg-runner.configName" -}} +{{- if not (and .Values.sg.organization .Values.sg.runnerGroup) }} +{{- fail "sg.organization and sg.runnerGroup are required: the runner group to register into" }} +{{- end }} +{{- printf "%s-config" (include "sg-runner.fullname" .) }} +{{- end }} + +{{/* +The runner group settings and node token as files in one directory +(/etc/sg-runner/credentials), as runner.sh and the scaler read them: the +ConfigMap plus only the SG_NODE_TOKEN key of the Secret. Pass the root context +as "ctx" and the file mode as "mode". +*/}} +{{- define "sg-runner.credentialsVolume" -}} +- name: sg-credentials + projected: + defaultMode: {{ .mode }} + sources: + - secret: + name: {{ include "sg-runner.credentialsSecretName" .ctx }} + items: [{ key: SG_NODE_TOKEN, path: SG_NODE_TOKEN }] + - configMap: + name: {{ include "sg-runner.configName" .ctx }} +{{- end }} + +{{/* +Runner size: what the runner advertises to SG (cpu, memory) and the dockerd +container's resources (only run controllers run there). A preset or, with +size=custom, runner.advertise and runner.controllerResources. Prints YAML. +*/}} +{{- define "sg-runner.size" -}} +{{- $presets := dict + "small" (dict "cpu" 2 "memory" "4Gi" "resources" (dict "requests" (dict "cpu" "500m" "memory" "1Gi") "limits" (dict "cpu" "2" "memory" "2Gi"))) + "medium" (dict "cpu" 4 "memory" "8Gi" "resources" (dict "requests" (dict "cpu" "1" "memory" "2Gi") "limits" (dict "cpu" "4" "memory" "4Gi"))) + "large" (dict "cpu" 8 "memory" "16Gi" "resources" (dict "requests" (dict "cpu" "2" "memory" "4Gi") "limits" (dict "cpu" "8" "memory" "8Gi"))) +}} +{{- $size := .Values.runner.size }} +{{- if eq $size "custom" }} +{{- $a := default dict .Values.runner.advertise }} +{{- $res := default dict .Values.runner.controllerResources }} +{{- if not (and $a.cpu $a.memory) }} +{{- fail "runner.size=custom needs runner.advertise.cpu (whole CPUs) and .memory (Mi or Gi): what the runner advertises to SG for workflows" }} +{{- end }} +{{- $limits := default dict $res.limits }} +{{- if not (and $limits.cpu $limits.memory) }} +{{- fail "runner.size=custom needs runner.controllerResources.limits.cpu and .memory: the run controllers' room" }} +{{- end }} +{{- toYaml (dict "cpu" $a.cpu "memory" $a.memory "resources" $res) }} +{{- else if hasKey $presets $size }} +{{- toYaml (get $presets $size) }} +{{- else }} +{{- fail (printf "runner.size must be small, medium, large or custom, got %q" $size) }} +{{- end }} +{{- end }} + +{{/* +Resources of the dockerd container (run controllers only). +*/}} +{{- define "sg-runner.dockerdResources" -}} +{{- toYaml (include "sg-runner.size" . | fromYaml).resources }} +{{- end }} + +{{/* +The steps broker image, runner.steps.broker.image when its repository is set, +the scaler image otherwise (it ships /sg-runner-steps). +*/}} +{{- define "sg-runner.brokerImage" -}} +{{- $img := .Values.runner.steps.broker.image }} +{{- if $img.repository }} +{{- printf "%s:%s" $img.repository (default .Chart.AppVersion $img.tag | toString) }} +{{- else }} +{{- include "sg-runner.scalerImage" . }} +{{- end }} +{{- end }} + +{{/* +The advertised memory (Mi or Gi) in MiB, what the ECS agent advertises +(SG_TASK_MEMORY_LIMIT_MIB). +*/}} +{{- define "sg-runner.advertiseMemoryMiB" -}} +{{- $m := toString (include "sg-runner.size" . | fromYaml).memory }} +{{- if regexMatch "^[1-9][0-9]*Gi$" $m }} +{{- mul (trimSuffix "Gi" $m | atoi) 1024 }} +{{- else if regexMatch "^[1-9][0-9]*Mi$" $m }} +{{- trimSuffix "Mi" $m | atoi }} +{{- else }} +{{- fail (printf "runner.advertise.memory must be Mi or Gi, got %q" $m) }} +{{- end }} +{{- end }} + +{{/* +Runner pod affinity: runner.affinity plus a required pod anti-affinity term +(one runner per node: steps run on the runner's node and mount its RWO +workspace), appended to any terms of the user's own. +*/}} +{{- define "sg-runner.runnerAffinity" -}} +{{- $aff := deepCopy (default dict .Values.runner.affinity) }} +{{- $selector := include "sg-runner.selectorLabels" (dict "ctx" . "component" "runner") | fromYaml }} +{{- $term := dict "topologyKey" "kubernetes.io/hostname" "labelSelector" (dict "matchLabels" $selector) }} +{{- $anti := default dict $aff.podAntiAffinity }} +{{- $required := default list $anti.requiredDuringSchedulingIgnoredDuringExecution }} +{{- $_ := set $anti "requiredDuringSchedulingIgnoredDuringExecution" (append $required $term) }} +{{- $_ := set $aff "podAntiAffinity" $anti }} +{{- toYaml $aff }} +{{- end }} + +{{/* +What the ECS agent advertises (runner.sh sizes the agent from it), with +nothing kept back for dockerd: steps run as pods next to the runner. On +ecs-agent and the register init container, which re-registers a runner whose +size changed. +*/}} +{{- define "sg-runner.advertiseEnv" -}} +- { name: SG_TASK_CPU_LIMIT, value: {{ (include "sg-runner.size" . | fromYaml).cpu | quote }} } +- { name: SG_TASK_MEMORY_LIMIT_MIB, value: {{ include "sg-runner.advertiseMemoryMiB" . | quote }} } +- { name: SG_DOCKERD_OVERHEAD_MIB, value: "0" } +{{- end }} diff --git a/k8s/chart/sg-runner/templates/config.yaml b/k8s/chart/sg-runner/templates/config.yaml new file mode 100644 index 0000000..91bf432 --- /dev/null +++ b/k8s/chart/sg-runner/templates/config.yaml @@ -0,0 +1,14 @@ +# The runner group to register into: not secret. Mounted next to the node +# token as one file per key; runner.sh and the scaler read these names. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "sg-runner.configName" . }} + labels: + {{- include "sg-runner.labels" (dict "ctx" . "component" "runner") | nindent 4 }} +data: + SG_ORGANIZATION: {{ .Values.sg.organization | quote }} + SG_RUNNER_GROUP: {{ .Values.sg.runnerGroup | quote }} + {{- with .Values.sg.apiUrl }} + SG_BASE_API: {{ . | quote }} + {{- end }} diff --git a/k8s/chart/sg-runner/templates/credentials-secret.yaml b/k8s/chart/sg-runner/templates/credentials-secret.yaml new file mode 100644 index 0000000..00b9f3c --- /dev/null +++ b/k8s/chart/sg-runner/templates/credentials-secret.yaml @@ -0,0 +1,14 @@ +{{- $name := include "sg-runner.credentialsSecretName" . }} +{{- if .Values.credentials.create }} +# The runner group's node token, the only secret. Mounted as a file named +# SG_NODE_TOKEN, next to the runner group ConfigMap's keys. +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" (dict "ctx" . "component" "runner") | nindent 4 }} +type: Opaque +stringData: + SG_NODE_TOKEN: {{ required "credentials.nodeToken is required with credentials.create=true" .Values.credentials.nodeToken | quote }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/networkpolicy.yaml b/k8s/chart/sg-runner/templates/networkpolicy.yaml new file mode 100644 index 0000000..0518594 --- /dev/null +++ b/k8s/chart/sg-runner/templates/networkpolicy.yaml @@ -0,0 +1,71 @@ +{{- if .Values.networkPolicy.enabled }} +{{- $np := .Values.networkPolicy }} +{{- $component := dict "ctx" . "component" "runner" }} +{{- $v4 := list }} +{{- $v6 := list }} +{{- range $np.clusterCIDRs }} +{{- if contains ":" . }}{{ $v6 = append $v6 . }}{{ else }}{{ $v4 = append $v4 . }}{{ end }} +{{- end }} +{{- if not $v4 }} +{{- fail "networkPolicy.clusterCIDRs needs the IPv4 pod, service and node CIDRs (and the API server endpoint if outside them) with networkPolicy.enabled=true" }} +{{- end }} +# Workflow isolation for runner pods (run controllers in the nested Docker, +# whose traffic leaves through the pod's IP, and the agents), step pods +# (component=step) and reap Job pods (component=reaper, set by the scaler): no +# cluster-internal destinations, no cloud metadata, everything else (SG API, +# AWS endpoints, VCS, registries, workflow targets) allowed. +# Needs a CNI that enforces NetworkPolicy (Calico, Cilium, ...); others ignore +# it without warning. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "sg-runner.runnerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "sg-runner.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + matchExpressions: + - { key: app.kubernetes.io/component, operator: In, values: [runner, reaper, step] } + policyTypes: ["Ingress", "Egress"] + # No ingress rules: runners only dial out. + ingress: [] + egress: + {{- if $np.allowClusterDNS }} + # Cluster DNS only; the CIDR rule below excludes it otherwise. + - to: + - {{ toYaml $np.dnsSelector | nindent 10 | trim }} + ports: + - { protocol: UDP, port: 53 } + - { protocol: TCP, port: 53 } + {{- end }} + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + {{- range $v4 }} + - {{ . }} + {{- end }} + {{- if $np.blockMetadata }} + - 169.254.169.254/32 + {{- end }} + {{- /* IPv6 egress only when the cluster's IPv6 CIDRs are listed (dual + stack); otherwise this policy allows no IPv6 at all. */}} + {{- if $v6 }} + - to: + - ipBlock: + cidr: ::/0 + except: + {{- range $v6 }} + - {{ . }} + {{- end }} + {{- if $np.blockMetadataIPv6 }} + - fd00:ec2::254/128 + {{- end }} + {{- end }} + {{- with $np.extraEgress }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/poddisruptionbudget.yaml b/k8s/chart/sg-runner/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..7726b62 --- /dev/null +++ b/k8s/chart/sg-runner/templates/poddisruptionbudget.yaml @@ -0,0 +1,16 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{- $component := dict "ctx" . "component" "runner" }} +# Keeps voluntary disruptions (node drains) from evicting the last runner: SG +# pauses workflows while no runner is ACTIVE. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "sg-runner.runnerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/runner-rbac.yaml b/k8s/chart/sg-runner/templates/runner-rbac.yaml new file mode 100644 index 0000000..874c374 --- /dev/null +++ b/k8s/chart/sg-runner/templates/runner-rbac.yaml @@ -0,0 +1,48 @@ +{{- $name := include "sg-runner.runnerName" . }} +{{- $component := dict "ctx" . "component" "runner" }} +# The steps broker's identity. Only the broker container mounts a token for +# it. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +automountServiceAccountToken: false +--- +# Step pods and their Secrets in the release namespace. Secrets are write +# only: the broker never reads them back (they hold the steps' env and +# registry credentials), so a leaked token can't list the namespace's Secrets. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + # patch: the outcome label on finished step pods. + verbs: ["create", "get", "list", "watch", "patch", "delete"] + # Step output, for the run controllers. + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} diff --git a/k8s/chart/sg-runner/templates/runner-service.yaml b/k8s/chart/sg-runner/templates/runner-service.yaml new file mode 100644 index 0000000..d605930 --- /dev/null +++ b/k8s/chart/sg-runner/templates/runner-service.yaml @@ -0,0 +1,14 @@ +{{- $component := dict "ctx" . "component" "runner" }} +# Headless Service backing the StatefulSet's serviceName. Runners need no +# inbound traffic; it gives pods their . FQDN, which the scaler +# uses to map SG runners (by hostname) to ordinals. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sg-runner.runnerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + clusterIP: None + selector: + {{- include "sg-runner.selectorLabels" $component | nindent 4 }} diff --git a/k8s/chart/sg-runner/templates/runner-statefulset.yaml b/k8s/chart/sg-runner/templates/runner-statefulset.yaml new file mode 100644 index 0000000..83c0219 --- /dev/null +++ b/k8s/chart/sg-runner/templates/runner-statefulset.yaml @@ -0,0 +1,322 @@ +{{- $name := include "sg-runner.runnerName" . }} +{{- $secret := include "sg-runner.credentialsSecretName" . }} +{{- $config := include "sg-runner.configName" . }} +{{- $image := include "sg-runner.image" . }} +{{- $component := dict "ctx" . "component" "runner" }} +{{- $steps := .Values.runner.steps }} +# Each replica is one registered runner. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + serviceName: {{ $name }} + {{- if not .Values.scaler.enabled }} + replicas: {{ .Values.runner.replicas }} + {{- end }} + {{- /* With the scaler, no replicas: it owns them, and Helm's 3-way merge + keeps the live value on upgrade. A new StatefulSet starts with 1. */}} + # Park and wake runners side by side, not one at a time (each park can wait + # for running workflows). + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + template: + metadata: + # Selector labels only: a chart version label here would roll (drain) + # every runner on each chart upgrade. + labels: + {{- include "sg-runner.selectorLabels" $component | nindent 8 }} + {{- with .Values.runner.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runner.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: false + # For the steps broker only: the token is mounted in that container alone. + serviceAccountName: {{ $name }} + {{- with include "sg-runner.imagePullSecrets" . }} + imagePullSecrets: + {{- . | nindent 8 }} + {{- end }} + # Upper bound on how long a terminating runner waits for running + # workflows (the ecs-agent preStop hook) before it is killed. + terminationGracePeriodSeconds: {{ .Values.runner.terminationGracePeriodSeconds }} + initContainers: + - name: register + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ['register'] + # Only what registration needs. + env: + - name: SG_NODE_TOKEN + valueFrom: + secretKeyRef: { name: {{ $secret }}, key: SG_NODE_TOKEN } + - name: SG_ORGANIZATION + valueFrom: + configMapKeyRef: { name: {{ $config }}, key: SG_ORGANIZATION } + - name: SG_RUNNER_GROUP + valueFrom: + configMapKeyRef: { name: {{ $config }}, key: SG_RUNNER_GROUP } + - name: SG_BASE_API + valueFrom: + configMapKeyRef: { name: {{ $config }}, key: SG_BASE_API, optional: true } + # A runner registered with another size than the agent will + # advertise (below) registers again (runner.reregisterOnResize). + - { name: SG_REREGISTER_ON_RESIZE, value: {{ .Values.runner.reregisterOnResize | quote }} } + {{- include "sg-runner.advertiseEnv" . | nindent 12 }} + volumeMounts: + - { name: state, mountPath: /etc/ecs, subPath: ecs-config } + # Agent state, wiped with the rest when it registers again. + - { name: state, mountPath: /var/lib/ecs/data, subPath: ecs-data } + - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } + - { name: state, mountPath: /var/log/registration, subPath: registration } + {{- $broker := $steps.broker }} + {{- $step := $steps.step }} + # Runs each workflow step as a pod on this node, for the run + # controllers (k8s/docs/steps-api.md), and guards dockerd (authorization + # plugin). The only container with API access. A native sidecar: it + # starts before dockerd and stops after every other container, so it + # outlives the drain. + - name: steps + image: {{ include "sg-runner.brokerImage" . }} + imagePullPolicy: {{ $broker.image.pullPolicy }} + restartPolicy: Always + command: ['/sg-runner-steps'] + args: ['serve'] + env: + - name: RUNNER_POD_NAME + valueFrom: + fieldRef: { fieldPath: metadata.name } + - name: RUNNER_POD_UID + valueFrom: + fieldRef: { fieldPath: metadata.uid } + - name: NODE_NAME + valueFrom: + fieldRef: { fieldPath: spec.nodeName } + - name: NAMESPACE + valueFrom: + fieldRef: { fieldPath: metadata.namespace } + - { name: SOCKET_PATH, value: /run/sg-steps/steps.sock } + # dockerd's authorization plugin, on the docker-plugins volume. + - { name: GUARD_SOCKET, value: /run/sg-guard/sg-steps-guard.sock } + # Its own image: step pods' wrapper init container. + - { name: BROKER_IMAGE, value: {{ include "sg-runner.brokerImage" . | quote }} } + - { name: BROKER_IMAGE_PULL_POLICY, value: {{ $broker.image.pullPolicy | quote }} } + {{- with include "sg-runner.pullSecretNames" . }} + - { name: BROKER_IMAGE_PULL_SECRETS, value: {{ . | quote }} } + {{- end }} + - { name: WORKSPACES_DIR, value: /var/tmp/workspace } + # The workspace volume's PVC (generic ephemeral: -). + - { name: WORKSPACE_CLAIM, value: $(RUNNER_POD_NAME)-workspace } + - name: STEP_LABELS + value: {{ include "sg-runner.selectorLabels" (dict "ctx" . "component" "step") | fromYaml | toJson | quote }} + - name: STEP_NODE_SELECTOR + value: {{ default .Values.runner.nodeSelector $step.nodeSelector | default dict | toJson | quote }} + - name: STEP_TOLERATIONS + value: {{ default .Values.runner.tolerations $step.tolerations | default list | toJson | quote }} + - { name: SCHEDULING_TIMEOUT, value: {{ $steps.schedulingTimeout | quote }} } + - { name: START_TIMEOUT, value: {{ $steps.startTimeout | quote }} } + - { name: FINISHED_TTL, value: {{ $steps.finishedTTL | quote }} } + {{- with $step.imagePullPolicy }} + - { name: STEP_IMAGE_PULL_POLICY, value: {{ . }} } + {{- end }} + {{- with $broker.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + # dockerd starts once this passes: its guard socket must exist. + startupProbe: + exec: + command: ['/sg-runner-steps', 'healthcheck'] + periodSeconds: 2 + timeoutSeconds: 6 + failureThreshold: 60 + readinessProbe: + exec: + command: ['/sg-runner-steps', 'healthcheck'] + periodSeconds: 10 + timeoutSeconds: 6 + {{- with $broker.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + volumeMounts: + - { name: steps-socket, mountPath: /run/sg-steps } + - { name: docker-plugins, mountPath: /run/sg-guard } + - { name: steps-api-access, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true } + containers: + - name: dockerd + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ['dockerd'] + {{- with .Values.runner.dockerd.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.runner.dockerd.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + # Only run controllers run inside this container (their steps are + # pods); the ECS agent advertises the runner's size (SG_TASK_*). + resources: + {{- include "sg-runner.dockerdResources" . | nindent 12 }} + lifecycle: + preStop: + exec: + command: ['/opt/sg-runner/runner.sh', 'wait-drained'] + readinessProbe: + exec: + command: ['docker', 'network', 'inspect', 'sg-net'] + periodSeconds: 10 + volumeMounts: + - { name: docker-run, mountPath: /var/run } + - { name: docker, mountPath: /var/lib/docker } + # Where dockerd finds the steps broker's authorization plugin. + - { name: docker-plugins, mountPath: /run/docker/plugins } + # Same path the agent's ECS_HOST_DATA_DIR points at. + - { name: state, mountPath: /var/lib/ecs/data, subPath: ecs-data } + # Run controller containers bind-mount this path from this daemon's + # filesystem; step pods mount the same PVC. + - { name: workspace, mountPath: /var/tmp/workspace } + # The steps broker's socket, where run controllers look for it. + - { name: steps-socket, mountPath: /var/tmp/workspace/.sg-runner } + - name: ssm-agent + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ['ssm-agent'] + # Keeps rotating the agent's credentials while a drain runs. + lifecycle: + preStop: + exec: + command: ['/opt/sg-runner/runner.sh', 'wait-drained'] + volumeMounts: + - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } + - { name: aws-creds, mountPath: /root/.aws } + - { name: docker-run, mountPath: /var/run } + - name: ecs-agent + image: {{ $image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ['ecs-agent'] + env: + {{- include "sg-runner.advertiseEnv" . | nindent 12 }} + {{- with .Values.runner.ecsAgent.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + lifecycle: + preStop: + exec: + command: ['/opt/sg-runner/runner.sh', 'drain'] + livenessProbe: + exec: + command: ['ecs-agent', '--healthcheck'] + initialDelaySeconds: 120 + periodSeconds: 30 + volumeMounts: + - { name: state, mountPath: /etc/ecs, subPath: ecs-config } + - { name: state, mountPath: /data, subPath: ecs-data } + - { name: state, mountPath: /var/lib/ecs/data, subPath: ecs-data } + - { name: state, mountPath: /log, subPath: ecs-log } + - { name: state, mountPath: /var/lib/amazon/ssm, subPath: ssm } + - { name: aws-creds, mountPath: /rotatingcreds, readOnly: true } + - { name: docker-run, mountPath: /var/run } + # Setting the runner's status through SG needs the node token and + # runner group. + - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true } + volumes: + - name: docker-run + emptyDir: {} + - name: aws-creds + emptyDir: + medium: Memory + # A PVC per pod (-workspace), created and deleted with it, that + # step pods mount too. + - name: workspace + ephemeral: + volumeClaimTemplate: + metadata: + labels: + {{- include "sg-runner.selectorLabels" $component | nindent 18 }} + spec: + accessModes: {{ toJson $steps.workspace.accessModes }} + {{- with $steps.workspace.storageClassName }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ $steps.workspace.size }} + - name: steps-socket + emptyDir: {} + # The broker's authorization plugin socket, for dockerd. + - name: docker-plugins + emptyDir: {} + # What the pod's automounted token would be (kube-api-access-*), for + # the steps container only. + - name: steps-api-access + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3607 + - configMap: + name: kube-root-ca.crt + items: [{ key: ca.crt, path: ca.crt }] + - downwardAPI: + items: + - path: namespace + fieldRef: { apiVersion: v1, fieldPath: metadata.namespace } + {{- include "sg-runner.credentialsVolume" (dict "ctx" . "mode" "0400") | nindent 8 }} + {{- with .Values.runner.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runner.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with include "sg-runner.runnerAffinity" . }} + affinity: + {{- . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + # Runner identity (ecs.config, SSM registration, agent state): survives + # pod restarts so the runner doesn't re-register. Kept on scale-down too: + # a parked runner wakes from it. Not managed by Helm, so uninstall keeps it. + - metadata: + name: state + spec: + accessModes: ['ReadWriteOnce'] + {{- with .Values.runner.storage.state.storageClassName }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.runner.storage.state.size }} + # Docker data root: the image cache survives parking. + - metadata: + name: docker + spec: + accessModes: ['ReadWriteOnce'] + {{- with .Values.runner.storage.docker.storageClassName }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.runner.storage.docker.size }} diff --git a/k8s/chart/sg-runner/templates/scaler-deployment.yaml b/k8s/chart/sg-runner/templates/scaler-deployment.yaml new file mode 100644 index 0000000..6c1d243 --- /dev/null +++ b/k8s/chart/sg-runner/templates/scaler-deployment.yaml @@ -0,0 +1,114 @@ +{{- if .Values.scaler.enabled }} +{{- $s := .Values.scaler }} +{{- $name := include "sg-runner.scalerName" . }} +{{- $component := dict "ctx" . "component" "scaler" }} +{{- if and (gt (int $s.replicas) 1) (not $s.leaderElection.enabled) }} +{{- fail "scaler.replicas > 1 needs scaler.leaderElection.enabled: two scalers without a lease would fight over the runner StatefulSet's replicas" }} +{{- end }} +# Scaler for the runner StatefulSet. It owns spec.replicas. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + replicas: {{ $s.replicas }} + strategy: + {{- if $s.leaderElection.enabled }} + # The lease keeps a single active scaler while old and new pods overlap. + type: RollingUpdate + {{- else }} + # No lease: never run two scalers at once. + type: Recreate + {{- end }} + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + template: + metadata: + labels: + {{- include "sg-runner.selectorLabels" $component | nindent 8 }} + {{- with $s.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ $name }} + {{- with include "sg-runner.imagePullSecrets" . }} + imagePullSecrets: + {{- . | nindent 8 }} + {{- end }} + {{- with $s.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: scaler + image: {{ include "sg-runner.scalerImage" . }} + imagePullPolicy: {{ $s.image.pullPolicy }} + {{- with $s.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $s.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - { name: SCALER_LEADER_ELECT, value: {{ $s.leaderElection.enabled | quote }} } + - { name: SCALER_MIN, value: {{ $s.min | quote }} } + - { name: SCALER_MAX, value: {{ $s.max | quote }} } + # % of awake runners' capacity to keep free (warm spare); 0 = off. + - { name: SCALER_HEADROOM_PCT, value: {{ $s.headroomPct | quote }} } + - { name: SCALER_IDLE_SECONDS, value: {{ $s.idleSeconds | quote }} } + - { name: SCALER_PARK_TTL, value: {{ $s.parkTTL | quote }} } + - { name: SCALER_STUCK_SECONDS, value: {{ $s.stuckSeconds | quote }} } + - { name: SCALER_SETTLE_TIMEOUT, value: {{ $s.settleTimeout | quote }} } + - { name: SCALER_INTERVAL, value: {{ $s.interval | quote }} } + - { name: SCALER_NUDGE, value: {{ $s.nudge.enabled | quote }} } + - { name: SCALER_NUDGE_IMAGE, value: {{ $s.nudge.image | quote }} } + - { name: SCALER_NUDGE_COOLDOWN, value: {{ $s.nudge.cooldownSeconds | quote }} } + {{- include "sg-runner.reapEnv" . | nindent 12 }} + # Serves /metrics, /healthz and /readyz. + - { name: SCALER_METRICS_ADDR, value: {{ printf ":%v" $s.metrics.port | quote }} } + {{- with $s.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + # Serves /metrics, /healthz and /readyz. + ports: + - { name: metrics, containerPort: {{ $s.metrics.port }}, protocol: TCP } + livenessProbe: + httpGet: { path: /healthz, port: metrics } + periodSeconds: 20 + failureThreshold: 3 + readinessProbe: + httpGet: { path: /readyz, port: metrics } + periodSeconds: 10 + {{- with $s.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - { name: sg-credentials, mountPath: /etc/sg-runner/credentials, readOnly: true } + volumes: + # 0440, not the runners' 0400: the files are owned by root, and the + # scaler runs as a non-root UID. The pod's fsGroup makes them + # group-owned by its GID, so group read is what lets it read them. + {{- include "sg-runner.credentialsVolume" (dict "ctx" . "mode" "0440") | nindent 8 }} + {{- with $s.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/scaler-metrics.yaml b/k8s/chart/sg-runner/templates/scaler-metrics.yaml new file mode 100644 index 0000000..b8078b1 --- /dev/null +++ b/k8s/chart/sg-runner/templates/scaler-metrics.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.scaler.enabled .Values.scaler.metrics.enabled }} +{{- $m := .Values.scaler.metrics }} +{{- $component := dict "ctx" . "component" "scaler" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sg-runner.scalerName" . }}-metrics + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +spec: + selector: + {{- include "sg-runner.selectorLabels" $component | nindent 4 }} + ports: + - { name: metrics, port: {{ $m.port }}, targetPort: metrics, protocol: TCP } +{{- if and $m.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "sg-runner.scalerName" . }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} + {{- with $m.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "sg-runner.selectorLabels" $component | nindent 6 }} + endpoints: + - port: metrics + interval: {{ $m.serviceMonitor.interval }} +{{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/scaler-rbac.yaml b/k8s/chart/sg-runner/templates/scaler-rbac.yaml new file mode 100644 index 0000000..4b2eb48 --- /dev/null +++ b/k8s/chart/sg-runner/templates/scaler-rbac.yaml @@ -0,0 +1,80 @@ +{{- if .Values.scaler.enabled }} +{{- $name := include "sg-runner.scalerName" . }} +{{- $sts := include "sg-runner.runnerName" . }} +{{- $component := dict "ctx" . "component" "scaler" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +--- +# Only the runner StatefulSet and its scale, PVCs, reap Jobs, its own lease and +# events. Runner pods' steps brokers have their own Role ({{ $sts }}). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +rules: + # Read replicas; patch for the scaler's timer annotations. + - apiGroups: ["apps"] + resources: ["statefulsets"] + resourceNames: [{{ $sts | quote }}] + verbs: ["get", "patch"] + - apiGroups: ["apps"] + resources: ["statefulsets/scale"] + resourceNames: [{{ $sts | quote }}] + verbs: ["get", "patch"] + # Parked-at stamps, and deleting a reaped runner's PVCs. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "get", "patch", "delete"] + # Reap Jobs deregister long-parked runners from their PVC; the scaler lists + # them to find reaps in progress. Placeholder Jobs ask the cluster + # autoscaler for nodes (scaler.nudge). + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "delete"] + {{- if .Values.scaler.nudge.enabled }} + # Read only: runner pods the scheduler can't place, and whether the + # cluster autoscaler refused to add a node for them. + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list"] + {{- end }} + {{- if .Values.scaler.leaderElection.enabled }} + # Leader election: one active scaler per StatefulSet. create can't be + # restricted by name (the object doesn't exist yet), get/update can. + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + resourceNames: [{{ printf "%s-scaler" $sts | quote }}] + verbs: ["get", "update"] + {{- end }} + # Scaling decisions as Kubernetes events. + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/k8s/chart/sg-runner/templates/uninstall-hook.yaml b/k8s/chart/sg-runner/templates/uninstall-hook.yaml new file mode 100644 index 0000000..5b16f4e --- /dev/null +++ b/k8s/chart/sg-runner/templates/uninstall-hook.yaml @@ -0,0 +1,144 @@ +{{- if .Values.uninstall.deregister }} +{{- $u := .Values.uninstall }} +{{- $s := .Values.scaler }} +{{- $name := printf "%s-uninstall" (include "sg-runner.fullname" .) }} +{{- $sts := include "sg-runner.runnerName" . }} +{{- $scaler := include "sg-runner.scalerName" . }} +{{- $component := dict "ctx" . "component" "uninstall" }} +{{- $deadline := $u.activeDeadlineSeconds | default (add .Values.runner.terminationGracePeriodSeconds 900) }} +# `helm uninstall` cleanup: before Helm deletes the release, park every runner, +# deregister each from StackGuardian and delete the runners' PVCs (which +# aren't part of the release). Its own ServiceAccount and Role, created as +# hooks too, so it works with the scaler disabled. Skipped by +# `helm uninstall --no-hooks`. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: + {{- if $s.enabled }} + # Stop the scaler first, so it doesn't wake or park runners meanwhile. + - apiGroups: ["apps"] + resources: ["deployments/scale"] + resourceNames: [{{ $scaler | quote }}] + verbs: ["get", "patch"] + {{- end }} + # Park every runner. + - apiGroups: ["apps"] + resources: ["statefulsets", "statefulsets/scale"] + resourceNames: [{{ $sts | quote }}] + verbs: ["get", "patch"] + # Wait for the scaler and runner pods to be gone. + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + # Find the runners' PVCs; delete them once deregistered. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "delete"] + # Reap Jobs deregister each runner from its state PVC. + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +# Kept when it fails, for its logs: kubectl logs job/{{ $name }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + labels: + {{- include "sg-runner.labels" $component | nindent 4 }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 0 + # Running workflows finish first (runner.terminationGracePeriodSeconds), + # then the reaps. + activeDeadlineSeconds: {{ $deadline }} + template: + metadata: + labels: + {{- include "sg-runner.selectorLabels" $component | nindent 8 }} + spec: + serviceAccountName: {{ $name }} + restartPolicy: Never + {{- with include "sg-runner.imagePullSecrets" . }} + imagePullSecrets: + {{- . | nindent 8 }} + {{- end }} + {{- with $s.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + # The scaler's binary, `teardown` mode. + - name: uninstall + image: {{ include "sg-runner.scalerImage" . }} + imagePullPolicy: {{ $s.image.pullPolicy }} + {{- with $s.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + args: ["teardown"] + {{- with $s.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- include "sg-runner.reapEnv" . | nindent 12 }} + - { name: TEARDOWN_RUNNER_SELECTOR, value: {{ include "sg-runner.selectorLabels" (dict "ctx" . "component" "runner") | fromYaml | include "sg-runner.selectorString" | quote }} } + {{- if $s.enabled }} + - { name: TEARDOWN_SCALER_DEPLOYMENT, value: {{ $scaler | quote }} } + - { name: TEARDOWN_SCALER_SELECTOR, value: {{ include "sg-runner.selectorLabels" (dict "ctx" . "component" "scaler") | fromYaml | include "sg-runner.selectorString" | quote }} } + {{- end }} + {{- with $u.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $s.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $s.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/k8s/chart/sg-runner/values.schema.json b/k8s/chart/sg-runner/values.schema.json new file mode 100644 index 0000000..920630e --- /dev/null +++ b/k8s/chart/sg-runner/values.schema.json @@ -0,0 +1,603 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "sg-runner chart values", + "type": "object", + "definitions": { + "quantity": { + "type": [ + "string", + "integer" + ], + "pattern": "^[0-9]+(\\.[0-9]+)?(m|k|Ki|M|Mi|G|Gi|T|Ti|P|Pi|E|Ei)?$" + }, + "storage": { + "type": "object", + "required": [ + "size" + ], + "properties": { + "size": { + "$ref": "#/definitions/quantity" + }, + "storageClassName": { + "type": "string" + } + } + }, + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string" + }, + "tag": { + "type": [ + "string", + "number" + ] + }, + "pullPolicy": { + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + } + }, + "positiveInt": { + "type": "integer", + "minimum": 1 + }, + "duration": { + "type": "string", + "pattern": "^([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$" + }, + "envList": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1 + } + } + } + }, + "resources": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/quantity" + } + }, + "limits": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/quantity" + } + } + } + } + }, + "properties": { + "nameOverride": { + "type": "string" + }, + "fullnameOverride": { + "type": "string", + "maxLength": 52 + }, + "image": { + "allOf": [ + { + "$ref": "#/definitions/image" + }, + { + "required": [ + "repository" + ], + "properties": { + "repository": { + "minLength": 1 + } + } + } + ] + }, + "imagePullSecrets": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1 + } + } + } + ] + } + }, + "sg": { + "type": "object", + "properties": { + "organization": { + "type": "string" + }, + "runnerGroup": { + "type": "string" + }, + "apiUrl": { + "type": "string", + "pattern": "^(https?://.+)?$" + } + } + }, + "credentials": { + "type": "object", + "properties": { + "existingSecret": { + "type": "string" + }, + "create": { + "type": "boolean" + }, + "nodeToken": { + "type": "string" + } + }, + "additionalProperties": false + }, + "runner": { + "type": "object", + "required": [ + "size" + ], + "properties": { + "replicas": { + "type": "integer", + "minimum": 0 + }, + "size": { + "enum": [ + "small", + "medium", + "large", + "custom" + ] + }, + "storage": { + "type": "object", + "properties": { + "state": { + "$ref": "#/definitions/storage" + }, + "docker": { + "$ref": "#/definitions/storage" + } + } + }, + "dockerd": { + "type": "object", + "properties": { + "securityContext": { + "type": "object" + }, + "extraEnv": { + "$ref": "#/definitions/envList" + } + } + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "ecsAgent": { + "type": "object", + "properties": { + "extraEnv": { + "$ref": "#/definitions/envList" + } + } + }, + "reregisterOnResize": { + "type": "boolean" + }, + "advertise": { + "type": "object", + "properties": { + "cpu": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "memory": { + "type": "string", + "pattern": "^([1-9][0-9]*(Mi|Gi))?$" + } + }, + "additionalProperties": false + }, + "controllerResources": { + "$ref": "#/definitions/resources" + }, + "steps": { + "type": "object", + "properties": { + "workspace": { + "type": "object", + "required": [ + "size" + ], + "properties": { + "storageClassName": { + "type": "string" + }, + "size": { + "$ref": "#/definitions/quantity" + }, + "accessModes": { + "type": "array", + "minItems": 1, + "items": { + "enum": [ + "ReadWriteOnce", + "ReadWriteOncePod", + "ReadWriteMany" + ] + } + } + } + }, + "schedulingTimeout": { + "$ref": "#/definitions/duration" + }, + "startTimeout": { + "$ref": "#/definitions/duration" + }, + "finishedTTL": { + "$ref": "#/definitions/duration" + }, + "step": { + "type": "object", + "properties": { + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "imagePullPolicy": { + "enum": [ + "", + "Always", + "IfNotPresent", + "Never" + ] + } + } + }, + "broker": { + "type": "object", + "properties": { + "image": { + "$ref": "#/definitions/image" + }, + "resources": { + "$ref": "#/definitions/resources" + }, + "extraEnv": { + "$ref": "#/definitions/envList" + } + } + } + } + } + }, + "if": { + "properties": { + "size": { + "const": "custom" + } + } + }, + "then": { + "required": [ + "advertise", + "controllerResources" + ], + "properties": { + "advertise": { + "required": [ + "cpu", + "memory" + ], + "properties": { + "cpu": { + "type": "integer" + }, + "memory": { + "minLength": 1 + } + } + }, + "controllerResources": { + "required": [ + "limits" + ], + "properties": { + "limits": { + "required": [ + "cpu", + "memory" + ] + } + } + } + } + }, + "additionalProperties": false + }, + "scaler": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "image": { + "allOf": [ + { + "$ref": "#/definitions/image" + }, + { + "required": [ + "repository" + ], + "properties": { + "repository": { + "minLength": 1 + } + } + } + ] + }, + "command": { + "type": "array", + "items": { + "type": "string" + } + }, + "min": { + "$ref": "#/definitions/positiveInt" + }, + "max": { + "$ref": "#/definitions/positiveInt" + }, + "headroomPct": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "idleSeconds": { + "type": "integer", + "minimum": 0 + }, + "parkTTL": { + "$ref": "#/definitions/positiveInt" + }, + "stuckSeconds": { + "type": "integer", + "minimum": 0 + }, + "settleTimeout": { + "type": "integer", + "minimum": 0 + }, + "interval": { + "$ref": "#/definitions/positiveInt" + }, + "extraEnv": { + "type": "array" + }, + "metrics": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "serviceMonitor": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "interval": { + "type": "string" + }, + "labels": { + "type": "object" + } + } + } + } + }, + "resources": { + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "replicas": { + "type": "integer", + "minimum": 1 + }, + "leaderElection": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "nudge": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "image": { + "type": "string", + "minLength": 1 + }, + "cooldownSeconds": { + "type": "integer", + "minimum": 30 + } + } + } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "clusterCIDRs": { + "type": "array", + "items": { + "type": "string", + "anyOf": [ + { + "pattern": "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" + }, + { + "pattern": "^[0-9a-fA-F:]+/[0-9]{1,3}$" + } + ] + } + }, + "blockMetadata": { + "type": "boolean" + }, + "allowClusterDNS": { + "type": "boolean" + }, + "dnsSelector": { + "type": "object" + }, + "extraEgress": { + "type": "array" + }, + "blockMetadataIPv6": { + "type": "boolean" + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + }, + "required": [ + "enabled" + ] + }, + "then": { + "properties": { + "clusterCIDRs": { + "minItems": 1 + } + } + } + }, + "podDisruptionBudget": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "minAvailable": { + "type": [ + "integer", + "string" + ] + } + } + }, + "uninstall": { + "type": "object", + "properties": { + "deregister": { + "type": "boolean" + }, + "activeDeadlineSeconds": { + "type": [ + "integer", + "null" + ], + "minimum": 60 + }, + "resources": { + "type": "object" + } + } + } + } +} diff --git a/k8s/chart/sg-runner/values.yaml b/k8s/chart/sg-runner/values.yaml new file mode 100644 index 0000000..d642f1b --- /dev/null +++ b/k8s/chart/sg-runner/values.yaml @@ -0,0 +1,325 @@ +# sg-runner chart (prototype). Each runner StatefulSet replica is one +# registered StackGuardian Private Runner; the scaler owns the replica count. +# +# The namespace must allow privileged pods (nested Docker): +# pod-security.kubernetes.io/enforce: privileged +# The chart doesn't create the namespace. + +nameOverride: "" +# Also renames the runner StatefulSet, so its pods and PVCs. Changing it on an +# existing install orphans the runners registered from the old PVCs. +fullnameOverride: "" + +# Runner image (register, dockerd, ssm-agent, ecs-agent; reap Jobs too). +image: + repository: ghcr.io/stackguardian/sg-runner-k8s + # Empty = the chart's appVersion (the release's version in a published + # chart). + tag: "" + pullPolicy: IfNotPresent + +# Secret names or {name: ...} objects, e.g. [regcred, {name: other}]. +imagePullSecrets: [] + +# The runner group to register into. Not secret: the chart puts these in a +# ConfigMap (-config). +sg: + # Required. + organization: "" + runnerGroup: "" + # Empty = https://api.app.stackguardian.io/api/v1. + apiUrl: "" + +# The runner group's node token, the only secret: used to register, set runner +# status and (scaler) read the runner group. Set exactly one of existingSecret +# or create=true. +credentials: + # Secret with key SG_NODE_TOKEN; other keys in it are ignored. + existingSecret: "" + # Let the chart create the Secret from nodeToken. The token then also ends up + # in the Helm release (a Secret in the namespace); prefer existingSecret. + create: false + nodeToken: "" + +runner: + # Only used with scaler.enabled=false. With the scaler the chart leaves + # spec.replicas out: the scaler owns it, and upgrades keep the live value. + replicas: 1 + + # Each runner takes a node of its own (required pod anti-affinity, added to + # runner.affinity). Run controllers run in the runner pod's nested Docker; + # each workflow step runs as a pod of its own on the runner's node, created + # by the `steps` broker sidecar (k8s/docs/steps-api.md). + # + # The size sets what the runner advertises to SG for workflows (step pods + # and run controllers together), and the dockerd container's resources, + # which only hold the run controllers: + # + # preset advertised controllers (dockerd) requests / limits + # small 2 CPU / 4Gi 500m / 1Gi 2 / 2Gi + # medium 4 CPU / 8Gi 1 / 2Gi 4 / 4Gi + # large 8 CPU / 16Gi 2 / 4Gi 8 / 8Gi + # custom runner.advertise and runner.controllerResources (required) + # + # A workflow reserves 0.5 vCPU / 512 MiB for its run controller plus its own + # resource setting (default 0.5 vCPU / 1 GiB), so a default workflow takes + # 1 vCPU / 1.5 GiB; the presets are CPU-bound. The runner must be at least + # the largest workflow + 0.5 vCPU / 512 MiB, else that workflow stays queued + # forever (the scaler warns about it). Size the controllers' memory limit at + # ~512Mi per concurrent workflow run. + # + # The node needs the advertised capacity free for step pods, plus the + # runner pod's own requests. The agent can't advertise more than the node + # has, and node memory minus the advertised memory must stay under 64 GiB + # (the agent's reservation is a uint16 of MiB; the runner refuses to start + # otherwise). + size: small + # size=custom only. Whole CPUs; memory in Mi or Gi (e.g. 6Gi, 3072Mi). + advertise: + cpu: + memory: "" + # size=custom only, e.g.: + # requests: { cpu: 500m, memory: 1Gi } + # limits: { cpu: "2", memory: 2Gi } + controllerResources: {} + + # Workflow steps as pods. + steps: + # Workspace PVC per runner pod (-workspace, a generic ephemeral + # volume, created and deleted with the pod) at /var/tmp/workspace on + # dockerd, mounted by the steps. Steps share the runner's node, so + # ReadWriteOnce is enough. + workspace: + # Empty = the cluster's default StorageClass. + storageClassName: "" + size: 20Gi + accessModes: [ReadWriteOnce] + # A step pod that can't be scheduled for this long fails (Unschedulable). + schedulingTimeout: 5m + # A step pod still pending (pulling, creating) after this long fails. + startTimeout: 30m + # Finished step pods are kept for this long, then deleted; their Secrets + # are deleted when they finish. + finishedTTL: 1h + # Scheduling of step pods (pinned to the runner's node anyway). Empty = + # runner.nodeSelector / runner.tolerations, so steps tolerate the runner + # node's taints. + step: + nodeSelector: {} + tolerations: [] + # Pull policy of step images. Empty = automatic: Always for `latest` or + # no tag, else IfNotPresent. IfNotPresent or Never for air-gapped + # clusters or images preloaded on the nodes. + imagePullPolicy: "" + # The `steps` sidecar (Go, distroless, non-root). + broker: + # Empty repository = the scaler image (it ships /sg-runner-steps). + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { memory: 256Mi } + extraEnv: [] + + storage: + # Runner identity (ecs.config, SSM registration, agent state). Survives + # restarts and parking so a runner doesn't register again. + state: + size: 1Gi + # Empty = the cluster's default StorageClass. + storageClassName: "" + # Docker data root (/var/lib/docker): run controller images cached across + # parking. + docker: + size: 20Gi + storageClassName: "" + + dockerd: + # Nested Docker needs a privileged container for now (see docs/TODO.md). + securityContext: + privileged: true + # Extra env vars for the dockerd container (advanced/testing), e.g. turn + # off the nested Docker prune loop: + # - { name: SG_PRUNE_INTERVAL_SECONDS, value: "0" } + extraEnv: [] + + ecsAgent: + # Extra env vars for the ecs-agent container (advanced/testing), e.g. + # - { name: ECS_IMAGE_PULL_BEHAVIOR, value: prefer-cached } + extraEnv: [] + + # ECS keeps a runner's CPU and memory from its first registration, so a + # new size (runner.size, runner.advertise) would otherwise not show until + # the runner registers anew. With this on, a runner whose registered size + # differs from what it now advertises deregisters and registers again when + # its pod starts, if nothing runs on it (else it keeps the old size and logs + # a warning). It gets a new runner ID in SG. + reregisterOnResize: true + + # Upper bound on how long a terminating runner waits for running workflows + # (the ecs-agent preStop drain) before it is killed. + terminationGracePeriodSeconds: 3600 + + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +scaler: + # Deployment that owns the runner StatefulSet's replicas: wakes parked + # runners or adds new ones when workflows queue, parks idle ones, and reaps + # runners parked longer than parkTTL. + enabled: true + + # The scaler's own image (Go, distroless, non-root; build with + # `make build-scaler`). It also ships the steps broker + # (runner.steps.broker.image defaults to it). + image: + repository: ghcr.io/stackguardian/sg-runner-scaler + # Empty = the chart's appVersion. + tag: "" + pullPolicy: IfNotPresent + # Empty = the image's entrypoint. + command: [] + + # Two replicas are safe with leader election: only the lease holder scales, + # the other takes over when it goes away. Without it, keep 1 (two scalers + # would fight over replicas). + replicas: 1 + leaderElection: + # Lease -scaler in the release namespace. + enabled: true + + # Runners kept awake. At least 1: SG pauses workflows while no runner is + # ACTIVE. 2 avoids that pause while a single runner restarts. + min: 1 + # Also capped by the runner group's MaxNumberOfRunners. + max: 10 + # % of awake runners' CPU or memory to keep free (warm spare); 0 = off. + headroomPct: 0 + # Park the highest awake runner after it has been idle this long. + idleSeconds: 60 + # Deregister runners parked longer than this (12h) and delete their PVCs. + parkTTL: 43200 + # Queued workflows with an idle awake runner for this long are treated as + # too big for any runner: warn and stop scaling out for them. + stuckSeconds: 180 + # How long to wait for a woken or new runner to connect before deciding + # without it. + settleTimeout: 300 + # Seconds between scaling decisions. + interval: 15 + # Some cluster autoscalers (GKE's, at least) never add a node for a pod with + # a persistent disk, so a runner that needs a new node stays Pending. When + # the autoscaler says so (event NotTriggerScaleUp on the runner pod), the + # scaler starts a placeholder Job with the runner's requests and scheduling + # rules but no volumes. As soon as the autoscaler accepts it + # (TriggeredScaleUp), the scaler deletes the placeholder, and the runner + # takes the new node when it joins. Does nothing where the autoscaler scales + # up for runners by itself. + nudge: + enabled: true + # Only sleeps; mirror it for private registries (imagePullSecrets apply). + image: registry.k8s.io/pause:3.10 + # Wait before asking again for a runner that's still Pending. Must outlast + # a node's boot, or a second node gets requested while the first boots. + cooldownSeconds: 180 + extraEnv: [] + + # The scaler always listens on this port (container port `metrics`, + # SCALER_METRICS_ADDR=:): /metrics, plus /healthz and /readyz for the + # probes. + # metrics.enabled only adds a Service (and optionally a ServiceMonitor). + metrics: + enabled: false + port: 9090 + # Needs the Prometheus Operator CRDs; skipped when the API isn't there. + serviceMonitor: + enabled: false + interval: 30s + labels: {} + + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { memory: 128Mi } + podAnnotations: {} + podLabels: {} + # The Go image is distroless and runs as non-root. runAsUser is explicit: + # the kubelet can't verify runAsNonRoot against a named USER (distroless + # `nonroot` = 65532). fsGroup makes the credentials Secret's files group-owned + # by that GID, so they're readable with mode 0440 (see the Deployment). + podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + nodeSelector: {} + tolerations: [] + affinity: {} + +# Egress isolation for runner pods (run controllers' nested traffic leaves +# through the pod IP, so they and the agents alike), step pods and reap Job +# pods (component=reaper, set by the scaler). Needs a CNI that enforces +# NetworkPolicy (Calico, Cilium, ...); others silently ignore it. It blocks +# the Kubernetes API too, which the steps broker needs: allow the API server's +# endpoint in extraEgress. +# +# IPv4 by default: no IPv6 egress is allowed at all. On dual-stack clusters, +# list the IPv6 pod/service/node CIDRs in clusterCIDRs too, which opens IPv6 +# egress except those. +networkPolicy: + enabled: false + # Pod, service and node CIDRs, plus the Kubernetes API endpoint if it's + # outside them. Required when enabled: workflows must not reach the cluster. + clusterCIDRs: [] + # Block the cloud metadata endpoint (169.254.169.254). + blockMetadata: true + # Also block AWS's IPv6 metadata endpoint (fd00:ec2::254). Only matters when + # IPv6 egress is open (IPv6 entries in clusterCIDRs). + blockMetadataIPv6: false + # Allow DNS to the cluster resolver. Off = workflows and agents must resolve + # through an external resolver. + allowClusterDNS: true + dnsSelector: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + # Extra egress rules (NetworkPolicyEgressRule list), added as-is. + extraEgress: [] + +# Keeps node drains from evicting the last runner: SG pauses workflows while no +# runner is ACTIVE. With a single awake runner it blocks drains until the +# scaler wakes another; use scaler.min >= 2 alongside it. +podDisruptionBudget: + enabled: false + minAvailable: 1 + +# `helm uninstall` cleans up after itself: a pre-delete hook Job stops the +# scaler, parks every runner (each waits for its running workflows), then +# deregisters every runner from StackGuardian from its state PVC and deletes +# the runners' PVCs. If a runner can't be deregistered, the hook fails and +# keeps that runner's PVCs, so nothing is orphaned; fix it and uninstall again. +# +# Runners finish running workflows first: pass a `helm uninstall --timeout` +# that covers them (Helm's default is 5m). `helm uninstall --no-hooks` skips +# the cleanup: runners stay registered and parked, PVCs stay. +uninstall: + deregister: true + # Upper bound for the whole cleanup; empty = runner.terminationGracePeriodSeconds + 15 minutes. + activeDeadlineSeconds: + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { memory: 128Mi } diff --git a/k8s/docs/README.md b/k8s/docs/README.md new file mode 100644 index 0000000..b15ed09 --- /dev/null +++ b/k8s/docs/README.md @@ -0,0 +1,65 @@ +# sg-runner on Kubernetes: handoff notes + +Working notes for whoever continues the Kubernetes runner prototype: what was +built, why, what's verified, what isn't, and what's still open. The +user-facing description lives in [`../README.md`](../README.md) and the chart's +[`README`](../chart/sg-runner/README.md); this folder is the background behind +them. + +> No secrets, account IDs or internal hostnames in these files: the repository +> may be public. QA specifics (runner group, API URL, token) live only in the +> gitignored `k8s/.env`. + +| File | What's in it | +|---|---| +| [open-items.md](open-items.md) | Everything not done yet, by priority, plus the known caveats | +| [testing.md](testing.md) | What has been verified and how, what still needs testing, step-by-step test plans | +| [decisions.md](decisions.md) | Design decisions with the reasons and rejected alternatives; facts learned from StackGuardian's code and AWS | +| [dev-notes.md](dev-notes.md) | Environment gotchas and working practices for the next session | +| [gke-one-runner-per-node.md](gke-one-runner-per-node.md) | The GKE layout with one runner per node: sizing, how parking and waking behave there, suggested settings | +| [native-steps.md](native-steps.md) | How the Kubernetes runner runs workflow steps as pods (the only mode): requirements, the step broker and guard, sizing, known behaviour | +| [steps-api.md](steps-api.md) | The step broker's API (v1): the contract between the run controller and the broker | + +## Status + +Branch `feat/k8s-prototype`, not pushed. Prototype quality: everything +listed below works, most of it was tested against the QA platform, but it is +not a supported product. + +| Area | State | +|---|---| +| Runner pod (`Dockerfile`, `runner.sh`) | Registers, advertises its real size, parks and wakes, drains through SG, prunes its Docker cache. Tested on QA. | +| Scaler (`scaler/`, Go) | Scales out on queue, parks idle runners, reaps long-parked ones, drains hard-killed ones; Events, metrics, leader election, persisted timers. Unit-tested and smoke-tested against a **fake** SG. The rules were validated on QA in the shell version it replaced; **the Go version has not run against real SG yet.** | +| Helm chart (`chart/sg-runner`) | Runner StatefulSet, scaler, credentials, size presets, workspace volume, optional NetworkPolicy and PDB, values schema. `helm lint --strict`, kubeconform, kind install with a fake API. | +| CI (`.github/workflows/k8s.yml`) | Go, chart (Helm 3 and 4), multi-arch image builds, Trivy, kind smoke. Passed locally; **never run on GitHub yet.** | +| Native steps (`scaler/internal/steps`, `internal/guard`, `internal/wrapper`) | The only mode: step broker as a native sidecar, `dockerd` guard, step wrapper; the run controller's broker client is in sg-run-controller (same branch name). Unit tests, kind smoke stage, and end to end on QA with an overlaid controller (OrbStack). Needs SG to ship the controller change. | +| `main.sh` fixes on this branch | Private-IP detection; error reports no longer retry themselves into a stack overflow. Both also affect VM runners. | + +## Picking it up + +```sh +cd k8s +cp .env.example .env # a throwaway QA runner group's credentials +make build build-scaler # sg-runner-k8s:dev, sg-runner-scaler:dev +make secret up # OrbStack Kubernetes (kubectl context "orbstack") +kubectl -n sg-runner logs -f deploy/sg-runner-scaler | jq -c 'select(.msg != "loop")' +make nuke # always end with this: helm uninstall deregisters every runner +``` + +First thing to do next: the end-to-end run of the chart with the Go scaler +against QA ([testing.md](testing.md#1-end-to-end-chart--go-scaler-on-qa)). + +## History (commits on the branch) + +| Step | Commits | +|---|---| +| Runner pod prototype, sizing, graceful drain | `b4faa67`, `76a1595` | +| Parking, drain through SG instead of ECS | `7e07b82` | +| Shell scaler (autoscaling, stuck-queue, settle timeout, headroom) | `76ed5d2`, `9d4fef1` | +| Private-IP fix in `main.sh` | `1464dca` | +| Docker prune, version pins, multi-arch, runner tests | `f58f5a9`, `42579e4`, `2d51eb0` | +| Helm chart | `e0ae7fd`, `02021e6`, `c5c1d5d`, `0087d30` | +| Go scaler | `b23ddfc`, `7352f4f`, `2a2c3e2`, `00e1b30`, `85e0c7f`, `b7f9dd6`, `4949366` | +| Docs | `46a54fa`, `7f1ada4` | +| CI | `29f5873`, `387cb8b` | +| Error-report recursion fix in `main.sh`, Debian updates in the image | `25f059c`, `ef01b6f` | diff --git a/k8s/docs/decisions.md b/k8s/docs/decisions.md new file mode 100644 index 0000000..b7f5a31 --- /dev/null +++ b/k8s/docs/decisions.md @@ -0,0 +1,107 @@ +# Decisions and background + +## Facts the design rests on + +From StackGuardian's own code (the backend and the run controller), the ECS +agent's source, and tests on QA. Re-check them if behavior seems off. + +- **ECS Anywhere is the dispatch path.** SG places each workflow run as an + ECS task (the "run controller") on a runner group's container instances, + matched by instance attributes (organization, runner group, and for newer + runner groups a signature). The run controller then starts the workflow + steps itself through `/var/run/docker.sock`. Both layers need a Docker API, + which is why a runner pod runs a nested `dockerd`. +- **Workflow size reaches ECS.** The run task is started with a task-level + CPU/memory override of 512 + the workflow's own setting each (default + 512 / 1024, so 1024 CPU units / 1536 MiB in total). ECS reserves that + against what the runner advertises. +- **Step limits.** The run controller starts steps with a hard memory limit + (the workflow's memory setting) but only a CPU *weight*, so a step can use + all of the pod's CPU. +- **No `ACTIVE` runner means paused, not queued.** When ECS can't place a run + and no runner in the group is `ACTIVE` and connected, SG sets the run + queued and the workflow paused, without retrying. A separate handler is + meant to resume it. With an `ACTIVE` connected runner, SG retries. +- **`QueuedWorkflowsCount`** (runner group API) counts the group's workflows + whose latest active run is queued. `EcsTaskFailAgentCount` is the + equivalent the shared-runner autoscaler uses. +- **The node token is enough.** With the runner group's node token: read the + group with `?getActiveWorkflows=true`, `POST .../runner_status/` + (`DRAINING`/`ACTIVE`, really changes the ECS instance), and deregister. The + group GET response contains secrets (the group's runner token): never log + it. +- **The ECS agent sizes itself from the node**: CPU = + `runtime.NumCPU() × 1024` (affinity, not cgroup quota), memory = `MemTotal` + minus `ECS_RESERVED_MEMORY` (parsed as `uint16`: values above 65535 silently + become 0). +- **The runner's instance role can't change its own ECS status** on QA (the + policy names the cluster ARN, the action is authorized against the + container-instance ARN). Irrelevant now: status changes go through SG. + +- **The run controller's mounts.** Its ECS task bind-mounts the host's + `/var/run/docker.sock` and `/var/tmp/workspace` at the same paths + (`WORKSPACES_DIR=/var/tmp/workspace`), runs as root, and its image is set by + tag, not digest. Steps get `.../wfruns//shared-workspace` at + `/mnt/sg_workspace` plus the AWS credentials files at their own paths. + Verified with `docker inspect` on QA. +- **ECS keeps a runner's registered size.** `registeredResources` are fixed + at the container instance's first registration; an agent restart with a + different size changes nothing (seen on QA: 2048/3840 stayed after the agent + advertised 4096/6144). The runner therefore re-registers when its size + changes (see below). +- **Exit codes 11 and 12 are signals.** A step exits 11 when it needs approval + and 12 to finish the run early; the run controller acts on the code. + +## Decisions + +| Decision | Why | Rejected alternatives | +|---|---|---| +| One pod = one runner host; nested `dockerd` | ECS agent and run controller both need a Docker API; real clusters run containerd | Host Docker socket (not on modern clusters) | +| Native steps: steps as pods through a step broker in the runner pod; run controller unchanged otherwise ([native-steps.md](native-steps.md)) | User code out of the privileged pod; Kubernetes owns scheduling and images; the broker, not the controller, holds cluster credentials and builds the pod spec | A Docker API shim in front of `dockerd` (emulates Docker semantics, works with an unmodified controller); the controller calling the Kubernetes API itself; a new run controller (12k lines of logic to fork) | +| Broker client duck-types docker-py in the controller | `main()` stays unchanged apart from one line; the existing exit-code and status logic and its tests keep working | A `StepRunner` refactor of `main()` | +| Native-only on Kubernetes, with a `dockerd` authorization plugin refusing step containers | The Kubernetes runner isn't released, so non-native mode has no users; one mode halves the code and tests; the guard makes an old controller fail clearly instead of running steps in a `dockerd` sized for controllers | Keep non-native as a fallback (old controllers, arm64 emulation, host mounts) | +| Broker as a native sidecar | Starts before `dockerd` (the guard must answer first) and stops after the main containers (no drain-marker tricks) | Regular container with preStop wait and SIGTERM handling (the first version) | +| A run on the broker never falls back to Docker | A silent fallback would run user code in a `dockerd` sized for controllers only | Fallback with a warning | +| Workspace as a generic ephemeral PVC, steps pinned to the runner's node | Same lifetime as the emptyDir it replaces; toggling is a normal rollout; works on any storage class | volumeClaimTemplate (immutable, needs scaler/teardown changes); hostPath; RWX (later option) | +| Wrap each step (`sg-step-wrapper`, PID 1 init) so exits 11/12 end `Completed`, with the real code in the termination message; read ENTRYPOINT/CMD from the image's registry config; run unwrapped if that fails | SG's step images signal approval and early finish with exit codes, which Kubernetes shows as failures; the controller keeps reading 11/12 unchanged | Delete finished pods ~10s after the result is read (the first version: hid the pods, lost `kubectl logs`); an outcome label only; step images signalling without exit codes (SG-wide change) | +| Keep finished step pods 1h with an outcome label; delete their Secrets at once | Debuggable with `kubectl`; credentials don't outlive the step | Keep Secrets with the pod | +| Re-register an idle runner whose advertised size changed (`runner.reregisterOnResize`) | ECS never updates a registered size; without it a resized runner keeps its old capacity silently | Document a manual reap; refuse to start (blocks rollouts) | +| Broker can create/delete Secrets but not read or list them | The node token is a Secret in the same namespace | Secret list for garbage collection | +| Run the ECS agent binary directly, not `ecs-init` | `ecs-init` needs systemd | Running the installer (systemd/packages) | +| Advertise the `dockerd` container's limits | Workflows run inside it; otherwise ECS over-places | Pod limits alone (the agent ignores cgroup quotas) | +| Never call ECS from the runner | Customers' runners have no ECS permissions; SG's `runner_status` API does it with SG's | Direct `UpdateContainerInstancesState` (also denied on QA) | +| Secret holds only the node token; organization, runner group and API URL are chart values in a ConfigMap | They aren't secret; visible and editable like any config, and secret stores only need the token | Everything in one Secret (the prototype's first layout) | +| Scaler nudges the cluster autoscaler with a volume-less placeholder Job | GKE's autoscaler won't add a node for a pod with a PD (template node lacks `topology.gke.io/zone`, and GKE refuses that node label); a placeholder with the runner's requests and affinity gets the node, which the runner then takes | Node pool label (rejected by GKE); always-on spare nodes (idle cost) | +| Node token only, no SG API key | Every call the runner and scaler make works with it; one credential instead of two | A separate org API key (tried, removed) | +| Park instead of deregister on scale-down | Waking takes 8s vs 20s for a fresh registration, keeps the image cache, avoids SSM/ECS churn | Stateless Deployment that re-registers every time | +| StatefulSet + PVC per runner | Stable identity and cache per runner; parking needs it | Deployment with pod-deletion-cost (stateless) | +| Drain on every shutdown, reactivate only the pod's own drain | No new work during restarts/rollouts; drains from the SG UI or an autoscaler must stick | Drain only on scale-down; reactivate unconditionally | +| Scaler as its own small loop, not KEDA | Must choose which runner to park (highest ordinal, only when idle), reap parked runners, detect stuck queues | KEDA / HPA (replica count only, extra install) | +| Scale out on queue only when no awake runner is fully idle | With an idle runner, SG's next retry places queued work there if it fits; a same-size runner can't help if it doesn't | Scale out on any queue (overshoots, wakes every runner for an oversized workflow) | +| Headroom optional, default off | Waking is fast enough; a warm spare is a cost choice | Always-on 80% reserved target (kept one idle runner per workload) | +| Timers: park after 60s idle, deregister after 12h parked | Fast scale-in, cheap re-wake through the working day | 300s / 24h (earlier defaults) | +| Go for the scaler | client-go, Events, metrics, leader election, small static non-root image; path to an operator | Python (team's main language), bash (prototype) | +| Helm chart, Helm 3 and 4 compatible | Customers install with either | Raw manifests (removed) | +| Size presets (`small`/`medium`/`large`/`custom`) | Customers pick runner size at install | Fixed size | +| Prune the image cache only under disk pressure | Docker filters images by creation time, not last use; age-only pruning empties the wake-up cache | Age-only pruning; no pruning | +| Trivy gates the scaler image only | Scaler findings are actionable; runner findings mostly come from bundled upstream binaries | Gating both (permanently red CI) | + +## The scaler's rules (summary) + +Every 15s, from SG's runner group data and the StatefulSet: + +- max = min(`scaler.max`, `MaxNumberOfRunners`); min ≥ 1. +- **Settling**: an awake runner not yet connected blocks decisions for up to + 300s; after that it's reported (`RunnerNotConnected`), drained, and ignored. +- **Stuck**: work queued while an awake runner is fully idle for 180s → + `StuckQueue`, the queue is ignored for scaling. +- **Out**: queue with no fully idle awake runner, or free capacity below + `headroomPct` → +1 (wakes a parked runner if its PVC exists). +- **In**: no queue (or stuck), highest awake runner idle ≥ 60s, rest keeps the + headroom → −1 (parks it). +- **Parked but `ACTIVE`** (hard kill) → `DRAINING` + PVC mark; reactivated on + wake only if marked. +- **Reap**: parked ≥ 12h → Job runs `runner.sh reap` against the PVC + (deregisters), then both PVCs are deleted. +- Timers persist as StatefulSet annotations; errors back off up to 5 minutes; + Events and `sg_runner_scaler_*` metrics report everything. diff --git a/k8s/docs/dev-notes.md b/k8s/docs/dev-notes.md new file mode 100644 index 0000000..52dfe09 --- /dev/null +++ b/k8s/docs/dev-notes.md @@ -0,0 +1,83 @@ +# Notes for the next session + +## Environment (macOS, OrbStack) + +- **Tools on PATH**: OrbStack's `docker` and `kubectl` live in + `/usr/local/bin`, which a non-login shell may not include + (`export PATH=/usr/local/bin:$PATH`). Helm 4 and kind come from Homebrew. +- **Always pin the kube context.** The Makefile passes `--context orbstack` to + every command. kind switches (and on delete, unsets) the kubeconfig's + current context; give kind its own `KUBECONFIG`, as + `.github/scripts/kind-smoke.sh` does. +- **OrbStack Kubernetes** is single-node, arm64, Docker runtime (cri-dockerd), + `local-path` storage. It uses local images directly: `imagePullPolicy: + IfNotPresent` with `:dev` tags, no registry. It can't test user namespaces + or NetworkPolicy enforcement. +- **ECR Public login**: if a stored `public.ecr.aws` login has expired, pulls + from there fail. The Dockerfile uses the ECS agent image from Docker Hub + instead. +- **Shell is zsh** for ad-hoc commands: arrays start at 1, `$var` doesn't + word-split, `k()` aliases may collide. macOS `head`/`sed` differ from GNU + (`head -n -1` doesn't exist, `sed -i ''`). +- **Helm 4 upgrades use server-side apply.** A field changed with `kubectl` + (e.g. `kubectl set env` on the scaler) makes the next `helm upgrade` fail + with a field-manager conflict, after applying part of the release. Change + test settings through values (`--set scaler.min=...`), or pass + `--force-conflicts` to take the fields back. +- **Don't upgrade with `--reuse-values`**: it keeps the *previous* chart's + defaults, so values a newer chart added are missing (template nil pointer). + Use `--reset-then-reuse-values`: new chart defaults, then the last + release's values (the token included), then `-f`/`--set`. +- **Multi-arch builds** need a docker-container buildx builder; + `make build-multiarch` creates `sg-runner-multiarch` on first use. + +## Working with QA + +- Use a throwaway runner group; its credentials go in `k8s/.env` (gitignored). + Registering creates real SSM managed instances and ECS container instances. +- **Always finish with `helm uninstall`** (or `make nuke`): its pre-delete + hook deregisters every runner, awake or parked, before deleting the PVCs. + Deleting PVCs or the namespace first orphans runners; so does + `helm uninstall --no-hooks`. +- The runner group's storage backend must point at a bucket its integration + role can use, or every workflow fails at the storage check (not a runner + problem). +- `MaxNumberOfRunners` on the group caps the scaler, and parked runners count + against it. + +## Secrets hygiene (learned the hard way) + +Two QA credentials leaked into a transcript during this work. To avoid it: + +- Never `cat`, `grep` or `jq .` the runner group API response or `.env`. + Select only the fields you need with `jq`, or mask values (`sed 's/=.*/=/'`). +- Never print `k8s/.env`; check keys with `awk -F= '{print $1, length($2)}'`. +- The Go scaler and the shell code never log response bodies; keep it that + way (there are tests for it). + +## Code conventions + +- Shell: `main.sh` style (2-space indent, `#{{{ name ... #}}}: name` folds, + `[[ ]]`, comments explain why). `make lint` enforces shellcheck plus the + `[[ ]]` / no-backticks gate on `main.sh` and `k8s/runner.sh`. +- `runner.sh` sources `main.sh` (found next to it in the image, one level up + in the repo) and has a `BASH_SOURCE` guard so tests can source it. +- Go scaler: rules live in the pure `internal/decide` package; change and + test them there first. `go vet`, `gofmt`, `go test -race`. +- Chart: values documented in `values.yaml` comments and the chart README; + the schema (`values.schema.json`) must follow any new value. +- Commit messages: `feat:` / `fix:` / `docs:`, short, plain. + +## Parallel work that went well + +The hardening round was split across separate worktrees (runtime, chart, +Go scaler, CI, a `main.sh` fix), each with a clear file boundary, then merged +into `feat/k8s-prototype`. What helped: + +- Define the interfaces up front (env var names, mounts, RBAC, ports) and + send changes to every agent they touch. +- Tell agents which namespaces, image tags and kube contexts are off limits + (a live QA deployment, `:dev` images, the `orbstack` context). +- Review every result yourself before merging (rerun tests, read the diff): + the CI agent's kind run found a real `main.sh` bug, and a reviewer caught a + missing RBAC verb and an unimplemented env contract. diff --git a/k8s/docs/gke-one-runner-per-node.md b/k8s/docs/gke-one-runner-per-node.md new file mode 100644 index 0000000..0fc8b9e --- /dev/null +++ b/k8s/docs/gke-one-runner-per-node.md @@ -0,0 +1,62 @@ +# GKE: one runner per node + +The layout tested on a zonal GKE Standard cluster: every runner gets a whole +node, nodes come and go with runners. Notes on how it behaves and the settings +we'd use. + +## Setup + +| Piece | Setting | Why | +|---|---|---| +| Node pool | `e2-custom-8-16384` (8 vCPU / 16 GB), autoscaling 1–4, on-demand | A default workflow needs 1 vCPU : 1.5 GiB; `e2-standard-8` (1 : 4) left over half its memory unused | +| Runner size | `runner.size=custom`, requests 6500m / 11Gi, limits 7 / 11Gi | What's left of ~7.9 CPU / ~13 GiB allocatable after GKE's DaemonSets (~0.5 CPU / 1 GiB) and the system pods on the first node. SG sees 7 CPU / ~10.7 GiB = 7 default workflows | +| Memory | request = limit | Memory can't be reclaimed; never overcommit it | +| One per node | required pod anti-affinity on `kubernetes.io/hostname` (runner labels) | Large requests already fit only one; the anti-affinity makes it explicit | +| Scaler | `scaler.max` = node pool `max_nodes` | One runner per node | +| Nudge | `scaler.nudge.enabled=true` (default) | GKE's cluster autoscaler never adds a node for a pod with a persistent disk (see [open-items.md](open-items.md)); the scaler's placeholder gets the node | + +## How runners move + +| Event | What happens | Time | +|---|---|---| +| Scale out, new runner | Pod Pending → autoscaler refuses (`NotTriggerScaleUp`) → nudge placeholder → node added → runner scheduled, registers | node boot (~45 s) + nudge (~15 s) + registration (~20 s) + image pulls | +| Park (idle `scaler.idleSeconds`) | Runner drains and its pod goes; volumes and SG registration stay; the empty node is removed by the autoscaler about 10 min later | 14 s park | +| Wake within ~10 min | The empty node is still there: the runner lands on it | ~8 s, as before | +| Wake after the node is gone | Nudge → new node → disk attaches → runner reconnects with its identity and image cache | ~70 s to scheduled (measured), plus reconnect | +| Reap (parked ≥ `scaler.parkTTL`) | Reap Job (no CPU/memory requests) mounts the state disk on any remaining node, deregisters, both disks deleted; no new node needed | seconds | + +A parked runner costs disk only: ~21 Gi `pd-balanced` (estimate: about +$2–3/month) plus a slot in the runner group's `MaxNumberOfRunners`. Parking +pays for the image cache and avoids SSM/ECS registration churn; the node boot +is the same either way. + +## Suggested settings + +1. **Keep `scaler.parkTTL` at 12 h.** Image caches stay warm through a working + day; parked runners are reaped overnight. Go to 2–4 h only if parked + runners crowd the SG UI or `MaxNumberOfRunners`. +2. **Keep `scaler.idleSeconds` at 60.** Parking fast is cheap: the node lingers + ~10 min (the autoscaler's scale-down delay), so a runner woken in that + window is back in seconds. +3. **Stay on GKE's default autoscaling profile, not `optimize-utilization`.** + With one runner per node it would only remove empty nodes sooner, losing + that warm window for little saving (an empty node for ~10 min costs cents). +4. **Set the runner group's `MaxNumberOfRunners` in SG** to cover awake *and* + parked runners, at least `scaler.max`; otherwise the scaler can't register + new runners while old ones are parked. +5. **Optional: `scaler.headroomPct`** keeps a spare awake runner, here a spare + node: no wait for the first queued workflow of a burst, at the price of one + idle node. +6. **`scaler.nudge.cooldownSeconds` (180)** must stay above a node's boot time, + or a second node is requested while the first boots. + +## Still to verify + +- Waking after a long park (hours, overnight): SSM and the ECS agent + reconnecting after a long disconnect ([testing.md](testing.md), plan 3). + Matters more here, since parked runners routinely sit for hours. +- The nudge releasing its placeholder at `TriggeredScaleUp` (chart 0.7.0) on + GKE; the earlier release-when-running version is verified. +- Actual workflow usage vs the 1 vCPU / 1.5 GiB SG reserves: if far lower, the + runner could advertise more CPU than its limit (CPU only, never memory) to + fit more workflows per node. diff --git a/k8s/docs/native-steps.md b/k8s/docs/native-steps.md new file mode 100644 index 0000000..25d4e5a --- /dev/null +++ b/k8s/docs/native-steps.md @@ -0,0 +1,199 @@ +# Native steps: workflow steps as pods + +How the Kubernetes runner runs workflows: the run controller stays an ECS task +in the runner pod's nested `dockerd`, registered through ECS Anywhere as +before; the workflow **steps** it starts are Kubernetes pods. This is the only +mode of the Kubernetes runner. + +## Requirements + +- **A run controller with the step broker client** (sg-run-controller, + `external_runner/step_broker.py`). SG's task definition picks the + controller, so SG must release it before Kubernetes runners can run + workflows. A controller without it is refused by the runner's guard (below): + its steps fail with a clear error instead of running in `dockerd`. +- **Kubernetes 1.29 or newer** (the broker is a native sidecar). +- **amd64 nodes** for SG's step images (amd64-only); arm64 only with + preloaded images (`runner.steps.step.imagePullPolicy: IfNotPresent`). +- **One runner per node** (rendered as a required anti-affinity; see Sizing). +- **No host bind mounts in steps.** Step `mountPoints` whose source is + outside the run's workspace (host paths, `docker.sock`) are rejected. Not + yet known whether any SG step template uses them (open item). + +Contract: [steps-api.md](steps-api.md). Tests: [testing.md](testing.md) +(plan 8, and the kind smoke's native-steps stage). + +## Why + +- **Security.** User code leaves the privileged runner pod. A step that + escapes its container lands in an unprivileged pod (no service account + token, `RuntimeDefault` seccomp, no privilege), not next to a privileged + `dockerd`. +- **Visibility.** Steps are pods: scheduling, image pulls, OOM kills and exit + codes show up as Kubernetes state and events. +- **Images.** Pulled by the kubelet and cached per node, shared by every + runner on it. +- **Resources.** Each step requests its own CPU and memory; the runner pod + only reserves what its run controllers need. + +## How it works + +``` +runner pod (StatefulSet) step pods (same node) +┌─────────────────────────────────────────────┐ +│ dockerd ── run controller (ECS task) │ ┌──────────────────────┐ +│ │ step_client(): unix socket │ │ step (user image) │ +│ ▼ │ create │ /mnt/sg_workspace = │ +│ steps (broker, Go) ── Kubernetes API ───────┼───────▶│ subPath of the │ +│ │ │ runner's workspace │ +│ workspace: ephemeral PVC -workspace ◀──┼────────┤ PVC │ +│ ecs-agent, ssm-agent │ └──────────────────────┘ +└─────────────────────────────────────────────┘ +``` + +- The run controller mounts the runner's `/var/tmp/workspace` (verified with + `docker inspect`), so it finds the broker's socket at + `/var/tmp/workspace/.sg-runner/steps.sock`: an emptyDir nested inside the + workspace mount, shared with the broker container. Steps mount only their + own run's directory and can't reach it. +- The controller's `step_client()` returns docker-py's client or, when that + socket exists, a broker client that duck-types the docker-py calls + `external.py` makes (`login`, `images.pull`, `containers.run` and the + container's `status`, `attrs`, `reload`, `wait`, `kill`, `logs`). `main()` + is unchanged apart from that one line. The choice is made once per run; a + run on the broker **never falls back to Docker**: a broker failure fails the + step with a clear error. +- The broker (`sg-runner-steps`, in the scaler image) builds the step pod + itself, from a fixed template: + - env in a Secret (`envFrom`), registry credentials in a pull Secret; + - started through `sg-step-wrapper` (below): the container's command is + the wrapper followed by the image's ENTRYPOINT and the step's command (or + the image's CMD), as Docker would run them; `$` is escaped so the kubelet + never expands `$(VAR)`; + - CPU request = Docker CPU shares / 1024, no CPU limit (Docker's shares were + a weight); memory request = limit; + - each Docker bind mount becomes a `subPath` of the runner's workspace PVC; + sources outside the run's own directory are rejected; + - pinned to the runner's node (required node affinity), owned by the runner + pod, no service account token, `RuntimeDefault` seccomp, unprivileged. +- **Exit codes 11 and 12 end `Completed`.** SG's step images exit 11 to ask + for approval and 12 to finish the run early; Kubernetes would show both as + `Failed` / `Error`. An init container copies `sg-step-wrapper` (a static + binary from the broker's image) into the step pod, and the step runs under + it as PID 1: it forwards signals, reaps zombies (like `docker run --init`), + and turns 11 and 12 into exit 0 with the real code in the container's + termination message, which the broker reports to the controller. Other + codes pass through, so real failures still show `Error`. To build that + command the broker reads the image's ENTRYPOINT and CMD from its registry + (manifest and config only, never layers; the step's own registry + credentials; cached 10 min). If that fails the step runs unwrapped, exactly + as before (annotation `sg-runner.stackguardian.io/wrapped: "false"` with the + reason), and 11/12 show as `Error` for that step only. +- **Finished steps stay** for `finishedTTL` (1h), labelled with their outcome + (`sg-runner.stackguardian.io/outcome`: `succeeded`, `approval-required`, + `completed-early`, `failed`, `stopped`), then are deleted. Their env and + pull Secrets (the run's credentials) are deleted as soon as they finish. + `kubectl get pods -l app.kubernetes.io/component=step -L + sg-runner.stackguardian.io/outcome` shows what happened. +- **Guard.** `dockerd` runs with an authorization plugin served by the + broker (`--authorization-plugin=sg-steps-guard`, socket shared through an + emptyDir at `/run/docker/plugins`). It refuses to create any container + labelled `wfrunId`, which is how the run controller labels steps, so a + controller without the broker client can't run user code in `dockerd`; its + step fails with "This runner runs workflow steps as Kubernetes pods and needs + a run controller with native step support". Everything else (the ECS agent + starting run controllers, pulls, logs) passes. Fail closed: while the + guard doesn't answer, `dockerd` refuses every call, so a crash-looping broker + blocks the runner. +- **Lifecycle.** The broker is a native sidecar (init container with + `restartPolicy: Always`): it starts and passes its health check (both + sockets) before `dockerd` and the agents start, and stops only after they + have exited, so a draining runner's controllers keep their steps running to + the end. +- **RBAC.** The runner's ServiceAccount can create, read, patch (the outcome + label) and delete pods, read pod logs, and create and delete Secrets. It can't read or list Secrets (the + node token is one) or exec into pods. The token is mounted in the broker + container only. + +## Sizing: one runner per node + +The runner pod requests only what its run controllers need +(derived from `runner.size`, or `runner.controllerResources` with `custom`; +roughly 512 Mi per concurrent run). The ECS agent advertises the size's +workflow capacity (`small` 2 CPU / 4 GiB, `medium` 4 / 8, `large` 8 / 16, or +`runner.advertise` with `custom`) +instead: that's what SG places workflows against, and the step pods consume +it. Nothing reserves that capacity on the node, so the supported layout is one +runner per node (required anti-affinity, rendered by the chart) with +`advertise` ≤ node allocatable − runner pod − DaemonSets. + +ECS keeps the CPU and memory a runner registered with. When `advertise` (or +the runner size) changes, the pod rolls, and its register step compares what +SG has registered with what the agent will advertise: an idle runner +deregisters and registers again with the new size (new runner ID, same image +cache); one with runs placed on it keeps the old size and logs a warning +(`runner.reregisterOnResize`, default on). + +## Workspace + +A generic ephemeral PVC (`-workspace`, RWO), created and deleted with the +runner pod, like the emptyDir it replaces; step pods mount it through +`subPath`. RWO allows several pods on the same node, which is why steps are +pinned there. Turning native steps on or off is a normal rollout. + +Later option: an RWX storage class, so steps can schedule on any node and the +cluster autoscaler adds nodes for steps (no PD on step pods, no nudge). Costs +an RWX prerequisite (Filestore, EFS, NFS) and slower small-file I/O. + +## Known behaviour and limits + +- **Step processes aren't PID 1.** The wrapper is; a step that relied on + PID 1's signal semantics (e.g. ignoring SIGTERM) now stops on SIGTERM, as + under `docker run --init`. +- **Unwrapped steps** (registry config not readable, e.g. a registry the + broker can't reach) still show `Failed` / `Error` for exit 11/12; their + outcome label is right regardless. Monitoring on failed pods should use the + outcome label for step pods. +- **Registry access.** The broker calls each step image's registry once per + 10 minutes per image (HTTPS); the first step with a new image waits up to + 20s for that lookup. +- **Architecture.** SG's step images are amd64-only. Nested Docker could run + them on arm64 under emulation; the kubelet refuses to pull them. On arm64 + nodes they must be preloaded with `runner.steps.step.imagePullPolicy: + IfNotPresent` (OrbStack testing); production runs on amd64 nodes. +- **Log bursts.** A step writing tens of MiB within a second can lose lines + when the kubelet rotates its log file (10 MiB by default) faster than the + stream reads it. At ~1 MiB/s nothing is lost. Raise the kubelet's + `containerLogMaxSize` if steps log heavily. +- **Pending time** (scheduling, image pull) isn't part of the step's timeout + in the run controller; the broker bounds it (`schedulingTimeout`, + `startTimeout`). +- **NetworkPolicy.** With `networkPolicy.enabled`, the policy also covers + step pods, but its cluster-CIDR exclusion blocks the broker's access to the + Kubernetes API. Workaround: allow the API endpoint through + `networkPolicy.extraEgress` (opens it to the whole runner pod). +- **Mount checks are lexical.** A symlink inside one run's directory pointing + into another run's would be followed by the kubelet. Mount sources are + chosen by the run controller, not users. +- **Node autoscaling.** The scaler's nudge placeholder copies the runner pod's + requests, i.e. controller-sized, so on GKE a new node may be too small for + the steps. Not handled yet. + +## Testing a controller that isn't released yet + +SG's task definition picks the run controller image. Until the broker client +ships in it, it's tested by overlaying the client onto the official image tag +inside the runner's `dockerd`, with the ECS agent set to +`ECS_IMAGE_PULL_BEHAVIOR=prefer-cached` (`runner.ecsAgent.extraEnv`). Steps in +[testing.md](testing.md), plan 8. + +## Open + +- Step templates with host `mountPoints`: check SG's templates for any; + they can't run on Kubernetes runners. +- Drift and policy-discovery runs (their workspace directories differ from a + normal run's): not tested yet. +- ValidatingAdmissionPolicy limiting what the runner's ServiceAccount may + create (the namespace is privileged for `dockerd`). +- RWX workspace; nudge sized for steps; NetworkPolicy value for the broker's + API access; GKE validation (PD attach, multi-node). diff --git a/k8s/docs/open-items.md b/k8s/docs/open-items.md new file mode 100644 index 0000000..35cb453 --- /dev/null +++ b/k8s/docs/open-items.md @@ -0,0 +1,121 @@ +# Open items and caveats + +## Do first + +1. **End-to-end run of the chart with the Go scaler on QA.** Nothing since + the Go rewrite has talked to real StackGuardian. Plan: + [testing.md](testing.md#1-end-to-end-chart--go-scaler-on-qa). +2. **Rotate the leaked QA credentials.** During development a QA runner + group's node token and an SG API key were printed into a session + transcript. Rotate both. The API key is no longer used anywhere; remove it + from `k8s/.env` too. After rotating the node token, restart the scaler: it + reads credentials once at start. +3. **Port the error-report recursion fix to `main`** (`25f059c`). On VM + runners, an unreachable SG API makes `err()` → `update_runner_group` → + `api_call` → `err()` loop until bash overflows its stack, sending one + authenticated request per round (857 in 36s in the CI smoke test). The fix + is self-contained (`main.sh` + `test/unit/update_runner_group.bats`) and + deliberately left off `main` for now. + +## Needs testing (details in [testing.md](testing.md)) + +- **Long parking**: wake a runner after 12–24h parked. The whole parking + design depends on SSM and the ECS agent reconnecting after a long + disconnect; only seconds have been tested. +- **Hard kills against real SG**: the scaler's "drain a parked runner that's + still `ACTIVE`, reactivate on wake" path (3.2) is only tested with a fake SG. +- **Do paused workflows resume?** SG pauses (doesn't queue) workflows + submitted while no runner is `ACTIVE` and connected. Restarting a single + runner leaves a ~10s window. Unknown whether SG resumes them when a runner + reconnects. +- **Queue latency**: how long a queued workflow waits for SG's retry after a + runner wakes. Decides whether headroom should default on. +- **Overshoot under real load** with the Go scaler (the shell version once + woke one runner too many; the queue rule was fixed afterwards). +- **NetworkPolicy on an enforcing CNI** (Calico, Cilium, VPC CNI policy mode). +- **CI on GitHub**: the gha build cache, QEMU arm64 build time, running + trivy-action twice in one job, the job summary size, kind on ubuntu-latest. + +## Decisions or input needed + +- **Largest workflow size customers configure** (workflow CPU/memory + settings). The runner size must exceed it plus 0.5 vCPU / 512 MiB, or that + workflow stays queued forever. Product data, not something the code can tell. +- **Default runner size** for the chart (`small` = 2 vCPU / 4Gi today). +- **Minimum runners**: `scaler.min=1` leaves the restart window above; + `scaler.min=2` avoids it at the cost of an idle pod. +- **Where images are published** (and multi-arch push in CI). Nothing pushes + today. +- **Push and review the branch.** It includes `main.sh` changes that affect VM + runners (private-IP detection, the recursion fix); reviewers should know. + +## Follow-up work + +| Item | Notes | +|---|---| +| Run without `privileged` (3.6) | Spike done on OrbStack: capabilities-only works but isn't a real security gain; rootless Docker doesn't fit; user namespaces need containerd 2.x or CRI-O (OrbStack's cri-dockerd can't); Sysbox untested. Next: a real cluster with `hostUsers: false` and `procMount: Unmasked`. | +| Docker past 27.5.1 | The runner image's remaining fixable CRITICALs (Trivy) are Go stdlib/grpc in the bundled static Docker binaries. A newer major raises the minimum Docker API version; check the ECS agent still works. | +| Rollouts without the restart window | Scaler-managed rollout (`updateStrategy: OnDelete`, wake a spare first, then restart outdated pods one by one). Only needed if paused workflows don't resume. | +| Mixed runner sizes | Two StatefulSets (e.g. small and large) in one runner group; ECS places each workflow where it fits. The scaler would need to know about pools and wake a large runner on a stuck queue. | +| Operator / `RunnerPool` CRD | Natural next step for the Go scaler (kubebuilder) if this becomes a product. | +| Proxy support | `main.sh`'s `--http-proxy` path isn't wired into the pod (env on the agents, Docker daemon proxy config). | +| `sg-net` bypass (known) | Workflow containers run on Docker's default bridge, so `main.sh`'s `DOCKER-USER` DROP rules for `sg-net` never apply to them, on VMs too. Known and parked. | +| SG release of the run controller change (**blocks Kubernetes runners**) | sg-run-controller branch `feat/k8s-run-controller` (`step_broker.py` + one line in `external.py`). Kubernetes runners are native-only: until it ships, the guard refuses the official controller's steps, so workflows only run with an overlaid controller image (testing.md plan 8). | +| Step templates with host `mountPoints` | Rejected on Kubernetes runners (sources outside the run's workspace, `docker.sock`). Check whether any SG template uses them. | +| Native steps: drift and policy-discovery runs | Not tested; their workspace directories (`shared-workspace-drift`, `-discovery`) differ from a normal run's. | +| Native steps: NetworkPolicy and the broker | The policy's cluster-CIDR exclusion blocks the broker's API access; needs a dedicated value instead of the `extraEgress` workaround. | +| Native steps: GKE | Nudge placeholder sized for steps (`advertise`), PD attach and node pinning on multi-node clusters, NetworkPolicy enforcement. | +| Native steps: hardening | ValidatingAdmissionPolicy on what the runner's ServiceAccount may create; symlink-safe mount checks. | +| Native steps: RWX workspace | Steps on any node, autoscaler adds nodes for steps; needs an RWX storage class. | +| Image and chart distribution on Amazon ECR Public | Move image and chart distribution to Amazon ECR Public (AWS-native, OIDC role, public.ecr.aws alias); GHCR is the beta choice. | +| Scaler doc for operators | Metrics reference, Event reasons, alert suggestions (StuckQueue, RunnerNotConnected, SGUnreachable). | + +## Caveats (current behavior to be aware of) + +- **GKE's cluster autoscaler never adds a node for a runner pod** (its PVCs, + new or bound, make the template node fail `NodeAffinity`; a plain pod of + the same size scales up). Verified on GKE 1.36 with throwaway pods. GKE + refuses the `topology.gke.io/zone` node label that would fix the template. + The scaler's nudge (placeholder Job without volumes) works around it, + verified on GKE for a woken parked runner (about 70 s from nudge to + scheduled) and a new one. The placeholder is released at + `TriggeredScaleUp`, so the runner takes the node as soon as it joins. + + +- **`helm uninstall` waits for running workflows** (the hook parks runners + first). Helm's default `--timeout` of 5m can expire first; the Job keeps + going, and running `helm uninstall` again restarts it. +- **A runner already deregistered elsewhere** (SG UI) but with its PVC still + there makes the uninstall hook fail on that runner; delete its PVCs by hand + and uninstall again. Not tested against real SG yet. +- **`helm uninstall --no-hooks`**, or deleting the namespace first, skips the + cleanup: runners stay registered with nothing left to deregister them. + +- **Privileged pod.** `dockerd` runs privileged; clusters enforcing + restricted Pod Security reject it. The namespace needs + `pod-security.kubernetes.io/enforce=privileged`; the chart doesn't create it. +- **Workflows larger than a runner never run.** The scaler emits + `StuckQueue` and stops reacting to that queue instead of waking every + runner, but it can't fix it. +- **Single-runner restarts pause new workflows** for ~10s (see above). + `podDisruptionBudget.enabled` protects against node drains only. +- **Parked runners count against `MaxNumberOfRunners`** and show in SG's UI + as disconnected and `DRAINING`. +- **Credentials in pods.** The node token is mounted into `ecs-agent` (status + changes) and the scaler. The scaler reads it once; restart after rotating. +- **A scaler-drained runner later drained from the SG UI** while the + scaler's mark is on its PVC gets reactivated on wake by mistake. +- **No per-task CPU/memory cgroups** (`ECS_ENABLE_TASK_CPU_MEM_LIMIT=false`); + the `dockerd` container's limits cap all workflows together. Step memory is + still hard-limited by the run controller; step CPU is only a weight. +- **Workspaces are scratch** (`emptyDir`, 20Gi default limit): a run doesn't + survive its pod, and exceeding the limit evicts the pod. +- **`ecs-init` is bypassed**: its agent defaults are copied into `runner.sh` + and may drift with new ECS agent releases. Agent and SSM versions are + pinned in `Dockerfile`. +- **Pods end in `Error`** after a clean drain (non-zero exit on SIGTERM); + cosmetic. +- **SSM Run Command** would land in the `ssm-agent` container, not on a host. + Unknown whether SG uses it. +- **CI's `k8s.yml` is path-filtered**: if made a required check, PRs that + don't touch `k8s/**` or `main.sh` leave it pending. diff --git a/k8s/docs/steps-api.md b/k8s/docs/steps-api.md new file mode 100644 index 0000000..5a4e0a1 --- /dev/null +++ b/k8s/docs/steps-api.md @@ -0,0 +1,302 @@ +# Step broker API (v1) + +Contract between the run controller (`sg-run-controller`, +`external_runner/step_broker.py`) and the step broker (`sg-runner-steps`, a +native sidecar of the runner pod). The broker runs each workflow step as a +pod; the controller only asks it to. The runner runs steps only this way: the +broker's [guard](#guard) keeps dockerd from running them. See +[native-steps.md](native-steps.md) for why. + +## Transport and discovery + +- HTTP/1.1 with JSON bodies over a unix socket. No authentication: only + processes that can reach the socket file can call it (run controllers, which + mount the workspace root; steps mount only their run's directory). +- The broker listens on `SOCKET_PATH` (in its container + `/run/sg-steps/steps.sock`). The same emptyDir is mounted in the `dockerd` + container at `/var/tmp/workspace/.sg-runner`, so run controllers see it at + **`${WORKSPACES_DIR}/.sg-runner/steps.sock`** + (`/var/tmp/workspace/.sg-runner/steps.sock`). +- The broker removes a stale socket file before listening and makes the + socket mode 0666. +- The controller uses the broker when that path is a socket + (`stat.S_ISSOCK`), decided once per run at the first step. It then calls + `GET /healthz`; any failure or an `api` other than `v1` fails the step. A + controller that uses the broker never falls back to Docker; one without + broker support is stopped by the [guard](#guard). + +## Errors + +Every non-2xx response has a JSON body: + +```json +{"reason": "InvalidMount", "message": "mount source /var/tmp/workspace is outside the run's workspace"} +``` + +| Status | Reasons | Meaning | +|---|---|---| +| 400 | `InvalidRequest`, `InvalidMount`, `InvalidEnv`, `EnvTooLarge` | Fix the request; retrying won't help | +| 404 | `NotFound` | No such step (never existed, or its pod is gone) | +| 500 | `KubernetesError` | The cluster API call failed; the message says which | + +## Endpoints + +### `GET /healthz` + +`200 {"api": "v1"}`. + +### `POST /v1/steps` + +Creates the step's Secrets and pod. Returns once the pod object exists, not +when it runs. + +```json +{ + "name": "example-org-wfgrps-x-wfs-y-wfruns-z-plan-0-1a2b", + "org": "/orgs/example-org", + "run": "/wfgrps/x/wfs/y/wfruns/z", + "stepIdx": 0, + "image": "registry.example.com/team/image:1.2", + "args": ["terraform", "plan"], + "env": {"SG_WORKFLOW_RUN_PAYLOAD": "…", "AWS_SHARED_CREDENTIALS_FILE": "…"}, + "mounts": [ + {"source": "/var/tmp/workspace/orgs/example-org/wfgrps/x/wfs/y/wfruns/z/shared-workspace", + "target": "/mnt/sg_workspace", "readOnly": false} + ], + "cpuShares": 512, + "memoryMiB": 1024, + "registryAuth": {"registry": "registry.example.com", "username": "u", "password": "p"}, + "timeoutSeconds": 2100 +} +``` + +| Field | Rules | +|---|---| +| `name` | Docker-style container name the controller built; stored as an annotation only | +| `org`, `run` | The controller's `orgId` and `workflowRunId` (the Docker labels `orgId` / `wfrunId`). Required. The run's workspace is `WORKSPACES_DIR + org + run` | +| `stepIdx` | Integer, for labels and names | +| `image` | Required. Pull policy `Always` when the tag is `latest` or absent, else `IfNotPresent` | +| `args` | Optional. Overrides the image's CMD and keeps its ENTRYPOINT (Docker's `command=`). A wrapped step gets it after the ENTRYPOINT in the container's `command`, an unwrapped one in its `args` (see [Step wrapper](#step-wrapper)); either way `$` is escaped as `$$`, so the kubelet never expands `$(VAR)`. `null`/absent/empty = the image's CMD | +| `env` | Map. Keys must match `[-._a-zA-Z0-9]+` (`InvalidEnv`); total size of keys and values ≤ 1,000,000 bytes (`EnvTooLarge`). Goes into a Secret, loaded with `envFrom` | +| `mounts` | Each `source` must be strictly under the run's workspace after path cleaning (no `..`, not the workspace itself, not the workspace root): `InvalidMount` otherwise. Becomes a `subPath` of the runner's workspace claim at `target` | +| `cpuShares` | Docker CPU shares; CPU **request** = `cpuShares/1024` cores (millicores rounded up, minimum 1m). No CPU limit | +| `memoryMiB` | Memory request = limit | +| `registryAuth` | Optional. Becomes a `kubernetes.io/dockerconfigjson` pull Secret for `registry` (a host, e.g. `123.dkr.ecr.eu-central-1.amazonaws.com`, or `https://index.docker.io/v1/` for Docker Hub) | +| `timeoutSeconds` | The step's own timeout. The pod's `activeDeadlineSeconds` = `timeoutSeconds + START_TIMEOUT + 300` bounds steps whose controller is gone | + +`201 {"id": "sg-runner-0-s-7f3k9q2x"}`. The id is the pod name. + +### `GET /v1/steps/{id}` + +```json +{"id": "sg-runner-0-s-7f3k9q2x", "state": "exited", "exitCode": 11, + "reason": "Completed", "message": "", "startedAt": "2026-10-14T10:00:00Z", + "finishedAt": "2026-10-14T10:05:00Z"} +``` + +| `state` | When | `exitCode` | +|---|---|---| +| `pending` | Scheduling, pulling, creating the container | `null` | +| `running` | Container running | `null` | +| `exited` | Container terminated (any code, including OOMKilled = 137 with reason `OOMKilled`; for a wrapped step 11 and 12 as the step exited, see [Step wrapper](#step-wrapper)) | int | +| `failed` | The step never ran or was lost; `reason` says why (below) | `null` | + +`failed` reasons: `Unschedulable` (longer than `SCHEDULING_TIMEOUT`), +`ImagePullBackOff`, `ErrImagePull` (not found / unauthorized only; otherwise +it stays `pending` until BackOff), `InvalidImageName`, `ErrImageNeverPull`, +`StartTimeout` (pending longer than `START_TIMEOUT`), `Evicted`, +`DeadlineExceeded`, `PodDeleted` (pod deleted without a stop). +`CreateContainerConfigError` is transient (`pending`) until `START_TIMEOUT`. + +A terminal state (`exited` or `failed`) is the step's for good once the broker +saw it (on a GET, a stop or its sweep): later GETs return it from memory, also +after the pod is gone, for 1 hour after the pod's deletion. The pod itself +stays `FINISHED_TTL` (1h) after it ended, its logs readable, labeled with its +outcome; its Secrets go as soon as it's terminal (see +[Retention](#retention-and-garbage-collection)). A stopped step whose pod is +gone before the broker saw it end returns the last state the broker saw +(`exited`, its code or 137). + +The broker keeps all this in memory: after a broker restart a step whose pod +is gone is `404`. A client that already got the terminal state keeps it; one +that asked for a stop treats `404` as `exited` with 137; any other treats it +as `failed`/`PodDeleted`. + +### `GET /v1/steps/{id}/logs?follow=1[&sinceTime=]` + +`200`, `Content-Type: text/plain`, chunked. Blocks until the container has +started or the step has finished; a step that `failed` before starting ends the +stream with no lines. Each line is ` \n` (the +kubelet's `timestamps=true` format); the client strips the timestamp and keeps +the last one. The stream ends when the container terminates. A stream that +ends while the step is still `running` was broken: reconnect with +`sinceTime=` (lines with exactly that timestamp may repeat; +the client drops a repeated first line). + +### `POST /v1/steps/{id}/stop` + +Body `{"graceSeconds": 30}` (0 = kill now). Records the stop, then deletes the +pod with that grace period: SIGTERM, SIGKILL after the grace. `202`, or `404`. + +### `DELETE /v1/steps/{id}` + +Best effort cleanup of a step: deletes the pod and its Secrets. `204` (also +when already gone). A finished step's pod goes anyway `FINISHED_TTL` after it +ended (see [Retention](#retention-and-garbage-collection)). + +## What the broker creates per step + +- Secret `-env` (the env), and `-pull` when `registryAuth` is set, + created before the pod and owned by the runner pod. +- Pod `` (`-s-<8 random [a-z0-9]>`), owned by the runner pod + (UID from the downward API, `blockOwnerDeletion` unset), labels from + `STEP_LABELS` plus `app.kubernetes.io/component: step`, + `sg-runner.stackguardian.io/runner: ` and + `sg-runner.stackguardian.io/run: `, and + once terminal `sg-runner.stackguardian.io/outcome` (below); annotations with + the raw `org`, `run`, `stepIdx` (`sg-runner.stackguardian.io/step-idx`) and + `name`, and `sg-runner.stackguardian.io/wrapped: "true"` or `"false"` (then + with `sg-runner.stackguardian.io/not-wrapped-reason`). +- Pod spec: container `step`; `restartPolicy: Never`; + `terminationGracePeriodSeconds: 30`; `automountServiceAccountToken: false`; + `enableServiceLinks: false`; `seccompProfile: RuntimeDefault`; + `privileged: false`; required node affinity `matchFields metadata.name = + NODE_NAME`; `STEP_NODE_SELECTOR` / `STEP_TOLERATIONS`; image pull secrets + `-pull` when present; volume `workspace` = PVC `WORKSPACE_CLAIM`, one + `volumeMount` per mount with `subPath = WORKSPACE_SUBPATH_PREFIX + + (source relative to WORKSPACES_DIR)`; termination message + `/dev/termination-log` (policy `File`). +- A wrapped step also gets the init container `sg-wrapper-install` (image + `BROKER_IMAGE`, pull policy `BROKER_IMAGE_PULL_POLICY`, command + `/sg-step-wrapper install /sg-wrapper/sg-step-wrapper`, UID/GID 65532, + non-root, read-only root filesystem, no privilege escalation, all + capabilities dropped, requests 10m CPU / 16Mi memory, limit 64Mi), the + emptyDir `sg-wrapper` (64Mi) mounted read-write there and read-only at + `/sg-wrapper` in `step`, and `BROKER_IMAGE_PULL_SECRETS` among its image + pull secrets. + +## Step wrapper + +A pod whose container exits non-zero ends `Failed`, but StackGuardian steps +exit 11 (approval required) and 12 (finished early) on purpose. So steps run +under `sg-step-wrapper`, an init process (like `tini` / `docker run --init`) +from the broker's image: + +- The broker reads the step image's config (the manifest and config blob for + its own platform, `linux/amd64` or the first linux image of an index + without it; never the layers) with the request's `registryAuth` when it is + for the image's registry, else anonymously; 20s timeout, cached 10 minutes + per image. +- The step's `command` is `["/sg-wrapper/sg-step-wrapper", "--", + , ]` (no `args`), each element with `$` escaped as `$$`. +- The wrapper runs that as its child (looked up in `PATH`), forwards SIGTERM, + SIGINT, SIGHUP, SIGQUIT, SIGUSR1 and SIGUSR2, reaps zombies, and leaves + stdio alone. A child exit of 11 or 12 writes `sg-exit-code=` to + `/dev/termination-log` and exits 0, so the pod ends `Completed`; any other + code is its own, a signal 128+n; a command it can't run is 126 or 127. +- `GET` reports a wrapped pod's exit 0 with exactly that termination message + as `exited` with 11 or 12. Nothing else is read from the message. + +A step runs unwrapped, exactly as without the wrapper (its 11 and 12 then end +the pod `Failed`), when `BROKER_IMAGE` is unset, the lookup fails, or neither +the image nor the request has anything to run. The broker logs a warning and +annotates the pod `wrapped: "false"` with the reason (no Event: the broker +can't create Events). + +## Retention and garbage collection + +A step is finished once the broker sees it terminal (on a GET, a stop's +follow-up, or its sweep every minute). It then, once: + +- labels the pod `sg-runner.stackguardian.io/outcome`: `succeeded` (exit 0), + `approval-required` (11), `completed-early` (12), `failed` (any other code, + or `failed`), `stopped` (a client stopped it, whatever the code); a pod + that has the label keeps it; +- deletes its Secrets: they hold credentials, and the container has its env. + +Every minute the broker also: + +- deletes finished step pods `FINISHED_TTL` (1h) after they ended + (`finishedAt`, else when the broker first saw them terminal), read or not, + unless a log stream is open; their final state stays for GETs 1 hour after; +- retries finishing (label, Secrets) that failed; +- deletes, by name, the Secrets of steps whose pod went without the broker + (a stop, or someone else); +- forgets steps gone for over an hour. + +It never lists or reads Secrets (that returns their data, and the runner's +node token shares the namespace): step Secrets go by name with their step, +or with the runner pod that owns them (ownerReference), like step pods. + +## Guard + +The broker also serves dockerd the authorization plugin `sg-steps-guard`, so +a run controller without broker support (an older one) can't run steps in +dockerd: they would run inside the runner pod, outside every pod limit. + +- It listens on `GUARD_SOCKET` (`/run/sg-guard/sg-steps-guard.sock`). The + chart mounts that directory into `dockerd` at `/run/docker/plugins`, where + dockerd finds the plugin by its socket name, and starts dockerd with + `--authorization-plugin=sg-steps-guard`. +- Docker plugin protocol (`application/vnd.docker.plugins.v1.2+json`): + `POST /Plugin.Activate` answers `{"Implements":["authz"]}`; + `POST /AuthZPlugin.AuthZReq` decides; `POST /AuthZPlugin.AuthZRes` always + allows. +- A request is refused (`{"Allow":false,"Msg":"This runner runs workflow + steps as Kubernetes pods and needs a run controller with native step support + (step broker client); refusing to start the step in dockerd."}`) when its + `RequestMethod` is `POST`, the path of its `RequestUri` matches + `^(/v[0-9.]+)?/containers/create$`, and its JSON body's `Labels` has the key + `wfrunId` (the run controller's label on step containers). Everything else + is allowed, the ECS agent creating run controllers (`com.amazonaws.ecs.*` + labels) included. +- A container create without a body (dockerd sends only JSON bodies, within + its size limit) or with one it can't parse is allowed, with a warning in the + broker's log. An unreadable authorization request gets `Err` (dockerd + refuses the call). +- While the guard doesn't answer, dockerd refuses every API call: the broker + starts before dockerd (a native sidecar) and stops after it. + +## RBAC + +The broker's ServiceAccount, in its namespace: pods `create`, `get`, `list`, +`delete`, `patch` (the outcome label); `pods/log` `get`; secrets `create`, +`delete`. The registry lookup needs no RBAC (it uses the request's +`registryAuth`). + +## Broker configuration (env) + +| Variable | Default | Meaning | +|---|---|---| +| `SOCKET_PATH` | `/run/sg-steps/steps.sock` | Listen socket | +| `WORKSPACES_DIR` | `/var/tmp/workspace` | The controllers' workspace root (paths in requests) | +| `WORKSPACE_CLAIM` | required | PVC holding `WORKSPACES_DIR` (the runner's ephemeral `-workspace`) | +| `WORKSPACE_SUBPATH_PREFIX` | empty | Prefix inside the claim (for a shared RWX claim later) | +| `RUNNER_POD_NAME`, `RUNNER_POD_UID`, `NODE_NAME`, `NAMESPACE` | required | Downward API | +| `STEP_LABELS` | `{}` | JSON map of labels for step pods (chart selector labels) | +| `STEP_NODE_SELECTOR` | `{}` | JSON map | +| `STEP_TOLERATIONS` | `[]` | JSON list of tolerations | +| `STEP_IMAGE_PULL_POLICY` | empty | `Always`, `IfNotPresent` or `Never` for every step pod (e.g. images preloaded on the node, air-gapped clusters); empty = `Always` for a `latest` or absent tag, else `IfNotPresent` | +| `SCHEDULING_TIMEOUT` | `5m` | Go duration | +| `START_TIMEOUT` | `30m` | Go duration | +| `FINISHED_TTL` | `1h` | Go duration: how long a finished step's pod stays after it ended | +| `BROKER_IMAGE` | empty | The broker's own image (has `/sg-step-wrapper`) for the wrapper's init container; empty = steps run unwrapped | +| `BROKER_IMAGE_PULL_POLICY` | empty | `Always`, `IfNotPresent` or `Never` for that init container; empty = the cluster's default | +| `BROKER_IMAGE_PULL_SECRETS` | empty | Comma-separated pull Secrets for `BROKER_IMAGE`, added to wrapped step pods | +| `GUARD_SOCKET` | `/run/sg-guard/sg-steps-guard.sock` | The guard's socket; its directory is dockerd's `/run/docker/plugins`. Must differ from `SOCKET_PATH` | + +## Lifecycle in the runner pod + +The broker is a native sidecar: an init container with `restartPolicy: +Always`. Kubernetes starts it, and waits for its startup probe, before the +runner's containers (dockerd needs the guard from its first API call, run +controllers the broker), and stops it only after they all exited. So on +SIGTERM it just shuts down: it stops listening and gives open requests up to +10s. + +- `sg-runner-steps serve` (the default) runs the broker. +- `sg-runner-steps healthcheck` is its startup and readiness probe (the image + has no shell): exit 0 when `GET /healthz` on `SOCKET_PATH` answers `api` + `v1` and `POST /Plugin.Activate` on `GUARD_SOCKET` answers `authz`, else + 1. It reads only those two variables. diff --git a/k8s/docs/testing.md b/k8s/docs/testing.md new file mode 100644 index 0000000..703ddd1 --- /dev/null +++ b/k8s/docs/testing.md @@ -0,0 +1,231 @@ +# Testing: verified, not verified, and how to test + +## Verified + +| What | How | Result | +|---|---|---| +| Runner pod registers and runs workflows | QA, OrbStack Kubernetes (arm64) | Registration, SSM activation, ECS agent connects, run controller + steps run nested; amd64 workflow images run on arm64 via emulation | +| Sizing | QA, ECS `describe-container-instances` | Advertised 2048 CPU units / 3840 MiB for a 2 vCPU / 4Gi `dockerd` limit (was the whole node: 12288 / 12015) | +| Wake / fresh / park timings | QA, `kubectl scale` | Wake **8s** (same container instance), fresh registration 20s (+ ~10s image pull on first workflow), park 14s (idle) | +| Drain through SG (`runner_status`) | QA, ECS status observed | `ACTIVE` → `DRAINING` on shutdown → `ACTIVE` after restart; node token accepted and really changes ECS state | +| External drain survives restart | Unit test (stubs) | Already-`DRAINING` runner stays drained after a restart | +| Shell scaler: scale out, park, reap | QA with real workflows | 1→3 under load, parked to 1 after idle, reaped after TTL; SG left clean | +| Stuck queue, settle timeout, headroom | Simulation of the shell scaler; Go unit tests | Oversized workflow → no scale-out, warning; broken runner ignored after 300s; headroom 0 → 2 runners, 20% → 3 | +| Go scaler | `go test -race`, fake clientset, fake SG; OrbStack smoke with pause pods | Coverage `decide` 97%; leader failover keeps timers; back-off on SG outage; no token in logs/errors/Events | +| Chart | `helm lint --strict`, kubeconform (k8s 1.37), schema rejections, server-side dry run, kind install | Helm 3.22 and 4.3 | +| Docker prune | Privileged container with short timers | Fresh containers survive, old ones removed, running containers and their images kept | +| Recursion fix | bats (counts curl calls), runner image against an unreachable API | One attempt + one report, exit 1 in <1s | +| `make nuke` with parked runners (old Makefile flow, before the uninstall hook) | QA | Woke 2 parked runners, deregistered all 3, no SSM/ECS leftovers | +| Workflow logs sidecar | Runner image with nested dockerd, look-alike containers; then a real workflow on GKE (QA) | Concurrent runs tagged apart, stderr included, no replay after a sidecar restart, restarts not duplicated, JSON mode, exits at once on SIGTERM. On QA: run controllers carry `com.amazonaws.ecs.task-arn`, steps mount `.../wfruns//...`, and the run controller relays every step line (hence steps only) | +| `helm uninstall` hook | Go unit tests (fake clientset); kind smoke with a fake API | Stops the scaler, parks, reaps every state PVC (never-registered ones succeed), deletes PVCs and Jobs; a failed reap keeps that runner's PVCs and fails the hook | +| Native steps: broker on a real cluster | OrbStack, `curl` over the socket from `dockerd` | Exit codes 0/1/11/12, `$(VAR)` stays literal, workspace writes visible, mounts outside the run rejected, bad image fails at once, Unschedulable after the timeout, stop keeps the step's exit code (143), 503,329/503,329 log lines at ~1 MiB/s across rotations, runner SA can't read Secrets or exec | +| Native steps: controller client | OrbStack, client inside the controller image in nested `dockerd`, real broker | Both observation paths (exited / wait), failing exit, bad image → `AdminJobException`, cancel via `stop_container` (graceful and SIGKILL), `DockerLogger` relays step output | +| Native-only runner + guard | QA, OrbStack (chart 0.9.0) | Broker as native sidecar, `dockerd` started with `--authorization-plugin=sg-steps-guard`; overlaid controller: step pod Completed / `completed-early`, run COMPLETED; SG's official controller: step create refused with the guard's message (403), run ERRORED, nothing in `dockerd`, no step pod | +| Native steps end to end | QA, OrbStack, QA controller image + broker client overlaid ([plan 8](#8-native-steps-end-to-end-runbook)) | Scenarios 1–9, 11, 12, 14 pass: plan/apply, approval + resume, exit 12 (output of a 1s step kept), exit 3, cancel, step timeout, bad image, command override/env/workspace, two concurrent runs isolated, drain mid-step, step pods gone with their runner pod, nothing runs in `dockerd`. Scenario 15 (step wrapper): approval and exit-12 step pods `Completed`, outcome `approval-required` / `completed-early`, wrapped (image config read from SG's private ECR with the controller's login), Secrets deleted, pods kept | + +## Not verified + +- Go scaler against real SG (all of it). +- Scaler nudge's release at `TriggeredScaleUp` (chart 0.7.0) on GKE; the + earlier release-when-running version was verified there. +- `helm uninstall` hook against real SG: deregistering awake and parked + runners, a runner with a running workflow (waits, then deregisters), a + runner already deregistered from the SG UI (expected: the reap fails, the + hook fails and keeps its PVCs). +- Waking after a long park (hours/days). +- Hard kill → scaler drain → reactivate, against real SG/ECS. +- Whether SG resumes workflows it paused while no runner was `ACTIVE`. +- Queue wait after a wake (SG retry interval). +- Whether SG ever lists two container instances for one runner pod (the Go + scaler tolerates it). +- NetworkPolicy enforcement. +- Unprivileged operation on a real cluster. +- CI on GitHub. +- Multi-node clusters: `local-path`-style node-local PVCs pin a runner to its + node; a lost node keeps the StatefulSet pod `Terminating` until the node is + removed. +- Native steps: drift and policy-discovery runs (separate workspace dirs); + NetworkPolicy for step pods; persistent-disk attach and node pinning on a + multi-node cluster; node autoscaling sized for steps (the nudge copies only + the runner's requests). +- Native steps with the real (released) run controller: tested only with the + broker client overlaid on the QA image. + +## Test plans + +### 1. End-to-end: chart + Go scaler on QA + +```sh +cd k8s && make build build-scaler && make secret up +helm --kube-context orbstack -n sg-runner upgrade sg-runner chart/sg-runner --reuse-values \ + --set scaler.idleSeconds=30 --set scaler.parkTTL=90 +kubectl -n sg-runner logs -f deploy/sg-runner-scaler | jq -c 'select(.msg != "loop")' +kubectl -n sg-runner get events -w --field-selector involvedObject.name=sg-runner +kubectl -n sg-runner port-forward deploy/sg-runner-scaler 9090 & +curl -s localhost:9090/metrics | grep '^sg_runner_scaler' +``` + +Trigger 2–3 workflows. Expect: `ScaledOut` without an extra idle runner +(headroom 0), `Parked` 30s after idle, `Woke` on the next workflow (~8s), +`Reaped` 90s after parking. End with `helm uninstall` (with a runner parked +and one awake) and check SG lists no runners for the group and no PVCs are +left. + +### 2. Hard kill (3.2) + +With at least two runners awake and idle: + +```sh +kubectl -n sg-runner scale sts sg-runner --replicas=1 +kubectl -n sg-runner delete pod sg-runner-1 --grace-period=0 --force +kubectl -n sg-runner get pvc state-sg-runner-1 -o jsonpath='{.metadata.annotations}' +``` + +Expect a `DrainedParkedRunner` event and the +`sg-runner.stackguardian.io/drained-by-scaler` annotation. Trigger a workflow +so it wakes: `Woke`, then `Reactivated`, annotation gone, runner `ACTIVE` in +SG. + +### 3. Long parking + +Default timers (park TTL 12h). Let the scaler park a runner, leave it +overnight, then trigger workflows so it wakes. Check: the register init +container says "Already registered", the ECS agent restores from its +checkpoint and connects, the runner is `ACTIVE`, a workflow runs on it. If it +fails, the park TTL must stay below the point where reconnecting breaks. + +### 4. Restart window + +With one runner: `kubectl -n sg-runner delete pod sg-runner-0` and submit a +workflow during the ~10s gap. Does SG resume it once the runner reconnects, +or does it stay paused? Decides between `scaler.min=2`, a PDB only, or +scaler-managed rollouts. + +### 5. Queue latency and overshoot + +Trigger N workflows that each fill a runner. Measure from submission to +placement on a freshly woken runner, and check the scaler never wakes more +runners than queued work needs. Repeat with `scaler.headroomPct=20`. + +### 6. NetworkPolicy + +On a cluster with an enforcing CNI, `networkPolicy.enabled=true` with the +cluster's pod/service/node CIDRs. From a workflow step: the Kubernetes API, +another pod and `169.254.169.254` must be unreachable; SG, AWS, VCS and +registries reachable. + +### 7. Unprivileged `dockerd` + +Real cluster with containerd 2.x or CRI-O. Try `hostUsers: false` with +`procMount: Unmasked` (and Sysbox if installable). The OrbStack spike's +checklist: dockerd starts, nested container with egress, `sg-net` + iptables +rules, `route_localnet`, the 169.254.170.2 → 127.0.0.1:51679 DNAT, a +container mounting docker.sock. + +### 8. Native steps end to end (runbook) + +Steps run as pods through the step broker +([native-steps.md](native-steps.md), [steps-api.md](steps-api.md)). Until SG +ships a run controller with the broker client, it is tested by overlaying the +client onto the official controller image inside the runner's `dockerd`. + +**Setup** (OrbStack, a throwaway runner group in `.env`, nothing else +registered in it): + +1. Values file (OrbStack): + + ```yaml + runner: + size: custom + advertise: { cpu: 4, memory: 6144Mi } + controllerResources: + limits: { cpu: "2", memory: 2Gi } + steps: + schedulingTimeout: 2m + startTimeout: 5m + step: + imagePullPolicy: IfNotPresent # arm64 only: preloaded amd64 images + ecsAgent: + extraEnv: + - { name: ECS_IMAGE_PULL_BEHAVIOR, value: prefer-cached } + - { name: ECS_DISABLE_IMAGE_CLEANUP, value: "true" } + dockerd: + extraEnv: + - { name: SG_PRUNE_INTERVAL_SECONDS, value: "0" } + ``` + +2. `make build build-scaler && make secret up HELM_ARGS="-f native.yaml"`. + A runner registered with another size registers again on its own when + idle (`runner.reregisterOnResize`). +3. Run one workflow so the official controller image is pulled into `dockerd` + (its `.Config.Image` must be a tag, not a digest). Its steps are refused by + the guard (scenario 16), which is expected. Then overlay: copy + `src/run_controller` out of the image, add `external_runner/step_broker.py` + and the one-line `step_client(...)` change to its `external.py`, and + commit it under the same tag: + + ```sh + IMG= + kubectl -n sg-runner exec sg-runner-0 -c dockerd -- docker tag $IMG $IMG-orig + tar -C -cf - run_controller | kubectl -n sg-runner exec -i sg-runner-0 -c dockerd -- \ + sh -c "c=\$(docker create $IMG-orig) && docker cp - \$c:/opt/stackguardian/src/ && docker commit \$c $IMG && docker rm \$c" + ``` + + Patch the image's own code, not sg-run-controller's main: the deployed + controller may differ from main. The overlay lives on the `docker` PVC and + survives pod restarts, not `make nuke` (`docker save` it first). +4. On arm64 (OrbStack), SG's step images are amd64-only: the kubelet refuses + to pull them, but runs them under emulation when they are already local. + Copy each one from `dockerd` into the host image store after a first run: + `kubectl exec ... -c dockerd -- docker save | docker load`. +5. Two workflows on the runner group: the Terraform one, and the same with a + custom pre-plan step using public `alpine:3.20` with a command override + (multi-arch, nothing to preload). + +**Watch** during every run: + +```sh +kubectl -n sg-runner get pods -l app.kubernetes.io/component=step -L sg-runner.stackguardian.io/outcome -w +kubectl -n sg-runner logs sg-runner-0 -c steps -f +# must stay empty: steps never fall back to dockerd +kubectl -n sg-runner exec sg-runner-0 -c dockerd -- docker ps -a --filter label=wfrunId +``` + +**Scenarios** + +| # | Do | Expect | +|---|---|---| +| 1 | Terraform plan + apply | Each step a pod on the runner's node; run log in the SG UI; COMPLETED | +| 2 | Plan → approval → approve | Plan pod exits 11, run `APPROVAL_REQUIRED`; a new controller resumes (cache restored into the workspace PVC) and runs apply | +| 3 | Alpine: `sh -c 'echo hi; exit 12'` | COMPLETED, later steps skipped, `hi` in the run log even for a 1s step | +| 4 | Alpine: `sh -c 'echo boom >&2; exit 3'` | ERRORED, "failed with exit code: 3", `boom` in the log | +| 5 | Cancel in the UI during a step | Step pod deleted with a 30s grace, run CANCELLED (not ERRORED) | +| 6 | Alpine: `sleep 600`, step timeout 1m | "exceeded timeout of 1.0m", pod stopped, ERRORED, Terraform not run | +| 7 | Alpine image `alpine:does-not-exist` | ERRORED within seconds: "failed to run: ErrImagePull … manifest unknown" | +| 8 | Alpine: `sh -c 'echo "payload=${#SG_WORKFLOW_RUN_PAYLOAD}"; ls /mnt/sg_workspace; id'` | Payload length > 0, workspace listing (`user`, `artifacts`, …), image's own user; COMPLETED | +| 9 | Two **different** workflows at once (SG queues runs of one workflow) | Two controllers and two step pods at once; in one pod, `/var/tmp/workspace` holds only its own run's paths | +| 10 | Drift or policy discovery run | `shared-workspace-drift` / `-discovery` mounts work (not run yet) | +| 11 | `kubectl delete pod sg-runner-0` while a step runs | The broker stays up through the drain, the step finishes in its pod, the run completes, then the runner restarts | +| 12 | Runner pod deleted | Its step pods are deleted with it (ownerReference) | +| 14 | Every run | `docker ps -a --filter label=wfrunId` in `dockerd` stays empty | +| 16 | Any run with the official controller (no broker client) | The step fails at once: "This runner runs workflow steps as Kubernetes pods and needs a run controller with native step support"; nothing runs in `dockerd` | +| 15 | Approval (exit 11) and exit 12 runs | Step pods `Completed`, outcome `approval-required` / `completed-early`, annotation `wrapped: "true"`; env Secret gone, pod kept for `finishedTTL` | + +**Known behaviour**: exit 11 and 12 end `Completed` through the step wrapper +(see native-steps.md); a step whose image config the broker can't read runs +unwrapped and shows `Error` for them. A step writing tens of MiB within a +second can lose lines to kubelet log rotation (raise `containerLogMaxSize`). + +## Automated checks + +```sh +make lint test # shellcheck + bats (176 tests) +(cd k8s/scaler && go vet ./... && go test -race ./...) +helm lint --strict k8s/chart/sg-runner --set credentials.existingSecret=x +.github/scripts/chart-check.sh # needs kubeconform on PATH +.github/scripts/kind-smoke.sh run # kind cluster, fake API; uses its own kubeconfig; includes the native-steps broker stage +CONTROLLER_REPO= .github/scripts/kind-smoke.sh run # + the controller client's live test (unit/test_step_broker_live.py) in the broker test pod +``` + +Trivy (local, last run): the scaler image has 0 CRITICAL; the runner image's +fixable CRITICALs are all in the bundled Docker 27.5.1 binaries. diff --git a/k8s/runner.sh b/k8s/runner.sh new file mode 100755 index 0000000..76c8a43 --- /dev/null +++ b/k8s/runner.sh @@ -0,0 +1,660 @@ +#!/bin/bash +# +# Kubernetes entrypoint for the sg-runner pod (prototype). +# +# One pod plays the part of one registered runner host. Each subcommand backs +# one container of the StatefulSet, standing in for what systemd and +# ecs-anywhere-install.sh do on a VM: +# +# register init container: SG API registration + SSM hybrid activation +# dockerd privileged: nested Docker daemon for the run controllers + +# main.sh network setup; guarded by the steps broker's +# authorization plugin +# ssm-agent amazon-ssm-agent; shares rotating creds via /root/.aws +# ecs-agent ECS agent binary, configured from ecs.config (replaces ecs-init) +# drain ecs-agent preStop: DRAINING, wait for workflows, park (keep registration) +# wait-drained dockerd/ssm-agent preStop: outlive the drain +# deregister run via `kubectl exec -i`; reads SG credentials from stdin +# reap scaler Job: deregister a runner parked too long, from its PVC +# +# main.sh is sourced for the API, config and network helpers; its +# BASH_SOURCE guard keeps main() from running. + +set -o pipefail + +# Next to this script in the image (/opt/sg-runner), one level up in the repo +# (k8s/runner.sh, main.sh), where the tests source it from. +RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -r "$RUNNER_DIR/main.sh" ]]; then + # shellcheck source-path=SCRIPTDIR/.. source=main.sh + source "$RUNNER_DIR/main.sh" +else + # shellcheck source-path=SCRIPTDIR/.. source=main.sh + source "$RUNNER_DIR/../main.sh" +fi + +readonly SSM_DIR="/var/lib/amazon/ssm" +readonly SSM_REGISTRATION_FILE="${SSM_DIR}/Vault/Store/RegistrationKey" +readonly AGENT_CREDS_FILE="/rotatingcreds/credentials" +readonly SG_CRED_KEYS="SG_NODE_TOKEN SG_ORGANIZATION SG_RUNNER_GROUP SG_BASE_API" +readonly SG_CREDS_DIR="/etc/sg-runner/credentials" +# On the emptyDir shared by every container: tells the others the drain is over. +readonly DRAINED_MARKER="/var/run/sg-runner-drained" +# On the state PVC: this pod drained the runner when it parked, so the next +# start may set it ACTIVE again. Absent when someone else drained it on purpose. +readonly SELF_DRAIN_MARKER="$ECS_CONFIG_DIR/.parked" +readonly AGENT_API="http://localhost:51678/v1" + +# Memory kept back for dockerd/containerd themselves, out of the dockerd +# container's limit, when sizing what the agent advertises. +SG_DOCKERD_OVERHEAD_MIB="${SG_DOCKERD_OVERHEAD_MIB:-256}" + +# Nested Docker cleanup (prune_loop), in place of the VM's `main.sh prune` +# cron. Every interval (0 = off), remove stopped containers, dangling images +# and build cache older than SG_PRUNE_UNTIL, and unused volumes. Unused images +# are the wake-up cache, so they only go once the Docker volume is +# SG_PRUNE_IMAGES_ABOVE_PCT full (0 = every run). +SG_PRUNE_INTERVAL_SECONDS="${SG_PRUNE_INTERVAL_SECONDS:-14400}" +SG_PRUNE_UNTIL="${SG_PRUNE_UNTIL:-24h}" +SG_PRUNE_IMAGES_ABOVE_PCT="${SG_PRUNE_IMAGES_ABOVE_PCT:-70}" +DOCKER_DATA_ROOT="${DOCKER_DATA_ROOT:-/var/lib/docker}" + +# dockerd's authorization plugin: the steps broker sidecar's guard. dockerd +# never starts without it (fail closed): it waits this long for the socket, +# which the sidecar creates before dockerd starts. +SG_AUTHZ_PLUGIN="${SG_AUTHZ_PLUGIN:-sg-steps-guard}" +SG_AUTHZ_PLUGIN_SOCKET="${SG_AUTHZ_PLUGIN_SOCKET:-/run/docker/plugins/${SG_AUTHZ_PLUGIN}.sock}" +SG_AUTHZ_PLUGIN_WAIT_SECONDS="${SG_AUTHZ_PLUGIN_WAIT_SECONDS:-60}" +SG_AUTHZ_PLUGIN_POLL_SECONDS="${SG_AUTHZ_PLUGIN_POLL_SECONDS:-2}" + +# What the agent sizing reads; overridable so tests can feed fake files. +PROC_STATUS_FILE="${PROC_STATUS_FILE:-/proc/self/status}" +MEMINFO_FILE="${MEMINFO_FILE:-/proc/meminfo}" + +CONTAINER_ORCHESTRATOR="docker" + +wait_for() { #{{{ + # Usage: wait_for + local what="$1" + shift + until "$@" >&/dev/null; do + info "Waiting for" "$what" + sleep 3 + done +} +#}}}: wait_for + +load_sg_credentials() { #{{{ + # Map the k8s-friendly names onto the globals main.sh expects. + SG_NODE_TOKEN="${SG_NODE_TOKEN:-}" + ORGANIZATION_ID="${SG_ORGANIZATION:-}" + RUNNER_GROUP_ID="${SG_RUNNER_GROUP:-}" + check_sg_args +} +#}}}: load_sg_credentials + +is_registered() { + [[ -s "$ECS_CONFIG_DIR/ecs.config" && -s "$SSM_REGISTRATION_FILE" ]] +} + +registered_resources() { #{{{ + # Prints " " of this + # runner's ECS container instance as SG reports it. The response holds + # secrets (the group's runner token): only these numbers leave here. + local runner_id + runner_id="$(configured_runner_id)" + validate_runner_id "$runner_id" >/dev/null 2>&1 || return 1 + sg_api GET "" | jq -er --arg r "$runner_id" ' + first(.msg.ContainerInstances[]? | select(.runnerId == $r)) + | [(.registeredResources[]? | select(.name == "CPU") | .integerValue), + (.registeredResources[]? | select(.name == "MEMORY") | .integerValue), + ((.runningTasksCount // 0) + (.pendingTasksCount // 0))] + | select(length == 3 and all(type == "number")) + | map(tostring) | join(" ")' 2>/dev/null +} +#}}}: registered_resources + +resized_while_idle() { #{{{ + # ECS keeps a container instance's resources from its first registration: + # an agent restarted with a new size still offers the old one. True when + # this runner's registration has another size than the agent will advertise + # and nothing runs on it, so it can register again. Never fails the + # registration: anything unknown keeps the existing one. + [[ "${SG_REREGISTER_ON_RESIZE:-true}" == "true" ]] || return 1 + local want got cpu mem tasks + if ! want="$(advertised_resources)"; then + info "Can't tell the advertised size;" "keeping the registration" + return 1 + fi + if ! got="$(registered_resources)" || [[ -z "$got" ]]; then + info "Can't read the registered size from SG;" "keeping the registration" + return 1 + fi + read -r cpu mem tasks <<<"$got" + [[ "$cpu $mem" == "$want" ]] && return 1 + if ((tasks > 0)); then + info "Registered CPU/memory $cpu/$mem, now ${want/ //}:" "$tasks task(s) running or pending, keeping the old size" + return 1 + fi + info "Registered CPU/memory $cpu/$mem, now ${want/ //}:" "re-registering" +} +#}}}: resized_while_idle + +cmd_register() { #{{{ + if is_registered; then + if ! resized_while_idle; then + info "Already registered, reusing state from" "$ECS_CONFIG_DIR/ecs.config" + return 0 + fi + # As cmd_reap: deregister through SG and wipe the identity. If that + # fails, keep the registration rather than block the runner. + if ! (deregister_runner); then + info "Could not deregister;" "keeping the old registration" + return 0 + fi + fi + if [[ -s "$ECS_CONFIG_DIR/ecs.config" || -s "$SSM_REGISTRATION_FILE" ]]; then + die "Partial registration state found." "Deregister and wipe the PVC before retrying." + fi + + load_sg_credentials + fetch_organization_info + + info "Registering SSM agent in" "$LOCAL_AWS_DEFAULT_REGION" + if ! amazon-ssm-agent -register -y \ + -code "$SSM_ACTIVATION_CODE" \ + -id "$SSM_ACTIVATION_ID" \ + -region "$LOCAL_AWS_DEFAULT_REGION" >>"$LOG_FILE" 2>&1; then + log_err + die "SSM registration failed." "See $LOG_FILE" + fi + info "SSM managed instance" "$(jq -r '.instanceID' "$SSM_REGISTRATION_FILE")" + + # Written last: its presence marks a complete registration. + configure_local_data + save_registration_details +} +#}}}: cmd_register + +cmd_dockerd() { #{{{ + # cgroup v2 nesting, as in moby's hack/dind: move our processes out of the + # root cgroup so controllers can be delegated to the inner containers. + if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then + mkdir -p /sys/fs/cgroup/init + xargs -rn1 /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || : + sed -e 's/ / +/g' -e 's/^/+/' /sys/fs/cgroup/cgroup.subtree_control + fi + + wait_for_authz_plugin + local args=() + mapfile -t args < <(dockerd_args) + dockerd "${args[@]}" & + local dockerd_pid=$! prune_pid="" + trap 'kill -TERM $prune_pid "$dockerd_pid" 2>/dev/null; wait "$dockerd_pid"' TERM INT + + wait_for "dockerd" docker info + # Same bridge, IMDS block and task-credential DNAT as on a VM. Rules land in + # the pod's network namespace, not the node's. + configure_local_network + info "Docker ready, network" "$SG_DOCKER_NETWORK" + + if prune_enabled; then + prune_loop & + prune_pid=$! + else + info "Docker prune disabled:" "SG_PRUNE_INTERVAL_SECONDS=$SG_PRUNE_INTERVAL_SECONDS" + fi + + wait "$dockerd_pid" +} +#}}}: cmd_dockerd + +wait_for_authz_plugin() { #{{{ + local deadline=$((SECONDS + SG_AUTHZ_PLUGIN_WAIT_SECONDS)) + until [[ -S "$SG_AUTHZ_PLUGIN_SOCKET" ]]; do + if ((SECONDS >= deadline)); then + die "No authorization plugin:" "$SG_AUTHZ_PLUGIN_SOCKET missing after ${SG_AUTHZ_PLUGIN_WAIT_SECONDS}s, not starting dockerd unguarded" + fi + info "Waiting for" "authorization plugin $SG_AUTHZ_PLUGIN_SOCKET" + sleep "$SG_AUTHZ_PLUGIN_POLL_SECONDS" + done + info "Authorization plugin" "$SG_AUTHZ_PLUGIN ($SG_AUTHZ_PLUGIN_SOCKET)" +} +#}}}: wait_for_authz_plugin + +dockerd_args() { #{{{ + # One per line. + printf '%s\n' --host=unix:///var/run/docker.sock "--authorization-plugin=$SG_AUTHZ_PLUGIN" +} +#}}}: dockerd_args + +prune_enabled() { + [[ "$SG_PRUNE_INTERVAL_SECONDS" =~ ^[0-9]+$ ]] && ((SG_PRUNE_INTERVAL_SECONDS > 0)) +} + +docker_disk_used_pct() { #{{{ + # Prints how full the filesystem holding Docker's data is, as a bare number. + df -P "$DOCKER_DATA_ROOT" 2>/dev/null | awk 'NR == 2 {sub(/%/, "", $5); print $5}' +} +#}}}: docker_disk_used_pct + +should_prune_images() { #{{{ + # Usage: should_prune_images + # Unused images are kept as the wake-up cache until the disk is filling up. + # An unknown usage counts as full: better a cold cache than a full PVC. + local used="$1" threshold="$SG_PRUNE_IMAGES_ABOVE_PCT" + [[ "$threshold" =~ ^[0-9]+$ ]] || threshold=70 + [[ "$used" =~ ^[0-9]+$ ]] || return 0 + ((threshold == 0 || used >= threshold)) +} +#}}}: should_prune_images + +docker_prune() { #{{{ + # Usage: docker_prune + # Runs one prune and prints the space it reclaimed (0B when docker reports + # none: `builder prune` prints "Total:", the others "Total reclaimed space:"). + local out space + if ! out="$(docker "$@" 2>&1)"; then + info "docker $* failed:" "$(tail -n1 <<<"$out")" >&2 + printf 'failed' + return 1 + fi + space="$(sed -nE 's/^Total( reclaimed space)?:[[:space:]]*//p' <<<"$out" | tail -n1)" + printf '%s' "${space:-0B}" +} +#}}}: docker_prune + +prune_docker() { #{{{ + # One cleanup pass over the nested Docker. Never a bare `system prune`: it + # removes every non-running container regardless of age, including ones the + # ECS agent has created but not started yet (see main.sh prune()). Every + # container-touching sweep carries the until= age guard. + local filter="until=$SG_PRUNE_UNTIL" used all=() containers images build volumes + containers="$(docker_prune container prune -f --filter "$filter")" + + used="$(docker_disk_used_pct)" + # -a: unused images too, not just dangling ones. `until` is the image's + # creation time, not when it was pulled or last used, so it's no LRU: under + # pressure, every unused image older than that goes. + should_prune_images "$used" && all=(-a) + images="$(docker_prune image prune -f "${all[@]}" --filter "$filter")" + + build="$(docker_prune builder prune -f --filter "$filter")" + # Anonymous volumes no container (in any state) refers to. Separate from the + # container sweep on purpose: this one can't remove a container. + volumes="$(docker_prune volume prune -f)" + + info "Docker prune ($filter, disk ${used:-?}% used${all[*]:+, unused images too}) reclaimed:" \ + "containers $containers, images $images, build cache $build, volumes $volumes" +} +#}}}: prune_docker + +prune_loop() { #{{{ + # Runs once right away, then every interval: a pod may park within minutes + # of waking and never see a full interval, so its first pass can't wait for + # one. Each pass runs in a subshell so a failing docker call or a stray exit + # can't end the loop. + local sleep_pid="" + trap 'kill $sleep_pid 2>/dev/null; exit 0' TERM INT + info "Docker prune every" "${SG_PRUNE_INTERVAL_SECONDS}s, keeping anything newer than $SG_PRUNE_UNTIL" + while :; do + (prune_docker) || info "Docker prune failed;" "retrying in ${SG_PRUNE_INTERVAL_SECONDS}s" + sleep "$SG_PRUNE_INTERVAL_SECONDS" & + sleep_pid=$! + wait "$sleep_pid" + done +} +#}}}: prune_loop + +cmd_ssm_agent() { #{{{ + wait_for "SSM registration" test -s "$SSM_REGISTRATION_FILE" + exec amazon-ssm-agent +} +#}}}: cmd_ssm_agent + +load_ecs_config() { #{{{ + # ecs-init normally turns ecs.config into the agent's environment. Read it + # line by line: ECS_INSTANCE_ATTRIBUTES holds JSON that `source` would mangle. + local line + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" == \#* ]] && continue + export "${line%%=*}=${line#*=}" + done <"$ECS_CONFIG_DIR/ecs.config" +} +#}}}: load_ecs_config + +mem_total_mib() { + echo $(($(awk '/^MemTotal:/ {print $2}' "$MEMINFO_FILE") / 1024)) +} + +reserved_memory_mib() { #{{{ + # The agent advertises MemTotal from /proc/meminfo, which is the whole node. + # Tasks run inside the dockerd container, so advertise its limit (downward + # API) minus dockerd's own overhead, by reserving the difference. The chart + # sets the limit to the runner's advertised size and the overhead to 0 + # (steps run as pods, not in dockerd). More than the node advertises the + # node. + # Prints the reservation; fails without a numeric limit (1, nothing printed) + # or when the agent can't take it (2). + local limit="${SG_TASK_MEMORY_LIMIT_MIB:-}" total available reserved + [[ "$limit" =~ ^[0-9]+$ ]] || return 1 + + total=$(mem_total_mib) + available=$((limit - SG_DOCKERD_OVERHEAD_MIB)) + reserved=$((total > available ? total - available : 0)) + echo "$reserved" + # The agent parses this as uint16: anything larger silently becomes 0, which + # would advertise the whole node again. + ((reserved <= 65535)) || return 2 +} +#}}}: reserved_memory_mib + +size_agent_memory() { #{{{ + local reserved rc=0 total + reserved=$(reserved_memory_mib) || rc=$? + ((rc == 1)) && return 0 + if ((rc == 2)); then + die "Can't size agent memory:" "reservation of ${reserved}MiB exceeds 65535MiB (advertise more memory or use a smaller node)" + fi + total=$(mem_total_mib) + export ECS_RESERVED_MEMORY="$reserved" + info "Advertising memory (MiB):" "$((total - reserved)) of $total" +} +#}}}: size_agent_memory + +allowed_cpus() { #{{{ + # Prints the CPUs this process may run on, space-separated. + local range first last cpu cpus=() ranges=() + IFS=, read -ra ranges <<<"$(awk '/^Cpus_allowed_list:/ {print $2}' "$PROC_STATUS_FILE")" + for range in "${ranges[@]}"; do + first="${range%-*}" + last="${range#*-}" + for ((cpu = first; cpu <= last; cpu++)); do + cpus+=("$cpu") + done + done + echo "${cpus[*]}" +} +#}}}: allowed_cpus + +cpu_affinity_list() { #{{{ + # Usage: cpu_affinity_list + # Prints the first n CPUs this process may run on as a taskset list, or + # nothing if n already covers all of them. + local n="$1" cpus=() + read -ra cpus <<<"$(allowed_cpus)" + ((n >= ${#cpus[@]})) && return 0 + ( + IFS=, + echo "${cpus[*]:0:n}" + ) +} +#}}}: cpu_affinity_list + +agent_cpu_list() { #{{{ + # The CPUs cmd_ecs_agent pins the agent to (SG_TASK_CPU_LIMIT of them), or + # nothing to leave it on all of them. + [[ "${SG_TASK_CPU_LIMIT:-}" =~ ^[0-9]+$ ]] || return 0 + cpu_affinity_list "$SG_TASK_CPU_LIMIT" +} +#}}}: agent_cpu_list + +advertised_resources() { #{{{ + # Prints " " as the ECS agent started by + # cmd_ecs_agent registers them: NumCPU of its CPU set x 1024, and MemTotal + # minus ECS_RESERVED_MEMORY. Fails when the agent wouldn't start (memory + # reservation too large). + local cpu_list cpus=() reserved=0 rc=0 + cpu_list="$(agent_cpu_list)" + if [[ -n "$cpu_list" ]]; then + IFS=, read -ra cpus <<<"$cpu_list" + else + read -ra cpus <<<"$(allowed_cpus)" + fi + reserved=$(reserved_memory_mib) || rc=$? + ((rc == 2)) && return 1 + ((rc == 1)) && reserved=0 + echo "$((${#cpus[@]} * 1024)) $(($(mem_total_mib) - reserved))" +} +#}}}: advertised_resources + +cmd_ecs_agent() { #{{{ + wait_for "ecs.config" test -s "$ECS_CONFIG_DIR/ecs.config" + wait_for "dockerd" docker info + wait_for "SSM rotating credentials" test -s "$AGENT_CREDS_FILE" + + load_ecs_config + + # Defaults ecs-init would set (see ecs-init/docker/docker.go). + export ECS_AVAILABLE_LOGGING_DRIVERS='["json-file","syslog","awslogs","fluentd","none"]' + export ECS_UPDATES_ENABLED=false + # Where dockerd sees the agent's data dir, for bind-mounting task metadata. + export ECS_HOST_DATA_DIR=/var/lib/ecs + # The agent can't create task cgroups from an unprivileged container. + export ECS_ENABLE_TASK_CPU_MEM_LIMIT=false + + size_agent_memory + + # The agent advertises runtime.NumCPU() x 1024 units. NumCPU follows CPU + # affinity, not the cgroup quota, so pin the agent to as many CPUs as the + # dockerd container's limit allows. + local cpu_list + cpu_list="$(agent_cpu_list)" + reactivate_when_registered & + + if [[ -n "$cpu_list" ]]; then + info "Advertising CPUs:" "$SG_TASK_CPU_LIMIT (agent pinned to $cpu_list)" + exec taskset -c "$cpu_list" ecs-agent + elif [[ -n "${SG_TASK_CPU_LIMIT:-}" ]]; then + info "Advertising CPUs:" "all this node allows (limit $SG_TASK_CPU_LIMIT)" + fi + exec ecs-agent +} +#}}}: cmd_ecs_agent + +read_sg_credentials_from_stdin() { #{{{ + # KEY=VALUE lines (the k8s/.env format); only known keys are taken, so the + # token never has to live in the pod spec or on a command line. + local key value + while IFS='=' read -r key value || [[ -n "$key" ]]; do + [[ " $SG_CRED_KEYS " == *" $key "* ]] || continue + value="${value%\"}" + value="${value#\"}" + printf -v "$key" '%s' "$value" + done +} +#}}}: read_sg_credentials_from_stdin + +load_sg_credentials_from_dir() { #{{{ + # The credentials Secret, mounted as one file per key. + local key + for key in $SG_CRED_KEYS; do + [[ -r "$SG_CREDS_DIR/$key" ]] && printf -v "$key" '%s' "$(<"$SG_CREDS_DIR/$key")" + done +} +#}}}: load_sg_credentials_from_dir + +configured_runner_id() { #{{{ + # Prints the SG runner ID recorded in ecs.config at registration. + [[ -s "$ECS_CONFIG_DIR/ecs.config" ]] || die "Not registered:" "$ECS_CONFIG_DIR/ecs.config missing" + grep ECS_INSTANCE_ATTRIBUTES "$ECS_CONFIG_DIR/ecs.config" | cut -d "=" -f2- | + jq -r '.sg_runner_id' +} +#}}}: configured_runner_id + +deregister_runner() { #{{{ + local url + load_sg_credentials + + RUNNER_ID="$(configured_runner_id)" + validate_runner_id "$RUNNER_ID" + + url="${SG_BASE_API}/orgs/${ORGANIZATION_ID}/runnergroups/${RUNNER_GROUP_ID}/deregister/" + if ! api_call "POST" "$url" "{ \"RunnerId\": \"${RUNNER_ID}\" }"; then + die "Could not deregister runner." "$status_code" "$message" + fi + info "Deregistered runner" "$RUNNER_ID" + + # Wipe the identity so a restarted pod can't come back as a ghost runner. + find "$ECS_CONFIG_DIR" "$ECS_DATA_DIR" "$SSM_DIR" -mindepth 1 -delete + info "Local state wiped." +} +#}}}: deregister_runner + +cmd_deregister() { #{{{ + [[ -t 0 ]] && die "Pipe SG credentials on stdin:" "kubectl exec -i ... deregister < k8s/.env" + read_sg_credentials_from_stdin + deregister_runner +} +#}}}: cmd_deregister + +cmd_reap() { #{{{ + # One-off Job the scaler (parked longer than its TTL) or the uninstall hook + # starts: the pod is gone, so deregister using the identity on the mounted + # state PVC. + load_sg_credentials_from_dir + # Never registered (registration failed, or already deregistered and + # wiped): nothing in SG to remove, and the PVC can go. + if [[ ! -s "$ECS_CONFIG_DIR/ecs.config" ]]; then + info "Not registered:" "nothing to deregister" + return 0 + fi + deregister_runner +} +#}}}: cmd_reap + + + +sg_api() { #{{{ + # Usage: sg_api [payload] + # Calls the runner group's SG API with the node token and prints the + # response. Subshell: api_call exits on an empty response. A failure logs + # only a string message: the runner group's body holds secrets. + local method="$1" path="$2" payload="${3:-}" + ( + if ! api_call "$method" \ + "${SG_BASE_API}/orgs/${SG_ORGANIZATION}/runnergroups/${SG_RUNNER_GROUP}/${path}" \ + "$payload"; then + info "SG API $method $path failed ($status_code):" \ + "$(jq -r '[.msg, .message] | map(strings) | first // "no message"' <<<"$response" 2>/dev/null)" >&2 + exit 1 + fi + printf '%s' "$response" + ) +} +#}}}: sg_api + +set_sg_runner_status() { #{{{ + # Usage: set_sg_runner_status DRAINING|ACTIVE + # SG's runner_status API, as used by sg-runner-autoscaler: SG changes the + # ECS container instance state on its side, with its own permissions. + local runner_id + runner_id="$(configured_runner_id)" + validate_runner_id "$runner_id" + sg_api POST runner_status/ \ + "$(jq -nc --arg s "$1" --arg r "$runner_id" '{Status: $s, RunnerId: $r}')" >/dev/null +} +#}}}: set_sg_runner_status + +set_runner_status() { #{{{ + # Usage: set_runner_status DRAINING|ACTIVE + # Only through SG: a customer's runner has no ECS permissions of its own. + load_sg_credentials_from_dir + set_sg_runner_status "$1" +} +#}}}: set_runner_status + +current_runner_status() { #{{{ + # Prints this runner's status as SG reports it (ACTIVE, DRAINING, ...). + # The response holds secrets; only the status leaves this function. + local runner_id + load_sg_credentials_from_dir + runner_id="$(configured_runner_id)" + sg_api GET "" | jq -r --arg r "$runner_id" \ + '.msg.ContainerInstances[]? | select(.runnerId == $r) | .status' +} +#}}}: current_runner_status + +reactivate_when_registered() { #{{{ + # Wake a parked runner: undo the drain this pod did on its way down + # (cmd_drain); it comes back with the same container instance. A drain anyone else + # started (SG UI, an autoscaler) leaves no marker and is kept. + [[ -e "$SELF_DRAIN_MARKER" ]] || return 0 + local attempt + for ((attempt = 0; attempt < 120; attempt++)); do + if curl -fsS --max-time 5 "$AGENT_API/metadata" 2>/dev/null | + jq -e '.ContainerInstanceArn // empty' >/dev/null; then + if set_runner_status ACTIVE; then + rm -f "$SELF_DRAIN_MARKER" + info "Runner" "ACTIVE" + else + info "Could not set ACTIVE;" "a restarted runner may stay DRAINING" + fi + return + fi + sleep 5 + done +} +#}}}: reactivate_when_registered + +cmd_drain() { #{{{ + # preStop hook of the ecs-agent container: park the runner. Every shutdown + # (scale-down, restart, rollout, eviction) sets it DRAINING so no new + # workflows land, waits for the running ones (bounded by + # terminationGracePeriodSeconds), and keeps the registration on the state + # PVC. Scaling back up wakes the same runner in seconds + # (reactivate_when_registered); deregistering parked runners is left to + # `make deregister` / the scaler. + # Hook output isn't logged by the kubelet, so write to the container's. + exec >/proc/1/fd/1 2>&1 + # Release the other containers whatever happens below. + trap 'touch "$DRAINED_MARKER"' EXIT + + info "Parking runner.." + # Already gone after a manual deregister (make nuke). + if [[ -s "$ECS_CONFIG_DIR/ecs.config" ]]; then + load_ecs_config + if [[ "$(current_runner_status)" == "DRAINING" ]]; then + # Drained on purpose by someone else: keep it that way on wake. + info "Runner already" "DRAINING" + elif set_runner_status DRAINING; then + info "Runner" "DRAINING" + touch "$SELF_DRAIN_MARKER" + else + info "Could not set DRAINING;" "new tasks may still land" + fi + fi + + until [[ -z "$(docker ps -q 2>/dev/null)" ]]; do + info "Waiting for running workflow containers.." + sleep 10 + done + info "Parked:" "no workflow containers running" +} +#}}}: cmd_drain + +cmd_wait_drained() { #{{{ + until [[ -e "$DRAINED_MARKER" ]]; do + sleep 5 + done +} +#}}}: cmd_wait_drained + +# Only dispatch when executed directly; sourcing exposes the functions (tests). +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + case "${1:-}" in + register) cmd_register ;; + dockerd) cmd_dockerd ;; + ssm-agent) cmd_ssm_agent ;; + ecs-agent) cmd_ecs_agent ;; + drain) cmd_drain ;; + wait-drained) cmd_wait_drained ;; + deregister) cmd_deregister ;; + reap) cmd_reap ;; + *) + echo "Usage: $(basename "$0") register|dockerd|ssm-agent|ecs-agent|drain|wait-drained|deregister|reap" >&2 + exit 2 + ;; + esac +fi diff --git a/k8s/scaler/.dockerignore b/k8s/scaler/.dockerignore new file mode 100644 index 0000000..dc9070a --- /dev/null +++ b/k8s/scaler/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +.dockerignore +*.test +*.out diff --git a/k8s/scaler/Dockerfile b/k8s/scaler/Dockerfile new file mode 100644 index 0000000..c373622 --- /dev/null +++ b/k8s/scaler/Dockerfile @@ -0,0 +1,26 @@ +# sg-runner-scaler: sizes the runner StatefulSet (see k8s/README.md, +# "Autoscaling"). The image also carries /sg-runner-steps, the step broker +# running in each runner pod, and /sg-step-wrapper, which step pods' init +# container installs (see k8s/docs/steps-api.md). Build from k8s/scaler: +# docker build -t sg-runner-scaler:dev . +# docker buildx build --platform linux/amd64,linux/arm64 -t sg-runner-scaler:dev . + +ARG GO_VERSION=1.27 + +# Cross-compiles on the build host's platform: no emulation needed. +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS build +ARG TARGETOS TARGETARCH +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/sg-runner-scaler ./cmd/sg-runner-steps ./cmd/sg-step-wrapper + +FROM gcr.io/distroless/static:nonroot +COPY --from=build /out/sg-runner-scaler /sg-runner-scaler +COPY --from=build /out/sg-runner-steps /sg-runner-steps +COPY --from=build /out/sg-step-wrapper /sg-step-wrapper +USER 65532:65532 +EXPOSE 9090 +ENTRYPOINT ["/sg-runner-scaler"] diff --git a/k8s/scaler/cmd/sg-runner-scaler/main.go b/k8s/scaler/cmd/sg-runner-scaler/main.go new file mode 100644 index 0000000..27b07fb --- /dev/null +++ b/k8s/scaler/cmd/sg-runner-scaler/main.go @@ -0,0 +1,241 @@ +// sg-runner-scaler sizes the sg-runner StatefulSet from the runner group's +// state in StackGuardian, parks idle runners and deregisters runners parked +// for too long. See k8s/README.md, "Autoscaling". +// +// `sg-runner-scaler teardown` instead deregisters every runner and deletes +// their PVCs: the Helm chart's pre-delete hook, so `helm uninstall` leaves +// nothing behind. +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "log/slog" + "net/http" + "os" + "os/signal" + "sync/atomic" + "syscall" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/config" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/controller" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/teardown" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + slog.SetDefault(log) + klog.SetSlogLogger(log.With("source", "client-go")) + run := run + if len(os.Args) > 1 && os.Args[1] == "teardown" { + run = runTeardown + } + if err := run(log); err != nil { + log.Error("exiting", "err", err) + os.Exit(1) + } +} + +func run(log *slog.Logger) error { + cfg, err := config.Load(nil) + if err != nil { + return err + } + if cfg.MinRaised { + log.Warn("SCALER_MIN raised to 1: SG pauses workflows without an ACTIVE runner", "given", cfg.MinGiven) + } + creds, err := sg.LoadCredentials(cfg.CredentialsDir) + if err != nil { + return err + } + + client, err := kubeClient() + if err != nil { + return err + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + m := metrics.New() + broadcaster := record.NewBroadcaster(record.WithContext(ctx)) + broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(cfg.Namespace)}) + defer broadcaster.Shutdown() + recorder := broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "sg-runner-scaler"}) + + k := kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret, cfg.ConfigMap) + k.ReapPod = cfg.ReapPod + k.NudgeImage = cfg.NudgeImage + ctrl := &controller.Controller{ + Config: cfg.Rules, + Interval: cfg.Interval, + SG: sg.New(creds, nil), + Kube: k, + Events: recorder, + Metrics: m, + Log: log, + Nudge: cfg.Nudge, + } + ctrl.NudgeCooldown = cfg.NudgeCooldown + + var leading atomic.Bool + srv := serve(cfg, m, ctrl, &leading, log) + defer func() { + shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutdown) + }() + + lead := func(ctx context.Context) { + leading.Store(true) + m.Leader.Set(1) + _ = ctrl.Run(ctx) + } + if !cfg.LeaderElect { + lead(ctx) + return nil + } + + id, err := identity() + if err != nil { + return err + } + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{Name: cfg.StatefulSet + "-scaler", Namespace: cfg.Namespace}, + Client: client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{Identity: id}, + } + log.Info("waiting for the leader lease", "lease", lock.LeaseMeta.Name, "identity", id) + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + Name: lock.LeaseMeta.Name, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + log.Info("leading", "identity", id) + lead(ctx) + }, + OnStoppedLeading: func() { + leading.Store(false) + m.Leader.Set(0) + log.Info("stopped leading", "identity", id) + }, + OnNewLeader: func(current string) { + if current != id { + log.Info("following", "leader", current) + } + }, + }, + }) + if ctx.Err() == nil { + // Lost the lease without being asked to stop: restart clean. + return errors.New("lost the leader lease") + } + return nil +} + +// runTeardown is the pre-delete hook: deregister every runner, delete PVCs. +// It runs until done or until the Job's activeDeadlineSeconds stops it. +func runTeardown(log *slog.Logger) error { + cfg, err := config.Load(nil) + if err != nil { + return err + } + client, err := kubeClient() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + k := kube.New(client, cfg.Namespace, cfg.StatefulSet, cfg.Image, cfg.CredentialsSecret, cfg.ConfigMap) + k.ReapPod = cfg.ReapPod + td := &teardown.Teardown{ + Kube: k, + ScalerDeployment: os.Getenv("TEARDOWN_SCALER_DEPLOYMENT"), + ScalerSelector: os.Getenv("TEARDOWN_SCALER_SELECTOR"), + RunnerSelector: os.Getenv("TEARDOWN_RUNNER_SELECTOR"), + Log: log, + } + if td.RunnerSelector == "" || (td.ScalerDeployment != "" && td.ScalerSelector == "") { + return errors.New("teardown needs TEARDOWN_RUNNER_SELECTOR, and TEARDOWN_SCALER_SELECTOR with TEARDOWN_SCALER_DEPLOYMENT") + } + return td.Run(ctx) +} + +// kubeClient connects in cluster, or with the usual kubeconfig out of +// cluster (development). +func kubeClient() (kubernetes.Interface, error) { + restCfg, err := rest.InClusterConfig() + if err != nil { + restCfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, err + } + } + restCfg.Timeout = 15 * time.Second + restCfg.UserAgent = "sg-runner-scaler" + return kubernetes.NewForConfig(restCfg) +} + +func identity() (string, error) { + host, err := os.Hostname() + if err != nil { + return "", err + } + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + return "", err + } + return host + "_" + hex.EncodeToString(b), nil +} + +// serve exposes /metrics, /healthz and /readyz. Not leading is healthy and +// ready (a standby); a leader whose loop stopped running is not healthy. +func serve(cfg config.Config, m *metrics.Metrics, ctrl *controller.Controller, leading *atomic.Bool, log *slog.Logger) *http.Server { + stale := 2*controller.DefaultMaxBackoff + cfg.Interval + time.Minute + mux := http.NewServeMux() + mux.Handle("/metrics", m.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if leading.Load() { + if last := ctrl.LastLoop(); !last.IsZero() && time.Since(last) > stale { + http.Error(w, "scaling loop stalled", http.StatusServiceUnavailable) + return + } + } + _, _ = w.Write([]byte("ok\n")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok\n")) + }) + srv := &http.Server{Addr: cfg.MetricsAddr, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error("metrics server", "err", err) + os.Exit(1) + } + }() + return srv +} diff --git a/k8s/scaler/cmd/sg-runner-steps/main.go b/k8s/scaler/cmd/sg-runner-steps/main.go new file mode 100644 index 0000000..62eaf5a --- /dev/null +++ b/k8s/scaler/cmd/sg-runner-steps/main.go @@ -0,0 +1,150 @@ +// sg-runner-steps is the step broker: a native sidecar of the runner pod that +// runs each workflow step of the run controllers as a pod of its own, and +// serves dockerd the sg-steps-guard authorization plugin, which refuses step +// containers in dockerd. See k8s/docs/steps-api.md. +// +// `sg-runner-steps healthcheck` is its startup and readiness probe: it exits +// 0 when both sockets answer. +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/guard" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/steps" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + slog.SetDefault(log) + klog.SetSlogLogger(log.With("source", "client-go")) + run := serve + if len(os.Args) > 1 { + switch os.Args[1] { + case "serve": + case "healthcheck": + run = healthcheck + default: + log.Error("unknown command: want serve or healthcheck", "command", os.Args[1]) + os.Exit(2) + } + } + if err := run(log); err != nil { + log.Error("exiting", "err", err) + os.Exit(1) + } +} + +// healthcheck checks both sockets; it reads only their paths, so it works +// whatever else the configuration says. +func healthcheck(*slog.Logger) error { + env := func(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return steps.Healthcheck(ctx, env("SOCKET_PATH", steps.DefaultSocketPath), + env("GUARD_SOCKET", steps.DefaultGuardSocket), 2*time.Second) +} + +func serve(log *slog.Logger) error { + cfg, err := steps.LoadConfig(nil) + if err != nil { + return err + } + client, err := kubeClient(15 * time.Second) + if err != nil { + return err + } + // Followed log streams last as long as their step. + streams, err := kubeClient(0) + if err != nil { + return err + } + + s := steps.NewServer(cfg, client, log) + s.Images = steps.NewRegistry() + if cfg.BrokerImage == "" { + log.Warn("BROKER_IMAGE is not set: steps run unwrapped, their exits 11 and 12 end the pod Failed") + } + s.Logs = func(ctx context.Context, ns, pod string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { + return streams.CoreV1().Pods(ns).GetLogs(pod, opts).Stream(ctx) + } + served := make(chan error, 2) + var servers []*http.Server + for _, sock := range []struct { + path string + handler http.Handler + }{ + {cfg.SocketPath, s.Handler()}, + {cfg.GuardSocket, guard.Handler(log)}, + } { + l, err := steps.Listen(sock.path) + if err != nil { + return fmt.Errorf("listen on %s: %w", sock.path, err) + } + srv := &http.Server{Handler: sock.handler, ReadHeaderTimeout: 10 * time.Second} + servers = append(servers, srv) + go func() { served <- srv.Serve(l) }() + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go s.RunGC(ctx, time.Minute) + log.Info("serving", "socket", cfg.SocketPath, "guard", cfg.GuardSocket, "runner", cfg.RunnerPodName, "node", cfg.NodeName) + + // A native sidecar gets SIGTERM only once the runner's containers (dockerd + // and the run controllers in it) are gone. + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) + select { + case err := <-served: + return fmt.Errorf("serving: %w", err) + case sig := <-sigs: + log.Info("shutting down", "signal", sig.String()) + } + + cancel() + shutdown, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + for _, srv := range servers { + if err := srv.Shutdown(shutdown); err != nil && !errors.Is(err, context.DeadlineExceeded) { + return err + } + } + return nil +} + +// kubeClient connects in cluster, or with the usual kubeconfig out of +// cluster (development); timeout 0 is none. +func kubeClient(timeout time.Duration) (kubernetes.Interface, error) { + restCfg, err := rest.InClusterConfig() + if err != nil { + restCfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, err + } + } + restCfg.Timeout = timeout + restCfg.UserAgent = "sg-runner-steps" + return kubernetes.NewForConfig(restCfg) +} diff --git a/k8s/scaler/cmd/sg-step-wrapper/main.go b/k8s/scaler/cmd/sg-step-wrapper/main.go new file mode 100644 index 0000000..db2da92 --- /dev/null +++ b/k8s/scaler/cmd/sg-step-wrapper/main.go @@ -0,0 +1,33 @@ +//go:build unix + +// sg-step-wrapper is the init process of a step container (see +// internal/wrapper): +// +// sg-step-wrapper -- [args...] run the step's command +// sg-step-wrapper install copy itself to dest (an init container) +// +// It writes nothing to stdout: that's the step's. +package main + +import ( + "fmt" + "os" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/wrapper" +) + +func main() { + args := os.Args[1:] + switch { + case len(args) >= 2 && args[0] == "--": + os.Exit(wrapper.Run(args[1:], wrapper.Options{})) + case len(args) == 2 && args[0] == "install": + if err := wrapper.Install(args[1]); err != nil { + fmt.Fprintf(os.Stderr, "sg-step-wrapper: install: %v\n", err) + os.Exit(1) + } + default: + fmt.Fprintln(os.Stderr, "usage: sg-step-wrapper -- [args...] | sg-step-wrapper install ") + os.Exit(2) + } +} diff --git a/k8s/scaler/go.mod b/k8s/scaler/go.mod new file mode 100644 index 0000000..59a348c --- /dev/null +++ b/k8s/scaler/go.mod @@ -0,0 +1,71 @@ +module github.com/StackGuardian/sg-runner/k8s/scaler + +go 1.24.0 + +require ( + github.com/google/go-containerregistry v0.20.7 + github.com/prometheus/client_golang v1.23.2 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.18.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/cli v29.0.3+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.1 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/vbatts/tar-split v0.12.2 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.34.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/k8s/scaler/go.sum b/k8s/scaler/go.sum new file mode 100644 index 0000000..4ce4e5f --- /dev/null +++ b/k8s/scaler/go.sum @@ -0,0 +1,202 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= +github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v29.0.3+incompatible h1:8J+PZIcF2xLd6h5sHPsp5pvvJA+Sr2wGQxHkRl53a1E= +github.com/docker/cli v29.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.7 h1:24VGNpS0IwrOZ2ms2P1QE3Xa5X9p4phx0aUgzYzHW6I= +github.com/google/go-containerregistry v0.20.7/go.mod h1:Lx5LCZQjLH1QBaMPeGwsME9biPeo1lPx6lbGj/UmzgM= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= +github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/k8s/scaler/internal/config/config.go b/k8s/scaler/internal/config/config.go new file mode 100644 index 0000000..5b44ef7 --- /dev/null +++ b/k8s/scaler/internal/config/config.go @@ -0,0 +1,190 @@ +// Package config reads the scaler's settings from the environment, with the +// names and defaults the original bash scaler used. +package config + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" +) + +const serviceAccountNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + +// Config is everything the scaler is told from outside. +type Config struct { + Rules decide.Config + Interval time.Duration + StatefulSet string + Image string + CredentialsSecret string + ConfigMap string + CredentialsDir string + Namespace string + MetricsAddr string + LeaderElect bool + // Nudge starts placeholder pods when the cluster autoscaler won't add a + // node for a runner pod; NudgeImage is their image. + Nudge bool + NudgeImage string + NudgeCooldown time.Duration + // ReapPod is extra pod settings for reap Jobs. + ReapPod kube.ReapPodOptions + // MinRaised reports a SCALER_MIN below 1 (MinGiven) raised to 1. + MinRaised bool + MinGiven int +} + +// Load reads the environment through getenv (os.Getenv when nil). +func Load(getenv func(string) string) (Config, error) { + if getenv == nil { + getenv = os.Getenv + } + var errs []string + str := func(key, def string) string { + if v := strings.TrimSpace(getenv(key)); v != "" { + return v + } + return def + } + num := func(key string, def, lowest int) int { + raw := str(key, "") + if raw == "" { + return def + } + v, err := strconv.Atoi(raw) + if err != nil || v < lowest { + errs = append(errs, fmt.Sprintf("%s=%q: want an integer >= %d", key, raw, lowest)) + return def + } + return v + } + secs := func(key string, def, lowest int) time.Duration { + return time.Duration(num(key, def, lowest)) * time.Second + } + + c := Config{ + Interval: secs("SCALER_INTERVAL", 15, 1), + StatefulSet: str("SCALER_STATEFULSET", "sg-runner"), + Image: str("SCALER_IMAGE", "sg-runner-k8s:dev"), + CredentialsSecret: str("SCALER_CREDENTIALS_SECRET", "sg-runner-credentials"), + ConfigMap: str("SCALER_CONFIGMAP", "sg-runner-config"), + CredentialsDir: str("SG_CREDENTIALS_DIR", "/etc/sg-runner/credentials"), + MetricsAddr: str("SCALER_METRICS_ADDR", ":9090"), + Rules: decide.Config{ + Min: num("SCALER_MIN", 1, -1<<31), + Max: num("SCALER_MAX", 10, 1), + HeadroomPct: num("SCALER_HEADROOM_PCT", 0, 0), + IdleAfter: secs("SCALER_IDLE_SECONDS", 60, 0), + ParkTTL: secs("SCALER_PARK_TTL", 43200, 0), + StuckAfter: secs("SCALER_STUCK_SECONDS", 180, 0), + SettleTimeout: secs("SCALER_SETTLE_TIMEOUT", 300, 0), + }, + } + if c.Rules.HeadroomPct > 100 { + errs = append(errs, fmt.Sprintf("SCALER_HEADROOM_PCT=%d: want 0-100", c.Rules.HeadroomPct)) + } + // SG pauses workflows without an ACTIVE, connected runner. + if c.Rules.Min < 1 { + c.MinRaised, c.MinGiven = true, c.Rules.Min + c.Rules.Min = 1 + } + if c.Rules.Min > c.Rules.Max { + errs = append(errs, fmt.Sprintf("SCALER_MIN=%d is above SCALER_MAX=%d", c.Rules.Min, c.Rules.Max)) + } + switch v := strings.ToLower(str("SCALER_LEADER_ELECT", "true")); v { + case "true", "1", "yes": + c.LeaderElect = true + case "false", "0", "no": + default: + errs = append(errs, fmt.Sprintf("SCALER_LEADER_ELECT=%q: want true or false", v)) + } + + switch v := strings.ToLower(str("SCALER_NUDGE", "true")); v { + case "true", "1", "yes": + c.Nudge = true + case "false", "0", "no": + default: + errs = append(errs, fmt.Sprintf("SCALER_NUDGE=%q: want true or false", v)) + } + c.NudgeImage = str("SCALER_NUDGE_IMAGE", kube.DefaultNudgeImage) + c.NudgeCooldown = secs("SCALER_NUDGE_COOLDOWN", 180, 30) + + c.ReapPod, errs = reapPod(str, errs) + + c.Namespace = str("POD_NAMESPACE", "") + if c.Namespace == "" { + if b, err := os.ReadFile(serviceAccountNamespace); err == nil { + c.Namespace = strings.TrimSpace(string(b)) + } + } + if c.Namespace == "" { + errs = append(errs, "no namespace: set POD_NAMESPACE") + } + if len(errs) > 0 { + return c, fmt.Errorf("invalid configuration: %s", strings.Join(errs, "; ")) + } + return c, nil +} + +// strictJSON decodes raw into v, rejecting unknown fields and trailing data. +func strictJSON(raw string, v any) error { + dec := json.NewDecoder(bytes.NewReader([]byte(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if dec.More() { + return fmt.Errorf("trailing data") + } + return nil +} + +// reapPod reads the reap Job pod settings the Helm chart passes. +func reapPod(str func(string, string) string, errs []string) (kube.ReapPodOptions, []string) { + var o kube.ReapPodOptions + if raw := str("SCALER_REAP_POD_LABELS", ""); raw != "" { + if err := strictJSON(raw, &o.Labels); err != nil { + errs = append(errs, fmt.Sprintf("SCALER_REAP_POD_LABELS: want a JSON object of strings: %v", err)) + } + for k, v := range o.Labels { + for _, msg := range append(validation.IsQualifiedName(k), validation.IsValidLabelValue(v)...) { + errs = append(errs, fmt.Sprintf("SCALER_REAP_POD_LABELS: label %q=%q: %s", k, v, msg)) + } + } + } + if raw := str("SCALER_REAP_IMAGE_PULL_SECRETS", ""); raw != "" { + for _, name := range strings.Split(raw, ",") { + name = strings.TrimSpace(name) + if name == "" { + continue + } + for _, msg := range validation.IsDNS1123Subdomain(name) { + errs = append(errs, fmt.Sprintf("SCALER_REAP_IMAGE_PULL_SECRETS: %q: %s", name, msg)) + } + o.ImagePullSecrets = append(o.ImagePullSecrets, name) + } + } + if raw := str("SCALER_REAP_NODE_SELECTOR", ""); raw != "" { + if err := strictJSON(raw, &o.NodeSelector); err != nil { + errs = append(errs, fmt.Sprintf("SCALER_REAP_NODE_SELECTOR: want a JSON object of strings: %v", err)) + } + } + if raw := str("SCALER_REAP_TOLERATIONS", ""); raw != "" { + var t []corev1.Toleration + if err := strictJSON(raw, &t); err != nil { + errs = append(errs, fmt.Sprintf("SCALER_REAP_TOLERATIONS: want a JSON array of tolerations: %v", err)) + } + o.Tolerations = t + } + return o, errs +} diff --git a/k8s/scaler/internal/config/config_test.go b/k8s/scaler/internal/config/config_test.go new file mode 100644 index 0000000..56dcf82 --- /dev/null +++ b/k8s/scaler/internal/config/config_test.go @@ -0,0 +1,109 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +func env(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestDefaults(t *testing.T) { + c, err := Load(env(map[string]string{"POD_NAMESPACE": "sg-runner"})) + if err != nil { + t.Fatal(err) + } + r := c.Rules + if c.Interval != 15*time.Second || r.Min != 1 || r.Max != 10 || r.HeadroomPct != 0 || + r.IdleAfter != 60*time.Second || r.ParkTTL != 43200*time.Second || r.StuckAfter != 180*time.Second || + r.SettleTimeout != 300*time.Second || c.StatefulSet != "sg-runner" || c.Image != "sg-runner-k8s:dev" || + c.CredentialsSecret != "sg-runner-credentials" || c.ConfigMap != "sg-runner-config" || + c.CredentialsDir != "/etc/sg-runner/credentials" || + c.MetricsAddr != ":9090" || !c.LeaderElect || c.Namespace != "sg-runner" || c.MinRaised || + !c.Nudge || c.NudgeImage != "registry.k8s.io/pause:3.10" || c.NudgeCooldown != 180*time.Second { + t.Errorf("defaults %+v", c) + } +} + +func TestOverrides(t *testing.T) { + c, err := Load(env(map[string]string{ + "POD_NAMESPACE": "ns", "SCALER_INTERVAL": "5", "SCALER_MIN": "0", "SCALER_MAX": "3", + "SCALER_HEADROOM_PCT": "20", "SCALER_LEADER_ELECT": "false", "SG_CREDENTIALS_DIR": "/c", + "SCALER_STATEFULSET": "r", "SCALER_IMAGE": "i:1", + })) + if err != nil { + t.Fatal(err) + } + if c.Interval != 5*time.Second || c.Rules.Min != 1 || !c.MinRaised || c.MinGiven != 0 || c.Rules.Max != 3 || + c.Rules.HeadroomPct != 20 || c.LeaderElect || c.CredentialsDir != "/c" || c.StatefulSet != "r" || c.Image != "i:1" { + t.Errorf("overrides %+v", c) + } +} + +func TestInvalid(t *testing.T) { + for _, m := range []map[string]string{ + {"SCALER_MAX": "x"}, + {"SCALER_MAX": "0"}, + {"SCALER_INTERVAL": "0"}, + {"SCALER_HEADROOM_PCT": "101"}, + {"SCALER_MIN": "5", "SCALER_MAX": "2"}, + {"SCALER_LEADER_ELECT": "maybe"}, + } { + m["POD_NAMESPACE"] = "ns" + if _, err := Load(env(m)); err == nil || !strings.Contains(err.Error(), "invalid configuration") { + t.Errorf("%v: want an error, got %v", m, err) + } + } +} + +func TestReapPod(t *testing.T) { + c, err := Load(env(map[string]string{ + "POD_NAMESPACE": "ns", + "SCALER_REAP_POD_LABELS": `{"app.kubernetes.io/part-of":"sg-runner","np":"egress"}`, + "SCALER_REAP_IMAGE_PULL_SECRETS": "regcred, other ,", + "SCALER_REAP_NODE_SELECTOR": `{"kubernetes.io/os":"linux"}`, + "SCALER_REAP_TOLERATIONS": `[{"key":"dedicated","operator":"Equal","value":"runners","effect":"NoSchedule"}]`, + })) + if err != nil { + t.Fatal(err) + } + o := c.ReapPod + if o.Labels["np"] != "egress" || len(o.Labels) != 2 || strings.Join(o.ImagePullSecrets, ",") != "regcred,other" || + o.NodeSelector["kubernetes.io/os"] != "linux" || len(o.Tolerations) != 1 || o.Tolerations[0].Value != "runners" { + t.Errorf("reap pod %+v", o) + } + for k, v := range map[string]string{ + "SCALER_REAP_POD_LABELS": `["a"]`, + "SCALER_REAP_IMAGE_PULL_SECRETS": "Bad_Name", + "SCALER_REAP_NODE_SELECTOR": `{"a":1}`, + "SCALER_REAP_TOLERATIONS": `[{"keyy":"x"}]`, + } { + _, err := Load(env(map[string]string{"POD_NAMESPACE": "ns", k: v})) + if err == nil || !strings.Contains(err.Error(), k) { + t.Errorf("%s=%s: want an error naming it, got %v", k, v, err) + } + } + if _, err := Load(env(map[string]string{"POD_NAMESPACE": "ns", "SCALER_REAP_POD_LABELS": `{"bad key!":"x"}`})); err == nil { + t.Error("want an invalid label key rejected") + } +} + +// The Helm chart renders unset values as "", {} and []. +func TestReapPodChartDefaults(t *testing.T) { + c, err := Load(env(map[string]string{ + "POD_NAMESPACE": "ns", + "SCALER_REAP_POD_LABELS": `{}`, + "SCALER_REAP_IMAGE_PULL_SECRETS": "", + "SCALER_REAP_NODE_SELECTOR": `{}`, + "SCALER_REAP_TOLERATIONS": `[]`, + })) + if err != nil { + t.Fatal(err) + } + o := c.ReapPod + if len(o.Labels) != 0 || len(o.ImagePullSecrets) != 0 || len(o.NodeSelector) != 0 || len(o.Tolerations) != 0 { + t.Errorf("want no reap pod settings, got %+v", o) + } +} diff --git a/k8s/scaler/internal/controller/controller.go b/k8s/scaler/internal/controller/controller.go new file mode 100644 index 0000000..f70836d --- /dev/null +++ b/k8s/scaler/internal/controller/controller.go @@ -0,0 +1,612 @@ +// Package controller runs the scaling loop: observe SG and the cluster, +// decide, act, report. +package controller + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "regexp" + "strconv" + "sync/atomic" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +// SG is the part of the SG API the loop uses. +type SG interface { + RunnerGroup(ctx context.Context) (*sg.RunnerGroup, error) + SetRunnerStatus(ctx context.Context, runnerID, status string) error +} + +// Defaults for the loop's back-off and event rate limits. +const ( + DefaultMaxBackoff = 5 * time.Minute + DefaultFailureEvents = 3 + DefaultWarnEvery = 10 * time.Minute +) + +// Controller is the scaling loop for one runner StatefulSet. +type Controller struct { + Config decide.Config + Interval time.Duration + SG SG + Kube *kube.Kube + Events record.EventRecorder + Metrics *metrics.Metrics + Log *slog.Logger + Now func() time.Time + + // MaxBackoff caps the delay after failed loops. + MaxBackoff time.Duration + // FailureEvents is how many failed loops in a row raise a Warning. + FailureEvents int + // WarnEvery rate-limits repeats of the same Warning event. + WarnEvery time.Duration + // Nudge starts placeholder pods for runner pods the cluster autoscaler + // won't add a node for (see kube.NudgeJob). + Nudge bool + // NudgeCooldown spaces placeholders for the same runner + // (DefaultNudgeCooldown when zero). + NudgeCooldown time.Duration + + state *decide.State + persisted map[string]*string + failures int + warned map[string]time.Time + // conditions are the keys of the warnings decide raised last loop. + conditions map[string]bool + // lastLoop is the Unix nano time of the last loop attempt. + lastLoop atomic.Int64 + ordinal *regexp.Regexp + // nudges are each unschedulable runner's placeholder attempts. + nudges map[int]nudgeAttempt +} + +// nudgeAttempt is when a runner's last placeholder started, and how many it +// had since it became unschedulable. +type nudgeAttempt struct { + at time.Time + attempts int +} + +// DefaultNudgeCooldown spaces placeholders for the same runner. It must +// outlast a node's boot: a runner still Pending sooner is just waiting for +// the node already requested. +const DefaultNudgeCooldown = 3 * time.Minute + +// NudgeWarnAfter is how many placeholders a runner may need before the +// scaler reports it as blocked. +const NudgeWarnAfter = 2 + +func (c *Controller) init() { + if c.Now == nil { + c.Now = time.Now + } + if c.Log == nil { + c.Log = slog.Default() + } + if c.MaxBackoff == 0 { + c.MaxBackoff = DefaultMaxBackoff + } + if c.FailureEvents == 0 { + c.FailureEvents = DefaultFailureEvents + } + if c.WarnEvery == 0 { + c.WarnEvery = DefaultWarnEvery + } + if c.warned == nil { + c.warned = map[string]time.Time{} + } + if c.nudges == nil { + c.nudges = map[int]nudgeAttempt{} + } + if c.NudgeCooldown == 0 { + c.NudgeCooldown = DefaultNudgeCooldown + } + if c.ordinal == nil { + c.ordinal = regexp.MustCompile(`^` + regexp.QuoteMeta(c.Kube.StatefulSet) + `-(\d+)\.`) + } +} + +// LastLoop is when the last loop started; zero before the first. +func (c *Controller) LastLoop() time.Time { + ns := c.lastLoop.Load() + if ns == 0 { + return time.Time{} + } + return time.Unix(0, ns) +} + +// Run loops until ctx is done. +func (c *Controller) Run(ctx context.Context) error { + c.init() + c.Log.Info("scaling", "statefulset", c.Kube.StatefulSet, "namespace", c.Kube.Namespace, + "min", c.Config.Min, "max", c.Config.Max, "headroom_pct", c.Config.HeadroomPct, + "idle", c.Config.IdleAfter.String(), "park_ttl", c.Config.ParkTTL.String(), + "stuck_after", c.Config.StuckAfter.String(), "settle_timeout", c.Config.SettleTimeout.String(), + "interval", c.Interval.String()) + // Timers are re-read from the StatefulSet when (re)starting, e.g. after + // taking over the lease. + c.state = nil + for { + _ = c.Once(ctx) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.Delay()): + } + } +} + +// Delay is the wait before the next loop: the interval, doubled for every +// failed loop in a row, up to MaxBackoff. +func (c *Controller) Delay() time.Duration { + d := c.Interval + for i := 0; i < c.failures && d < c.MaxBackoff; i++ { + d *= 2 + } + return min(d, max(c.MaxBackoff, c.Interval)) +} + +// loopError marks which API failed a loop. +type loopError struct { + reason string + err error +} + +func (e *loopError) Error() string { return e.err.Error() } +func (e *loopError) Unwrap() error { return e.err } + +// Once runs one loop. On an SG or Kubernetes API error it makes no +// decision and returns the error; the next loop waits longer. +func (c *Controller) Once(ctx context.Context) error { + c.init() + start := c.Now() + c.lastLoop.Store(start.UnixNano()) + defer func() { c.Metrics.LoopDuration.Observe(c.Now().Sub(start).Seconds()) }() + + sts, err := c.loop(ctx, start) + if err == nil { + if c.failures > 0 { + c.Log.Info("recovered", "after_failures", c.failures) + } + c.failures = 0 + c.Metrics.ConsecutiveFailures.Set(0) + delete(c.warned, "SGUnreachable") + delete(c.warned, "KubernetesAPIError") + return nil + } + c.failures++ + c.Metrics.ConsecutiveFailures.Set(float64(c.failures)) + reason := "KubernetesAPIError" + var le *loopError + if errors.As(err, &le) { + reason = le.reason + } + c.Log.Error("loop failed, skipping decisions", "reason", reason, "err", err, + "consecutive_failures", c.failures, "next_in", c.Delay().String()) + if c.failures >= c.FailureEvents && sts != nil { + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: reason, Key: reason, + Message: fmt.Sprintf("%d loops failed in a row, not scaling: %v", c.failures, err)}) + } + return err +} + +func (c *Controller) sgErr(err error) error { + c.Metrics.SGErrors.Inc() + return &loopError{reason: "SGUnreachable", err: err} +} + +func (c *Controller) kubeErr(what string, err error) error { + c.Metrics.KubeErrors.Inc() + return &loopError{reason: "KubernetesAPIError", err: fmt.Errorf("%s: %w", what, err)} +} + +// loop returns the StatefulSet when it got that far, for events. +func (c *Controller) loop(ctx context.Context, now time.Time) (*appsv1.StatefulSet, error) { + sts, err := c.Kube.GetStatefulSet(ctx) + if err != nil { + return nil, c.kubeErr("get statefulset", err) + } + group, err := c.SG.RunnerGroup(ctx) + if err != nil { + return sts, c.sgErr(err) + } + claims, err := c.Kube.Claims(ctx) + if err != nil { + return sts, c.kubeErr("list pvcs", err) + } + jobs, err := c.Kube.ReapJobs(ctx) + if err != nil { + return sts, c.kubeErr("list reap jobs", err) + } + if c.state == nil { + s := decide.DecodeState(sts.Annotations) + c.state = &s + c.persisted = decide.EncodeState(s) + } + + obs := decide.Observation{ + Now: now, Replicas: kube.Replicas(sts), Queued: group.QueuedWorkflowsCount, + Runners: c.runners(group), Claims: claims, ReapJobs: jobs, + } + if group.MaxNumberOfRunners != nil { + obs.MaxRunners = *group.MaxNumberOfRunners + } + d := decide.Decide(c.Config, obs, *c.state) + + c.Log.Info("loop", "replicas", obs.Replicas, "max", d.Max, "queued", obs.Queued, + "reserved_pct", d.ReservedPct, "free_pct", d.FreePct, "settling", d.Settling, + "unconnected", len(d.Unconnected), "stuck", d.Stuck, "scale", d.Scale.String(), "why", d.Why) + + active := map[string]bool{} + for _, e := range d.Events { + active[e.Key] = true + c.Log.Warn(e.Message, "reason", e.Reason) + c.warn(sts, e) + } + // A condition that cleared and comes back is reported again right away. + for k := range c.conditions { + if !active[k] { + delete(c.warned, k) + } + } + c.conditions = active + + if d.Scale != decide.ScaleNone { + if err := c.Kube.Scale(ctx, d.Replicas); err != nil { + return sts, c.kubeErr("scale statefulset", err) + } + c.Metrics.ScaleEvents.WithLabelValues(d.Scale.String()).Inc() + reason, msg := "ScaledOut", fmt.Sprintf("replicas %d -> %d: %s", obs.Replicas, d.Replicas, d.Why) + if d.Scale == decide.ScaleIn { + reason = "ScaledIn" + msg = fmt.Sprintf("parking runner %d (replicas %d -> %d): %s", d.Replicas, obs.Replicas, d.Replicas, d.Why) + } else if _, parked := claims[obs.Replicas]; parked { + msg += fmt.Sprintf("; waking parked runner %d", obs.Replicas) + } else { + msg += fmt.Sprintf("; registering new runner %d", obs.Replicas) + } + c.Log.Info(msg, "reason", reason) + c.Events.Event(sts, corev1.EventTypeNormal, reason, msg) + } + c.state = &d.State + c.persistState(ctx) + + for _, sc := range d.StatusChanges { + c.changeStatus(ctx, sts, sc, now) + } + for _, name := range d.ClearDrainMarks { + if err := c.Kube.AnnotateClaim(ctx, name, map[string]*string{kube.AnnotationDrainedByScaler: nil}); err != nil { + c.Metrics.KubeErrors.Inc() + c.Log.Error("clear drain mark", "pvc", name, "err", err) + } + } + c.reap(ctx, sts, d.Reap, now) + c.nudge(ctx, sts, d.Replicas, now) + + parked := 0 + for n := range claims { + if n >= d.Replicas { + parked++ + } + } + m := c.Metrics + m.Queued.Set(float64(obs.Queued)) + m.ReservedPct.Set(float64(d.ReservedPct)) + m.FreePct.Set(float64(d.FreePct)) + m.Replicas.Set(float64(d.Replicas)) + m.Awake.Set(float64(obs.Replicas)) + m.Parked.Set(float64(parked)) + m.Max.Set(float64(d.Max)) + m.Unconnected.Set(float64(len(d.Unconnected))) + m.Settling.Set(float64(d.Settling)) + m.Stuck.Set(b2f(d.Stuck)) + return sts, nil +} + +func b2f(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// runners maps SG runners to StatefulSet ordinals through their host name +// (the pod's FQDN); others (VM runners of the same group) are left out. +func (c *Controller) runners(g *sg.RunnerGroup) []decide.Runner { + var out []decide.Runner + for _, i := range g.ContainerInstances { + m := c.ordinal.FindStringSubmatch(i.ComputerName()) + if m == nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + out = append(out, decide.Runner{ + Ordinal: n, ID: i.RunnerID, Status: i.Status, Connected: i.AgentConnected, + Busy: i.RunningTasksCount + i.PendingTasksCount, + CPU: sg.ResourceValue(i.RegisteredResources, "CPU"), + CPUFree: sg.ResourceValue(i.RemainingResources, "CPU"), + Mem: sg.ResourceValue(i.RegisteredResources, "MEMORY"), + MemFree: sg.ResourceValue(i.RemainingResources, "MEMORY"), + }) + } + return out +} + +// persistState writes changed timers to the StatefulSet. A failure is +// retried next loop; the timers stay in memory meanwhile. +func (c *Controller) persistState(ctx context.Context) { + want := decide.EncodeState(*c.state) + if maps.EqualFunc(want, c.persisted, func(a, b *string) bool { + return (a == nil && b == nil) || (a != nil && b != nil && *a == *b) + }) { + return + } + if err := c.Kube.AnnotateStatefulSet(ctx, want); err != nil { + c.Metrics.KubeErrors.Inc() + c.Log.Error("persist timers on the statefulset", "err", err) + return + } + c.persisted = want +} + +// changeStatus drains or reactivates a runner through SG. A drain is marked +// on the runner's PVC first, so a drain is never left without its mark +// (which would keep the runner DRAINING for good); a reactivation clears +// the mark after SG accepted it. +func (c *Controller) changeStatus(ctx context.Context, sts *appsv1.StatefulSet, sc decide.StatusChange, now time.Time) { + log := c.Log.With("runner", sc.Ordinal, "status", sc.Status, "reason", sc.Reason) + fail := func(what string, err error) { + c.Metrics.StatusChanges.WithLabelValues(sc.Reason, "failed").Inc() + log.Error(what, "err", err) + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: "StatusChangeFailed", + Key: fmt.Sprintf("StatusChangeFailed/%d/%s", sc.Ordinal, sc.Status), + Message: fmt.Sprintf("could not set runner %d %s (%s): %v", sc.Ordinal, sc.Status, sc.Reason, err)}) + } + if sc.Status == decide.StatusDraining { + v := strconv.FormatInt(now.Unix(), 10) + if err := c.Kube.AnnotateClaim(ctx, sc.Claim, map[string]*string{kube.AnnotationDrainedByScaler: &v}); err != nil { + c.Metrics.KubeErrors.Inc() + fail("mark drain on pvc", err) + return + } + } + if err := c.SG.SetRunnerStatus(ctx, sc.RunnerID, sc.Status); err != nil { + c.Metrics.SGErrors.Inc() + fail("set runner status", err) + return + } + if sc.Status == decide.StatusActive { + if err := c.Kube.AnnotateClaim(ctx, sc.Claim, map[string]*string{kube.AnnotationDrainedByScaler: nil}); err != nil { + // Retried next loop: an ACTIVE runner's mark is cleared as stale. + c.Metrics.KubeErrors.Inc() + log.Error("clear drain mark", "err", err) + } + } + c.Metrics.StatusChanges.WithLabelValues(sc.Reason, "ok").Inc() + var msg string + switch sc.Reason { + case "DrainedParkedRunner": + msg = fmt.Sprintf("runner %d is parked but SG had it ACTIVE (its pod went away without draining): set DRAINING", sc.Ordinal) + case "DrainedUnconnectedRunner": + msg = fmt.Sprintf("runner %d is awake but not connected past the settle timeout: set DRAINING until it connects", sc.Ordinal) + default: + msg = fmt.Sprintf("runner %d awake and connected again: set ACTIVE (undoing the scaler's drain)", sc.Ordinal) + } + log.Info(msg) + c.Events.Event(sts, corev1.EventTypeNormal, sc.Reason, msg) +} + +func (c *Controller) reap(ctx context.Context, sts *appsv1.StatefulSet, actions []decide.ReapAction, now time.Time) { + for _, a := range actions { + log := c.Log.With("runner", a.Ordinal, "pvc", a.Claim) + kerr := func(what string, err error) { + c.Metrics.KubeErrors.Inc() + log.Error(what, "err", err) + } + switch a.Kind { + case decide.ReapStamp: + v := strconv.FormatInt(now.Unix(), 10) + if err := c.Kube.AnnotateClaim(ctx, a.Claim, map[string]*string{kube.AnnotationParkedAt: &v}); err != nil { + kerr("stamp parked-at", err) + continue + } + log.Info("runner parked") + c.Events.Eventf(sts, corev1.EventTypeNormal, "Parked", "runner %d parked (pod gone, registration kept on %s)", a.Ordinal, a.Claim) + case decide.ReapClearStamp: + if err := c.Kube.AnnotateClaim(ctx, a.Claim, map[string]*string{kube.AnnotationParkedAt: nil}); err != nil { + kerr("clear parked-at", err) + continue + } + log.Info("runner awake") + c.Events.Eventf(sts, corev1.EventTypeNormal, "Woke", "runner %d awake again", a.Ordinal) + case decide.ReapStartJob: + if err := c.Kube.CreateReapJob(ctx, a.Ordinal); err != nil { + kerr("create reap job", err) + continue + } + c.Metrics.Reaps.WithLabelValues("started").Inc() + log.Info("deregistering parked runner", "parked_for", a.ParkedFor.Round(time.Second)) + c.Events.Eventf(sts, corev1.EventTypeNormal, "Reaping", + "runner %d parked for %s: deregistering it (job/%s)", a.Ordinal, a.ParkedFor.Round(time.Second), c.Kube.JobName(a.Ordinal)) + case decide.ReapCleanup: + if err := c.Kube.DeleteClaim(ctx, a.Claim); err != nil { + kerr("delete state pvc", err) + continue + } + if err := c.Kube.DeleteClaim(ctx, c.Kube.DockerClaimName(a.Ordinal)); err != nil { + kerr("delete docker pvc", err) + continue + } + if err := c.Kube.DeleteReapJob(ctx, a.Ordinal); err != nil { + kerr("delete reap job", err) + } + c.Metrics.Reaps.WithLabelValues("succeeded").Inc() + log.Info("runner deregistered, PVCs deleted") + c.Events.Eventf(sts, corev1.EventTypeNormal, "Reaped", "runner %d deregistered from SG and its PVCs deleted", a.Ordinal) + case decide.ReapRetry: + if err := c.Kube.DeleteReapJob(ctx, a.Ordinal); err != nil { + kerr("delete failed reap job", err) + continue + } + c.Metrics.Reaps.WithLabelValues("failed").Inc() + log.Warn("deregistration failed, retrying next loop") + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: "ReapFailed", + Key: fmt.Sprintf("ReapFailed/%d", a.Ordinal), + Message: fmt.Sprintf("deregistering parked runner %d failed; retrying (identity kept on %s)", a.Ordinal, a.Claim)}) + } + } +} + +// nudge gets nodes for awake runners whose pods the cluster autoscaler +// refused to add a node for: a placeholder without volumes asks for one, and +// is deleted as soon as the autoscaler accepts (TriggeredScaleUp), so the +// runner takes the node the moment it joins. A runner still unschedulable a +// cooldown later gets another placeholder; repeated attempts are reported. +// Best effort: errors are logged and retried next loop. +func (c *Controller) nudge(ctx context.Context, sts *appsv1.StatefulSet, replicas int, now time.Time) { + if !c.Nudge { + return + } + kerr := func(what string, err error) { + c.Metrics.KubeErrors.Inc() + c.Log.Error(what, "err", err) + } + sel, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector) + if err != nil { + c.Log.Error("runner selector", "err", err) + return + } + pods, err := c.Kube.RunnerPods(ctx, sel.String()) + if err != nil { + kerr("list runner pods", err) + return + } + jobs, err := c.Kube.NudgeJobs(ctx) + if err != nil { + kerr("list placeholder jobs", err) + return + } + blocked := func(n int, why string) { + c.warn(sts, decide.Event{Type: decide.EventWarning, Reason: "ScaleUpBlocked", + Key: fmt.Sprintf("ScaleUpBlocked/%d", n), + Message: fmt.Sprintf("runner %d can't get a node: %s; retrying in %s", n, why, c.NudgeCooldown)}) + } + + for n, j := range jobs { + p := pods[n] + job := c.Kube.NudgeJobName(n) + log := c.Log.With("runner", n, "job", job) + remove := func() bool { + if err := c.Kube.DeleteNudgeJob(ctx, n); err != nil { + kerr("delete placeholder job", err) + return false + } + return true + } + if n >= replicas || p == nil || !kube.Unschedulable(p) { + // Placed, parked or gone: nothing left to ask for. + if remove() { + log.Info("placeholder no longer needed, deleted") + } + continue + } + reason, msg, err := c.Kube.NudgeOutcome(ctx, n, j.Created) + if err != nil { + kerr("list placeholder events", err) + continue + } + switch { + case reason == kube.EventScaleUpTriggered || j.State == kube.NudgeRunning: + // A node is coming (or there): free it for the runner now. + if !remove() { + continue + } + c.Metrics.Nudges.WithLabelValues("node_requested").Inc() + log.Info("cluster autoscaler is adding a node; placeholder deleted so the runner takes it") + c.Events.Eventf(sts, corev1.EventTypeNormal, "NodeRequested", + "runner %d: the cluster autoscaler is adding a node for job/%s; deleted the placeholder so the runner takes the node", n, job) + case reason == kube.EventScaleUpRefused: + if !remove() { + continue + } + c.Metrics.Nudges.WithLabelValues("refused").Inc() + log.Warn("cluster autoscaler refused the placeholder too", "message", msg) + blocked(n, "the cluster autoscaler won't add a node even without volumes ("+msg+")") + case j.State == kube.NudgeFailed: + if !remove() { + continue + } + c.Metrics.Nudges.WithLabelValues("refused").Inc() + log.Warn("no answer from the cluster autoscaler within the placeholder's deadline") + blocked(n, fmt.Sprintf("no node for its placeholder within %s", time.Duration(kube.NudgeDeadline)*time.Second)) + } + } + + for n, p := range pods { + if n >= replicas || !kube.Unschedulable(p) { + delete(c.nudges, n) + continue + } + if _, ok := jobs[n]; ok { + continue + } + last := c.nudges[n] + if last.attempts > 0 && now.Sub(last.at) < c.NudgeCooldown { + continue + } + refused, err := c.Kube.ScaleUpRefused(ctx, p) + if err != nil { + kerr("list autoscaler events", err) + continue + } + if !refused { + continue + } + if last.attempts >= NudgeWarnAfter { + blocked(n, fmt.Sprintf("still unschedulable after %d node requests (the new node went to another pod, or never came)", last.attempts)) + } + if err := c.Kube.CreateNudgeJob(ctx, n, p); err != nil { + kerr("create placeholder job", err) + continue + } + c.nudges[n] = nudgeAttempt{at: now, attempts: last.attempts + 1} + c.Metrics.Nudges.WithLabelValues("started").Inc() + c.Log.Info("cluster autoscaler won't add a node for the runner pod; asking with a placeholder", + "runner", n, "job", c.Kube.NudgeJobName(n), "attempt", last.attempts+1) + c.Events.Eventf(sts, corev1.EventTypeNormal, "NudgingAutoscaler", + "runner %d can't be scheduled and the cluster autoscaler won't add a node for it (its volumes): job/%s asks for one", + n, c.Kube.NudgeJobName(n)) + } + for n := range c.nudges { + if _, ok := pods[n]; !ok { + delete(c.nudges, n) + } + } +} + +// warn emits a Warning event unless the same one went out within WarnEvery. +func (c *Controller) warn(sts *appsv1.StatefulSet, e decide.Event) { + now := c.Now() + if last, ok := c.warned[e.Key]; ok && now.Sub(last) < c.WarnEvery { + return + } + c.warned[e.Key] = now + c.Events.Event(sts, e.Type, e.Reason, e.Message) +} diff --git a/k8s/scaler/internal/controller/controller_test.go b/k8s/scaler/internal/controller/controller_test.go new file mode 100644 index 0000000..4ebd83b --- /dev/null +++ b/k8s/scaler/internal/controller/controller_test.go @@ -0,0 +1,473 @@ +package controller + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/metrics" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +const ns = "sg" + +var t0 = time.Unix(1_700_000_000, 0) + +type fakeSG struct { + group *sg.RunnerGroup + err error + statuses []string +} + +func (f *fakeSG) RunnerGroup(context.Context) (*sg.RunnerGroup, error) { + if f.err != nil { + return nil, f.err + } + return f.group, nil +} + +func (f *fakeSG) SetRunnerStatus(_ context.Context, id, status string) error { + f.statuses = append(f.statuses, id+"="+status) + return nil +} + +func inst(n int, status string, connected bool, busy int) sg.Instance { + i := sg.Instance{ + RunnerID: fmt.Sprintf("r-%d", n), Status: status, AgentConnected: connected, RunningTasksCount: busy, + RegisteredResources: []sg.Resource{{Name: "CPU", IntegerValue: 2048}, {Name: "MEMORY", IntegerValue: 4096}}, + RemainingResources: []sg.Resource{{Name: "CPU", IntegerValue: 2048}, {Name: "MEMORY", IntegerValue: 4096}}, + } + if busy > 0 { + i.RemainingResources = []sg.Resource{{Name: "CPU", IntegerValue: 0}, {Name: "MEMORY", IntegerValue: 0}} + } + i.InstanceDetails = append(i.InstanceDetails, struct { + ComputerName string `json:"ComputerName"` + }{fmt.Sprintf("sg-runner-%d.sg-runner.sg.svc.cluster.local", n)}) + return i +} + +func statefulSet(replicas int32) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-runner", Namespace: ns}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(replicas)}, + } +} + +func claim(name string, ann map[string]string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Annotations: ann}} +} + +type env struct { + t *testing.T + ctrl *Controller + sg *fakeSG + client *fake.Clientset + events *record.FakeRecorder + now time.Time + logs *bytes.Buffer +} + +func newEnv(t *testing.T, s SG, objs ...runtime.Object) *env { + t.Helper() + e := &env{t: t, client: fake.NewClientset(objs...), events: record.NewFakeRecorder(100), now: t0, logs: &bytes.Buffer{}} + if f, ok := s.(*fakeSG); ok { + e.sg = f + } + e.ctrl = e.newController(s) + return e +} + +func (e *env) newController(s SG) *Controller { + return &Controller{ + Config: decide.Config{Min: 1, Max: 10, IdleAfter: time.Minute, StuckAfter: 3 * time.Minute, + SettleTimeout: 5 * time.Minute, ParkTTL: 12 * time.Hour}, + Interval: 15 * time.Second, + SG: s, + Kube: kube.New(e.client, ns, "sg-runner", "img:1", "creds", "cfg"), + Events: e.events, + Metrics: metrics.New(), + Log: slog.New(slog.NewTextHandler(e.logs, nil)), + Now: func() time.Time { return e.now }, + } +} + +func (e *env) once() error { + e.t.Helper() + return e.ctrl.Once(context.Background()) +} + +func (e *env) sts() *appsv1.StatefulSet { + s, err := e.client.AppsV1().StatefulSets(ns).Get(context.Background(), "sg-runner", metav1.GetOptions{}) + if err != nil { + e.t.Fatal(err) + } + return s +} + +func (e *env) claim(name string) (*corev1.PersistentVolumeClaim, error) { + return e.client.CoreV1().PersistentVolumeClaims(ns).Get(context.Background(), name, metav1.GetOptions{}) +} + +func (e *env) drainEvents() []string { + var out []string + for { + select { + case ev := <-e.events.Events: + out = append(out, ev) + default: + return out + } + } +} + +func hasEvent(events []string, prefix string) bool { + for _, ev := range events { + if strings.HasPrefix(ev, prefix) { + return true + } + } + return false +} + +func TestScaleOut(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{QueuedWorkflowsCount: 1, ContainerInstances: []sg.Instance{ + inst(0, "ACTIVE", true, 1), + {RunnerID: "vm", Status: "ACTIVE", AgentConnected: true}, // VM runner of the same group: ignored + }}} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil)) + if err := e.once(); err != nil { + t.Fatal(err) + } + if r := kube.Replicas(e.sts()); r != 2 { + t.Fatalf("replicas = %d, want 2", r) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal ScaledOut replicas 1 -> 2") || !strings.Contains(strings.Join(ev, "|"), "registering new runner 1") { + t.Errorf("events %v", ev) + } + if v := testutil.ToFloat64(e.ctrl.Metrics.ScaleEvents.WithLabelValues("out")); v != 1 { + t.Errorf("scale_events_total{out} = %v", v) + } + if v := testutil.ToFloat64(e.ctrl.Metrics.Queued); v != 1 { + t.Errorf("queued = %v", v) + } +} + +// Timers live on the StatefulSet: a new scaler (restart, new leader) keeps +// counting where the old one stopped. +func TestTimersSurviveRestart(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 0), inst(1, "ACTIVE", true, 0)}}} + e := newEnv(t, f, statefulSet(2), claim("state-sg-runner-0", nil), claim("state-sg-runner-1", nil)) + if err := e.once(); err != nil { + t.Fatal(err) + } + if got := e.sts().Annotations[decide.AnnotationIdleSince]; got != fmt.Sprintf(`{"1":%d}`, t0.Unix()) { + t.Fatalf("idle-since annotation = %q", got) + } + // Restarted scaler, 60s later. + e.ctrl = e.newController(f) + e.now = t0.Add(time.Minute) + if err := e.once(); err != nil { + t.Fatal(err) + } + s := e.sts() + if kube.Replicas(s) != 1 { + t.Fatalf("replicas = %d, want 1 (parked with the persisted timer)", kube.Replicas(s)) + } + if _, ok := s.Annotations[decide.AnnotationIdleSince]; ok { + t.Errorf("idle-since should be cleared after parking: %v", s.Annotations) + } + // The parked runner's PVC gets stamped. + p, _ := e.claim("state-sg-runner-1") + if p.Annotations[kube.AnnotationParkedAt] != fmt.Sprint(e.now.Unix()) { + t.Errorf("parked-at = %v", p.Annotations) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal ScaledIn") || !hasEvent(ev, "Normal Parked") { + t.Errorf("events %v", ev) + } +} + +// A runner parked without its preStop drain is drained and marked; on +// wake the scaler reactivates it and clears the mark. Unmarked drains stick. +func TestDrainParkedAndReactivate(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{ + inst(0, "ACTIVE", true, 1), inst(1, "ACTIVE", false, 0), inst(2, "DRAINING", false, 0), + }}} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil), claim("state-sg-runner-1", nil), claim("state-sg-runner-2", nil)) + if err := e.once(); err != nil { + t.Fatal(err) + } + if strings.Join(f.statuses, ",") != "r-1=DRAINING" { + t.Fatalf("status calls = %v", f.statuses) + } + p, _ := e.claim("state-sg-runner-1") + if p.Annotations[kube.AnnotationDrainedByScaler] != fmt.Sprint(t0.Unix()) { + t.Fatalf("drain mark = %v", p.Annotations) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal DrainedParkedRunner") { + t.Errorf("events %v", ev) + } + + // Woken (someone scales to 3), both connected again, DRAINING. + f.statuses = nil + if err := e.ctrl.Kube.Scale(context.Background(), 3); err != nil { + t.Fatal(err) + } + f.group.ContainerInstances = []sg.Instance{inst(0, "ACTIVE", true, 1), inst(1, "DRAINING", true, 0), inst(2, "DRAINING", true, 0)} + e.now = t0.Add(time.Minute) + if err := e.once(); err != nil { + t.Fatal(err) + } + if strings.Join(f.statuses, ",") != "r-1=ACTIVE" { + t.Fatalf("status calls = %v (runner 2 was drained by someone else)", f.statuses) + } + p, _ = e.claim("state-sg-runner-1") + if _, ok := p.Annotations[kube.AnnotationDrainedByScaler]; ok { + t.Errorf("drain mark not cleared: %v", p.Annotations) + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal Reactivated") { + t.Errorf("events %v", ev) + } + if v := testutil.ToFloat64(e.ctrl.Metrics.StatusChanges.WithLabelValues("Reactivated", "ok")); v != 1 { + t.Errorf("status changes metric = %v", v) + } +} + +func TestReap(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 1), inst(1, "DRAINING", false, 0)}}} + parkedAt := map[string]string{kube.AnnotationParkedAt: fmt.Sprint(t0.Unix())} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil), + claim("state-sg-runner-1", parkedAt), claim("docker-sg-runner-1", nil)) + ctx := context.Background() + jobs := e.client.BatchV1().Jobs(ns) + + // Within the TTL: nothing. + e.now = t0.Add(11 * time.Hour) + if err := e.once(); err != nil { + t.Fatal(err) + } + if l, _ := jobs.List(ctx, metav1.ListOptions{}); len(l.Items) != 0 { + t.Fatal("job started before the TTL") + } + + // Past it: a reap Job, which blocks waking that ordinal. + e.now = t0.Add(12 * time.Hour) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, err := jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if j.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != "state-sg-runner-1" { + t.Error("job mounts the wrong PVC") + } + f.group.QueuedWorkflowsCount = 1 + if err := e.once(); err != nil { + t.Fatal(err) + } + if kube.Replicas(e.sts()) != 1 { + t.Fatal("woke a runner being deregistered") + } + f.group.QueuedWorkflowsCount = 0 + + // Failed: deleted for a retry, warned. + j.Status.Failed = 1 + if _, err := jobs.UpdateStatus(ctx, j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.drainEvents() + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}); err == nil { + t.Fatal("failed job not deleted") + } + if ev := e.drainEvents(); !hasEvent(ev, "Warning ReapFailed") { + t.Errorf("events %v", ev) + } + + // Retried, succeeds: PVCs and Job deleted. + if err := e.once(); err != nil { + t.Fatal(err) + } + j, _ = jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}) + j.Status.Succeeded = 1 + if _, err := jobs.UpdateStatus(ctx, j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := e.once(); err != nil { + t.Fatal(err) + } + for _, name := range []string{"state-sg-runner-1", "docker-sg-runner-1"} { + if _, err := e.claim(name); err == nil { + t.Errorf("%s not deleted", name) + } + } + if _, err := jobs.Get(ctx, "sg-runner-reap-1", metav1.GetOptions{}); err == nil { + t.Error("job not deleted") + } + if _, err := e.claim("state-sg-runner-0"); err != nil { + t.Error("awake runner's PVC deleted") + } + if ev := e.drainEvents(); !hasEvent(ev, "Normal Reaped") { + t.Errorf("events %v", ev) + } + m := e.ctrl.Metrics.Reaps + if testutil.ToFloat64(m.WithLabelValues("started")) != 2 || testutil.ToFloat64(m.WithLabelValues("failed")) != 1 || + testutil.ToFloat64(m.WithLabelValues("succeeded")) != 1 { + t.Error("reap metrics") + } +} + +func TestBackoffAndFailureEvents(t *testing.T) { + f := &fakeSG{err: errors.New("SG API GET /: HTTP 503")} + e := newEnv(t, f, statefulSet(2)) + var delays []time.Duration + for i := 0; i < 7; i++ { + if err := e.once(); err == nil { + t.Fatal("want an error") + } + delays = append(delays, e.ctrl.Delay()) + } + want := []time.Duration{30, 60, 120, 240, 300, 300, 300} + for i := range want { + if delays[i] != want[i]*time.Second { + t.Fatalf("delays = %v", delays) + } + } + if kube.Replicas(e.sts()) != 2 { + t.Error("scaled while SG was unreachable") + } + ev := e.drainEvents() + if len(ev) != 1 || !hasEvent(ev, "Warning SGUnreachable") { + t.Errorf("want one rate-limited SGUnreachable warning, got %v", ev) + } + if testutil.ToFloat64(e.ctrl.Metrics.SGErrors) != 7 || testutil.ToFloat64(e.ctrl.Metrics.ConsecutiveFailures) != 7 { + t.Error("error metrics") + } + // Recovered. + f.err = nil + f.group = &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 1), inst(1, "ACTIVE", true, 1)}} + if err := e.once(); err != nil { + t.Fatal(err) + } + if e.ctrl.Delay() != 15*time.Second || testutil.ToFloat64(e.ctrl.Metrics.ConsecutiveFailures) != 0 { + t.Error("back-off not reset") + } +} + +func TestKubeErrorSkipsDecisions(t *testing.T) { + f := &fakeSG{group: &sg.RunnerGroup{QueuedWorkflowsCount: 1, ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 1)}}} + e := newEnv(t, f) // no StatefulSet + if err := e.once(); err == nil { + t.Fatal("want an error") + } + if testutil.ToFloat64(e.ctrl.Metrics.KubeErrors) != 1 { + t.Error("kube error not counted") + } +} + +// Identical warnings go out once per WarnEvery while the condition lasts, +// and again as soon as it comes back after clearing. +func TestWarningRateLimit(t *testing.T) { + stuck := &sg.RunnerGroup{QueuedWorkflowsCount: 1, ContainerInstances: []sg.Instance{inst(0, "ACTIVE", true, 0)}} + f := &fakeSG{group: stuck} + e := newEnv(t, f, statefulSet(1), claim("state-sg-runner-0", nil)) + count := func() int { + n := 0 + for _, ev := range e.drainEvents() { + if strings.HasPrefix(ev, "Warning StuckQueue") { + n++ + } + } + return n + } + for at := time.Duration(0); at <= 12*time.Minute; at += 15 * time.Second { + e.now = t0.Add(at) + if err := e.once(); err != nil { + t.Fatal(err) + } + } + // Stuck from 3m: warned at 3m and 13m would be next; 3m..12m is one. + if n := count(); n != 1 { + t.Fatalf("StuckQueue warnings = %d, want 1", n) + } + if testutil.ToFloat64(e.ctrl.Metrics.Stuck) != 1 { + t.Error("stuck metric") + } + f.group = &sg.RunnerGroup{ContainerInstances: stuck.ContainerInstances} + e.now = e.now.Add(15 * time.Second) + e.once() + f.group = stuck + for at := time.Duration(0); at <= 3*time.Minute; at += 15 * time.Second { + e.now = e.now.Add(15 * time.Second) + e.once() + } + if n := count(); n != 1 { + t.Fatalf("StuckQueue warnings after the condition came back = %d, want 1", n) + } +} + +// End to end with the real SG client: the runner group's token must not +// reach logs, events or errors. +func TestNoSecretsInLogs(t *testing.T) { + const secret = "sgrt_TOPSECRET_runner_token" + body := `{"msg": {"RunnerToken": "` + secret + `", "QueuedWorkflowsCount": 1, "ContainerInstances": [ + {"runnerId": "r-0", "status": "ACTIVE", "agentConnected": true, "runningTasksCount": 1, + "registeredResources": [{"name": "CPU", "integerValue": 2048}, {"name": "MEMORY", "integerValue": 4096}], + "remainingResources": [{"name": "CPU", "integerValue": 0}, {"name": "MEMORY", "integerValue": 0}], + "instanceDetails": [{"ComputerName": "sg-runner-0.sg-runner.sg.svc"}]}]}}` + fail := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fail { + w.WriteHeader(500) + } + io.WriteString(w, body) + })) + defer srv.Close() + client := sg.New(sg.Credentials{Token: "node-tok", Organization: "o", RunnerGroup: "g", BaseAPI: srv.URL}, nil) + e := newEnv(t, client, statefulSet(1), claim("state-sg-runner-0", nil)) + e.ctrl.FailureEvents = 1 + if err := e.once(); err != nil { + t.Fatal(err) + } + fail = true + err := e.once() + if err == nil { + t.Fatal("want an error") + } + all := e.logs.String() + strings.Join(e.drainEvents(), "\n") + err.Error() + if strings.Contains(all, secret) || strings.Contains(all, "node-tok") { + t.Errorf("secret leaked:\n%s", all) + } + if !strings.Contains(all, "HTTP 500") { + t.Errorf("failure not reported:\n%s", all) + } +} + +func TestDelayCap(t *testing.T) { + c := &Controller{Interval: 10 * time.Minute, MaxBackoff: 5 * time.Minute} + c.failures = 3 + if c.Delay() != 10*time.Minute { + t.Errorf("an interval above the cap is kept: %v", c.Delay()) + } +} diff --git a/k8s/scaler/internal/controller/nudge_test.go b/k8s/scaler/internal/controller/nudge_test.go new file mode 100644 index 0000000..9b3b5ba --- /dev/null +++ b/k8s/scaler/internal/controller/nudge_test.go @@ -0,0 +1,263 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/sg" +) + +var runnerLabels = map[string]string{"app.kubernetes.io/component": "runner"} + +func runnerPod(n string, node string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-" + n, Namespace: ns, UID: types.UID("uid-" + n), Labels: runnerLabels}, + Spec: corev1.PodSpec{ + NodeName: node, + Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{MatchLabels: runnerLabels}}}}}, + Containers: []corev1.Container{{Name: "dockerd", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("6500m"), corev1.ResourceMemory: resource.MustParse("11Gi")}}}}, + Volumes: []corev1.Volume{{Name: "state"}}, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + if node == "" { + p.Status = corev1.PodStatus{Phase: corev1.PodPending, Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable}}} + } + return p +} + +func refusedEvent(pod string) *corev1.Event { + return &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: pod + ".1", Namespace: ns}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: pod, Namespace: ns}, + Reason: "NotTriggerScaleUp", Source: corev1.EventSource{Component: "cluster-autoscaler"}, + } +} + +// nudgeEnv: runner 0 runs and is busy, runner 1 (awake, new) can't be +// scheduled. +func nudgeEnv(t *testing.T, objs ...runtime.Object) *env { + sts := statefulSet(2) + sts.Spec.Selector = &metav1.LabelSelector{MatchLabels: runnerLabels} + objs = append(objs, sts, runnerPod("0", "node-a"), runnerPod("1", ""), + claim("state-sg-runner-0", nil), claim("state-sg-runner-1", nil)) + e := newEnv(t, &fakeSG{group: &sg.RunnerGroup{ContainerInstances: []sg.Instance{inst(0, decide.StatusActive, true, 1)}}}, objs...) + e.ctrl.Nudge = true + return e +} + +func (e *env) job(name string) (*batchv1.Job, error) { + return e.client.BatchV1().Jobs(ns).Get(context.Background(), name, metav1.GetOptions{}) +} + +func TestNudgeStartsPlaceholderWhenAutoscalerRefuses(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, err := e.job("sg-runner-nudge-1") + if err != nil { + t.Fatalf("no placeholder job: %v\n%s", err, e.logs) + } + ps := j.Spec.Template.Spec + req := ps.Containers[0].Resources.Requests + if req.Cpu().String() != "6500m" || req.Memory().String() != "11Gi" { + t.Errorf("placeholder requests %v, want the runner's", req) + } + if len(ps.Volumes) != 0 { + t.Errorf("placeholder has volumes %v", ps.Volumes) + } + if ps.Affinity == nil || ps.Affinity.PodAntiAffinity == nil { + t.Errorf("placeholder lost the runner's anti-affinity") + } + if j.Labels["app.kubernetes.io/component"] == "runner" { + t.Errorf("placeholder carries runner labels: %v", j.Labels) + } + if !hasEvent(e.drainEvents(), "Normal NudgingAutoscaler runner 1") { + t.Error("no NudgingAutoscaler event") + } + if _, err := e.job("sg-runner-nudge-0"); err == nil { + t.Error("placeholder for the scheduled runner 0") + } +} + +func TestNudgeWaitsForTheAutoscalersVerdict(t *testing.T) { + // No NotTriggerScaleUp: the autoscaler may still add a node itself. + e := nudgeEnv(t) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("placeholder started without the autoscaler refusing") + } +} + +// autoscalerSays records the cluster autoscaler's answer on the runner's +// current placeholder pod. +func (e *env) autoscalerSays(n, reason string) { + e.t.Helper() + ev := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "ph-" + n + "-" + reason + e.now.String(), Namespace: ns}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: "sg-runner-nudge-" + n + "-xyz", Namespace: ns}, + Reason: reason, Message: "max node group size reached", LastTimestamp: metav1.NewTime(e.now), + } + if _, err := e.client.CoreV1().Events(ns).Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { + e.t.Fatal(err) + } +} + +func TestNudgeReleasesThePlaceholderOnceTheAutoscalerAccepts(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + e.autoscalerSays("1", "TriggeredScaleUp") + e.drainEvents() + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Fatal("placeholder kept after the autoscaler accepted") + } + if !hasEvent(e.drainEvents(), "Normal NodeRequested runner 1") { + t.Error("no NodeRequested event") + } + if v := testutil.ToFloat64(e.ctrl.Metrics.Nudges.WithLabelValues("node_requested")); v != 1 { + t.Errorf("node_requested = %v, want 1", v) + } + // The node is booting and the runner still Pending: no new placeholder + // within the cooldown. + e.now = e.now.Add(time.Minute) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("new placeholder within the cooldown") + } + // Past it, still unschedulable: ask again, and that's the attempt + // after which it's reported. + e.now = e.now.Add(e.ctrl.NudgeCooldown) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err != nil { + t.Fatalf("no placeholder after the cooldown: %v", err) + } + if hasEvent(e.drainEvents(), "Warning ScaleUpBlocked") { + t.Error("blocked after only one earlier attempt") + } + _ = e.client.BatchV1().Jobs(ns).Delete(context.Background(), "sg-runner-nudge-1", metav1.DeleteOptions{}) + e.now = e.now.Add(e.ctrl.NudgeCooldown) + if err := e.once(); err != nil { + t.Fatal(err) + } + if !hasEvent(e.drainEvents(), "Warning ScaleUpBlocked runner 1 can't get a node: still unschedulable after 2") { + t.Error("no ScaleUpBlocked after repeated attempts") + } +} + +func TestNudgeHandsOverARunningPlaceholder(t *testing.T) { + // The placeholder found room without a scale-up (a node freed up). + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, _ := e.job("sg-runner-nudge-1") + j.Status.Ready = ptr.To[int32](1) + if _, err := e.client.BatchV1().Jobs(ns).UpdateStatus(context.Background(), j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("running placeholder kept") + } +} + +func TestNudgeReportsAPlaceholderTheAutoscalerRefuses(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + e.autoscalerSays("1", "NotTriggerScaleUp") + e.drainEvents() + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("refused placeholder kept") + } + if !hasEvent(e.drainEvents(), "Warning ScaleUpBlocked runner 1 can't get a node: the cluster autoscaler won't add a node even without volumes (max node group size reached)") { + t.Error("no ScaleUpBlocked warning with the autoscaler's reason") + } +} + +func TestNudgeDeletesPlaceholderOnceTheRunnerIsPlaced(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.client.CoreV1().Pods(ns).Update(context.Background(), runnerPod("1", "node-b"), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("placeholder kept after the runner was placed") + } +} + +func TestNudgeReportsAPlaceholderPastItsDeadline(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + if err := e.once(); err != nil { + t.Fatal(err) + } + j, _ := e.job("sg-runner-nudge-1") + j.Status.Failed = 1 + if _, err := e.client.BatchV1().Jobs(ns).UpdateStatus(context.Background(), j, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + e.drainEvents() + e.now = e.now.Add(e.ctrl.Interval) + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("failed placeholder kept") + } + if !hasEvent(e.drainEvents(), "Warning ScaleUpBlocked runner 1") { + t.Error("no ScaleUpBlocked warning") + } +} + +func TestNudgeOff(t *testing.T) { + e := nudgeEnv(t, refusedEvent("sg-runner-1")) + e.ctrl.Nudge = false + if err := e.once(); err != nil { + t.Fatal(err) + } + if _, err := e.job("sg-runner-nudge-1"); err == nil { + t.Error("placeholder with nudging off") + } +} diff --git a/k8s/scaler/internal/decide/decide.go b/k8s/scaler/internal/decide/decide.go new file mode 100644 index 0000000..ac88e9a --- /dev/null +++ b/k8s/scaler/internal/decide/decide.go @@ -0,0 +1,482 @@ +// Package decide holds the scaler's rules as a pure function: an observation +// of SG and the cluster, the timers kept between loops and the configuration +// go in; the replicas to set, runner status changes, reap steps, warnings and +// the updated timers come out. Nothing here does I/O. +package decide + +import ( + "fmt" + "math" + "sort" + "time" +) + +// Runner statuses as SG reports them. +const ( + StatusActive = "ACTIVE" + StatusDraining = "DRAINING" +) + +// Event types, as Kubernetes names them. +const ( + EventNormal = "Normal" + EventWarning = "Warning" +) + +// Config holds the scaling rules' knobs. +type Config struct { + Min int + Max int + HeadroomPct int + IdleAfter time.Duration + StuckAfter time.Duration + SettleTimeout time.Duration + ParkTTL time.Duration +} + +// Runner is one SG container instance mapped to a StatefulSet ordinal. +type Runner struct { + Ordinal int + ID string + Status string + Connected bool + // Busy is running plus pending tasks. + Busy int + CPU float64 + CPUFree float64 + Mem float64 + MemFree float64 +} + +// FullyIdle reports a runner with nothing running and nothing reserved. +func (r Runner) FullyIdle() bool { + return r.Busy == 0 && r.CPUFree == r.CPU && r.MemFree == r.Mem +} + +// Size is the runner's registered CPU units / MiB, for messages. +func (r Runner) Size() string { + return fmt.Sprintf("%g/%g", r.CPU, r.Mem) +} + +// JobState is the state of a runner's reap Job. +type JobState int + +// Reap Job states. +const ( + JobNone JobState = iota + JobRunning + JobSucceeded + JobFailed +) + +// Claim is a runner's state PVC. +type Claim struct { + Name string + // ParkedAt is the parked-at stamp; zero when absent or unreadable. + ParkedAt time.Time + // Stamped reports a parked-at annotation present, readable or not. + Stamped bool + // DrainedByScaler reports that the scaler set this runner DRAINING. + DrainedByScaler bool +} + +// Observation is what one loop sees. +type Observation struct { + Now time.Time + Replicas int + Queued int + // MaxRunners is the runner group's MaxNumberOfRunners; 0 when unset. + MaxRunners int + // Runners are the SG runners that map to an ordinal of the StatefulSet. + Runners []Runner + // Claims are the state PVCs by ordinal. + Claims map[int]Claim + // ReapJobs are the reap Jobs by ordinal. + ReapJobs map[int]JobState +} + +// State is what the scaler remembers between loops. +type State struct { + // IdleSince: highest awake ordinal -> when it was first seen idle. + IdleSince map[int]time.Time + // StuckSince: since when workflows were queued while an awake runner + // sat fully idle; zero when not. + StuckSince time.Time + // UnconnectedSince: awake ordinal not connected to SG -> first seen so. + UnconnectedSince map[int]time.Time +} + +// Clone returns a deep copy. +func (s State) Clone() State { + c := State{StuckSince: s.StuckSince, IdleSince: map[int]time.Time{}, UnconnectedSince: map[int]time.Time{}} + for k, v := range s.IdleSince { + c.IdleSince[k] = v + } + for k, v := range s.UnconnectedSince { + c.UnconnectedSince[k] = v + } + return c +} + +// Scale is a replicas change. +type Scale int + +// Scale directions. +const ( + ScaleNone Scale = iota + ScaleOut + ScaleIn +) + +func (s Scale) String() string { + switch s { + case ScaleOut: + return "out" + case ScaleIn: + return "in" + } + return "none" +} + +// StatusChange sets a runner's status through SG. +type StatusChange struct { + Ordinal int + RunnerID string + // Claim is the runner's state PVC, which records drains by the scaler. + Claim string + Status string + // Reason is DrainedParkedRunner, DrainedUnconnectedRunner or Reactivated. + Reason string +} + +// ReapKind is one step of reaping parked runners. +type ReapKind int + +// Reap steps. +const ( + // ReapStamp stamps parked-at on a newly parked runner's PVC. + ReapStamp ReapKind = iota + // ReapClearStamp clears parked-at on a runner awake again. + ReapClearStamp + // ReapStartJob starts the Job deregistering a runner parked past the TTL. + ReapStartJob + // ReapCleanup deletes a deregistered runner's PVCs and its Job. + ReapCleanup + // ReapRetry deletes a failed Job so a later loop retries. + ReapRetry +) + +// ReapAction is one reap step for one ordinal. +type ReapAction struct { + Kind ReapKind + Ordinal int + Claim string + ParkedFor time.Duration +} + +// Event is something worth reporting as a Kubernetes Event. Key identifies +// repeats of the same condition so they can be rate-limited. +type Event struct { + Type string + Reason string + Key string + Message string +} + +// Decision is the outcome of one loop. +type Decision struct { + State State + + Max int + ReservedPct int + FreePct int + Awake int + Settling int + // Unconnected are the awake ordinals not connected to SG. + Unconnected []int + Stuck bool + + // Replicas to set; equal to the observed replicas when Scale is ScaleNone. + Replicas int + Scale Scale + // Why explains the scale decision (or why a scale out waits). + Why string + + StatusChanges []StatusChange + // ClearDrainMarks are state PVCs whose drained-by-scaler mark is stale. + ClearDrainMarks []string + Reap []ReapAction + Events []Event +} + +// ReservedPct is the share (0-100, floored) of CPU or memory, whichever is +// higher, reserved on ACTIVE and connected runners with an ordinal below +// the given one. 100 when there are none. +func ReservedPct(runners []Runner, below int) int { + var cpu, cpuFree, mem, memFree float64 + n := 0 + for _, r := range runners { + if r.Ordinal < below && r.Status == StatusActive && r.Connected { + n++ + cpu += r.CPU + cpuFree += r.CPUFree + mem += r.Mem + memFree += r.MemFree + } + } + if n == 0 { + return 100 + } + cpuPct := (cpu - cpuFree) * 100 / math.Max(cpu, 1) + memPct := (mem - memFree) * 100 / math.Max(mem, 1) + return int(math.Floor(math.Max(cpuPct, memPct))) +} + +// Decide applies the scaling rules to one observation. +func Decide(cfg Config, obs Observation, prev State) Decision { + now := obs.Now + st := prev.Clone() + d := Decision{State: st, Replicas: obs.Replicas} + replicas := obs.Replicas + + d.Max = cfg.Max + if obs.MaxRunners > 0 && obs.MaxRunners < d.Max { + d.Max = obs.MaxRunners + } + + // Undo the scaler's own drains once the runner is awake and connected + // again; drains by anyone else (no mark) stick. Decide as if they were + // ACTIVE already, or the runner reactivated this loop looks busy and + // another one gets woken for nothing. + var reactivate []StatusChange + runners := make([]Runner, len(obs.Runners)) + copy(runners, obs.Runners) + for i, r := range runners { + claim, ok := obs.Claims[r.Ordinal] + if !ok || !claim.DrainedByScaler || r.Ordinal >= replicas || !r.Connected || r.ID == "" { + continue + } + switch r.Status { + case StatusDraining: + reactivate = append(reactivate, StatusChange{ + Ordinal: r.Ordinal, RunnerID: r.ID, Claim: claim.Name, Status: StatusActive, + Reason: "Reactivated", + }) + runners[i].Status = StatusActive + case StatusActive: + // Active again (by someone, or a drain that failed): the mark is stale. + d.ClearDrainMarks = append(d.ClearDrainMarks, claim.Name) + } + } + obs.Runners = runners + + d.ReservedPct = ReservedPct(obs.Runners, replicas) + d.FreePct = 100 - d.ReservedPct + d.Awake = replicas + + byOrdinal := map[int][]Runner{} + for _, r := range obs.Runners { + byOrdinal[r.Ordinal] = append(byOrdinal[r.Ordinal], r) + } + connected := func(n int) bool { + for _, r := range byOrdinal[n] { + if r.Connected { + return true + } + } + return false + } + + // Settling: an awake runner not connected yet. Wait for it, or one queued + // workflow wakes several runners; past the settle timeout, decide without + // it, report it and drain it so SG places nothing there. + unconnected := map[int]bool{} + for n := 0; n < replicas; n++ { + if !connected(n) { + unconnected[n] = true + d.Unconnected = append(d.Unconnected, n) + } + } + for n := range st.UnconnectedSince { + if !unconnected[n] { + delete(st.UnconnectedSince, n) + } + } + for _, n := range d.Unconnected { + since, ok := st.UnconnectedSince[n] + if !ok { + since = now + st.UnconnectedSince[n] = now + } + waited := now.Sub(since) + if waited < cfg.SettleTimeout { + d.Settling++ + continue + } + d.Events = append(d.Events, Event{ + Type: EventWarning, Reason: "RunnerNotConnected", Key: fmt.Sprintf("RunnerNotConnected/%d", n), + Message: fmt.Sprintf("runner %d awake but not connected to SG for %s; scaling without it (check its pod)", + n, waited.Round(time.Second)), + }) + claim, hasClaim := obs.Claims[n] + for _, r := range byOrdinal[n] { + if r.Status == StatusActive && r.ID != "" && hasClaim { + d.StatusChanges = append(d.StatusChanges, StatusChange{ + Ordinal: n, RunnerID: r.ID, Claim: claim.Name, Status: StatusDraining, + Reason: "DrainedUnconnectedRunner", + }) + } + } + } + + // Stuck queue: queued while a runner sits fully idle past SG's retry + // delay means what's queued fits no runner of this size. + var idle *Runner + for i, r := range obs.Runners { + if r.Ordinal < replicas && r.Status == StatusActive && r.Connected && r.FullyIdle() { + idle = &obs.Runners[i] + break + } + } + if obs.Queued > 0 && idle != nil { + if st.StuckSince.IsZero() { + st.StuckSince = now + } + if waited := now.Sub(st.StuckSince); waited >= cfg.StuckAfter { + d.Stuck = true + d.Events = append(d.Events, Event{ + Type: EventWarning, Reason: "StuckQueue", Key: "StuckQueue", + Message: fmt.Sprintf("%d workflow(s) queued for %s while runner %d (CPU units/MiB %s) sits idle: "+ + "they need more than one runner pod offers. Ignoring them; raise the runner's resources to run them.", + obs.Queued, waited.Round(time.Second), idle.Ordinal, idle.Size()), + }) + } + } else { + st.StuckSince = time.Time{} + } + + // Scaling. + queueWantsOut := obs.Queued > 0 && idle == nil + headroomWantsOut := d.FreePct < cfg.HeadroomPct + belowMin := replicas < cfg.Min + switch { + case d.Settling > 0: + d.Why = fmt.Sprintf("waiting for %d runner(s) to connect", d.Settling) + case replicas < d.Max && (queueWantsOut || headroomWantsOut || belowMin): + switch { + case queueWantsOut: + d.Why = fmt.Sprintf("%d workflow(s) queued and no runner idle", obs.Queued) + case headroomWantsOut: + d.Why = fmt.Sprintf("%d%% free, below the %d%% headroom", d.FreePct, cfg.HeadroomPct) + default: + d.Why = fmt.Sprintf("below the minimum of %d", cfg.Min) + } + if obs.ReapJobs[replicas] != JobNone { + d.Why = fmt.Sprintf("scale out waits: runner %d is being deregistered", replicas) + break + } + d.Scale = ScaleOut + d.Replicas = replicas + 1 + st.IdleSince = map[int]time.Time{} + // The new runner starts idle: don't mistake it for a stuck queue. + st.StuckSince = time.Time{} + case replicas > cfg.Min && (obs.Queued == 0 || d.Stuck || replicas > d.Max): + top := replicas - 1 + for n := range st.IdleSince { + if n != top { + delete(st.IdleSince, n) + } + } + busy, seen := 0, len(byOrdinal[top]) > 0 + for _, r := range byOrdinal[top] { + busy += r.Busy + } + if !seen || busy > 0 { + delete(st.IdleSince, top) + break + } + since, ok := st.IdleSince[top] + if !ok { + since = now + st.IdleSince[top] = now + } + // Don't park into a scale out: the rest must keep the headroom. + freeWithoutTop := 100 - ReservedPct(obs.Runners, top) + if idleFor := now.Sub(since); idleFor >= cfg.IdleAfter && freeWithoutTop >= cfg.HeadroomPct { + d.Scale = ScaleIn + d.Replicas = top + d.Why = fmt.Sprintf("runner %d idle for %s", top, idleFor.Round(time.Second)) + delete(st.IdleSince, top) + } + } + + // Parked runners SG still reports ACTIVE: the pod went away without its + // preStop drain (node loss, force delete). Drain them so SG places + // nothing there, and mark the drain as the scaler's own. + for _, r := range obs.Runners { + if r.Ordinal < replicas || r.Status != StatusActive || r.Connected || r.ID == "" { + continue + } + claim, ok := obs.Claims[r.Ordinal] + if !ok { + continue + } + d.StatusChanges = append(d.StatusChanges, StatusChange{ + Ordinal: r.Ordinal, RunnerID: r.ID, Claim: claim.Name, Status: StatusDraining, + Reason: "DrainedParkedRunner", + }) + } + + // Not the runner parked this loop: its pod is on its way down. + for _, sc := range reactivate { + if sc.Ordinal < d.Replicas { + d.StatusChanges = append(d.StatusChanges, sc) + } + } + + d.Reap = PlanReap(obs.Claims, obs.ReapJobs, d.Replicas, now, cfg.ParkTTL) + d.State = st + return d +} + +// PlanReap stamps newly parked runners, clears stamps of runners awake +// again and deregisters runners parked longer than ttl. +func PlanReap(claims map[int]Claim, jobs map[int]JobState, replicas int, now time.Time, ttl time.Duration) []ReapAction { + ordinals := make([]int, 0, len(claims)) + for n := range claims { + ordinals = append(ordinals, n) + } + sort.Ints(ordinals) + + var out []ReapAction + for _, n := range ordinals { + c := claims[n] + if n < replicas { + if c.Stamped { + out = append(out, ReapAction{Kind: ReapClearStamp, Ordinal: n, Claim: c.Name}) + } + continue + } + if c.ParkedAt.IsZero() { + out = append(out, ReapAction{Kind: ReapStamp, Ordinal: n, Claim: c.Name}) + continue + } + parkedFor := now.Sub(c.ParkedAt) + if parkedFor < ttl { + continue + } + a := ReapAction{Ordinal: n, Claim: c.Name, ParkedFor: parkedFor} + switch jobs[n] { + case JobNone: + a.Kind = ReapStartJob + case JobSucceeded: + a.Kind = ReapCleanup + case JobFailed: + a.Kind = ReapRetry + default: + continue + } + out = append(out, a) + } + return out +} diff --git a/k8s/scaler/internal/decide/decide_test.go b/k8s/scaler/internal/decide/decide_test.go new file mode 100644 index 0000000..b7306ee --- /dev/null +++ b/k8s/scaler/internal/decide/decide_test.go @@ -0,0 +1,489 @@ +package decide + +import ( + "reflect" + "strings" + "testing" + "time" +) + +var t0 = time.Unix(1_700_000_000, 0) + +func cfg() Config { + return Config{ + Min: 1, Max: 10, HeadroomPct: 0, + IdleAfter: 60 * time.Second, StuckAfter: 180 * time.Second, + SettleTimeout: 300 * time.Second, ParkTTL: 12 * time.Hour, + } +} + +// Runner sizes as SG registers them: CPU units and MiB. +const cpu, mem = 2048, 4096 + +func idle(n int) Runner { + return Runner{Ordinal: n, ID: id(n), Status: StatusActive, Connected: true, CPU: cpu, CPUFree: cpu, Mem: mem, MemFree: mem} +} + +// full is a runner one workflow fills completely. +func full(n int) Runner { + r := idle(n) + r.Busy, r.CPUFree, r.MemFree = 1, 0, 0 + return r +} + +// half is a runner one workflow fills halfway. +func half(n int) Runner { + r := idle(n) + r.Busy, r.CPUFree, r.MemFree = 1, cpu/2, mem/2 + return r +} + +func unconnected(n int, status string) Runner { + r := idle(n) + r.Connected, r.Status = false, status + return r +} + +func id(n int) string { return "runner-id-" + string(rune('a'+n)) } + +func claims(ordinals ...int) map[int]Claim { + m := map[int]Claim{} + for _, n := range ordinals { + m[n] = Claim{Name: claimName(n)} + } + return m +} + +func claimName(n int) string { return "state-sg-runner-" + string(rune('0'+n)) } + +func obs(at time.Duration, replicas, queued int, runners ...Runner) Observation { + cl := map[int]Claim{} + for n := 0; n < replicas; n++ { + cl[n] = Claim{Name: claimName(n)} + } + return Observation{Now: t0.Add(at), Replicas: replicas, Queued: queued, Runners: runners, Claims: cl} +} + +func hasEvent(d Decision, reason string) bool { + for _, e := range d.Events { + if e.Reason == reason { + return true + } + } + return false +} + +func TestReservedPct(t *testing.T) { + tests := []struct { + name string + runners []Runner + below int + want int + }{ + {"none awake", nil, 3, 100}, + {"only unconnected", []Runner{unconnected(0, StatusActive)}, 1, 100}, + {"only draining", []Runner{func() Runner { r := idle(0); r.Status = StatusDraining; return r }()}, 1, 100}, + {"idle", []Runner{idle(0)}, 1, 0}, + {"full", []Runner{full(0)}, 1, 100}, + {"half of two", []Runner{full(0), idle(1)}, 2, 50}, + {"ordinal at or above excluded", []Runner{idle(0), full(1)}, 1, 0}, + {"memory higher than cpu", []Runner{func() Runner { r := idle(0); r.MemFree = mem / 4; return r }()}, 1, 75}, + {"floored", []Runner{func() Runner { r := idle(0); r.CPUFree = cpu * 2 / 3; return r }()}, 1, 33}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ReservedPct(tt.runners, tt.below); got != tt.want { + t.Errorf("ReservedPct = %d, want %d", got, tt.want) + } + }) + } +} + +// Single loops, each rule on its own. +func TestDecideRules(t *testing.T) { + tests := []struct { + name string + cfg func(*Config) + obs Observation + state State + wantScale Scale + wantRepl int + wantEvent string + }{ + {name: "queued, runner busy: out", obs: obs(0, 1, 1, full(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "queued, runner fully idle: wait for SG's retry", obs: obs(0, 1, 1, idle(0)), wantRepl: 1}, + {name: "queued, half-busy runner counts as not idle: out", obs: obs(0, 1, 1, half(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "nothing queued, no headroom: nothing", obs: obs(0, 1, 0, full(0)), wantRepl: 1}, + {name: "headroom: free below it: out", cfg: func(c *Config) { c.HeadroomPct = 60 }, obs: obs(0, 1, 0, half(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "headroom: free at it: nothing", cfg: func(c *Config) { c.HeadroomPct = 50 }, obs: obs(0, 1, 0, half(0)), wantRepl: 1}, + {name: "at SCALER_MAX: no out", cfg: func(c *Config) { c.Max = 2 }, obs: obs(0, 2, 1, full(0), full(1)), wantRepl: 2}, + {name: "at MaxNumberOfRunners: no out", obs: func() Observation { + o := obs(0, 2, 1, full(0), full(1)) + o.MaxRunners = 2 + return o + }(), wantRepl: 2}, + {name: "MaxNumberOfRunners above SCALER_MAX: SCALER_MAX wins", cfg: func(c *Config) { c.Max = 2 }, obs: func() Observation { + o := obs(0, 2, 1, full(0), full(1)) + o.MaxRunners = 5 + return o + }(), wantRepl: 2}, + {name: "below SCALER_MIN: out", cfg: func(c *Config) { c.Min = 2 }, obs: obs(0, 1, 0, idle(0)), wantScale: ScaleOut, wantRepl: 2}, + {name: "zero replicas: up to the minimum", obs: obs(0, 0, 0), wantScale: ScaleOut, wantRepl: 1}, + {name: "reap Job on next ordinal: out waits", obs: func() Observation { + o := obs(0, 1, 1, full(0)) + o.ReapJobs = map[int]JobState{1: JobRunning} + return o + }(), wantRepl: 1}, + {name: "reap Job on another ordinal: out", obs: func() Observation { + o := obs(0, 1, 1, full(0)) + o.ReapJobs = map[int]JobState{2: JobRunning} + return o + }(), wantScale: ScaleOut, wantRepl: 2}, + {name: "at SCALER_MIN, idle long: no in", obs: obs(time.Hour, 1, 0, idle(0)), state: State{IdleSince: map[int]time.Time{0: t0}}, wantRepl: 1}, + {name: "top idle long enough: in", obs: obs(60*time.Second, 2, 0, idle(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantScale: ScaleIn, wantRepl: 1}, + {name: "top idle not long enough: nothing", obs: obs(59*time.Second, 2, 0, idle(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "top busy: nothing", obs: obs(time.Hour, 2, 0, idle(0), full(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "top unknown to SG: not idle", obs: obs(time.Hour, 2, 0, idle(0)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "queued: no in", obs: obs(time.Hour, 2, 1, full(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "headroom without top too low: no in", cfg: func(c *Config) { c.HeadroomPct = 20 }, + obs: obs(time.Hour, 2, 0, full(0), idle(1)), state: State{IdleSince: map[int]time.Time{1: t0}}, wantRepl: 2}, + {name: "above max (MaxNumberOfRunners lowered), queued: idle top still parks", obs: func() Observation { + o := obs(time.Hour, 3, 1, full(0), full(1), idle(2)) + o.MaxRunners = 2 + return o + }(), state: State{IdleSince: map[int]time.Time{2: t0}}, wantScale: ScaleIn, wantRepl: 2}, + {name: "settling: no decision", obs: obs(0, 2, 1, full(0)), wantRepl: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := cfg() + if tt.cfg != nil { + tt.cfg(&c) + } + d := Decide(c, tt.obs, tt.state) + if d.Scale != tt.wantScale || d.Replicas != tt.wantRepl { + t.Errorf("got scale %v replicas %d (%s), want %v %d", d.Scale, d.Replicas, d.Why, tt.wantScale, tt.wantRepl) + } + if tt.wantEvent != "" && !hasEvent(d, tt.wantEvent) { + t.Errorf("missing event %s in %+v", tt.wantEvent, d.Events) + } + }) + } +} + +func TestScaleOutResetsTimers(t *testing.T) { + st := State{IdleSince: map[int]time.Time{0: t0}, StuckSince: t0} + d := Decide(cfg(), obs(10*time.Second, 1, 1, full(0)), st) + if d.Scale != ScaleOut { + t.Fatalf("want scale out, got %v", d.Scale) + } + if len(d.State.IdleSince) != 0 || !d.State.StuckSince.IsZero() { + t.Errorf("timers not reset: %+v", d.State) + } + if len(st.IdleSince) != 1 { + t.Error("Decide modified the previous state") + } +} + +// step runs one loop and carries the state and replicas forward. +type sim struct { + t *testing.T + cfg Config + state State + repl int +} + +func (s *sim) loop(at time.Duration, queued int, runners ...Runner) Decision { + s.t.Helper() + d := Decide(s.cfg, obs(at, s.repl, queued, runners...), s.state) + s.state, s.repl = d.State, d.Replicas + return d +} + +// A workflow bigger than any runner pod: the idle runner never takes it. +func TestOversizedWorkflow(t *testing.T) { + s := &sim{t: t, cfg: cfg(), repl: 2} + // Two runners awake, both idle, one oversized workflow queued. + if d := s.loop(0, 1, idle(0), idle(1)); d.Scale != ScaleNone || d.Stuck { + t.Fatalf("t=0: want nothing, got %v stuck=%v", d.Scale, d.Stuck) + } + if d := s.loop(179*time.Second, 1, idle(0), idle(1)); d.Stuck { + t.Fatal("stuck before 180s") + } + d := s.loop(180*time.Second, 1, idle(0), idle(1)) + if !d.Stuck || !hasEvent(d, "StuckQueue") { + t.Fatalf("want stuck + StuckQueue at 180s, got stuck=%v events=%+v", d.Stuck, d.Events) + } + if !strings.Contains(d.Events[0].Message, "2048/4096") { + t.Errorf("stuck message lacks the runner size: %s", d.Events[0].Message) + } + // Stuck: the queue is ignored, so the extra runner parks once idle long enough. + if d := s.loop(239*time.Second, 1, idle(0), idle(1)); d.Scale != ScaleNone { + t.Fatalf("parked before idle timeout: %v", d.Scale) + } + if d := s.loop(240*time.Second, 1, idle(0), idle(1)); d.Scale != ScaleIn || d.Replicas != 1 { + t.Fatalf("want park to 1 while stuck, got %v %d", d.Scale, d.Replicas) + } + // Never below the minimum, never out for the stuck workflow. + for at := 255 * time.Second; at < time.Hour; at += 15 * time.Second { + if d := s.loop(at, 1, idle(0)); d.Scale != ScaleNone { + t.Fatalf("t=%s: want nothing, got %v", at, d.Scale) + } + } + // Queue drained (cancelled): stuck clears. + if d := s.loop(time.Hour, 0, idle(0)); d.Stuck || !d.State.StuckSince.IsZero() { + t.Error("stuck not cleared once the queue is empty") + } +} + +// A queued workflow with the runner busy wakes exactly one more runner, and +// no second one while the new runner waits idle for SG's retry. +func TestNoOvershoot(t *testing.T) { + s := &sim{t: t, cfg: cfg(), repl: 1} + if d := s.loop(0, 1, full(0)); d.Scale != ScaleOut || d.Replicas != 2 { + t.Fatalf("want +1, got %v %d", d.Scale, d.Replicas) + } + // Runner 1 starting: not in SG yet, then registered but not connected. + if d := s.loop(15*time.Second, 1, full(0)); d.Scale != ScaleNone || d.Settling != 1 { + t.Fatalf("want settling, got %v settling=%d", d.Scale, d.Settling) + } + if d := s.loop(30*time.Second, 1, full(0), unconnected(1, StatusActive)); d.Scale != ScaleNone || d.Settling != 1 { + t.Fatalf("want settling, got %v settling=%d", d.Scale, d.Settling) + } + // Connected and idle: SG's retry will place the workflow there. + for at := 45 * time.Second; at < 150*time.Second; at += 15 * time.Second { + if d := s.loop(at, 1, full(0), idle(1)); d.Scale != ScaleNone { + t.Fatalf("t=%s: overshoot: %v to %d", at, d.Scale, d.Replicas) + } + } + // Placed. + if d := s.loop(150*time.Second, 0, full(0), full(1)); d.Scale != ScaleNone || d.Replicas != 2 { + t.Fatalf("want 2 runners, got %v %d", d.Scale, d.Replicas) + } +} + +// Two workflows that each fill a runner: headroom 0 ends with 2 runners, +// headroom 20 with a warm third. +func TestHeadroom(t *testing.T) { + for _, tc := range []struct { + headroom int + want int + }{{0, 2}, {20, 3}} { + c := cfg() + c.HeadroomPct = tc.headroom + s := &sim{t: t, cfg: c, repl: 1} + s.loop(0, 2, idle(0)) // runner 0 idle: SG places one + s.loop(15*time.Second, 1, full(0)) // one left queued: +1 + s.loop(30*time.Second, 1, full(0)) // runner 1 settling + s.loop(45*time.Second, 1, full(0), idle(1)) // idle: SG places the second + s.loop(60*time.Second, 0, full(0), full(1)) // both full + var runners = []Runner{full(0), full(1)} + if s.repl == 3 { + runners = append(runners, idle(2)) + s.loop(75*time.Second, 0, full(0), full(1)) // runner 2 settling + } + for at := 90 * time.Second; at < time.Hour; at += 15 * time.Second { + s.loop(at, 0, runners...) + } + if s.repl != tc.want { + t.Errorf("headroom %d: %d runners, want %d", tc.headroom, s.repl, tc.want) + } + } +} + +// A runner that never connects: waited for, then left out, warned about and +// drained; then it connects and the scaler undoes its own drain. +func TestSettleTimeout(t *testing.T) { + s := &sim{t: t, cfg: cfg(), repl: 2} + broken := unconnected(1, StatusActive) + d := s.loop(0, 1, full(0), broken) + if d.Scale != ScaleNone || d.Settling != 1 || len(d.StatusChanges) != 0 { + t.Fatalf("want settling, got %+v", d) + } + if d := s.loop(299*time.Second, 1, full(0), broken); d.Scale != ScaleNone || d.Settling != 1 { + t.Fatalf("decided before the settle timeout: %v", d.Scale) + } + d = s.loop(300*time.Second, 1, full(0), broken) + if d.Settling != 0 || !hasEvent(d, "RunnerNotConnected") { + t.Fatalf("want timed out + warning, got settling=%d events=%+v", d.Settling, d.Events) + } + want := []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusDraining, Reason: "DrainedUnconnectedRunner"}} + if !reflect.DeepEqual(d.StatusChanges, want) { + t.Fatalf("status changes = %+v, want %+v", d.StatusChanges, want) + } + if d.Scale != ScaleOut || d.Replicas != 3 { + t.Fatalf("broken runner should not block scaling: %v %d", d.Scale, d.Replicas) + } + // Already drained: no second drain, still warned. + drained := unconnected(1, StatusDraining) + d = Decide(s.cfg, obs(400*time.Second, 2, 0, full(0), drained), d.State) + if len(d.StatusChanges) != 0 || !hasEvent(d, "RunnerNotConnected") { + t.Fatalf("want warning only, got %+v", d) + } +} + +func TestParkedRunnerDrains(t *testing.T) { + mark := func(o Observation, n int) Observation { + c := o.Claims[n] + c.DrainedByScaler = true + o.Claims[n] = c + return o + } + withClaim := func(o Observation, n int) Observation { + o.Claims[n] = Claim{Name: claimName(n), ParkedAt: t0, Stamped: true} + return o + } + tests := []struct { + name string + obs Observation + want []StatusChange + wantClear []string + }{ + {name: "parked, ACTIVE, gone: drain", + obs: withClaim(obs(0, 1, 0, full(0), unconnected(1, StatusActive)), 1), + want: []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusDraining, Reason: "DrainedParkedRunner"}}}, + {name: "parked, ACTIVE, still connected (preStop running): leave it", + obs: withClaim(obs(0, 1, 0, full(0), idle(1)), 1)}, + {name: "parked, DRAINING: leave it", + obs: withClaim(obs(0, 1, 0, full(0), unconnected(1, StatusDraining)), 1)}, + {name: "parked, ACTIVE, no PVC (reaped): leave it", + obs: obs(0, 1, 0, full(0), unconnected(1, StatusActive))}, + {name: "parked, marked but still ACTIVE (drain failed): drain again", + obs: mark(withClaim(obs(0, 1, 0, full(0), unconnected(1, StatusActive)), 1), 1), + want: []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusDraining, Reason: "DrainedParkedRunner"}}}, + {name: "woke, DRAINING, marked: reactivate", + obs: mark(obs(0, 2, 0, full(0), func() Runner { r := full(1); r.Status = StatusDraining; return r }()), 1), + want: []StatusChange{{Ordinal: 1, RunnerID: id(1), Claim: claimName(1), Status: StatusActive, Reason: "Reactivated"}}}, + {name: "woke, DRAINING, not marked (drained by someone else): leave it", + obs: obs(0, 2, 0, full(0), func() Runner { r := full(1); r.Status = StatusDraining; return r }())}, + {name: "woke, DRAINING, marked, not connected yet: wait", + obs: mark(obs(0, 2, 0, full(0), unconnected(1, StatusDraining)), 1)}, + {name: "woke, ACTIVE, marked: clear the stale mark", + obs: mark(obs(0, 2, 0, full(0), full(1)), 1), wantClear: []string{claimName(1)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := Decide(cfg(), tt.obs, State{}) + if !reflect.DeepEqual(d.StatusChanges, tt.want) { + t.Errorf("status changes = %+v, want %+v", d.StatusChanges, tt.want) + } + if !reflect.DeepEqual(d.ClearDrainMarks, tt.wantClear) { + t.Errorf("clear marks = %v, want %v", d.ClearDrainMarks, tt.wantClear) + } + }) + } +} + +// The runner being parked this loop is not reactivated. +func TestNoReactivateWhileParking(t *testing.T) { + r := idle(1) + r.Status = StatusDraining + o := obs(time.Hour, 2, 0, idle(0), r) + o.Claims[1] = Claim{Name: claimName(1), DrainedByScaler: true} + d := Decide(cfg(), o, State{IdleSince: map[int]time.Time{1: t0}}) + if d.Scale != ScaleIn || len(d.StatusChanges) != 0 { + t.Errorf("got %v changes=%+v", d.Scale, d.StatusChanges) + } +} + +func TestIdleTimerTracksTopOnly(t *testing.T) { + d := Decide(cfg(), obs(0, 3, 0, idle(0), idle(1), full(2)), State{IdleSince: map[int]time.Time{1: t0.Add(-time.Hour)}}) + if len(d.State.IdleSince) != 0 { + t.Errorf("stale idle timers kept: %v", d.State.IdleSince) + } + d = Decide(cfg(), obs(0, 3, 0, idle(0), idle(1), idle(2)), State{}) + if !d.State.IdleSince[2].Equal(t0) || len(d.State.IdleSince) != 1 { + t.Errorf("idle timer for top not started: %v", d.State.IdleSince) + } +} + +func TestPlanReap(t *testing.T) { + ttl := 12 * time.Hour + now := t0.Add(ttl) + cl := map[int]Claim{ + 0: {Name: "s-0", Stamped: true, ParkedAt: t0}, // awake: clear + 1: {Name: "s-1"}, // awake: nothing + 2: {Name: "s-2"}, // parked: stamp + 3: {Name: "s-3", Stamped: true}, // unreadable stamp: restamp + 4: {Name: "s-4", Stamped: true, ParkedAt: now.Add(-time.Hour)}, // within TTL + 5: {Name: "s-5", Stamped: true, ParkedAt: t0}, // past TTL, no Job: start + 6: {Name: "s-6", Stamped: true, ParkedAt: t0}, // Job running + 7: {Name: "s-7", Stamped: true, ParkedAt: t0}, // Job done: cleanup + 8: {Name: "s-8", Stamped: true, ParkedAt: t0}, // Job failed: retry + } + jobs := map[int]JobState{6: JobRunning, 7: JobSucceeded, 8: JobFailed} + got := PlanReap(cl, jobs, 2, now, ttl) + want := []ReapAction{ + {Kind: ReapClearStamp, Ordinal: 0, Claim: "s-0"}, + {Kind: ReapStamp, Ordinal: 2, Claim: "s-2"}, + {Kind: ReapStamp, Ordinal: 3, Claim: "s-3"}, + {Kind: ReapStartJob, Ordinal: 5, Claim: "s-5", ParkedFor: ttl}, + {Kind: ReapCleanup, Ordinal: 7, Claim: "s-7", ParkedFor: ttl}, + {Kind: ReapRetry, Ordinal: 8, Claim: "s-8", ParkedFor: ttl}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("PlanReap =\n%+v\nwant\n%+v", got, want) + } +} + +// Reaping uses the replicas after this loop's scale in. +func TestDecideStampsJustParked(t *testing.T) { + o := obs(time.Hour, 2, 0, idle(0), idle(1)) + d := Decide(cfg(), o, State{IdleSince: map[int]time.Time{1: t0}}) + want := []ReapAction{{Kind: ReapStamp, Ordinal: 1, Claim: claimName(1)}} + if d.Scale != ScaleIn || !reflect.DeepEqual(d.Reap, want) { + t.Errorf("got %v reap=%+v", d.Scale, d.Reap) + } +} + +func TestStateRoundTrip(t *testing.T) { + s := State{ + IdleSince: map[int]time.Time{2: t0}, + StuckSince: t0.Add(time.Minute), + UnconnectedSince: map[int]time.Time{0: t0, 3: t0.Add(time.Second)}, + } + ann := map[string]string{} + for k, v := range EncodeState(s) { + if v != nil { + ann[k] = *v + } + } + if ann[AnnotationIdleSince] != `{"2":1700000000}` { + t.Errorf("idle annotation = %q", ann[AnnotationIdleSince]) + } + if got := DecodeState(ann); !reflect.DeepEqual(got, s) { + t.Errorf("round trip = %+v, want %+v", got, s) + } + empty := EncodeState(State{}) + for _, k := range StateAnnotations { + if v, ok := empty[k]; !ok || v != nil { + t.Errorf("empty state should remove %s", k) + } + } + bad := DecodeState(map[string]string{AnnotationIdleSince: "{", AnnotationStuckSince: "x", + AnnotationUnconnectedSince: `{"a":1,"-1":2,"4":0,"5":7}`}) + if len(bad.IdleSince) != 0 || !bad.StuckSince.IsZero() || len(bad.UnconnectedSince) != 1 { + t.Errorf("bad annotations decoded to %+v", bad) + } +} + +// A woken runner the scaler reactivates this loop counts as ACTIVE: an +// idle one takes the queued workflow, so no other runner is woken. +func TestReactivatedRunnerCountsAsActive(t *testing.T) { + r := idle(1) + r.Status = StatusDraining + o := obs(0, 2, 1, full(0), r) + o.Claims[1] = Claim{Name: claimName(1), DrainedByScaler: true} + d := Decide(cfg(), o, State{}) + if d.Scale != ScaleNone || len(d.StatusChanges) != 1 || d.StatusChanges[0].Status != StatusActive { + t.Errorf("got %v (%s) changes=%+v", d.Scale, d.Why, d.StatusChanges) + } + if o.Runners[1].Status != StatusDraining { + t.Error("Decide modified the observation") + } + // Drained by someone else: it stays DRAINING and doesn't count. + o.Claims[1] = Claim{Name: claimName(1)} + if d := Decide(cfg(), o, State{}); d.Scale != ScaleOut { + t.Errorf("want scale out past a runner drained on purpose, got %v", d.Scale) + } +} diff --git a/k8s/scaler/internal/decide/state.go b/k8s/scaler/internal/decide/state.go new file mode 100644 index 0000000..1f795ae --- /dev/null +++ b/k8s/scaler/internal/decide/state.go @@ -0,0 +1,77 @@ +package decide + +import ( + "encoding/json" + "strconv" + "time" +) + +// Annotations on the StatefulSet that keep the scaler's timers across +// restarts and leader changes. Values are Unix epochs; per-ordinal timers +// are JSON objects keyed by ordinal. +const ( + AnnotationIdleSince = "sg-runner.stackguardian.io/scaler-idle-since" + AnnotationStuckSince = "sg-runner.stackguardian.io/scaler-stuck-since" + AnnotationUnconnectedSince = "sg-runner.stackguardian.io/scaler-unconnected-since" +) + +// StateAnnotations lists every annotation the state is stored in. +var StateAnnotations = []string{AnnotationIdleSince, AnnotationStuckSince, AnnotationUnconnectedSince} + +// DecodeState reads the timers from StatefulSet annotations. Unreadable +// values are dropped: a timer that restarts is the safe failure. +func DecodeState(annotations map[string]string) State { + s := State{IdleSince: decodeTimers(annotations[AnnotationIdleSince]), + UnconnectedSince: decodeTimers(annotations[AnnotationUnconnectedSince])} + if v, err := strconv.ParseInt(annotations[AnnotationStuckSince], 10, 64); err == nil && v > 0 { + s.StuckSince = time.Unix(v, 0) + } + return s +} + +// EncodeState renders the timers as annotation values; an empty timer maps +// to nil (remove the annotation). +func EncodeState(s State) map[string]*string { + out := map[string]*string{ + AnnotationIdleSince: encodeTimers(s.IdleSince), + AnnotationUnconnectedSince: encodeTimers(s.UnconnectedSince), + AnnotationStuckSince: nil, + } + if !s.StuckSince.IsZero() { + v := strconv.FormatInt(s.StuckSince.Unix(), 10) + out[AnnotationStuckSince] = &v + } + return out +} + +func decodeTimers(raw string) map[int]time.Time { + out := map[int]time.Time{} + if raw == "" { + return out + } + var m map[string]int64 + if json.Unmarshal([]byte(raw), &m) != nil { + return out + } + for k, v := range m { + n, err := strconv.Atoi(k) + if err != nil || n < 0 || v <= 0 { + continue + } + out[n] = time.Unix(v, 0) + } + return out +} + +func encodeTimers(t map[int]time.Time) *string { + if len(t) == 0 { + return nil + } + m := make(map[string]int64, len(t)) + for n, at := range t { + m[strconv.Itoa(n)] = at.Unix() + } + b, _ := json.Marshal(m) // map keys are sorted: stable output + v := string(b) + return &v +} diff --git a/k8s/scaler/internal/guard/guard.go b/k8s/scaler/internal/guard/guard.go new file mode 100644 index 0000000..18aa282 --- /dev/null +++ b/k8s/scaler/internal/guard/guard.go @@ -0,0 +1,111 @@ +// Package guard is sg-steps-guard, a Docker authorization plugin the step +// broker serves to the runner's dockerd: it refuses to create workflow step +// containers there, as the runner runs steps as pods (a run controller +// without step broker support would otherwise run them in dockerd). See +// k8s/docs/steps-api.md, "Guard". +package guard + +import ( + "encoding/json" + "log/slog" + "net/http" + "net/url" + "regexp" +) + +// PluginName is the plugin's name for dockerd (--authorization-plugin): its +// socket is /.sock. +const PluginName = "sg-steps-guard" + +// MediaType is the Docker plugin protocol's content type. +const MediaType = "application/vnd.docker.plugins.v1.2+json" + +// StepLabel marks the containers the run controller creates for steps. +const StepLabel = "wfrunId" + +// DenyMessage is what dockerd tells the caller of a refused step create. +const DenyMessage = "This runner runs workflow steps as Kubernetes pods and needs a run controller with " + + "native step support (step broker client); refusing to start the step in dockerd." + +// Request is the part of dockerd's authorization request the guard reads +// (github.com/docker/docker/pkg/authorization.Request). +type Request struct { + RequestMethod string `json:"RequestMethod,omitempty"` + RequestURI string `json:"RequestUri,omitempty"` + RequestBody []byte `json:"RequestBody,omitempty"` +} + +// Response is the plugin's answer. +type Response struct { + Allow bool `json:"Allow"` + Msg string `json:"Msg,omitempty"` + Err string `json:"Err,omitempty"` +} + +// Manifest is the answer to Plugin.Activate. +type Manifest struct { + Implements []string +} + +var createRE = regexp.MustCompile(`^(/v[0-9.]+)?/containers/create$`) + +// Decide answers an authorization request; warning is set when it allowed a +// container create it couldn't read. +func Decide(r Request) (resp Response, warning string) { + allow := Response{Allow: true} + if r.RequestMethod != http.MethodPost { + return allow, "" + } + u, err := url.Parse(r.RequestURI) + if err != nil || !createRE.MatchString(u.Path) { + return allow, "" + } + // dockerd sends JSON bodies only (and no larger than its limit). + if len(r.RequestBody) == 0 { + return allow, "container create without a body: can't tell whether it's a step" + } + var body struct { + Labels map[string]any + } + if err := json.Unmarshal(r.RequestBody, &body); err != nil { + return allow, "container create with an unreadable body: " + err.Error() + } + if _, ok := body.Labels[StepLabel]; ok { + return Response{Allow: false, Msg: DenyMessage}, "" + } + return allow, "" +} + +// Handler serves the plugin protocol: activation, request authorization, and +// response authorization (always allowed). +func Handler(log *slog.Logger) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST /Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { + write(w, Manifest{Implements: []string{"authz"}}) + }) + mux.HandleFunc("POST /AuthZPlugin.AuthZReq", func(w http.ResponseWriter, r *http.Request) { + var req Request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + // Not dockerd's request: dockerd refuses the call on Err. + write(w, Response{Err: "sg-steps-guard: unreadable authorization request: " + err.Error()}) + return + } + resp, warning := Decide(req) + switch { + case warning != "": + log.Warn("guard: allowed "+warning, "method", req.RequestMethod, "uri", req.RequestURI) + case !resp.Allow: + log.Info("guard: refused a step container in dockerd", "uri", req.RequestURI) + } + write(w, resp) + }) + mux.HandleFunc("POST /AuthZPlugin.AuthZRes", func(w http.ResponseWriter, r *http.Request) { + write(w, Response{Allow: true}) + }) + return mux +} + +func write(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", MediaType) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/k8s/scaler/internal/guard/guard_test.go b/k8s/scaler/internal/guard/guard_test.go new file mode 100644 index 0000000..536ba59 --- /dev/null +++ b/k8s/scaler/internal/guard/guard_test.go @@ -0,0 +1,126 @@ +package guard + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// Bodies as the run controller (docker-py) and the ECS agent send them, +// trimmed. +const ( + stepCreate = `{"Image":"registry.example.com/team/image:1.2","Cmd":["terraform","plan"],` + + `"Env":["A=b"],"Labels":{"orgId":"/orgs/example-org","wfrunId":"/wfgrps/x/wfs/y/wfruns/z"},` + + `"HostConfig":{"Binds":["/var/tmp/workspace/orgs/example-org/wfgrps/x/wfs/y/wfruns/z/shared:/mnt/sg_workspace"]}}` + controllerCreate = `{"Image":"stackguardian/sg-run-controller:1","Labels":{` + + `"com.amazonaws.ecs.cluster":"c","com.amazonaws.ecs.task-arn":"arn:aws:ecs:x","com.amazonaws.ecs.container-name":"run"},` + + `"HostConfig":{"Binds":["/var/tmp/workspace:/var/tmp/workspace"]}}` +) + +// dockerdRequest is an authorization request as dockerd marshals it: the +// body base64, under the key RequestUri. +func dockerdRequest(method, uri, body string) string { + m := map[string]any{"User": "", "RequestMethod": method, "RequestUri": uri, + "RequestHeaders": map[string]string{"Content-Type": "application/json"}} + if body != "" { + m["RequestBody"] = base64.StdEncoding.EncodeToString([]byte(body)) + } + b, _ := json.Marshal(m) + return string(b) +} + +func authz(t *testing.T, h http.Handler, raw string) Response { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/AuthZPlugin.AuthZReq", strings.NewReader(raw)) + req.Header.Set("Accept", MediaType) + h.ServeHTTP(rec, req) + var resp Response + if rec.Code != 200 || rec.Header().Get("Content-Type") != MediaType || json.Unmarshal(rec.Body.Bytes(), &resp) != nil { + t.Fatalf("AuthZReq %d %q %s", rec.Code, rec.Header().Get("Content-Type"), rec.Body) + } + return resp +} + +func TestAuthZReq(t *testing.T) { + logs := &bytes.Buffer{} + h := Handler(slog.New(slog.NewJSONHandler(logs, nil))) + for _, c := range []struct { + name string + method, uri, body string + allow bool + }{ + {"step create", "POST", "/v1.41/containers/create", stepCreate, false}, + {"step create, named", "POST", "/v1.47/containers/create?name=example-org-plan-0-1a2b", stepCreate, false}, + {"step create, unversioned", "POST", "/containers/create", stepCreate, false}, + {"step create, minor version", "POST", "/v1.4/containers/create?name=x&platform=linux", stepCreate, false}, + {"controller create", "POST", "/v1.44/containers/create?name=ecs-run-1", controllerCreate, true}, + {"no labels", "POST", "/v1.41/containers/create", `{"Image":"alpine","Labels":null}`, true}, + {"wfrunId elsewhere", "POST", "/v1.41/containers/create", `{"Image":"alpine","Env":["wfrunId=x"],"Labels":{"x":"wfrunId"}}`, true}, + {"no body", "POST", "/v1.41/containers/create", "", true}, + {"unreadable body", "POST", "/v1.41/containers/create", "{not json", true}, + {"start", "POST", "/v1.41/containers/abc/start", "", true}, + {"step-labeled exec", "POST", "/v1.41/containers/abc/exec", stepCreate, true}, + {"inspect", "GET", "/v1.41/containers/create", "", true}, + {"look-alike path", "POST", "/v1.41/containers/create/x", stepCreate, true}, + {"prefix path", "POST", "/foo/containers/create", stepCreate, true}, + {"list", "GET", "/v1.41/containers/json?all=1&filters=%7B%22label%22%3A%5B%22wfrunId%22%5D%7D", "", true}, + {"image pull", "POST", "/v1.41/images/create?fromImage=alpine", "", true}, + } { + resp := authz(t, h, dockerdRequest(c.method, c.uri, c.body)) + if resp.Allow != c.allow || resp.Err != "" || (!resp.Allow && resp.Msg != DenyMessage) { + t.Errorf("%s: %+v, want allow=%v", c.name, resp, c.allow) + } + } + for _, want := range []string{"refused a step container", "without a body", "unreadable body"} { + if !strings.Contains(logs.String(), want) { + t.Errorf("no %q in the log: %s", want, logs) + } + } +} + +func TestDockerdShapes(t *testing.T) { + // The exact key casing dockerd uses, with every field it sends. + raw := `{"User":"","UserAuthNMethod":"","RequestMethod":"POST","RequestUri":"/v1.41/containers/create",` + + `"RequestBody":"` + base64.StdEncoding.EncodeToString([]byte(stepCreate)) + `",` + + `"RequestHeaders":{"Content-Type":"application/json","User-Agent":"docker-sdk-python/7.1.0"},` + + `"RequestPeerCertificates":null,"ResponseStatusCode":0}` + if resp := authz(t, Handler(slog.New(slog.NewJSONHandler(io.Discard, nil))), raw); resp.Allow { + t.Errorf("step create allowed: %+v", resp) + } + // The response: Allow always present. + b, _ := json.Marshal(Response{Allow: false}) + if string(b) != `{"Allow":false}` { + t.Errorf("response %s", b) + } +} + +func TestProtocol(t *testing.T) { + h := Handler(slog.New(slog.NewJSONHandler(io.Discard, nil))) + do := func(path, body string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest("POST", path, strings.NewReader(body))) + return rec + } + if rec := do("/Plugin.Activate", ""); rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != `{"Implements":["authz"]}` || + rec.Header().Get("Content-Type") != MediaType { + t.Errorf("activate %d %s", rec.Code, rec.Body) + } + // Responses are always allowed, even a step create's. + if rec := do("/AuthZPlugin.AuthZRes", dockerdRequest("POST", "/v1.41/containers/create", stepCreate)); strings.TrimSpace(rec.Body.String()) != `{"Allow":true}` { + t.Errorf("AuthZRes %s", rec.Body) + } + // Not dockerd: an error, which dockerd treats as a refusal. + if rec := do("/AuthZPlugin.AuthZReq", "nope"); !strings.Contains(rec.Body.String(), `"Allow":false,"Err":"sg-steps-guard: unreadable`) { + t.Errorf("garbage %s", rec.Body) + } + if rec := do("/Other", ""); rec.Code != 404 { + t.Errorf("unknown endpoint %d", rec.Code) + } +} diff --git a/k8s/scaler/internal/kube/kube.go b/k8s/scaler/internal/kube/kube.go new file mode 100644 index 0000000..797bdfc --- /dev/null +++ b/k8s/scaler/internal/kube/kube.go @@ -0,0 +1,330 @@ +// Package kube is the scaler's Kubernetes side: the runner StatefulSet, its +// state PVCs and the reap Jobs. +package kube + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" +) + +// Annotations on a runner's state PVC. +const ( + // AnnotationParkedAt is when the runner was first seen parked (epoch). + AnnotationParkedAt = "sg-runner.stackguardian.io/parked-at" + // AnnotationDrainedByScaler marks a drain the scaler did (epoch), so it + // undoes only its own drains on wake. + AnnotationDrainedByScaler = "sg-runner.stackguardian.io/drained-by-scaler" +) + +// ReapJobTTL deletes finished reap Jobs the scaler didn't get to. +const ReapJobTTL int32 = 24 * 3600 + +// ReapPodOptions are extra settings for reap Jobs, so a chart's +// NetworkPolicy, pull secrets and scheduling cover them. +type ReapPodOptions struct { + // Labels go on the Job and its pod, next to app: -reaper (which wins). + Labels map[string]string + ImagePullSecrets []string + NodeSelector map[string]string + Tolerations []corev1.Toleration +} + +// Kube talks to the API server for one runner StatefulSet. +type Kube struct { + Client kubernetes.Interface + Namespace string + StatefulSet string + // Image is the runner image reap Jobs run. + Image string + // CredentialsSecret holds the runner group's node token (key + // SG_NODE_TOKEN). + CredentialsSecret string + // ConfigMap holds the runner group (SG_ORGANIZATION, SG_RUNNER_GROUP, + // SG_BASE_API). + ConfigMap string + // ReapPod customizes reap Jobs (and placeholder Jobs' pull secrets). + ReapPod ReapPodOptions + // NudgeImage is the placeholder Jobs' image (DefaultNudgeImage when empty). + NudgeImage string + + claimRE *regexp.Regexp + dockerRE *regexp.Regexp + jobRE *regexp.Regexp +} + +// New returns the adapter for one StatefulSet. +func New(c kubernetes.Interface, namespace, statefulSet, image, secret, configMap string) *Kube { + q := regexp.QuoteMeta(statefulSet) + return &Kube{ + Client: c, Namespace: namespace, StatefulSet: statefulSet, Image: image, + CredentialsSecret: secret, ConfigMap: configMap, + claimRE: regexp.MustCompile(`^state-` + q + `-(\d+)$`), + dockerRE: regexp.MustCompile(`^docker-` + q + `-(\d+)$`), + jobRE: regexp.MustCompile(`^` + q + `-reap-(\d+)$`), + } +} + +// ClaimName is the state PVC of an ordinal. +func (k *Kube) ClaimName(n int) string { return fmt.Sprintf("state-%s-%d", k.StatefulSet, n) } + +// DockerClaimName is the Docker cache PVC of an ordinal. +func (k *Kube) DockerClaimName(n int) string { return fmt.Sprintf("docker-%s-%d", k.StatefulSet, n) } + +// JobName is the reap Job of an ordinal. +func (k *Kube) JobName(n int) string { return fmt.Sprintf("%s-reap-%d", k.StatefulSet, n) } + +// GetStatefulSet reads the runner StatefulSet. +func (k *Kube) GetStatefulSet(ctx context.Context) (*appsv1.StatefulSet, error) { + return k.Client.AppsV1().StatefulSets(k.Namespace).Get(ctx, k.StatefulSet, metav1.GetOptions{}) +} + +// Replicas is a StatefulSet's spec.replicas (1 when unset, as the API defaults). +func Replicas(s *appsv1.StatefulSet) int { + if s.Spec.Replicas == nil { + return 1 + } + return int(*s.Spec.Replicas) +} + +// Scale sets replicas through the scale subresource. +func (k *Kube) Scale(ctx context.Context, replicas int) error { + body := fmt.Sprintf(`{"spec":{"replicas":%d}}`, replicas) + _, err := k.Client.AppsV1().StatefulSets(k.Namespace).Patch(ctx, k.StatefulSet, + types.MergePatchType, []byte(body), metav1.PatchOptions{}, "scale") + return err +} + +func annotationsPatch(a map[string]*string) ([]byte, error) { + return json.Marshal(map[string]any{"metadata": map[string]any{"annotations": a}}) +} + +// AnnotateStatefulSet merges annotations into the StatefulSet's metadata; +// nil values remove them. +func (k *Kube) AnnotateStatefulSet(ctx context.Context, a map[string]*string) error { + body, err := annotationsPatch(a) + if err != nil { + return err + } + _, err = k.Client.AppsV1().StatefulSets(k.Namespace).Patch(ctx, k.StatefulSet, + types.MergePatchType, body, metav1.PatchOptions{}) + return err +} + +// AnnotateClaim merges annotations into a PVC; nil values remove them. +func (k *Kube) AnnotateClaim(ctx context.Context, name string, a map[string]*string) error { + body, err := annotationsPatch(a) + if err != nil { + return err + } + _, err = k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).Patch(ctx, name, + types.MergePatchType, body, metav1.PatchOptions{}) + return err +} + +// Claims lists the state PVCs by ordinal. +func (k *Kube) Claims(ctx context.Context) (map[int]decide.Claim, error) { + list, err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := map[int]decide.Claim{} + for _, p := range list.Items { + m := k.claimRE.FindStringSubmatch(p.Name) + if m == nil || p.DeletionTimestamp != nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + c := decide.Claim{Name: p.Name} + if v, ok := p.Annotations[AnnotationParkedAt]; ok { + c.Stamped = true + if at, err := strconv.ParseInt(v, 10, 64); err == nil && at > 0 { + c.ParkedAt = time.Unix(at, 0) + } + } + _, c.DrainedByScaler = p.Annotations[AnnotationDrainedByScaler] + out[n] = c + } + return out, nil +} + +// DockerClaims lists the ordinals that have a Docker cache PVC. +func (k *Kube) DockerClaims(ctx context.Context) ([]int, error) { + list, err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + var out []int + for _, p := range list.Items { + m := k.dockerRE.FindStringSubmatch(p.Name) + if m == nil || p.DeletionTimestamp != nil { + continue + } + if n, err := strconv.Atoi(m[1]); err == nil { + out = append(out, n) + } + } + return out, nil +} + +// ScaleDeployment sets a Deployment's replicas through the scale subresource. +func (k *Kube) ScaleDeployment(ctx context.Context, name string, replicas int) error { + body := fmt.Sprintf(`{"spec":{"replicas":%d}}`, replicas) + _, err := k.Client.AppsV1().Deployments(k.Namespace).Patch(ctx, name, + types.MergePatchType, []byte(body), metav1.PatchOptions{}, "scale") + return err +} + +// CountPods counts the pods matching a label selector. +func (k *Kube) CountPods(ctx context.Context, selector string) (int, error) { + list, err := k.Client.CoreV1().Pods(k.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return 0, err + } + return len(list.Items), nil +} + +// ReapJobExists tells whether the reap Job of an ordinal still exists. +func (k *Kube) ReapJobExists(ctx context.Context, n int) (bool, error) { + _, err := k.Client.BatchV1().Jobs(k.Namespace).Get(ctx, k.JobName(n), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil, err +} + +// DeleteClaim deletes a PVC; a missing one is fine. +func (k *Kube) DeleteClaim(ctx context.Context, name string) error { + err := k.Client.CoreV1().PersistentVolumeClaims(k.Namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + return err +} + +// ReapJobs lists the reap Jobs by ordinal. +func (k *Kube) ReapJobs(ctx context.Context) (map[int]decide.JobState, error) { + list, err := k.Client.BatchV1().Jobs(k.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=" + k.StatefulSet + "-reaper", + }) + if err != nil { + return nil, err + } + out := map[int]decide.JobState{} + for _, j := range list.Items { + m := k.jobRE.FindStringSubmatch(j.Name) + if m == nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + switch { + case j.Status.Succeeded > 0: + out[n] = decide.JobSucceeded + case j.Status.Failed > 0: + out[n] = decide.JobFailed + default: + out[n] = decide.JobRunning + } + } + return out, nil +} + +// ReapJob is the Job deregistering a parked runner from its state PVC +// (`runner.sh reap`). +func (k *Kube) ReapJob(n int) *batchv1.Job { + labels := map[string]string{} + for key, v := range k.ReapPod.Labels { + labels[key] = v + } + labels["app"] = k.StatefulSet + "-reaper" + var pullSecrets []corev1.LocalObjectReference + for _, name := range k.ReapPod.ImagePullSecrets { + pullSecrets = append(pullSecrets, corev1.LocalObjectReference{Name: name}) + } + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: k.JobName(n), Namespace: k.Namespace, Labels: labels}, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + TTLSecondsAfterFinished: ptr.To(ReapJobTTL), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: ptr.To(false), + ImagePullSecrets: pullSecrets, + NodeSelector: k.ReapPod.NodeSelector, + Tolerations: k.ReapPod.Tolerations, + Containers: []corev1.Container{{ + Name: "reap", Image: k.Image, ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{"reap"}, + VolumeMounts: []corev1.VolumeMount{ + {Name: "state", MountPath: "/etc/ecs", SubPath: "ecs-config"}, + {Name: "state", MountPath: "/var/lib/ecs/data", SubPath: "ecs-data"}, + {Name: "state", MountPath: "/var/lib/amazon/ssm", SubPath: "ssm"}, + {Name: "sg-credentials", MountPath: "/etc/sg-runner/credentials", ReadOnly: true}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "state", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: k.ClaimName(n)}}}, + // The node token and the runner group as files in one + // directory, as runner.sh reads them. + {Name: "sg-credentials", VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr.To[int32](0o400), + Sources: []corev1.VolumeProjection{ + {Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: k.CredentialsSecret}, + Items: []corev1.KeyToPath{{Key: "SG_NODE_TOKEN", Path: "SG_NODE_TOKEN"}}, + }}, + {ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: k.ConfigMap}, + }}, + }, + }}}, + }, + }, + }, + }, + } +} + +// CreateReapJob starts the reap Job of an ordinal; an existing one is fine. +func (k *Kube) CreateReapJob(ctx context.Context, n int) error { + _, err := k.Client.BatchV1().Jobs(k.Namespace).Create(ctx, k.ReapJob(n), metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +// DeleteReapJob deletes the reap Job of an ordinal and its pod. +func (k *Kube) DeleteReapJob(ctx context.Context, n int) error { + err := k.Client.BatchV1().Jobs(k.Namespace).Delete(ctx, k.JobName(n), metav1.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }) + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/k8s/scaler/internal/kube/kube_test.go b/k8s/scaler/internal/kube/kube_test.go new file mode 100644 index 0000000..d6da133 --- /dev/null +++ b/k8s/scaler/internal/kube/kube_test.go @@ -0,0 +1,221 @@ +package kube + +import ( + "context" + "reflect" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" +) + +const ns = "sg" + +func pvc(name string, ann map[string]string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Annotations: ann}} +} + +func newKube(objs ...runtime.Object) *Kube { + return New(fake.NewClientset(objs...), ns, "sg-runner", "img:1", "creds", "cfg") +} + +func TestClaims(t *testing.T) { + k := newKube( + pvc("state-sg-runner-0", nil), + pvc("state-sg-runner-1", map[string]string{AnnotationParkedAt: "1700000000", AnnotationDrainedByScaler: "1700000001"}), + pvc("state-sg-runner-2", map[string]string{AnnotationParkedAt: "garbage"}), + pvc("docker-sg-runner-1", nil), + pvc("state-sg-runner-other-3", nil), + pvc("state-sg-runner-x", nil), + ) + got, err := k.Claims(context.Background()) + if err != nil { + t.Fatal(err) + } + want := map[int]decide.Claim{ + 0: {Name: "state-sg-runner-0"}, + 1: {Name: "state-sg-runner-1", Stamped: true, ParkedAt: time.Unix(1700000000, 0), DrainedByScaler: true}, + 2: {Name: "state-sg-runner-2", Stamped: true}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Claims =\n%+v\nwant\n%+v", got, want) + } +} + +func TestAnnotateClaim(t *testing.T) { + k := newKube(pvc("state-sg-runner-1", map[string]string{"keep": "me", AnnotationParkedAt: "1"})) + ctx := context.Background() + v := "42" + if err := k.AnnotateClaim(ctx, "state-sg-runner-1", map[string]*string{AnnotationDrainedByScaler: &v, AnnotationParkedAt: nil}); err != nil { + t.Fatal(err) + } + p, _ := k.Client.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "state-sg-runner-1", metav1.GetOptions{}) + if want := map[string]string{"keep": "me", AnnotationDrainedByScaler: "42"}; !reflect.DeepEqual(p.Annotations, want) { + t.Errorf("annotations = %v, want %v", p.Annotations, want) + } +} + +func TestScaleAndAnnotateStatefulSet(t *testing.T) { + k := newKube(&appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-runner", Namespace: ns, Annotations: map[string]string{"helm": "x"}}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To[int32](1)}, + }) + ctx := context.Background() + if err := k.Scale(ctx, 3); err != nil { + t.Fatal(err) + } + v := `{"2":1}` + if err := k.AnnotateStatefulSet(ctx, map[string]*string{decide.AnnotationIdleSince: &v, decide.AnnotationStuckSince: nil}); err != nil { + t.Fatal(err) + } + s, err := k.GetStatefulSet(ctx) + if err != nil { + t.Fatal(err) + } + if Replicas(s) != 3 || s.Annotations["helm"] != "x" || s.Annotations[decide.AnnotationIdleSince] != v { + t.Errorf("statefulset = replicas %d annotations %v", Replicas(s), s.Annotations) + } + if Replicas(&appsv1.StatefulSet{}) != 1 { + t.Error("unset replicas should default to 1") + } +} + +func TestReapJobSpec(t *testing.T) { + k := New(nil, ns, "sg-runner", "img:1", "creds", "cfg") + j := k.ReapJob(4) + if j.Name != "sg-runner-reap-4" || j.Labels["app"] != "sg-runner-reaper" { + t.Errorf("metadata %+v", j.ObjectMeta) + } + if *j.Spec.BackoffLimit != 0 || *j.Spec.TTLSecondsAfterFinished != ReapJobTTL { + t.Error("backoffLimit/ttl") + } + ps := j.Spec.Template.Spec + if ps.RestartPolicy != corev1.RestartPolicyNever || *ps.AutomountServiceAccountToken { + t.Error("restart policy / token automount") + } + c := ps.Containers[0] + if c.Image != "img:1" || !reflect.DeepEqual(c.Args, []string{"reap"}) || c.ImagePullPolicy != corev1.PullIfNotPresent { + t.Errorf("container %+v", c) + } + wantMounts := []corev1.VolumeMount{ + {Name: "state", MountPath: "/etc/ecs", SubPath: "ecs-config"}, + {Name: "state", MountPath: "/var/lib/ecs/data", SubPath: "ecs-data"}, + {Name: "state", MountPath: "/var/lib/amazon/ssm", SubPath: "ssm"}, + {Name: "sg-credentials", MountPath: "/etc/sg-runner/credentials", ReadOnly: true}, + } + if !reflect.DeepEqual(c.VolumeMounts, wantMounts) { + t.Errorf("mounts %+v", c.VolumeMounts) + } + if ps.Volumes[0].PersistentVolumeClaim.ClaimName != "state-sg-runner-4" || + *ps.Volumes[1].Projected.DefaultMode != 0o400 { + t.Errorf("volumes %+v", ps.Volumes) + } + // Only the token from the Secret, the rest from the ConfigMap. + src := ps.Volumes[1].Projected.Sources + wantToken := []corev1.KeyToPath{{Key: "SG_NODE_TOKEN", Path: "SG_NODE_TOKEN"}} + if len(src) != 2 || src[0].Secret.Name != "creds" || !reflect.DeepEqual(src[0].Secret.Items, wantToken) || + src[1].ConfigMap.Name != "cfg" || src[1].ConfigMap.Items != nil { + t.Errorf("credential sources %+v", src) + } +} + +func TestReapJobLifecycle(t *testing.T) { + k := newKube( + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-5", Namespace: ns, Labels: map[string]string{"app": "sg-runner-reaper"}}, + Status: batchv1.JobStatus{Succeeded: 1}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-6", Namespace: ns, Labels: map[string]string{"app": "sg-runner-reaper"}}, + Status: batchv1.JobStatus{Failed: 1}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: ns, Labels: map[string]string{"app": "sg-runner-reaper"}}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-9", Namespace: ns}}, + ) + ctx := context.Background() + if err := k.CreateReapJob(ctx, 3); err != nil { + t.Fatal(err) + } + if err := k.CreateReapJob(ctx, 3); err != nil { + t.Fatalf("existing job should be fine: %v", err) + } + got, err := k.ReapJobs(ctx) + if err != nil { + t.Fatal(err) + } + want := map[int]decide.JobState{3: decide.JobRunning, 5: decide.JobSucceeded, 6: decide.JobFailed} + if !reflect.DeepEqual(got, want) { + t.Errorf("ReapJobs = %v, want %v", got, want) + } + if err := k.DeleteReapJob(ctx, 5); err != nil { + t.Fatal(err) + } + if err := k.DeleteReapJob(ctx, 5); err != nil { + t.Fatalf("missing job should be fine: %v", err) + } + if err := k.DeleteClaim(ctx, "nope"); err != nil { + t.Fatalf("missing pvc should be fine: %v", err) + } +} + +func TestReapJobPodOptions(t *testing.T) { + k := New(nil, ns, "sg-runner", "img:1", "creds", "cfg") + k.ReapPod = ReapPodOptions{ + Labels: map[string]string{"np": "egress", "app": "hijack"}, + ImagePullSecrets: []string{"regcred"}, + NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpExists}}, + } + j := k.ReapJob(1) + ps := j.Spec.Template.Spec + for _, l := range []map[string]string{j.Labels, j.Spec.Template.Labels} { + if l["np"] != "egress" || l["app"] != "sg-runner-reaper" { + t.Errorf("labels %v", l) + } + } + if len(ps.ImagePullSecrets) != 1 || ps.ImagePullSecrets[0].Name != "regcred" || + ps.NodeSelector["kubernetes.io/os"] != "linux" || len(ps.Tolerations) != 1 { + t.Errorf("pod spec %+v", ps) + } + if len(k.ReapPod.Labels) != 2 || k.ReapPod.Labels["app"] != "hijack" { + t.Error("ReapJob modified the options") + } +} + +// The Job as the API server stores it, with the chart's labels. +func TestCreateReapJobWithChartOptions(t *testing.T) { + k := newKube() + k.ReapPod = ReapPodOptions{ + Labels: map[string]string{"app.kubernetes.io/name": "sg-runner", "app.kubernetes.io/instance": "x", + "app.kubernetes.io/component": "reaper"}, + ImagePullSecrets: []string{"regcred"}, + NodeSelector: map[string]string{"kubernetes.io/arch": "arm64"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpEqual, Value: "runners", Effect: corev1.TaintEffectNoSchedule}}, + } + ctx := context.Background() + if err := k.CreateReapJob(ctx, 2); err != nil { + t.Fatal(err) + } + j, err := k.Client.BatchV1().Jobs(ns).Get(ctx, "sg-runner-reap-2", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"app": "sg-runner-reaper", "app.kubernetes.io/name": "sg-runner", + "app.kubernetes.io/instance": "x", "app.kubernetes.io/component": "reaper"} + if !reflect.DeepEqual(j.Labels, want) || !reflect.DeepEqual(j.Spec.Template.Labels, want) { + t.Errorf("labels job=%v pod=%v", j.Labels, j.Spec.Template.Labels) + } + ps := j.Spec.Template.Spec + if !reflect.DeepEqual(ps.ImagePullSecrets, []corev1.LocalObjectReference{{Name: "regcred"}}) || + ps.NodeSelector["kubernetes.io/arch"] != "arm64" || !reflect.DeepEqual(ps.Tolerations, k.ReapPod.Tolerations) { + t.Errorf("pod spec %+v", ps) + } + // Still found by the scaler's label selector. + if jobs, err := k.ReapJobs(ctx); err != nil || jobs[2] != decide.JobRunning { + t.Errorf("ReapJobs = %v, %v", jobs, err) + } +} diff --git a/k8s/scaler/internal/kube/nudge.go b/k8s/scaler/internal/kube/nudge.go new file mode 100644 index 0000000..7f30d9c --- /dev/null +++ b/k8s/scaler/internal/kube/nudge.go @@ -0,0 +1,288 @@ +package kube + +// Nudging the cluster autoscaler. Some autoscalers (GKE's, at least) never +// add a node for a pod with a persistent disk: the template node they +// simulate lacks the disk driver's zone label, so a runner with its PVCs +// stays Pending ("NotTriggerScaleUp ... didn't match Pod's node +// affinity/selector") while the same pod without volumes would scale up. A +// nudge is a placeholder Job with the runner pod's requests and scheduling +// rules but no volumes: the autoscaler adds a node for it, and once it runs, +// deleting it leaves that node to the runner. + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/utils/ptr" +) + +// DefaultNudgeImage is the placeholder's image: it only has to sleep. +const DefaultNudgeImage = "registry.k8s.io/pause:3.10" + +// NudgeDeadline bounds a placeholder that never gets a node (e.g. the node +// pool's maximum is reached). +const NudgeDeadline int64 = 15 * 60 + +// NudgeState is a placeholder Job's progress. +type NudgeState int + +// Placeholder progress. +const ( + // NudgeWaiting: created, its pod not running yet (a node is coming). + NudgeWaiting NudgeState = iota + // NudgeRunning: its pod runs, so the node is there. + NudgeRunning + // NudgeFailed: past its deadline without a node. + NudgeFailed +) + +func (k *Kube) nudgeApp() string { return k.StatefulSet + "-nudge" } + +// NudgeJobName is the placeholder Job of an ordinal. +func (k *Kube) NudgeJobName(n int) string { return fmt.Sprintf("%s-nudge-%d", k.StatefulSet, n) } + +// RunnerPods lists the runner StatefulSet's pods by ordinal. +func (k *Kube) RunnerPods(ctx context.Context, selector string) (map[int]*corev1.Pod, error) { + list, err := k.Client.CoreV1().Pods(k.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`^` + regexp.QuoteMeta(k.StatefulSet) + `-(\d+)$`) + out := map[int]*corev1.Pod{} + for i := range list.Items { + p := &list.Items[i] + m := re.FindStringSubmatch(p.Name) + if m == nil { + continue + } + if n, err := strconv.Atoi(m[1]); err == nil { + out[n] = p + } + } + return out, nil +} + +// Unschedulable reports a pod the scheduler can't place on any node. +func Unschedulable(p *corev1.Pod) bool { + if p.Spec.NodeName != "" || p.DeletionTimestamp != nil || p.Status.Phase != corev1.PodPending { + return false + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse && c.Reason == corev1.PodReasonUnschedulable { + return true + } + } + return false +} + +// Cluster autoscaler event reasons on a pending pod. +const ( + // EventScaleUpRefused: no node group can host the pod. + EventScaleUpRefused = "NotTriggerScaleUp" + // EventScaleUpTriggered: a node group is being resized for it. + EventScaleUpTriggered = "TriggeredScaleUp" +) + +// podEvents lists a reason's events on pods; the field selector is checked +// again here (fake clients ignore it). +func (k *Kube) podEvents(ctx context.Context, reason string, match func(corev1.ObjectReference) bool) ([]corev1.Event, error) { + sel := fields.Set{"involvedObject.kind": "Pod", "reason": reason}.AsSelector().String() + list, err := k.Client.CoreV1().Events(k.Namespace).List(ctx, metav1.ListOptions{FieldSelector: sel}) + if err != nil { + return nil, err + } + var out []corev1.Event + for _, e := range list.Items { + if e.InvolvedObject.Kind == "Pod" && e.Reason == reason && match(e.InvolvedObject) { + out = append(out, e) + } + } + return out, nil +} + +// ScaleUpRefused reports whether the cluster autoscaler said it won't add a +// node for this pod (event NotTriggerScaleUp). +func (k *Kube) ScaleUpRefused(ctx context.Context, p *corev1.Pod) (bool, error) { + ev, err := k.podEvents(ctx, EventScaleUpRefused, func(o corev1.ObjectReference) bool { + return o.Name == p.Name && (o.UID == "" || o.UID == p.UID) + }) + return len(ev) > 0, err +} + +// NudgeOutcome is what the cluster autoscaler said about an ordinal's +// current placeholder pod (events since the Job's creation; earlier ones +// belong to earlier placeholders): the event reason (EventScaleUpTriggered or +// EventScaleUpRefused) and its message, or empty while it hasn't decided. +func (k *Kube) NudgeOutcome(ctx context.Context, n int, since time.Time) (reason, message string, err error) { + prefix := k.NudgeJobName(n) + "-" + since = since.Truncate(time.Second) + isPlaceholder := func(o corev1.ObjectReference) bool { return strings.HasPrefix(o.Name, prefix) } + // A triggered scale-up wins: the node is coming even if the autoscaler + // also refused once (e.g. while another scale-up was in flight). + for _, r := range []string{EventScaleUpTriggered, EventScaleUpRefused} { + ev, err := k.podEvents(ctx, r, isPlaceholder) + if err != nil { + return "", "", err + } + for _, e := range ev { + if !eventTime(e).Before(since) { + return r, e.Message, nil + } + } + } + return "", "", nil +} + +// eventTime is when an event last happened. +func eventTime(e corev1.Event) time.Time { + switch { + case !e.LastTimestamp.IsZero(): + return e.LastTimestamp.Time + case !e.EventTime.IsZero(): + return e.EventTime.Time + default: + return e.FirstTimestamp.Time + } +} + +// Nudge is a placeholder Job: its progress and when it was created. +type Nudge struct { + State NudgeState + Created time.Time +} + +// NudgeJobs lists the placeholder Jobs by ordinal. +func (k *Kube) NudgeJobs(ctx context.Context) (map[int]Nudge, error) { + list, err := k.Client.BatchV1().Jobs(k.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "app=" + k.nudgeApp()}) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`^` + regexp.QuoteMeta(k.StatefulSet) + `-nudge-(\d+)$`) + out := map[int]Nudge{} + for _, j := range list.Items { + m := re.FindStringSubmatch(j.Name) + if m == nil || j.DeletionTimestamp != nil { + continue + } + n, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + nj := Nudge{State: NudgeWaiting, Created: j.CreationTimestamp.Time} + switch { + case j.Status.Failed > 0: + nj.State = NudgeFailed + case j.Status.Ready != nil && *j.Status.Ready > 0: + nj.State = NudgeRunning + } + out[n] = nj + } + return out, nil +} + +// PodRequests is what the scheduler reserves for a pod: the containers' +// requests summed, or an init container's if larger. +func PodRequests(p *corev1.Pod) corev1.ResourceList { + out := corev1.ResourceList{} + for _, c := range p.Spec.Containers { + for name, q := range c.Resources.Requests { + sum := out[name] + sum.Add(q) + out[name] = sum + } + } + for _, c := range p.Spec.InitContainers { + for name, q := range c.Resources.Requests { + if cur, ok := out[name]; !ok || q.Cmp(cur) > 0 { + out[name] = q.DeepCopy() + } + } + } + return out +} + +// NudgeJob is the placeholder for a runner pod: its requests, node +// selection, tolerations and affinity (so it asks for the same kind of node), +// no volumes. +func (k *Kube) NudgeJob(n int, runner *corev1.Pod) *batchv1.Job { + labels := map[string]string{"app": k.nudgeApp()} + var pullSecrets []corev1.LocalObjectReference + for _, name := range k.ReapPod.ImagePullSecrets { + pullSecrets = append(pullSecrets, corev1.LocalObjectReference{Name: name}) + } + image := k.NudgeImage + if image == "" { + image = DefaultNudgeImage + } + requests := PodRequests(runner) + // Nothing to reserve otherwise, and the pause container needs no more. + if len(requests) == 0 { + requests = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1m")} + } + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: k.NudgeJobName(n), Namespace: k.Namespace, Labels: labels}, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + ActiveDeadlineSeconds: ptr.To(NudgeDeadline), + TTLSecondsAfterFinished: ptr.To[int32](600), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: ptr.To(false), + TerminationGracePeriodSeconds: ptr.To[int64](0), + ImagePullSecrets: pullSecrets, + NodeSelector: runner.Spec.NodeSelector, + Tolerations: runner.Spec.Tolerations, + Affinity: runner.Spec.Affinity, + PriorityClassName: runner.Spec.PriorityClassName, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65535), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: "placeholder", Image: image, ImagePullPolicy: corev1.PullIfNotPresent, + Resources: corev1.ResourceRequirements{Requests: requests}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }, + }, + }, + } +} + +// CreateNudgeJob starts the placeholder for a runner pod; an existing one is +// fine. +func (k *Kube) CreateNudgeJob(ctx context.Context, n int, runner *corev1.Pod) error { + _, err := k.Client.BatchV1().Jobs(k.Namespace).Create(ctx, k.NudgeJob(n, runner), metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +// DeleteNudgeJob deletes a placeholder Job and its pod; a missing one is fine. +func (k *Kube) DeleteNudgeJob(ctx context.Context, n int) error { + err := k.Client.BatchV1().Jobs(k.Namespace).Delete(ctx, k.NudgeJobName(n), metav1.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }) + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/k8s/scaler/internal/kube/nudge_test.go b/k8s/scaler/internal/kube/nudge_test.go new file mode 100644 index 0000000..9cf1dca --- /dev/null +++ b/k8s/scaler/internal/kube/nudge_test.go @@ -0,0 +1,130 @@ +package kube + +import ( + "context" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestPodRequestsSumsContainersAndTakesTheLargerInit(t *testing.T) { + q := resource.MustParse + p := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: q("8"), corev1.ResourceMemory: q("1Gi")}}}}, + Containers: []corev1.Container{ + {Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: q("6500m"), corev1.ResourceMemory: q("11Gi")}}}, + {Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceMemory: q("32Mi")}}}, + {}, + }, + }} + got := PodRequests(p) + if got.Cpu().String() != "8" || got.Memory().String() != "11296Mi" { + t.Errorf("PodRequests = cpu %s memory %s, want 8 and 11296Mi", got.Cpu(), got.Memory()) + } +} + +func TestUnschedulable(t *testing.T) { + cond := func(reason string) []corev1.PodCondition { + return []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: reason}} + } + cases := map[string]struct { + pod corev1.Pod + want bool + }{ + "unschedulable": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: cond(corev1.PodReasonUnschedulable)}}, true}, + "not tried yet": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, false}, + "other reason": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: cond("SchedulingGated")}}, false}, + "bound": {corev1.Pod{Spec: corev1.PodSpec{NodeName: "n"}, Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: cond(corev1.PodReasonUnschedulable)}}, false}, + "running": {corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodRunning}}, false}, + } + for name, c := range cases { + if got := Unschedulable(&c.pod); got != c.want { + t.Errorf("%s: Unschedulable = %v, want %v", name, got, c.want) + } + } +} + +func TestNudgeJobs(t *testing.T) { + job := func(name string, st batchv1.JobStatus) *batchv1.Job { + return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"app": "sg-runner-nudge"}}, Status: st} + } + k := newKube( + job("sg-runner-nudge-1", batchv1.JobStatus{}), + job("sg-runner-nudge-2", batchv1.JobStatus{Ready: ptr.To[int32](1)}), + job("sg-runner-nudge-3", batchv1.JobStatus{Failed: 1}), + ) + got, err := k.NudgeJobs(context.Background()) + if err != nil { + t.Fatal(err) + } + want := map[int]NudgeState{1: NudgeWaiting, 2: NudgeRunning, 3: NudgeFailed} + for n, s := range want { + if got[n].State != s { + t.Errorf("ordinal %d: %v, want %v", n, got[n].State, s) + } + } + if len(got) != len(want) { + t.Errorf("NudgeJobs = %v", got) + } +} + +func TestNudgeJobIsAPlainPlaceholder(t *testing.T) { + k := newKube() + k.ReapPod.ImagePullSecrets = []string{"regcred"} + runner := &corev1.Pod{Spec: corev1.PodSpec{ + NodeSelector: map[string]string{"pool": "runners"}, + Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpExists}}, + Containers: []corev1.Container{{Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2")}}}}, + Volumes: []corev1.Volume{{Name: "state"}}, + }} + j := k.NudgeJob(4, runner) + ps := j.Spec.Template.Spec + if j.Name != "sg-runner-nudge-4" || j.Labels["app"] != "sg-runner-nudge" { + t.Errorf("name/labels %s %v", j.Name, j.Labels) + } + if ps.Containers[0].Image != DefaultNudgeImage || len(ps.Volumes) != 0 || *ps.AutomountServiceAccountToken { + t.Errorf("not a plain placeholder: %+v", ps) + } + if ps.NodeSelector["pool"] != "runners" || len(ps.Tolerations) != 1 || ps.ImagePullSecrets[0].Name != "regcred" { + t.Errorf("scheduling not copied: %+v", ps) + } + if *j.Spec.ActiveDeadlineSeconds != NudgeDeadline || *j.Spec.BackoffLimit != 0 { + t.Errorf("deadline/backoff %+v", j.Spec) + } +} + +func TestNudgeOutcomeReadsOnlyTheCurrentPlaceholder(t *testing.T) { + created := time.Unix(1_700_000_000, 0) + ev := func(name, pod, reason string, at time.Time) *corev1.Event { + return &corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: pod}, Reason: reason, Message: reason + " msg", + LastTimestamp: metav1.NewTime(at)} + } + k := newKube( + // An earlier placeholder for runner 2, and another runner's. + ev("e1", "sg-runner-nudge-2-old", EventScaleUpTriggered, created.Add(-5*time.Minute)), + ev("e2", "sg-runner-nudge-3-abc", EventScaleUpTriggered, created.Add(time.Minute)), + ev("e3", "sg-runner-2", EventScaleUpRefused, created.Add(time.Minute)), + ) + ctx := context.Background() + if r, _, err := k.NudgeOutcome(ctx, 2, created); err != nil || r != "" { + t.Fatalf("outcome before any answer = %q, %v; want none", r, err) + } + + _, _ = k.Client.CoreV1().Events(ns).Create(ctx, ev("e4", "sg-runner-nudge-2-new", EventScaleUpRefused, created.Add(time.Second)), metav1.CreateOptions{}) + if r, msg, _ := k.NudgeOutcome(ctx, 2, created); r != EventScaleUpRefused || msg != "NotTriggerScaleUp msg" { + t.Errorf("outcome = %q %q, want the refusal", r, msg) + } + // Triggered wins over an earlier refusal of the same placeholder. + _, _ = k.Client.CoreV1().Events(ns).Create(ctx, ev("e5", "sg-runner-nudge-2-new", EventScaleUpTriggered, created.Add(2*time.Second)), metav1.CreateOptions{}) + if r, _, _ := k.NudgeOutcome(ctx, 2, created); r != EventScaleUpTriggered { + t.Errorf("outcome = %q, want %q", r, EventScaleUpTriggered) + } +} diff --git a/k8s/scaler/internal/metrics/metrics.go b/k8s/scaler/internal/metrics/metrics.go new file mode 100644 index 0000000..94c9e07 --- /dev/null +++ b/k8s/scaler/internal/metrics/metrics.go @@ -0,0 +1,95 @@ +// Package metrics defines the scaler's Prometheus metrics. +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ns = "sg_runner_scaler" + +// Metrics are the scaler's gauges and counters, on their own registry. +type Metrics struct { + Registry *prometheus.Registry + + Queued prometheus.Gauge + ReservedPct prometheus.Gauge + FreePct prometheus.Gauge + Replicas prometheus.Gauge + Awake prometheus.Gauge + Parked prometheus.Gauge + Max prometheus.Gauge + Stuck prometheus.Gauge + Unconnected prometheus.Gauge + Settling prometheus.Gauge + Leader prometheus.Gauge + ConsecutiveFailures prometheus.Gauge + + ScaleEvents *prometheus.CounterVec + Reaps *prometheus.CounterVec + Nudges *prometheus.CounterVec + StatusChanges *prometheus.CounterVec + SGErrors prometheus.Counter + KubeErrors prometheus.Counter + LoopDuration prometheus.Histogram +} + +// New registers the metrics on a fresh registry. +func New() *Metrics { + g := func(name, help string) prometheus.Gauge { + return prometheus.NewGauge(prometheus.GaugeOpts{Namespace: ns, Name: name, Help: help}) + } + m := &Metrics{ + Registry: prometheus.NewRegistry(), + Queued: g("queued_workflows", "Workflows queued in the runner group."), + ReservedPct: g("reserved_percent", "Share of awake ACTIVE, connected runners' CPU or memory reserved (higher of the two)."), + FreePct: g("free_percent", "100 minus reserved_percent."), + Replicas: g("replicas", "Replicas of the runner StatefulSet after this loop."), + Awake: g("awake_runners", "Runners with a pod (ordinal below replicas) at the start of the loop."), + Parked: g("parked_runners", "Runners parked: a state PVC but no pod."), + Max: g("max_runners", "Effective maximum: the lower of SCALER_MAX and MaxNumberOfRunners."), + Stuck: g("stuck", "1 while queued workflows fit no runner (stuck queue)."), + Unconnected: g("unconnected_runners", "Awake runners not connected to SG."), + Settling: g("settling_runners", "Awake runners not connected yet, within the settle timeout."), + Leader: g("leader", "1 while this replica holds the leader lease."), + ConsecutiveFailures: g("consecutive_failures", "Loops failed in a row (SG or Kubernetes API errors)."), + ScaleEvents: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "scale_events_total", + Help: "Replica changes by direction (out, in)."}, []string{"direction"}), + Reaps: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "reaps_total", + Help: "Reap steps by result (started, succeeded, failed)."}, []string{"result"}), + Nudges: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "autoscaler_nudges_total", + Help: "Placeholder pods asking the cluster autoscaler for a node, by result (started, node_requested, refused)."}, []string{"result"}), + StatusChanges: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: ns, Name: "runner_status_changes_total", + Help: "Runner status changes through SG by reason and result."}, []string{"reason", "result"}), + SGErrors: prometheus.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "sg_api_errors_total", + Help: "Failed SG API calls."}), + KubeErrors: prometheus.NewCounter(prometheus.CounterOpts{Namespace: ns, Name: "kube_api_errors_total", + Help: "Failed Kubernetes API calls."}), + LoopDuration: prometheus.NewHistogram(prometheus.HistogramOpts{Namespace: ns, Name: "loop_duration_seconds", + Help: "Duration of a scaling loop.", Buckets: prometheus.ExponentialBuckets(0.05, 2, 10)}), + } + m.Registry.MustRegister( + collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + m.Queued, m.ReservedPct, m.FreePct, m.Replicas, m.Awake, m.Parked, m.Max, m.Stuck, m.Unconnected, + m.Settling, m.Leader, m.ConsecutiveFailures, + m.ScaleEvents, m.Reaps, m.Nudges, m.StatusChanges, m.SGErrors, m.KubeErrors, m.LoopDuration, + ) + for _, d := range []string{"out", "in"} { + m.ScaleEvents.WithLabelValues(d) + } + for _, r := range []string{"started", "succeeded", "failed"} { + m.Reaps.WithLabelValues(r) + } + for _, r := range []string{"started", "node_requested", "refused"} { + m.Nudges.WithLabelValues(r) + } + return m +} + +// Handler serves the registry. +func (m *Metrics) Handler() http.Handler { + return promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{Registry: m.Registry}) +} diff --git a/k8s/scaler/internal/sg/sg.go b/k8s/scaler/internal/sg/sg.go new file mode 100644 index 0000000..42196a5 --- /dev/null +++ b/k8s/scaler/internal/sg/sg.go @@ -0,0 +1,209 @@ +// Package sg is a minimal client for the StackGuardian runner group API, +// authenticated with the runner group's node token. +// +// The runner group response carries secrets (the group's runner token), so +// responses are decoded into structs holding only the fields the scaler +// needs, and neither bodies nor decoder messages end up in errors. +package sg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// DefaultBaseAPI is used when the credentials don't name one. +const DefaultBaseAPI = "https://api.app.stackguardian.io/api/v1" + +// maxBody caps what is read from a response. +const maxBody = 16 << 20 + +// Credentials identify the runner group. +type Credentials struct { + Token string + Organization string + RunnerGroup string + BaseAPI string +} + +// LoadCredentials reads the node token (Secret) and the runner group +// (ConfigMap), mounted together as one file per key. +func LoadCredentials(dir string) (Credentials, error) { + read := func(key string, required bool) (string, error) { + b, err := os.ReadFile(filepath.Join(dir, key)) + if err != nil { + if !required && errors.Is(err, os.ErrNotExist) { + return "", nil + } + return "", fmt.Errorf("read credential %s: %w", key, err) + } + v := strings.TrimSpace(string(b)) + if required && v == "" { + return "", fmt.Errorf("credential %s is empty", key) + } + return v, nil + } + var c Credentials + var err error + if c.Token, err = read("SG_NODE_TOKEN", true); err != nil { + return c, err + } + if c.Organization, err = read("SG_ORGANIZATION", true); err != nil { + return c, err + } + if c.RunnerGroup, err = read("SG_RUNNER_GROUP", true); err != nil { + return c, err + } + if c.BaseAPI, err = read("SG_BASE_API", false); err != nil { + return c, err + } + if c.BaseAPI == "" { + c.BaseAPI = DefaultBaseAPI + } + return c, nil +} + +// Client calls the runner group API. +type Client struct { + creds Credentials + http *http.Client +} + +// New returns a client; a nil http.Client gets a 10s timeout. +func New(creds Credentials, hc *http.Client) *Client { + if hc == nil { + hc = &http.Client{Timeout: 10 * time.Second} + } + creds.BaseAPI = strings.TrimRight(creds.BaseAPI, "/") + return &Client{creds: creds, http: hc} +} + +// Resource is a registered or remaining resource of a runner. +type Resource struct { + Name string `json:"name"` + IntegerValue float64 `json:"integerValue"` +} + +// Instance is a runner (ECS container instance) of the group. +type Instance struct { + RunnerID string `json:"runnerId"` + Status string `json:"status"` + AgentConnected bool `json:"agentConnected"` + RunningTasksCount int `json:"runningTasksCount"` + PendingTasksCount int `json:"pendingTasksCount"` + RegisteredResources []Resource `json:"registeredResources"` + RemainingResources []Resource `json:"remainingResources"` + InstanceDetails []struct { + ComputerName string `json:"ComputerName"` + } `json:"instanceDetails"` +} + +// ComputerName is the runner's host name: the pod's FQDN for k8s runners. +func (i Instance) ComputerName() string { + if len(i.InstanceDetails) == 0 { + return "" + } + return i.InstanceDetails[0].ComputerName +} + +// ResourceValue returns the named resource's value from a list, 0 if absent. +func ResourceValue(list []Resource, name string) float64 { + for _, r := range list { + if r.Name == name { + return r.IntegerValue + } + } + return 0 +} + +// RunnerGroup is the part of the runner group the scaler uses. +type RunnerGroup struct { + QueuedWorkflowsCount int `json:"QueuedWorkflowsCount"` + MaxNumberOfRunners *int `json:"MaxNumberOfRunners"` + ContainerInstances []Instance `json:"ContainerInstances"` +} + +// APIError is a non-2xx answer. It never holds the response body. +type APIError struct { + Method string + Path string + StatusCode int +} + +func (e *APIError) Error() string { + return fmt.Sprintf("SG API %s %s: HTTP %d", e.Method, e.Path, e.StatusCode) +} + +// RunnerGroup fetches the runner group with its active workflow counts. +func (c *Client) RunnerGroup(ctx context.Context) (*RunnerGroup, error) { + body, err := c.do(ctx, http.MethodGet, "/?getActiveWorkflows=true", nil) + if err != nil { + return nil, err + } + var resp struct { + Msg *RunnerGroup `json:"msg"` + } + if err := json.Unmarshal(body, &resp); err != nil { + // Decoder messages can quote input: report only where it failed. + var se *json.SyntaxError + if errors.As(err, &se) { + return nil, fmt.Errorf("SG API GET runner group: invalid JSON at offset %d", se.Offset) + } + return nil, errors.New("SG API GET runner group: unexpected response shape") + } + if resp.Msg == nil { + return nil, errors.New("SG API GET runner group: response has no msg") + } + return resp.Msg, nil +} + +// SetRunnerStatus sets a runner ACTIVE or DRAINING through SG, which changes +// the ECS container instance state on its side. +func (c *Client) SetRunnerStatus(ctx context.Context, runnerID, status string) error { + payload, err := json.Marshal(map[string]string{"Status": status, "RunnerId": runnerID}) + if err != nil { + return err + } + _, err = c.do(ctx, http.MethodPost, "/runner_status/", payload) + return err +} + +func (c *Client) do(ctx context.Context, method, path string, payload []byte) ([]byte, error) { + u := c.creds.BaseAPI + "/orgs/" + url.PathEscape(c.creds.Organization) + + "/runnergroups/" + url.PathEscape(c.creds.RunnerGroup) + path + var body io.Reader + if payload != nil { + body = bytes.NewReader(payload) + } + req, err := http.NewRequestWithContext(ctx, method, u, body) + if err != nil { + return nil, fmt.Errorf("SG API %s %s: %w", method, path, err) + } + req.Header.Set("Authorization", "apikey "+c.creds.Token) + req.Header.Set("Accept", "application/json") + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("SG API %s %s: %w", method, path, err) + } + defer resp.Body.Close() + b, err := io.ReadAll(io.LimitReader(resp.Body, maxBody)) + if err != nil { + return nil, fmt.Errorf("SG API %s %s: read response: %w", method, path, err) + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, &APIError{Method: method, Path: path, StatusCode: resp.StatusCode} + } + return b, nil +} diff --git a/k8s/scaler/internal/sg/sg_test.go b/k8s/scaler/internal/sg/sg_test.go new file mode 100644 index 0000000..4350968 --- /dev/null +++ b/k8s/scaler/internal/sg/sg_test.go @@ -0,0 +1,181 @@ +package sg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +const fakeRunnerToken = "sgrt_SUPERSECRET_runner_token_123" + +const groupJSON = `{"msg": { + "ResourceName": "rg", + "RunnerToken": "` + fakeRunnerToken + `", + "QueuedWorkflowsCount": 2, + "MaxNumberOfRunners": 4, + "ContainerInstances": [ + {"runnerId": "r-0", "status": "ACTIVE", "agentConnected": true, + "runningTasksCount": 1, "pendingTasksCount": 1, + "registeredResources": [{"name": "CPU", "integerValue": 2048}, {"name": "MEMORY", "integerValue": 4096}], + "remainingResources": [{"name": "CPU", "integerValue": 1024}, {"name": "MEMORY", "integerValue": 3584}], + "instanceDetails": [{"ComputerName": "sg-runner-0.sg-runner.sg-runner.svc.cluster.local"}]}, + {"runnerId": "vm-1", "status": "DRAINING", "agentConnected": false, + "instanceDetails": [{"ComputerName": "ip-10-0-0-1"}]} + ]}}` + +func fakeSG(t *testing.T, handler http.HandlerFunc) (*Client, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return New(Credentials{Token: "node-token", Organization: "org", RunnerGroup: "rg", BaseAPI: srv.URL + "/api/v1/"}, nil), srv +} + +func TestRunnerGroup(t *testing.T) { + c, _ := fakeSG(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/v1/orgs/org/runnergroups/rg/" || + r.URL.Query().Get("getActiveWorkflows") != "true" { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + } + if got := r.Header.Get("Authorization"); got != "apikey node-token" { + t.Errorf("Authorization = %q", got) + } + io.WriteString(w, groupJSON) + }) + g, err := c.RunnerGroup(context.Background()) + if err != nil { + t.Fatal(err) + } + if g.QueuedWorkflowsCount != 2 || g.MaxNumberOfRunners == nil || *g.MaxNumberOfRunners != 4 || len(g.ContainerInstances) != 2 { + t.Fatalf("decoded %+v", g) + } + i := g.ContainerInstances[0] + if i.RunnerID != "r-0" || !i.AgentConnected || i.RunningTasksCount+i.PendingTasksCount != 2 || + ResourceValue(i.RegisteredResources, "CPU") != 2048 || ResourceValue(i.RemainingResources, "MEMORY") != 3584 || + !strings.HasPrefix(i.ComputerName(), "sg-runner-0.") { + t.Errorf("instance %+v", i) + } + if g.ContainerInstances[1].ComputerName() != "ip-10-0-0-1" || ResourceValue(nil, "CPU") != 0 { + t.Error("second instance") + } +} + +func TestRunnerGroupNullMax(t *testing.T) { + c, _ := fakeSG(t, func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"msg": {"MaxNumberOfRunners": null, "ContainerInstances": null}}`) + }) + g, err := c.RunnerGroup(context.Background()) + if err != nil || g.MaxNumberOfRunners != nil || g.QueuedWorkflowsCount != 0 { + t.Fatalf("got %+v, %v", g, err) + } +} + +func TestSetRunnerStatus(t *testing.T) { + var got map[string]string + c, _ := fakeSG(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/orgs/org/runnergroups/rg/runner_status/" { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Error("missing content type") + } + json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(http.StatusCreated) + }) + if err := c.SetRunnerStatus(context.Background(), "r-1", "DRAINING"); err != nil { + t.Fatal(err) + } + if got["Status"] != "DRAINING" || got["RunnerId"] != "r-1" { + t.Errorf("payload %v", got) + } +} + +// Responses carry the runner token: no error may echo a body, whatever +// goes wrong. +func TestErrorsNeverLeakBodies(t *testing.T) { + cases := map[string]http.HandlerFunc{ + "http 500": func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + io.WriteString(w, groupJSON) + }, + "http 403": func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(403) + io.WriteString(w, `{"msg":"`+fakeRunnerToken+`"}`) + }, + "truncated json": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, groupJSON[:len(groupJSON)-40]) + }, + "wrong type": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"msg": {"QueuedWorkflowsCount": "`+fakeRunnerToken+`"}}`) + }, + "syntax error after secret": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"msg": {"RunnerToken": "`+fakeRunnerToken+`" x}}`) + }, + "no msg": func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"RunnerToken": "`+fakeRunnerToken+`"}`) + }, + } + for name, h := range cases { + t.Run(name, func(t *testing.T) { + var logs bytes.Buffer + log := slog.New(slog.NewJSONHandler(&logs, nil)) + c, _ := fakeSG(t, h) + _, err := c.RunnerGroup(context.Background()) + if err == nil { + t.Fatal("want an error") + } + log.Error("loop failed", "err", err) + serr := c.SetRunnerStatus(context.Background(), "r", "ACTIVE") + log.Error("status", "err", serr) + for _, s := range []string{err.Error(), logs.String()} { + if strings.Contains(s, fakeRunnerToken) || strings.Contains(s, "node-token") { + t.Errorf("secret leaked: %s", s) + } + } + }) + } + var apiErr *APIError + c, _ := fakeSG(t, cases["http 403"]) + if _, err := c.RunnerGroup(context.Background()); !errors.As(err, &apiErr) || apiErr.StatusCode != 403 { + t.Errorf("want APIError 403, got %v", err) + } +} + +func TestUnreachable(t *testing.T) { + c, srv := fakeSG(t, func(w http.ResponseWriter, r *http.Request) {}) + srv.Close() + if _, err := c.RunnerGroup(context.Background()); err == nil || strings.Contains(err.Error(), "node-token") { + t.Errorf("got %v", err) + } +} + +func TestLoadCredentials(t *testing.T) { + dir := t.TempDir() + write := func(k, v string) { os.WriteFile(filepath.Join(dir, k), []byte(v), 0o600) } + if _, err := LoadCredentials(dir); err == nil { + t.Error("want error without files") + } + write("SG_NODE_TOKEN", "tok\n") + write("SG_ORGANIZATION", "org\n") + write("SG_RUNNER_GROUP", "rg") + c, err := LoadCredentials(dir) + if err != nil || c != (Credentials{Token: "tok", Organization: "org", RunnerGroup: "rg", BaseAPI: DefaultBaseAPI}) { + t.Fatalf("got %+v, %v", c, err) + } + write("SG_BASE_API", "https://example.test/api/v1\n") + if c, _ := LoadCredentials(dir); c.BaseAPI != "https://example.test/api/v1" { + t.Errorf("base = %q", c.BaseAPI) + } + write("SG_RUNNER_GROUP", " \n") + if _, err := LoadCredentials(dir); err == nil { + t.Error("want error for an empty runner group") + } +} diff --git a/k8s/scaler/internal/steps/config.go b/k8s/scaler/internal/steps/config.go new file mode 100644 index 0000000..50b6204 --- /dev/null +++ b/k8s/scaler/internal/steps/config.go @@ -0,0 +1,201 @@ +// Package steps is the step broker: it runs each workflow step of the run +// controllers in a runner pod as a pod of its own. See k8s/docs/steps-api.md. +package steps + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" +) + +// Config is everything the broker is told from outside. +type Config struct { + // SocketPath is the API's socket, GuardSocket the dockerd authorization + // plugin's. + SocketPath string + GuardSocket string + // WorkspacesDir is the run controllers' workspace root: request paths + // start with it. + WorkspacesDir string + // WorkspaceClaim is the PVC holding WorkspacesDir, and + // WorkspaceSubPathPrefix the directory inside it ("" for its root). + WorkspaceClaim string + WorkspaceSubPathPrefix string + // The runner pod the broker runs in (downward API). + RunnerPodName string + RunnerPodUID types.UID + NodeName string + Namespace string + // Step pod settings: extra labels, node selector and tolerations. + StepLabels map[string]string + StepNodeSelector map[string]string + StepTolerations []corev1.Toleration + // StepImagePullPolicy overrides every step's pull policy ("" for the + // automatic one: Always for latest or no tag, else IfNotPresent). + StepImagePullPolicy corev1.PullPolicy + // SchedulingTimeout fails a step the scheduler can't place, StartTimeout + // one whose container hasn't started; FinishedTTL is how long finished + // step pods stay. + SchedulingTimeout time.Duration + StartTimeout time.Duration + FinishedTTL time.Duration + // BrokerImage is the broker's own image, carrying sg-step-wrapper for the + // steps' init container; its pull policy ("" for the cluster's default) + // and pull secrets (for the step pods). No BrokerImage: steps aren't + // wrapped. + BrokerImage string + BrokerImagePullPolicy corev1.PullPolicy + BrokerImagePullSecrets []string +} + +// Default sockets, also for healthcheck. +const ( + DefaultSocketPath = "/run/sg-steps/steps.sock" + DefaultGuardSocket = "/run/sg-guard/sg-steps-guard.sock" +) + +// LoadConfig reads the environment through getenv (os.Getenv when nil). +func LoadConfig(getenv func(string) string) (Config, error) { + if getenv == nil { + getenv = os.Getenv + } + var errs []string + str := func(key, def string) string { + if v := strings.TrimSpace(getenv(key)); v != "" { + return v + } + return def + } + required := func(key string) string { + v := str(key, "") + if v == "" { + errs = append(errs, key+" is required") + } + return v + } + duration := func(key, def string) time.Duration { + raw := str(key, def) + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + errs = append(errs, fmt.Sprintf("%s=%q: want a positive Go duration (e.g. 5m)", key, raw)) + } + return d + } + + c := Config{ + SocketPath: str("SOCKET_PATH", DefaultSocketPath), + GuardSocket: str("GUARD_SOCKET", DefaultGuardSocket), + WorkspacesDir: str("WORKSPACES_DIR", "/var/tmp/workspace"), + WorkspaceClaim: required("WORKSPACE_CLAIM"), + RunnerPodName: required("RUNNER_POD_NAME"), + RunnerPodUID: types.UID(required("RUNNER_POD_UID")), + NodeName: required("NODE_NAME"), + Namespace: required("NAMESPACE"), + SchedulingTimeout: duration("SCHEDULING_TIMEOUT", "5m"), + StartTimeout: duration("START_TIMEOUT", "30m"), + FinishedTTL: duration("FINISHED_TTL", "1h"), + BrokerImage: str("BROKER_IMAGE", ""), + } + + if c.GuardSocket == c.SocketPath { + errs = append(errs, "GUARD_SOCKET and SOCKET_PATH are the same") + } + if !path.IsAbs(c.WorkspacesDir) { + errs = append(errs, fmt.Sprintf("WORKSPACES_DIR=%q: want an absolute path", c.WorkspacesDir)) + } + c.WorkspacesDir = path.Clean(c.WorkspacesDir) + if c.WorkspacesDir == "/" { + errs = append(errs, "WORKSPACES_DIR: the filesystem root is not a workspace root") + } + if p := str("WORKSPACE_SUBPATH_PREFIX", ""); p != "" { + if path.IsAbs(p) || hasDotDot(p) { + errs = append(errs, fmt.Sprintf("WORKSPACE_SUBPATH_PREFIX=%q: want a relative path without ..", p)) + } + if p = path.Clean(p); p != "." { + c.WorkspaceSubPathPrefix = p + } + } + if c.WorkspaceClaim != "" { + for _, msg := range validation.IsDNS1123Subdomain(c.WorkspaceClaim) { + errs = append(errs, fmt.Sprintf("WORKSPACE_CLAIM=%q: %s", c.WorkspaceClaim, msg)) + } + } + // Step pods and the runner label carry the name: it has to be a label. + if c.RunnerPodName != "" { + for _, msg := range validation.IsDNS1123Label(c.RunnerPodName) { + errs = append(errs, fmt.Sprintf("RUNNER_POD_NAME=%q: %s", c.RunnerPodName, msg)) + } + } + + if err := strictJSON(str("STEP_LABELS", "{}"), &c.StepLabels); err != nil { + errs = append(errs, fmt.Sprintf("STEP_LABELS: want a JSON object of strings: %v", err)) + } + for k, v := range c.StepLabels { + for _, msg := range append(validation.IsQualifiedName(k), validation.IsValidLabelValue(v)...) { + errs = append(errs, fmt.Sprintf("STEP_LABELS: label %q=%q: %s", k, v, msg)) + } + } + if err := strictJSON(str("STEP_NODE_SELECTOR", "{}"), &c.StepNodeSelector); err != nil { + errs = append(errs, fmt.Sprintf("STEP_NODE_SELECTOR: want a JSON object of strings: %v", err)) + } + if err := strictJSON(str("STEP_TOLERATIONS", "[]"), &c.StepTolerations); err != nil { + errs = append(errs, fmt.Sprintf("STEP_TOLERATIONS: want a JSON array of tolerations: %v", err)) + } + + pullPolicy := func(key string) corev1.PullPolicy { + switch p := corev1.PullPolicy(str(key, "")); p { + case "", corev1.PullAlways, corev1.PullIfNotPresent, corev1.PullNever: + return p + default: + errs = append(errs, fmt.Sprintf("%s=%q: want Always, IfNotPresent or Never", key, p)) + return "" + } + } + c.StepImagePullPolicy = pullPolicy("STEP_IMAGE_PULL_POLICY") + c.BrokerImagePullPolicy = pullPolicy("BROKER_IMAGE_PULL_POLICY") + for _, name := range strings.Split(str("BROKER_IMAGE_PULL_SECRETS", ""), ",") { + if name = strings.TrimSpace(name); name == "" { + continue + } + for _, msg := range validation.IsDNS1123Subdomain(name) { + errs = append(errs, fmt.Sprintf("BROKER_IMAGE_PULL_SECRETS: %q: %s", name, msg)) + } + c.BrokerImagePullSecrets = append(c.BrokerImagePullSecrets, name) + } + + if len(errs) > 0 { + return c, fmt.Errorf("invalid configuration: %s", strings.Join(errs, "; ")) + } + return c, nil +} + +// strictJSON decodes raw into v, rejecting unknown fields and trailing data. +func strictJSON(raw string, v any) error { + dec := json.NewDecoder(bytes.NewReader([]byte(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if dec.More() { + return fmt.Errorf("trailing data") + } + return nil +} + +// hasDotDot reports a ".." segment in a slash-separated path. +func hasDotDot(p string) bool { + for _, s := range strings.Split(p, "/") { + if s == ".." { + return true + } + } + return false +} diff --git a/k8s/scaler/internal/steps/config_test.go b/k8s/scaler/internal/steps/config_test.go new file mode 100644 index 0000000..4f66124 --- /dev/null +++ b/k8s/scaler/internal/steps/config_test.go @@ -0,0 +1,96 @@ +package steps + +import ( + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func env(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +// required is the downward API and claim every configuration needs. +func required(extra map[string]string) map[string]string { + m := map[string]string{ + "WORKSPACE_CLAIM": "sg-runner-0-workspace", "RUNNER_POD_NAME": "sg-runner-0", + "RUNNER_POD_UID": "uid-0", "NODE_NAME": "node-a", "NAMESPACE": "sg", + } + for k, v := range extra { + m[k] = v + } + return m +} + +func TestConfigDefaults(t *testing.T) { + c, err := LoadConfig(env(required(nil))) + if err != nil { + t.Fatal(err) + } + if c.SocketPath != "/run/sg-steps/steps.sock" || c.WorkspacesDir != "/var/tmp/workspace" || + c.WorkspaceClaim != "sg-runner-0-workspace" || c.WorkspaceSubPathPrefix != "" || + c.RunnerPodName != "sg-runner-0" || c.RunnerPodUID != types.UID("uid-0") || c.NodeName != "node-a" || + c.Namespace != "sg" || len(c.StepLabels) != 0 || len(c.StepNodeSelector) != 0 || len(c.StepTolerations) != 0 || + c.SchedulingTimeout != 5*time.Minute || c.StartTimeout != 30*time.Minute || c.FinishedTTL != time.Hour || + c.GuardSocket != "/run/sg-guard/sg-steps-guard.sock" || c.BrokerImage != "" || c.BrokerImagePullPolicy != "" || + c.BrokerImagePullSecrets != nil || c.StepImagePullPolicy != "" { + t.Errorf("defaults %+v", c) + } +} + +func TestConfigOverrides(t *testing.T) { + c, err := LoadConfig(env(required(map[string]string{ + "SOCKET_PATH": "/s/x.sock", "WORKSPACES_DIR": "/w/", "WORKSPACE_SUBPATH_PREFIX": "runners/r0/", + "STEP_LABELS": `{"app.kubernetes.io/name":"sg-runner"}`, + "STEP_NODE_SELECTOR": `{"pool":"steps"}`, + "STEP_TOLERATIONS": `[{"key":"dedicated","operator":"Exists","effect":"NoSchedule"}]`, + "SCHEDULING_TIMEOUT": "90s", "START_TIMEOUT": "1h", "FINISHED_TTL": "1m", "GUARD_SOCKET": "/g/x.sock", + "STEP_IMAGE_PULL_POLICY": "Never", "BROKER_IMAGE": "reg/scaler:1", "BROKER_IMAGE_PULL_POLICY": "Always", + "BROKER_IMAGE_PULL_SECRETS": "regcred, other ,", + }))) + if err != nil { + t.Fatal(err) + } + if c.SocketPath != "/s/x.sock" || c.WorkspacesDir != "/w" || c.WorkspaceSubPathPrefix != "runners/r0" || + c.StepLabels["app.kubernetes.io/name"] != "sg-runner" || c.StepNodeSelector["pool"] != "steps" || + len(c.StepTolerations) != 1 || c.StepTolerations[0].Key != "dedicated" || + c.SchedulingTimeout != 90*time.Second || c.StartTimeout != time.Hour || c.FinishedTTL != time.Minute || + c.GuardSocket != "/g/x.sock" || c.StepImagePullPolicy != corev1.PullNever || c.BrokerImage != "reg/scaler:1" || + c.BrokerImagePullPolicy != corev1.PullAlways || strings.Join(c.BrokerImagePullSecrets, ",") != "regcred,other" { + t.Errorf("overrides %+v", c) + } +} + +func TestConfigInvalid(t *testing.T) { + for name, m := range map[string]map[string]string{ + "no claim": {"WORKSPACE_CLAIM": " "}, + "no runner pod": {"RUNNER_POD_NAME": ""}, + "no uid": {"RUNNER_POD_UID": ""}, + "no node": {"NODE_NAME": ""}, + "no namespace": {"NAMESPACE": ""}, + "runner not a label": {"RUNNER_POD_NAME": "Runner_0"}, + "claim not a name": {"WORKSPACE_CLAIM": "Bad Claim"}, + "relative workspaces": {"WORKSPACES_DIR": "var/tmp"}, + "root workspaces": {"WORKSPACES_DIR": "/"}, + "absolute prefix": {"WORKSPACE_SUBPATH_PREFIX": "/x"}, + "dotdot prefix": {"WORKSPACE_SUBPATH_PREFIX": "a/../../b"}, + "bad duration": {"START_TIMEOUT": "30"}, + "zero duration": {"FINISHED_TTL": "0s"}, + "labels not json": {"STEP_LABELS": "a=b"}, + "bad label": {"STEP_LABELS": `{"a":"not ok"}`}, + "selector trailing": {"STEP_NODE_SELECTOR": `{"a":"b"} {}`}, + "toleration unknown": {"STEP_TOLERATIONS": `[{"keyy":"a"}]`}, + "bad pull policy": {"STEP_IMAGE_PULL_POLICY": "never"}, + "zero start timeout": {"START_TIMEOUT": "0s"}, + "one socket for both": {"GUARD_SOCKET": "/run/sg-steps/steps.sock"}, + "bad broker policy": {"BROKER_IMAGE_PULL_POLICY": "sometimes"}, + "bad pull secret": {"BROKER_IMAGE_PULL_SECRETS": "Not_A_Name"}, + } { + if _, err := LoadConfig(env(required(m))); err == nil || !strings.Contains(err.Error(), "invalid configuration") { + t.Errorf("%s: want an error, got %v", name, err) + } + } +} diff --git a/k8s/scaler/internal/steps/finish.go b/k8s/scaler/internal/steps/finish.go new file mode 100644 index 0000000..86984e5 --- /dev/null +++ b/k8s/scaler/internal/steps/finish.go @@ -0,0 +1,106 @@ +package steps + +import ( + "context" + "encoding/json" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Outcome is the outcome label of a terminal state. +func Outcome(st State, stopped bool) string { + switch { + case stopped: + return OutcomeStopped + case st.State != StateExited || st.ExitCode == nil: + return OutcomeFailed + } + switch *st.ExitCode { + case 0: + return OutcomeSucceeded + case 11: + return OutcomeApprovalRequired + case 12: + return OutcomeCompletedEarly + } + return OutcomeFailed +} + +// wrap is how a step runs: under the wrapper when the broker's image is known +// and the step image's entrypoint and cmd (or the request's args) are. +func (s *Server) wrap(ctx context.Context, r *Request) Wrap { + switch { + case s.Config.BrokerImage == "": + return Wrap{Reason: "BROKER_IMAGE is not set"} + case s.Images == nil: + return Wrap{Reason: "no image config lookup"} + } + cfg, err := s.Images.Lookup(ctx, r.Image, r.RegistryAuth) + if err != nil { + return Wrap{Reason: lookupError(err)} + } + cmd := wrapCommand(cfg, r.Args) + if cmd == nil { + return Wrap{Reason: "the image has no entrypoint or cmd, and the step no args"} + } + return Wrap{Command: cmd} +} + +// terminalLocked records a terminal state the broker saw (the first one +// stays) and claims finishing the step: true makes the caller its only +// finisher. s.mu is held. +func (s *Server) terminalLocked(id string, st State) bool { + rec := s.recordLocked(id) + if rec.final == nil { + rec.final, rec.terminalAt = &st, s.now() + } + if rec.finishing || rec.finished { + return false + } + rec.finishing = true + return true +} + +func (s *Server) finishAsync(id string, p *corev1.Pod) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + s.finish(ctx, id, p) +} + +// finish labels a claimed terminal step's pod with its outcome (once: a +// labeled pod keeps its label) and deletes its Secrets, which hold +// credentials a finished pod doesn't need. p is nil when the pod is gone. A +// failure leaves it to the next GET or sweep. +func (s *Server) finish(ctx context.Context, id string, p *corev1.Pod) { + s.mu.Lock() + rec := s.steps[id] + outcome := Outcome(*rec.final, rec.stopped) + s.mu.Unlock() + + err := func() error { + // A pod being deleted takes metadata changes too (a stopped one). + if p != nil && p.Labels[LabelOutcome] == "" { + patch, _ := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]string{LabelOutcome: outcome}}}) + _, err := s.Client.CoreV1().Pods(s.Config.Namespace).Patch(ctx, id, types.MergePatchType, patch, metav1.PatchOptions{}) + if err := ignoreNotFound(err); err != nil { + return kubeError("label Pod "+id, err) + } + } + return s.deleteSecrets(ctx, id) + }() + + s.mu.Lock() + rec.finishing = false + if err == nil { + rec.finished, rec.secretsDeleted = true, true + } + s.mu.Unlock() + if err != nil { + s.Log.Warn("finishing a step", "id", id, "err", err) + return + } + s.Log.Info("step finished", "id", id, "outcome", outcome) +} diff --git a/k8s/scaler/internal/steps/finish_test.go b/k8s/scaler/internal/steps/finish_test.go new file mode 100644 index 0000000..11c45a5 --- /dev/null +++ b/k8s/scaler/internal/steps/finish_test.go @@ -0,0 +1,263 @@ +package steps + +import ( + "context" + "errors" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8stesting "k8s.io/client-go/testing" +) + +// waitFor polls cond for up to 5s. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + for deadline := time.Now().Add(5 * time.Second); !cond(); time.Sleep(5 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting: %s", what) + } + } +} + +// count counts the client's calls of a verb on a resource. +func count(hs *harness, verb, resource string) int { + n := 0 + for _, a := range hs.client.Actions() { + if a.GetVerb() == verb && a.GetResource().Resource == resource { + n++ + } + } + return n +} + +func stepSecrets(id string) []runtime.Object { + return []runtime.Object{ + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: id + "-env", Namespace: "sg"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: id + "-pull", Namespace: "sg"}}, + } +} + +func (hs *harness) finished(id string) bool { + hs.s.mu.Lock() + defer hs.s.mu.Unlock() + rec := hs.s.steps[id] + return rec != nil && rec.finished +} + +func TestOutcome(t *testing.T) { + code := func(c int32) *int32 { return &c } + for _, c := range []struct { + st State + stopped bool + want string + }{ + {State{State: StateExited, ExitCode: code(0)}, false, OutcomeSucceeded}, + {State{State: StateExited, ExitCode: code(11)}, false, OutcomeApprovalRequired}, + {State{State: StateExited, ExitCode: code(12)}, false, OutcomeCompletedEarly}, + {State{State: StateExited, ExitCode: code(1)}, false, OutcomeFailed}, + {State{State: StateExited, ExitCode: code(137)}, false, OutcomeFailed}, + {State{State: StateFailed, Reason: FailedImagePullBackOff}, false, OutcomeFailed}, + {State{State: StateExited, ExitCode: code(0)}, true, OutcomeStopped}, + {State{State: StateExited, ExitCode: code(143)}, true, OutcomeStopped}, + } { + if got := Outcome(c.st, c.stopped); got != c.want { + t.Errorf("%+v stopped=%v: %s, want %s", c.st, c.stopped, got, c.want) + } + } +} + +// TestTerminalGetFinishesTheStep: the first GET that sees a step ended labels +// its pod with the outcome and deletes its Secrets; the pod stays. +func TestTerminalGetFinishesTheStep(t *testing.T) { + for name, c := range map[string]struct { + pod *corev1.Pod + code string + outcome string + }{ + "approval": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=11"), wrapped), `"exitCode":11`, OutcomeApprovalRequired}, + "early finish": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=12"), wrapped), `"exitCode":12`, OutcomeCompletedEarly}, + "success": {pod(corev1.PodSucceeded, wrappedExit0(""), wrapped), `"exitCode":0`, OutcomeSucceeded}, + "error": {pod(corev1.PodFailed, terminated(3, "Error", t0, t0)), `"exitCode":3`, OutcomeFailed}, + // The outcome follows the exit code, wrapped or not. + "unwrapped 11": {pod(corev1.PodFailed, terminated(11, "Error", t0, t0)), `"exitCode":11`, OutcomeApprovalRequired}, + "unwrapped 12": {pod(corev1.PodFailed, terminated(12, "Error", t0, t0)), `"exitCode":12`, OutcomeCompletedEarly}, + "pull failed": {pod(corev1.PodPending, waiting("ImagePullBackOff", "")), `"state":"failed"`, OutcomeFailed}, + } { + hs := newHarness(t, append(stepSecrets(testID), c.pod)...) + first := hs.do("GET", "/v1/steps/"+testID, "") + if first.Code != 200 || !strings.Contains(first.Body.String(), c.code) { + t.Fatalf("%s: get %d %s", name, first.Code, first.Body) + } + waitFor(t, name+": finished", func() bool { return hs.finished(testID) }) + if got := hs.outcomes()["7f3k9q2x"]; got != c.outcome { + t.Errorf("%s: outcome %q, want %q", name, got, c.outcome) + } + if len(hs.secretNames()) != 0 || len(hs.pods()) != 1 { + t.Errorf("%s: secrets %v, pods %v", name, hs.secretNames(), hs.podNames()) + } + // Later GETs answer the same from memory; nothing is done twice. + for i := 0; i < 3; i++ { + if rec := hs.do("GET", "/v1/steps/"+testID, ""); rec.Body.String() != first.Body.String() { + t.Errorf("%s: later get %s, want %s", name, rec.Body, first.Body) + } + } + if err := hs.s.Sweep(context.Background()); err != nil { + t.Fatal(err) + } + if count(hs, "patch", "pods") != 1 || count(hs, "delete", "pods") != 0 || count(hs, "get", "pods") != 1 { + t.Errorf("%s: %d patches, %d pod deletes, %d gets", name, count(hs, "patch", "pods"), + count(hs, "delete", "pods"), count(hs, "get", "pods")) + } + } +} + +func TestNonTerminalGetsChangeNothing(t *testing.T) { + hs := newHarness(t, append(stepSecrets(testID), pod(corev1.PodRunning, running(t0)))...) + for i := 0; i < 3; i++ { + hs.do("GET", "/v1/steps/"+testID, "") + } + time.Sleep(20 * time.Millisecond) + if count(hs, "patch", "pods") != 0 || count(hs, "delete", "secrets") != 0 || len(hs.s.steps) != 0 { + t.Errorf("actions %v, records %v", hs.client.Actions(), hs.s.steps) + } +} + +func TestLabeledPodIsNotPatchedAgain(t *testing.T) { + p := pod(corev1.PodFailed, terminated(3, "Error", t0, t0)) + p.Labels = map[string]string{LabelOutcome: OutcomeFailed} + hs := newHarness(t, append(stepSecrets(testID), p)...) + hs.do("GET", "/v1/steps/"+testID, "") + waitFor(t, "finished", func() bool { return hs.finished(testID) }) + if count(hs, "patch", "pods") != 0 || len(hs.secretNames()) != 0 { + t.Errorf("%d patches, secrets %v", count(hs, "patch", "pods"), hs.secretNames()) + } +} + +func TestFailedFinishIsRetried(t *testing.T) { + hs := newHarness(t, append(stepSecrets(testID), + stepPod(testID, "sg-runner-0", t0, corev1.PodSucceeded, terminated(0, "Completed", t0, t0)))...) + var mu sync.Mutex + failing := true + hs.client.PrependReactor("delete", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + defer mu.Unlock() + if failing { + return true, nil, errors.New("connection refused") + } + return false, nil, nil + }) + hs.do("GET", "/v1/steps/"+testID, "") + waitFor(t, "the first attempt", func() bool { + hs.s.mu.Lock() + defer hs.s.mu.Unlock() + return count(hs, "delete", "secrets") == 1 && !hs.s.steps[testID].finishing + }) + if hs.finished(testID) { + t.Fatal("finished despite the failure") + } + mu.Lock() + failing = false + mu.Unlock() + if err := hs.s.Sweep(context.Background()); err != nil { + t.Fatal(err) + } + if !hs.finished(testID) || len(hs.secretNames()) != 0 || len(hs.pods()) != 1 { + t.Errorf("the sweep didn't retry: secrets %v", hs.secretNames()) + } +} + +// TestStoppedStepOutcome: a stopped pod (being deleted) gets the stopped +// outcome and loses its Secrets. +func TestStoppedStepOutcome(t *testing.T) { + hs := newHarness(t, append(stepSecrets(testID), pod(corev1.PodRunning, running(t0)))...) + hs.s.StopPollInterval = 5 * time.Millisecond + // A graceful delete keeps the pod until the kubelet is done. + hs.client.PrependReactor("delete", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + if rec := hs.do("POST", "/v1/steps/"+testID+"/stop", `{"graceSeconds":10}`); rec.Code != 202 { + t.Fatalf("stop %d %s", rec.Code, rec.Body) + } + p := pod(corev1.PodFailed, terminated(143, "Error", t0, t0.Add(time.Minute)), deleting) + if _, err := hs.client.CoreV1().Pods("sg").UpdateStatus(context.Background(), p, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + waitFor(t, "finished", func() bool { return hs.finished(testID) }) + if got := hs.outcomes()["7f3k9q2x"]; got != OutcomeStopped || len(hs.secretNames()) != 0 { + t.Errorf("outcome %q, secrets %v", got, hs.secretNames()) + } + if rec := hs.do("GET", "/v1/steps/"+testID, ""); !strings.Contains(rec.Body.String(), `"exitCode":143`) { + t.Errorf("get %s", rec.Body) + } +} + +// fakeImages answers lookups from a map and records them. +type fakeImages struct { + mu sync.Mutex + configs map[string]ImageConfig + auths []*RegistryAuth +} + +func (f *fakeImages) Lookup(_ context.Context, image string, auth *RegistryAuth) (ImageConfig, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.auths = append(f.auths, auth) + c, ok := f.configs[image] + if !ok { + return ImageConfig{}, errors.New("MANIFEST_UNKNOWN: manifest unknown") + } + return c, nil +} + +func TestCreateWraps(t *testing.T) { + images := &fakeImages{configs: map[string]ImageConfig{ + "registry.example.com/team/image:1.2": {Entrypoint: []string{"/entry"}, Cmd: []string{"default"}}, + "empty:1": {}, + }} + created := func(t *testing.T, cfg func(*Config), edit func(*Request)) *corev1.Pod { + t.Helper() + hs := newHarness(t) + if cfg != nil { + cfg(&hs.s.Config) + } + hs.s.Images = images + if rec := hs.do("POST", "/v1/steps", requestJSON(t, edit)); rec.Code != 201 { + t.Fatalf("create %d %s", rec.Code, rec.Body) + } + return &hs.pods()[0] + } + + p := created(t, nil, func(r *Request) { + r.RegistryAuth = &RegistryAuth{Registry: "registry.example.com", Username: "u", Password: "p"} + }) + c := p.Spec.Containers[0] + if p.Annotations[AnnotationWrapped] != "true" || len(p.Spec.InitContainers) != 1 || + strings.Join(c.Command, " ") != WrapperPath+" -- /entry sh -c echo $$(HOME) $$HOME" || c.Args != nil { + t.Errorf("wrapped pod: annotations %v, command %q, args %q", p.Annotations, c.Command, c.Args) + } + if a := images.auths[0]; a == nil || a.Username != "u" { + t.Errorf("lookup auth %+v", a) + } + + for name, c := range map[string]struct { + cfg func(*Config) + edit func(*Request) + reason string + }{ + "lookup fails": {nil, func(r *Request) { r.Image = "unknown:1" }, "image config lookup: MANIFEST_UNKNOWN"}, + "nothing to run": {nil, func(r *Request) { r.Image, r.Args = "empty:1", nil }, "the image has no entrypoint or cmd"}, + "no broker image": {func(c *Config) { c.BrokerImage = "" }, nil, "BROKER_IMAGE is not set"}, + } { + p := created(t, c.cfg, c.edit) + ct := p.Spec.Containers[0] + if p.Annotations[AnnotationWrapped] != "false" || !strings.HasPrefix(p.Annotations[AnnotationNotWrappedReason], c.reason) || + p.Spec.InitContainers != nil || ct.Command != nil { + t.Errorf("%s: annotations %v, command %q", name, p.Annotations, ct.Command) + } + } +} diff --git a/k8s/scaler/internal/steps/gc.go b/k8s/scaler/internal/steps/gc.go new file mode 100644 index 0000000..abd644c --- /dev/null +++ b/k8s/scaler/internal/steps/gc.go @@ -0,0 +1,128 @@ +package steps + +import ( + "context" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +// selector matches this runner's step pods. +func (c *Config) selector() string { + return labels.Set{LabelComponent: ComponentStep, LabelRunner: c.RunnerPodName}.String() +} + +// Sweep finishes terminal steps no GET saw end (outcome label, Secrets), +// deletes finished step pods FINISHED_TTL after they ended, deletes the +// Secrets of steps whose pod went without the broker, and forgets steps gone +// for an hour. It never lists Secrets (a list returns their data, the node +// token's too): step Secrets go by name with their step, or with the runner +// pod that owns them. +func (s *Server) Sweep(ctx context.Context) error { + now := s.now() + pods, err := s.Client.CoreV1().Pods(s.Config.Namespace).List(ctx, metav1.ListOptions{LabelSelector: s.Config.selector()}) + if err != nil { + return kubeError("list step pods", err) + } + + live := map[string]bool{} + for i := range pods.Items { + p := &pods.Items[i] + if !s.idRE.MatchString(p.Name) { + continue + } + live[p.Name] = true + if p.DeletionTimestamp != nil { + continue + } + s.mu.Lock() + rec := s.steps[p.Name] + var st State + if rec != nil && rec.final != nil { + st = *rec.final + } else { + stopped := rec != nil && rec.stopped + st = s.Config.StepState(p, now, stopped) + if stopped { + rec.see(st) + } + } + if !st.Terminal() { + s.mu.Unlock() + continue + } + finish := s.terminalLocked(p.Name, st) + ended := s.steps[p.Name].terminalAt + s.mu.Unlock() + if finish { + s.finish(ctx, p.Name, p) + } + + if st.FinishedAt != nil { + ended = *st.FinishedAt + } + if s.streaming(p.Name) || now.Sub(ended) <= s.Config.FinishedTTL { + continue + } + if err := s.deleteStep(ctx, p.Name); err != nil { + s.Log.Warn("gc: deleting a finished step", "id", p.Name, "err", err) + continue + } + s.mu.Lock() + rec = s.steps[p.Name] + rec.deleted, rec.deletedAt, rec.secretsDeleted = true, now, true + s.mu.Unlock() + s.Log.Info("gc: deleted a finished step", "id", p.Name, "state", st.State, "reason", st.Reason) + } + + // A stop, or someone else, deleted the pod only. + var gone []string + s.mu.Lock() + for id, rec := range s.steps { + if !live[id] && !rec.secretsDeleted && !rec.finishing { + gone = append(gone, id) + } + } + s.mu.Unlock() + for _, id := range gone { + if err := s.deleteSecrets(ctx, id); err != nil { + s.Log.Warn("gc: deleting the Secrets of a step without a pod", "id", id, "err", err) + continue + } + s.mu.Lock() + if rec := s.steps[id]; rec != nil { + rec.secretsDeleted = true + } + s.mu.Unlock() + } + + s.mu.Lock() + for id, rec := range s.steps { + since := rec.at + if rec.deleted { + since = rec.deletedAt + } + if !live[id] && !rec.finishing && now.Sub(since) > recordTTL { + delete(s.steps, id) + } + } + s.mu.Unlock() + return nil +} + +// RunGC sweeps every interval until ctx ends. +func (s *Server) RunGC(ctx context.Context, every time.Duration) { + t := time.NewTicker(every) + defer t.Stop() + for { + if err := s.Sweep(ctx); err != nil && ctx.Err() == nil { + s.Log.Warn("gc", "err", err) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} diff --git a/k8s/scaler/internal/steps/gc_test.go b/k8s/scaler/internal/steps/gc_test.go new file mode 100644 index 0000000..70764e3 --- /dev/null +++ b/k8s/scaler/internal/steps/gc_test.go @@ -0,0 +1,160 @@ +package steps + +import ( + "context" + "sort" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// stepPod is a labeled step pod of a runner, created at created. +func stepPod(name, runner string, created time.Time, phase corev1.PodPhase, cs *corev1.ContainerState) *corev1.Pod { + p := pod(phase, cs) + p.Name, p.CreationTimestamp = name, metav1.NewTime(created) + p.Labels = map[string]string{LabelComponent: ComponentStep, LabelRunner: runner} + return p +} + +func stepSecret(name, runner string, created time.Time) *corev1.Secret { + return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "sg", CreationTimestamp: metav1.NewTime(created), + Labels: map[string]string{LabelComponent: ComponentStep, LabelRunner: runner}}} +} + +func short(names []string) string { + out := make([]string, 0, len(names)) + for _, n := range names { + out = append(out, strings.TrimPrefix(n, "sg-runner-0-s-")) + } + sort.Strings(out) + return strings.Join(out, " ") +} + +func (hs *harness) podNames() []string { + var out []string + for _, p := range hs.pods() { + out = append(out, p.Name) + } + return out +} + +func (hs *harness) outcomes() map[string]string { + out := map[string]string{} + for _, p := range hs.pods() { + if o, ok := p.Labels[LabelOutcome]; ok { + out[strings.TrimPrefix(p.Name, "sg-runner-0-s-")] = o + } + } + return out +} + +func TestSweep(t *testing.T) { + now := t0.Add(3 * time.Hour) + ago := func(d time.Duration) time.Time { return now.Add(-d) } + id := func(s string) string { return "sg-runner-0-s-" + s } + var objs []runtime.Object + add := func(p *corev1.Pod, secrets bool) { + objs = append(objs, p) + if secrets { + objs = append(objs, stepSecret(p.Name+"-env", "sg-runner-0", p.CreationTimestamp.Time), + stepSecret(p.Name+"-pull", "sg-runner-0", p.CreationTimestamp.Time)) + } + } + // Ended over FINISHED_TTL (1h) ago: goes, with its Secrets. + add(stepPod(id("old00000"), "sg-runner-0", ago(2*time.Hour), corev1.PodFailed, terminated(3, "Error", ago(2*time.Hour), ago(61*time.Minute))), true) + // Ended within the TTL, or read by a stream: stays, finished. + add(stepPod(id("recent00"), "sg-runner-0", ago(2*time.Hour), corev1.PodSucceeded, terminated(0, "Completed", ago(2*time.Hour), ago(59*time.Minute))), true) + add(stepPod(id("streamed"), "sg-runner-0", ago(2*time.Hour), corev1.PodSucceeded, terminated(0, "Completed", ago(2*time.Hour), ago(2*time.Hour))), true) + // Running: untouched. + add(stepPod(id("running0"), "sg-runner-0", ago(time.Hour), corev1.PodRunning, running(ago(time.Hour))), true) + // Failed in the broker's eyes: its deadline, or (no time to tell) when + // the broker first saw it, counts as its end. + unsched := stepPod(id("unsched0"), "sg-runner-0", ago(2*time.Hour), corev1.PodPending, nil) + unscheduled(unsched) + unsched.CreationTimestamp = metav1.NewTime(ago(2 * time.Hour)) + add(unsched, false) + add(stepPod(id("pullfail"), "sg-runner-0", ago(2*time.Hour), corev1.PodPending, waiting("ImagePullBackOff", "")), true) + // Not ours to touch. + add(stepPod("sg-runner-1-s-other000", "sg-runner-1", ago(2*time.Hour), corev1.PodSucceeded, terminated(0, "Completed", ago(2*time.Hour), ago(2*time.Hour))), false) + add(stepPod("sg-runner-0-s-bad", "sg-runner-0", ago(2*time.Hour), corev1.PodSucceeded, terminated(0, "Completed", ago(2*time.Hour), ago(2*time.Hour))), false) + // Secrets of a stopped step whose pod is gone go; Secrets of a pod that + // vanished without the broker wait for the runner pod. + objs = append(objs, stepSecret(id("gone0000")+"-env", "sg-runner-0", ago(time.Hour)), + stepSecret(id("vanished")+"-env", "sg-runner-0", ago(time.Hour))) + + hs := newHarness(t, objs...) + hs.now = now + hs.s.steps[id("gone0000")] = &stepRecord{at: ago(61 * time.Minute), stopped: true} + hs.s.steps[id("gone0001")] = &stepRecord{at: ago(59 * time.Minute), stopped: true} + exited := State{ID: id("forgot00"), State: StateExited} + hs.s.steps[id("forgot00")] = &stepRecord{at: ago(3 * time.Hour), final: &exited, deleted: true, + deletedAt: ago(61 * time.Minute), secretsDeleted: true} + hs.s.openStream(id("streamed")) + + if err := hs.s.Sweep(context.Background()); err != nil { + t.Fatal(err) + } + // A Secret list or get would return their data, the node token's too. + for _, a := range hs.client.Actions() { + if a.GetResource().Resource == "secrets" && a.GetVerb() != "delete" { + t.Errorf("the GC did %s on secrets", a.GetVerb()) + } + } + if got := short(hs.podNames()); got != "bad pullfail recent00 running0 sg-runner-1-s-other000 streamed" { + t.Errorf("pods left: %s", got) + } + if got := short(hs.secretNames()); got != "running0-env running0-pull vanished-env" { + t.Errorf("secrets left: %s", got) + } + want := map[string]string{"pullfail": OutcomeFailed, "recent00": OutcomeSucceeded, "streamed": OutcomeSucceeded} + if got := hs.outcomes(); len(got) != len(want) || got["pullfail"] != want["pullfail"] || + got["recent00"] != want["recent00"] || got["streamed"] != want["streamed"] { + t.Errorf("outcomes %v", got) + } + var records []string + for k := range hs.s.steps { + records = append(records, k) + } + if got := short(records); got != "gone0001 old00000 pullfail recent00 streamed unsched0" { + t.Errorf("records left: %s", got) + } + // Deleted after FINISHED_TTL, the step still tells how it ended. + rec := hs.do("GET", "/v1/steps/"+id("old00000"), "") + if !strings.Contains(rec.Body.String(), `"state":"exited","exitCode":3`) { + t.Errorf("after the TTL: %d %s", rec.Code, rec.Body) + } + + // An hour on, the stream closed: the pull failure first seen an hour ago + // and the streamed step go; nothing is finished twice. + hs.client.ClearActions() + hs.s.closeStream(id("streamed")) + hs.now = now.Add(61 * time.Minute) + if err := hs.s.Sweep(context.Background()); err != nil { + t.Fatal(err) + } + if got := short(hs.podNames()); got != "bad running0 sg-runner-1-s-other000" { + t.Errorf("pods left an hour on: %s", got) + } + for _, a := range hs.client.Actions() { + if a.GetVerb() == "patch" { + t.Errorf("patched again: %v", a) + } + } +} + +func TestRunGCStopsWithItsContext(t *testing.T) { + hs := newHarness(t) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { hs.s.RunGC(ctx, time.Hour); close(done) }() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("RunGC didn't stop") + } +} diff --git a/k8s/scaler/internal/steps/health.go b/k8s/scaler/internal/steps/health.go new file mode 100644 index 0000000..ee4b005 --- /dev/null +++ b/k8s/scaler/internal/steps/health.go @@ -0,0 +1,57 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "slices" + "time" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/guard" +) + +// Healthcheck checks that both sockets answer: GET /healthz on the API (api +// v1), Plugin.Activate on the guard (authz). +func Healthcheck(ctx context.Context, socketPath, guardSocket string, timeout time.Duration) error { + client := func(sock string) *http.Client { + return &http.Client{Timeout: timeout, Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", sock) + }, + }} + } + get := func(sock, method, url string, out any) error { + req, err := http.NewRequestWithContext(ctx, method, url, nil) + if err != nil { + return err + } + req.Header.Set("Accept", guard.MediaType) + resp, err := client(sock).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s %s: %s", method, url, resp.Status) + } + return json.NewDecoder(resp.Body).Decode(out) + } + + var health struct{ API string } + if err := get(socketPath, http.MethodGet, "http://steps/healthz", &health); err != nil { + return fmt.Errorf("steps socket %s: %w", socketPath, err) + } + if health.API != APIVersion { + return fmt.Errorf("steps socket %s: api %q, want %q", socketPath, health.API, APIVersion) + } + var m guard.Manifest + if err := get(guardSocket, http.MethodPost, "http://guard/Plugin.Activate", &m); err != nil { + return fmt.Errorf("guard socket %s: %w", guardSocket, err) + } + if !slices.Contains(m.Implements, "authz") { + return fmt.Errorf("guard socket %s: implements %v, not authz", guardSocket, m.Implements) + } + return nil +} diff --git a/k8s/scaler/internal/steps/health_test.go b/k8s/scaler/internal/steps/health_test.go new file mode 100644 index 0000000..c3902d8 --- /dev/null +++ b/k8s/scaler/internal/steps/health_test.go @@ -0,0 +1,54 @@ +package steps + +import ( + "context" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/guard" +) + +// serveOn serves h on a unix socket until the test ends. +func serveOn(t *testing.T, sock string, h http.Handler) { + t.Helper() + l, err := Listen(sock) + if err != nil { + t.Fatal(err) + } + srv := &http.Server{Handler: h} + go func() { _ = srv.Serve(l) }() + t.Cleanup(func() { _ = srv.Close() }) +} + +func TestHealthcheck(t *testing.T) { + // macOS keeps unix socket paths under 104 bytes: a short temp dir. + dir, err := os.MkdirTemp("", "hc") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + stepsSock, guardSock := filepath.Join(dir, "steps.sock"), filepath.Join(dir, "guard.sock") + ctx := context.Background() + + if err := Healthcheck(ctx, stepsSock, guardSock, time.Second); err == nil || !strings.Contains(err.Error(), "steps socket") { + t.Errorf("nothing listening: %v", err) + } + serveOn(t, stepsSock, newHarness(t).s.Handler()) + if err := Healthcheck(ctx, stepsSock, guardSock, time.Second); err == nil || !strings.Contains(err.Error(), "guard socket") { + t.Errorf("no guard: %v", err) + } + serveOn(t, guardSock, guard.Handler(slog.New(slog.NewJSONHandler(io.Discard, nil)))) + if err := Healthcheck(ctx, stepsSock, guardSock, time.Second); err != nil { + t.Errorf("both up: %v", err) + } + // The sockets swapped: neither answers as expected. + if err := Healthcheck(ctx, guardSock, stepsSock, time.Second); err == nil { + t.Error("swapped sockets: want an error") + } +} diff --git a/k8s/scaler/internal/steps/imageconfig.go b/k8s/scaler/internal/steps/imageconfig.go new file mode 100644 index 0000000..06ab9e3 --- /dev/null +++ b/k8s/scaler/internal/steps/imageconfig.go @@ -0,0 +1,198 @@ +package steps + +import ( + "context" + "errors" + "fmt" + "net/http" + "runtime" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// ImageConfig is what a step image runs by default. +type ImageConfig struct { + Entrypoint []string + Cmd []string +} + +// ImageConfigs looks up step images' configs. +type ImageConfigs interface { + Lookup(ctx context.Context, image string, auth *RegistryAuth) (ImageConfig, error) +} + +// Registry reads image configs from their registries: the manifest and the +// config blob, never the layers. +type Registry struct { + // Platform picks an index's image: the broker's own, as it runs on the + // step's node. + Platform v1.Platform + Timeout time.Duration + TTL time.Duration + // Transport is for tests (remote's default when nil). + Transport http.RoundTripper + // Now is the clock (time.Now when nil). + Now func() time.Time + + mu sync.Mutex + cache map[string]cachedConfig +} + +type cachedConfig struct { + config ImageConfig + at time.Time +} + +// NewRegistry returns a lookup for this platform, with a 20s timeout and a +// 10 minute cache. +func NewRegistry() *Registry { + return &Registry{ + Platform: v1.Platform{OS: runtime.GOOS, Architecture: runtime.GOARCH}, + Timeout: 20 * time.Second, + TTL: 10 * time.Minute, + } +} + +func (r *Registry) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +// Lookup returns an image's entrypoint and cmd, from the cache when fresh. +// auth is used for the image's registry only, else access is anonymous. +func (r *Registry) Lookup(ctx context.Context, image string, auth *RegistryAuth) (ImageConfig, error) { + r.mu.Lock() + c, ok := r.cache[image] + r.mu.Unlock() + if ok && r.now().Sub(c.at) < r.TTL { + return c.config, nil + } + + ref, err := name.ParseReference(image) + if err != nil { + return ImageConfig{}, err + } + ctx, cancel := context.WithTimeout(ctx, r.Timeout) + defer cancel() + opts := []remote.Option{remote.WithContext(ctx), remote.WithAuth(authFor(ref.Context().Registry, auth))} + if r.Transport != nil { + opts = append(opts, remote.WithTransport(r.Transport)) + } + desc, err := remote.Get(ref, opts...) + if err != nil { + return ImageConfig{}, err + } + var img v1.Image + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + if err != nil { + return ImageConfig{}, err + } + m, err := idx.IndexManifest() + if err != nil { + return ImageConfig{}, err + } + d, err := pickPlatform(m.Manifests, r.Platform) + if err != nil { + return ImageConfig{}, err + } + if img, err = idx.Image(d.Digest); err != nil { + return ImageConfig{}, err + } + } else if img, err = desc.Image(); err != nil { + return ImageConfig{}, err + } + cf, err := img.ConfigFile() + if err != nil { + return ImageConfig{}, err + } + out := ImageConfig{Entrypoint: cf.Config.Entrypoint, Cmd: cf.Config.Cmd} + + r.mu.Lock() + if r.cache == nil { + r.cache = map[string]cachedConfig{} + } + for k, v := range r.cache { + if r.now().Sub(v.at) >= r.TTL { + delete(r.cache, k) + } + } + r.cache[image] = cachedConfig{config: out, at: r.now()} + r.mu.Unlock() + return out, nil +} + +// pickPlatform is an index's image for platform, else linux/amd64, else the +// first linux one (attestations and the like have no os). +func pickPlatform(ms []v1.Descriptor, want v1.Platform) (v1.Descriptor, error) { + match := func(os, arch string) (v1.Descriptor, bool) { + for _, d := range ms { + if d.Platform != nil && d.Platform.OS == os && (arch == "" || d.Platform.Architecture == arch) { + return d, true + } + } + return v1.Descriptor{}, false + } + for _, p := range []v1.Platform{want, {OS: "linux", Architecture: "amd64"}, {OS: "linux"}} { + if d, ok := match(p.OS, p.Architecture); ok { + return d, nil + } + } + return v1.Descriptor{}, errors.New("no linux image in the index") +} + +// authFor is auth when it's for reg, anonymous otherwise. +func authFor(reg name.Registry, auth *RegistryAuth) authn.Authenticator { + if auth == nil || registryHost(auth.Registry) != registryHost(reg.RegistryStr()) { + return authn.Anonymous + } + return authn.FromConfig(authn.AuthConfig{Username: auth.Username, Password: auth.Password}) +} + +// registryHost normalizes a registry as Docker credentials name it (a host, +// or https://index.docker.io/v1/ for Docker Hub). +func registryHost(r string) string { + r = strings.ToLower(strings.TrimSpace(r)) + r = strings.TrimPrefix(strings.TrimPrefix(r, "https://"), "http://") + if i := strings.Index(r, "/"); i >= 0 { + r = r[:i] + } + switch r { + case "docker.io", "registry-1.docker.io", "index.docker.io": + return name.DefaultRegistry + } + return r +} + +// wrapCommand is the step container's command under the wrapper: the image's +// entrypoint, then the request's args (Docker's command=) or the image's +// cmd. Empty when there's nothing to run. +func wrapCommand(cfg ImageConfig, args []string) []string { + argv := append([]string{}, cfg.Entrypoint...) + if len(args) > 0 { + argv = append(argv, args...) + } else { + argv = append(argv, cfg.Cmd...) + } + if len(argv) == 0 { + return nil + } + return append([]string{WrapperPath, "--"}, argv...) +} + +// lookupError shortens a lookup error for an annotation. +func lookupError(err error) string { + msg := fmt.Sprintf("image config lookup: %v", err) + if len(msg) > 512 { + msg = msg[:512] + } + return msg +} diff --git a/k8s/scaler/internal/steps/imageconfig_test.go b/k8s/scaler/internal/steps/imageconfig_test.go new file mode 100644 index 0000000..be6caac --- /dev/null +++ b/k8s/scaler/internal/steps/imageconfig_test.go @@ -0,0 +1,276 @@ +package steps + +import ( + "context" + "io" + "log" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// testRegistry is an in-memory registry that counts requests by path and can +// require basic auth. +type testRegistry struct { + srv *httptest.Server + host string + + mu sync.Mutex + requests []string + user string + password string +} + +func newTestRegistry(t *testing.T) *testRegistry { + r := &testRegistry{} + reg := registry.New(registry.Logger(log.New(io.Discard, "", 0))) + r.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + r.mu.Lock() + r.requests = append(r.requests, req.Method+" "+req.URL.Path) + user, password := r.user, r.password + r.mu.Unlock() + if user != "" { + if u, p, ok := req.BasicAuth(); !ok || u != user || p != password { + w.Header().Set("WWW-Authenticate", `Basic realm="test"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + } + reg.ServeHTTP(w, req) + })) + t.Cleanup(r.srv.Close) + r.host = strings.TrimPrefix(r.srv.URL, "http://") + return r +} + +func (r *testRegistry) seen() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string{}, r.requests...) +} + +// image is a random image with a layer, and entrypoint and cmd. +func image(t *testing.T, entrypoint, cmd []string, platform *v1.Platform) v1.Image { + img, err := random.Image(1024, 2) + if err != nil { + t.Fatal(err) + } + cf, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + cf = cf.DeepCopy() + cf.Config.Entrypoint, cf.Config.Cmd = entrypoint, cmd + if platform != nil { + cf.OS, cf.Architecture = platform.OS, platform.Architecture + } + img, err = mutate.ConfigFile(img, cf) + if err != nil { + t.Fatal(err) + } + return img +} + +func (r *testRegistry) push(t *testing.T, ref string, img v1.Image) { + t.Helper() + tag, err := name.ParseReference(r.host + "/" + ref) + if err != nil { + t.Fatal(err) + } + if err := remote.Write(tag, img); err != nil { + t.Fatal(err) + } +} + +func newLookup(platform v1.Platform) *Registry { + r := NewRegistry() + r.Platform = platform + return r +} + +var amd64, arm64 = v1.Platform{OS: "linux", Architecture: "amd64"}, v1.Platform{OS: "linux", Architecture: "arm64"} + +func TestLookupReadsOnlyManifestAndConfig(t *testing.T) { + reg := newTestRegistry(t) + img := image(t, []string{"/entry"}, []string{"run", "--x"}, nil) + reg.push(t, "team/step:1", img) + layers, _ := img.Layers() + + before := len(reg.seen()) + got, err := newLookup(amd64).Lookup(context.Background(), reg.host+"/team/step:1", nil) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, ImageConfig{Entrypoint: []string{"/entry"}, Cmd: []string{"run", "--x"}}) { + t.Errorf("config %+v", got) + } + cfgDigest, _ := img.ConfigName() + for _, req := range reg.seen()[before:] { + for _, l := range layers { + d, _ := l.Digest() + if strings.Contains(req, d.String()) { + t.Errorf("fetched a layer: %s", req) + } + } + if strings.Contains(req, "/blobs/") && !strings.Contains(req, cfgDigest.String()) { + t.Errorf("fetched another blob: %s", req) + } + } +} + +func TestLookupPicksThePlatform(t *testing.T) { + reg := newTestRegistry(t) + add := func(idx v1.ImageIndex, p v1.Platform, entry string) v1.ImageIndex { + return mutate.AppendManifests(idx, mutate.IndexAddendum{Add: image(t, []string{entry}, nil, &p), + Descriptor: v1.Descriptor{Platform: &p}}) + } + // Attestations come as "unknown/unknown" entries. + unknown := v1.Platform{OS: "unknown", Architecture: "unknown"} + multi := add(add(add(empty.Index, unknown, "/attestation"), amd64, "/amd64"), arm64, "/arm64") + armOnly := add(add(empty.Index, unknown, "/attestation"), arm64, "/arm64") + s390 := v1.Platform{OS: "linux", Architecture: "s390x"} + others := add(add(empty.Index, unknown, "/attestation"), s390, "/s390x") + windows := add(empty.Index, v1.Platform{OS: "windows", Architecture: "amd64"}, "/windows") + for ref, idx := range map[string]v1.ImageIndex{"multi:1": multi, "arm:1": armOnly, "others:1": others, "windows:1": windows} { + tag, _ := name.ParseReference(reg.host + "/" + ref) + if err := remote.WriteIndex(tag, idx); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + platform v1.Platform + ref string + want string + }{ + {arm64, "multi:1", "/arm64"}, + {amd64, "multi:1", "/amd64"}, + {v1.Platform{OS: "linux", Architecture: "riscv64"}, "multi:1", "/amd64"}, + {amd64, "arm:1", "/arm64"}, + {amd64, "others:1", "/s390x"}, + {amd64, "windows:1", ""}, + } { + got, err := newLookup(c.platform).Lookup(context.Background(), reg.host+"/"+c.ref, nil) + if c.want == "" { + if err == nil { + t.Errorf("%s on %s: want an error, got %+v", c.ref, c.platform.Architecture, got) + } + continue + } + if err != nil || len(got.Entrypoint) != 1 || got.Entrypoint[0] != c.want { + t.Errorf("%s on %s: %+v %v, want %s", c.ref, c.platform.Architecture, got, err, c.want) + } + } +} + +func TestLookupAuth(t *testing.T) { + reg := newTestRegistry(t) + reg.push(t, "private:1", image(t, []string{"/entry"}, nil, nil)) + reg.mu.Lock() + reg.user, reg.password = "u", "p" + reg.mu.Unlock() + ref := reg.host + "/private:1" + + if _, err := newLookup(amd64).Lookup(context.Background(), ref, nil); err == nil { + t.Error("anonymous: want an error") + } + // Credentials of another registry stay out of it. + other := &RegistryAuth{Registry: "registry.example.com", Username: "u", Password: "p"} + if _, err := newLookup(amd64).Lookup(context.Background(), ref, other); err == nil { + t.Error("another registry's auth: want an error") + } + for _, registry := range []string{reg.host, "http://" + reg.host + "/v2/"} { + auth := &RegistryAuth{Registry: registry, Username: "u", Password: "p"} + got, err := newLookup(amd64).Lookup(context.Background(), ref, auth) + if err != nil || got.Entrypoint[0] != "/entry" { + t.Errorf("auth for %s: %+v %v", registry, got, err) + } + } +} + +func TestLookupCaches(t *testing.T) { + reg := newTestRegistry(t) + reg.push(t, "step:1", image(t, nil, []string{"sh"}, nil)) + l := newLookup(amd64) + now := t0 + l.Now = func() time.Time { return now } + ref := reg.host + "/step:1" + + lookups := func() int { + n := 0 + for _, r := range reg.seen() { + if strings.Contains(r, "/manifests/") { + n++ + } + } + return n + } + start := lookups() + for _, at := range []time.Duration{0, time.Minute, 9 * time.Minute} { + now = t0.Add(at) + if got, err := l.Lookup(context.Background(), ref, nil); err != nil || got.Cmd[0] != "sh" { + t.Fatalf("lookup at %s: %+v %v", at, got, err) + } + } + if n := lookups() - start; n != 1 { + t.Errorf("%d manifest requests within 10 minutes, want 1", n) + } + now = t0.Add(11 * time.Minute) + if _, err := l.Lookup(context.Background(), ref, nil); err != nil { + t.Fatal(err) + } + if n := lookups() - start; n != 2 { + t.Errorf("%d manifest requests after the TTL, want 2", n) + } + // Failures aren't cached. + if _, err := l.Lookup(context.Background(), reg.host+"/missing:1", nil); err == nil { + t.Error("missing image: want an error") + } + if _, ok := l.cache[reg.host+"/missing:1"]; ok { + t.Error("a failure got cached") + } +} + +func TestLookupTimeout(t *testing.T) { + hang := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-hang })) + defer srv.Close() + defer close(hang) + l := newLookup(amd64) + l.Timeout = 100 * time.Millisecond + start := time.Now() + _, err := l.Lookup(context.Background(), strings.TrimPrefix(srv.URL, "http://")+"/step:1", nil) + if err == nil || time.Since(start) > 5*time.Second { + t.Errorf("err %v after %s", err, time.Since(start)) + } +} + +func TestRegistryHost(t *testing.T) { + for in, want := range map[string]string{ + "https://index.docker.io/v1/": "index.docker.io", + "docker.io": "index.docker.io", + "registry-1.docker.io": "index.docker.io", + "123.dkr.ecr.eu-central-1.amazonaws.com": "123.dkr.ecr.eu-central-1.amazonaws.com", + "https://Registry.Example.com:5000/path": "registry.example.com:5000", + " ghcr.io ": "ghcr.io", + } { + if got := registryHost(in); got != want { + t.Errorf("registryHost(%q) = %q, want %q", in, got, want) + } + } + hub, _ := name.ParseReference("ubuntu:24.04") + if registryHost(hub.Context().RegistryStr()) != "index.docker.io" { + t.Errorf("Docker Hub images: %s", hub.Context().RegistryStr()) + } +} diff --git a/k8s/scaler/internal/steps/podspec.go b/k8s/scaler/internal/steps/podspec.go new file mode 100644 index 0000000..07cc6de --- /dev/null +++ b/k8s/scaler/internal/steps/podspec.go @@ -0,0 +1,481 @@ +package steps + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "path" + "regexp" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" +) + +// Labels on step pods and their Secrets (next to STEP_LABELS, which lose). +const ( + LabelComponent = "app.kubernetes.io/component" + LabelRunner = "sg-runner.stackguardian.io/runner" + LabelRun = "sg-runner.stackguardian.io/run" + ComponentStep = "step" +) + +// Annotations on step pods: the request's raw values. +const ( + AnnotationOrg = "sg-runner.stackguardian.io/org" + AnnotationRun = "sg-runner.stackguardian.io/run" + AnnotationStepIdx = "sg-runner.stackguardian.io/step-idx" + AnnotationName = "sg-runner.stackguardian.io/name" +) + +// ContainerName is the step pod's container. +const ContainerName = "step" + +// The wrapper: the init container installs it from the broker's image into +// an emptyDir the step runs it from. +const ( + WrapperInitContainer = "sg-wrapper-install" + WrapperVolume = "sg-wrapper" + WrapperDir = "/sg-wrapper" + WrapperPath = WrapperDir + "/sg-step-wrapper" + // WrapperSource is the wrapper in the broker's image. + WrapperSource = "/sg-step-wrapper" +) + +// AnnotationWrapped is "true" on a pod whose step runs under the wrapper, +// "false" (with AnnotationNotWrappedReason) on one that doesn't. +const ( + AnnotationWrapped = "sg-runner.stackguardian.io/wrapped" + AnnotationNotWrappedReason = "sg-runner.stackguardian.io/not-wrapped-reason" +) + +// LabelOutcome is how a finished step ended, set once it's terminal. +const LabelOutcome = "sg-runner.stackguardian.io/outcome" + +// Outcomes. +const ( + OutcomeSucceeded = "succeeded" + OutcomeApprovalRequired = "approval-required" + OutcomeCompletedEarly = "completed-early" + OutcomeFailed = "failed" + OutcomeStopped = "stopped" +) + +// MaxEnvBytes caps the env's keys and values together (a Secret holds 1 MiB). +const MaxEnvBytes = 1_000_000 + +// StopSlack is added to a step's own timeout and START_TIMEOUT for the pod's +// activeDeadlineSeconds. +const StopSlack int64 = 300 + +// Mount is a directory of the run's workspace a step sees. +type Mount struct { + Source string `json:"source"` + Target string `json:"target"` + ReadOnly bool `json:"readOnly"` +} + +// RegistryAuth is the credential for pulling a step's image. +type RegistryAuth struct { + Registry string `json:"registry"` + Username string `json:"username"` + Password string `json:"password"` +} + +// Request is POST /v1/steps. +type Request struct { + Name string `json:"name"` + Org string `json:"org"` + Run string `json:"run"` + StepIdx int `json:"stepIdx"` + Image string `json:"image"` + Args []string `json:"args"` + Env map[string]string `json:"env"` + Mounts []Mount `json:"mounts"` + CPUShares int64 `json:"cpuShares"` + MemoryMiB int64 `json:"memoryMiB"` + RegistryAuth *RegistryAuth `json:"registryAuth"` + TimeoutSeconds int64 `json:"timeoutSeconds"` +} + +// Error is an API error: the response's status, reason and message. +type Error struct { + Status int `json:"-"` + Reason string `json:"reason"` + Message string `json:"message"` +} + +func (e *Error) Error() string { return e.Reason + ": " + e.Message } + +// API error reasons. +const ( + ReasonInvalidRequest = "InvalidRequest" + ReasonInvalidMount = "InvalidMount" + ReasonInvalidEnv = "InvalidEnv" + ReasonEnvTooLarge = "EnvTooLarge" + ReasonNotFound = "NotFound" + ReasonKubernetesError = "KubernetesError" +) + +func badRequest(reason, format string, a ...any) *Error { + return &Error{Status: 400, Reason: reason, Message: fmt.Sprintf(format, a...)} +} + +var envKeyRE = regexp.MustCompile(`^[-._a-zA-Z0-9]+$`) + +// Validate checks a request against the contract; the error is an *Error. +func (c *Config) Validate(r *Request) error { + if r.Org == "" || r.Run == "" { + return badRequest(ReasonInvalidRequest, "org and run are required") + } + if hasDotDot(r.Org) || hasDotDot(r.Run) { + return badRequest(ReasonInvalidRequest, "org and run must not contain ..") + } + if r.Image == "" || strings.ContainsAny(r.Image, " \t\r\n") { + return badRequest(ReasonInvalidRequest, "image %q: want an image reference", r.Image) + } + if r.StepIdx < 0 || r.CPUShares < 0 || r.MemoryMiB < 0 || r.TimeoutSeconds < 0 { + return badRequest(ReasonInvalidRequest, "stepIdx, cpuShares, memoryMiB and timeoutSeconds must not be negative") + } + if a := r.RegistryAuth; a != nil && a.Registry == "" { + return badRequest(ReasonInvalidRequest, "registryAuth.registry is required") + } + + size := 0 + for k, v := range r.Env { + // Secret keys are also at most 253 characters. + if !envKeyRE.MatchString(k) || len(validation.IsConfigMapKey(k)) > 0 { + return badRequest(ReasonInvalidEnv, "env key %q: want [-._a-zA-Z0-9]+, at most 253 characters", k) + } + size += len(k) + len(v) + } + if size > MaxEnvBytes { + return badRequest(ReasonEnvTooLarge, "env is %d bytes, over %d", size, MaxEnvBytes) + } + + ws := c.RunWorkspace(r) + if !strings.HasPrefix(ws, c.WorkspacesDir+"/") { + return badRequest(ReasonInvalidRequest, "run workspace %s is outside %s", ws, c.WorkspacesDir) + } + targets := map[string]bool{} + for _, m := range r.Mounts { + src := path.Clean(m.Source) + if !path.IsAbs(m.Source) || hasDotDot(m.Source) || !strings.HasPrefix(src, ws+"/") { + return badRequest(ReasonInvalidMount, "mount source %s is outside the run's workspace %s", m.Source, ws) + } + dst := path.Clean(m.Target) + if !path.IsAbs(m.Target) || dst == "/" { + return badRequest(ReasonInvalidMount, "mount target %q: want an absolute path below /", m.Target) + } + if targets[dst] { + return badRequest(ReasonInvalidMount, "mount target %s is used twice", dst) + } + targets[dst] = true + } + return nil +} + +// RunWorkspace is a request's run directory, as the controllers see it. +func (c *Config) RunWorkspace(r *Request) string { + return path.Join(c.WorkspacesDir, r.Org, r.Run) +} + +// subPath is where a validated mount source is inside the workspace claim. +func (c *Config) subPath(source string) string { + rel := strings.TrimPrefix(path.Clean(source), c.WorkspacesDir+"/") + if c.WorkspaceSubPathPrefix == "" { + return rel + } + return path.Join(c.WorkspaceSubPathPrefix, rel) +} + +const idAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + +// NewID is a step id (and pod name): `-s-<8 random [a-z0-9]>`. +func (c *Config) NewID(random io.Reader) (string, error) { + out := make([]byte, 0, 8) + b := make([]byte, 16) + for len(out) < 8 { + if _, err := io.ReadFull(random, b); err != nil { + return "", err + } + for _, x := range b { + // Rejection sampling: 252 is the largest multiple of 36 below 256. + if x < 252 && len(out) < 8 { + out = append(out, idAlphabet[x%36]) + } + } + } + return c.RunnerPodName + "-s-" + string(out), nil +} + +// idRE matches the ids of this runner's steps. +func (c *Config) idRE() *regexp.Regexp { + return regexp.MustCompile(`^` + regexp.QuoteMeta(c.RunnerPodName) + `-s-[a-z0-9]{8}$`) +} + +// EnvSecretName and PullSecretName are a step's Secrets. +func EnvSecretName(id string) string { return id + "-env" } +func PullSecretName(id string) string { return id + "-pull" } + +// RunLabel is a label value for a run: its sanitized id, cut to fit, and a +// short hash of org and run so cut or sanitized ids stay apart. +func RunLabel(org, run string) string { + sum := sha256.Sum256([]byte(org + run)) + hash := hex.EncodeToString(sum[:4]) + var b strings.Builder + for _, r := range run { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': + b.WriteRune(r) + default: + b.WriteByte('-') + } + } + // Label values start and end alphanumeric. + notAlnum := func(r rune) bool { return r == '-' || r == '_' || r == '.' } + text := strings.TrimLeftFunc(b.String(), notAlnum) + if n := validation.LabelValueMaxLength - len(hash) - 1; len(text) > n { + text = text[:n] + } + text = strings.TrimRightFunc(text, notAlnum) + if text == "" { + return hash + } + return text + "-" + hash +} + +// labels are a step's pod and Secret labels. +func (c *Config) labels(r *Request) map[string]string { + out := map[string]string{} + for k, v := range c.StepLabels { + out[k] = v + } + out[LabelComponent] = ComponentStep + out[LabelRunner] = c.RunnerPodName + out[LabelRun] = RunLabel(r.Org, r.Run) + return out +} + +// meta is a step object's metadata, owned by the runner pod so it goes with +// it. blockOwnerDeletion stays unset: it would need delete rights on pods' +// finalizers. +func (c *Config) meta(name string, r *Request) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, + Namespace: c.Namespace, + Labels: c.labels(r), + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "v1", Kind: "Pod", Name: c.RunnerPodName, UID: c.RunnerPodUID, + }}, + } +} + +// EnvSecret holds a step's env, loaded with envFrom. +func (c *Config) EnvSecret(id string, r *Request) *corev1.Secret { + data := make(map[string][]byte, len(r.Env)) + for k, v := range r.Env { + data[k] = []byte(v) + } + return &corev1.Secret{ + ObjectMeta: c.meta(EnvSecretName(id), r), + Type: corev1.SecretTypeOpaque, + Data: data, + } +} + +// PullSecret is the image pull Secret of a request with registryAuth, nil +// without. +func (c *Config) PullSecret(id string, r *Request) (*corev1.Secret, error) { + a := r.RegistryAuth + if a == nil { + return nil, nil + } + cfg, err := json.Marshal(map[string]any{"auths": map[string]any{ + a.Registry: map[string]string{ + "username": a.Username, + "password": a.Password, + "auth": base64.StdEncoding.EncodeToString([]byte(a.Username + ":" + a.Password)), + }, + }}) + if err != nil { + return nil, err + } + return &corev1.Secret{ + ObjectMeta: c.meta(PullSecretName(id), r), + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: cfg}, + }, nil +} + +// PullPolicy is Always for a latest or absent tag, as Docker pulls those. +func PullPolicy(image string) corev1.PullPolicy { + if strings.Contains(image, "@") { + return corev1.PullIfNotPresent + } + name := image[strings.LastIndex(image, "/")+1:] + i := strings.LastIndex(name, ":") + if i < 0 || name[i+1:] == "latest" { + return corev1.PullAlways + } + return corev1.PullIfNotPresent +} + +// CPURequest is Docker CPU shares as a request: shares/1024 cores, in +// millicores rounded up, at least 1m. +func CPURequest(shares int64) resource.Quantity { + m := (shares*1000 + 1023) / 1024 + if m < 1 { + m = 1 + } + return *resource.NewMilliQuantity(m, resource.DecimalSI) +} + +// Wrap is how a step pod runs its command: Command (wrapCommand) runs it +// under the wrapper; without, Reason says why not and the image's own +// entrypoint runs. +type Wrap struct { + Command []string + Reason string +} + +// escape keeps the kubelet from expanding $(VAR) in a command or args, as +// Docker never does. +func escape(in []string) []string { + var out []string + for _, a := range in { + out = append(out, strings.ReplaceAll(a, "$", "$$")) + } + return out +} + +// Pod is a step's pod. +func (c *Config) Pod(id string, r *Request, withPullSecret bool, wrap Wrap) *corev1.Pod { + meta := c.meta(id, r) + meta.Annotations = map[string]string{ + AnnotationOrg: r.Org, + AnnotationRun: r.Run, + AnnotationStepIdx: strconv.Itoa(r.StepIdx), + AnnotationName: r.Name, + AnnotationWrapped: "true", + } + command, args := escape(wrap.Command), escape(r.Args) + if command != nil { + args = nil + } else { + meta.Annotations[AnnotationWrapped] = "false" + meta.Annotations[AnnotationNotWrappedReason] = wrap.Reason + } + + res := corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: CPURequest(r.CPUShares)}} + if r.MemoryMiB > 0 { + mem := *resource.NewQuantity(r.MemoryMiB*1024*1024, resource.BinarySI) + res.Requests[corev1.ResourceMemory] = mem + res.Limits = corev1.ResourceList{corev1.ResourceMemory: mem} + } + + var mounts []corev1.VolumeMount + var volumes []corev1.Volume + for _, m := range r.Mounts { + mounts = append(mounts, corev1.VolumeMount{ + Name: "workspace", MountPath: path.Clean(m.Target), SubPath: c.subPath(m.Source), ReadOnly: m.ReadOnly, + }) + } + if len(mounts) > 0 { + volumes = []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: c.WorkspaceClaim}}}} + } + + var pullSecrets []corev1.LocalObjectReference + if withPullSecret { + pullSecrets = []corev1.LocalObjectReference{{Name: PullSecretName(id)}} + } + + var initContainers []corev1.Container + if command != nil { + for _, name := range c.BrokerImagePullSecrets { + pullSecrets = append(pullSecrets, corev1.LocalObjectReference{Name: name}) + } + volumes = append(volumes, corev1.Volume{Name: WrapperVolume, VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: ptr.To(resource.MustParse("64Mi"))}}}) + mounts = append(mounts, corev1.VolumeMount{Name: WrapperVolume, MountPath: WrapperDir, ReadOnly: true}) + initContainers = []corev1.Container{{ + Name: WrapperInitContainer, + Image: c.BrokerImage, + ImagePullPolicy: c.BrokerImagePullPolicy, + Command: []string{WrapperSource, "install", WrapperPath}, + VolumeMounts: []corev1.VolumeMount{{Name: WrapperVolume, MountPath: WrapperDir}}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("16Mi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("64Mi")}, + }, + SecurityContext: &corev1.SecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To[int64](65532), + RunAsGroup: ptr.To[int64](65532), + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }} + } + + pullPolicy := c.StepImagePullPolicy + if pullPolicy == "" { + pullPolicy = PullPolicy(r.Image) + } + + var deadline *int64 + if r.TimeoutSeconds > 0 { + deadline = ptr.To(r.TimeoutSeconds + int64(c.StartTimeout.Seconds()) + StopSlack) + } + + return &corev1.Pod{ + ObjectMeta: meta, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + TerminationGracePeriodSeconds: ptr.To[int64](30), + ActiveDeadlineSeconds: deadline, + AutomountServiceAccountToken: ptr.To(false), + EnableServiceLinks: ptr.To(false), + ImagePullSecrets: pullSecrets, + NodeSelector: c.StepNodeSelector, + Tolerations: c.StepTolerations, + // The workspace claim is the runner's, attached to its node. + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{ + Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{c.NodeName}, + }}}}, + }, + }}, + SecurityContext: &corev1.PodSecurityContext{ + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + InitContainers: initContainers, + Containers: []corev1.Container{{ + Name: ContainerName, + Image: r.Image, + ImagePullPolicy: pullPolicy, + Command: command, + Args: args, + // The wrapper leaves mapped exit codes here. + TerminationMessagePath: corev1.TerminationMessagePathDefault, + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + EnvFrom: []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: EnvSecretName(id)}}}}, + Resources: res, + VolumeMounts: mounts, + SecurityContext: &corev1.SecurityContext{Privileged: ptr.To(false)}, + }}, + Volumes: volumes, + }, + } +} diff --git a/k8s/scaler/internal/steps/podspec_test.go b/k8s/scaler/internal/steps/podspec_test.go new file mode 100644 index 0000000..435251b --- /dev/null +++ b/k8s/scaler/internal/steps/podspec_test.go @@ -0,0 +1,392 @@ +package steps + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + testOrg = "/orgs/example-org" + testRun = "/wfgrps/x/wfs/y/wfruns/z" + testWS = "/var/tmp/workspace/orgs/example-org/wfgrps/x/wfs/y/wfruns/z" + testID = "sg-runner-0-s-7f3k9q2x" +) + +func testConfig() Config { + return Config{ + SocketPath: "/run/sg-steps/steps.sock", WorkspacesDir: "/var/tmp/workspace", + WorkspaceClaim: "sg-runner-0-workspace", RunnerPodName: "sg-runner-0", RunnerPodUID: "uid-0", + NodeName: "node-a", Namespace: "sg", + StepLabels: map[string]string{"app.kubernetes.io/name": "sg-runner", LabelComponent: "overridden"}, + StepNodeSelector: map[string]string{"pool": "steps"}, + StepTolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpExists}}, + SchedulingTimeout: 5 * time.Minute, StartTimeout: 30 * time.Minute, FinishedTTL: time.Hour, + BrokerImage: "reg.example.com/sg-runner-scaler:1", BrokerImagePullPolicy: corev1.PullIfNotPresent, + BrokerImagePullSecrets: []string{"regcred"}, + } +} + +func testRequest() Request { + return Request{ + Name: "example-org-plan-0", Org: testOrg, Run: testRun, StepIdx: 2, + Image: "registry.example.com/team/image:1.2", + Args: []string{"sh", "-c", "echo $(HOME) $HOME"}, + Env: map[string]string{"SG_WORKFLOW_RUN_PAYLOAD": "{}", "a.b-c_D": "x"}, + Mounts: []Mount{ + {Source: testWS + "/shared-workspace", Target: "/mnt/sg_workspace"}, + {Source: testWS + "/artifacts/", Target: "/mnt/artifacts/", ReadOnly: true}, + }, + CPUShares: 512, MemoryMiB: 1024, TimeoutSeconds: 2100, + } +} + +func TestValidate(t *testing.T) { + cfg := testConfig() + cases := map[string]struct { + edit func(*Request) + reason string + }{ + "valid": {func(r *Request) {}, ""}, + "no mounts or env": {func(r *Request) { r.Mounts, r.Env, r.Args = nil, nil, nil }, ""}, + "no org": {func(r *Request) { r.Org = "" }, ReasonInvalidRequest}, + "no run": {func(r *Request) { r.Run = "" }, ReasonInvalidRequest}, + "dotdot run": {func(r *Request) { r.Run = "/../../../etc" }, ReasonInvalidRequest}, + "no image": {func(r *Request) { r.Image = "" }, ReasonInvalidRequest}, + "image with space": {func(r *Request) { r.Image = "a b" }, ReasonInvalidRequest}, + "negative cpu": {func(r *Request) { r.CPUShares = -1 }, ReasonInvalidRequest}, + "negative timeout": {func(r *Request) { r.TimeoutSeconds = -1 }, ReasonInvalidRequest}, + "auth without registry": {func(r *Request) { r.RegistryAuth = &RegistryAuth{Username: "u"} }, ReasonInvalidRequest}, + "bad env key": {func(r *Request) { r.Env["A=B"] = "" }, ReasonInvalidEnv}, + "empty env key": {func(r *Request) { r.Env[""] = "" }, ReasonInvalidEnv}, + "long env key": {func(r *Request) { r.Env[strings.Repeat("a", 254)] = "" }, ReasonInvalidEnv}, + "env too large": {func(r *Request) { + r.Env = map[string]string{"A": strings.Repeat("x", MaxEnvBytes)} + }, ReasonEnvTooLarge}, + "env at the limit": {func(r *Request) { + r.Env = map[string]string{"A": strings.Repeat("x", MaxEnvBytes-1)} + }, ""}, + "mount the run dir": {func(r *Request) { r.Mounts[0].Source = testWS }, ReasonInvalidMount}, + "mount the run dir slash": {func(r *Request) { r.Mounts[0].Source = testWS + "/" }, ReasonInvalidMount}, + "mount the root": {func(r *Request) { r.Mounts[0].Source = "/var/tmp/workspace" }, ReasonInvalidMount}, + "mount the socket dir": {func(r *Request) { r.Mounts[0].Source = "/var/tmp/workspace/.sg-runner" }, ReasonInvalidMount}, + "mount another run": {func(r *Request) { + r.Mounts[0].Source = "/var/tmp/workspace/orgs/example-org/wfgrps/x/wfs/y/wfruns/zz/shared" + }, ReasonInvalidMount}, + "mount with dotdot": {func(r *Request) { r.Mounts[0].Source = testWS + "/a/../../z/a" }, ReasonInvalidMount}, + "mount relative": {func(r *Request) { r.Mounts[0].Source = "orgs/example-org" }, ReasonInvalidMount}, + "mount elsewhere": {func(r *Request) { r.Mounts[0].Source = "/etc" }, ReasonInvalidMount}, + "target relative": {func(r *Request) { r.Mounts[0].Target = "mnt" }, ReasonInvalidMount}, + "target root": {func(r *Request) { r.Mounts[0].Target = "/" }, ReasonInvalidMount}, + "target twice": {func(r *Request) { r.Mounts[1].Target = "/mnt/sg_workspace/" }, ReasonInvalidMount}, + } + for name, c := range cases { + r := testRequest() + c.edit(&r) + err := cfg.Validate(&r) + if c.reason == "" { + if err != nil { + t.Errorf("%s: %v", name, err) + } + continue + } + var e *Error + if !errors.As(err, &e) || e.Reason != c.reason || e.Status != 400 { + t.Errorf("%s: got %v, want %s", name, err, c.reason) + } + } +} + +func TestNewID(t *testing.T) { + cfg := testConfig() + // 252 and up are skipped; the rest map onto a-z0-9. + random := bytes.NewReader(append([]byte{255, 0, 25, 26, 35, 36, 252, 71, 1, 2}, make([]byte, 22)...)) + id, err := cfg.NewID(random) + if err != nil { + t.Fatal(err) + } + if id != "sg-runner-0-s-az09a9bc" { + t.Errorf("NewID = %q", id) + } + if !cfg.idRE().MatchString(id) { + t.Errorf("%q doesn't match the id pattern", id) + } + if _, err := cfg.NewID(bytes.NewReader([]byte{1, 2})); err == nil { + t.Error("want an error from a short random source") + } +} + +func TestIDPattern(t *testing.T) { + cfg := testConfig() + re := cfg.idRE() + for id, want := range map[string]bool{ + testID: true, + "sg-runner-0-s-7f3k9q2": false, + "sg-runner-0-s-7F3K9Q2X": false, + "sg-runner-1-s-7f3k9q2x": false, + "sg-runner-0": false, + "x-sg-runner-0-s-7f3k9q2x": false, + "sg-runner-0-s-7f3k9q2x-env": false, + } { + if re.MatchString(id) != want { + t.Errorf("%q: match = %v, want %v", id, !want, want) + } + } +} + +func TestRunLabel(t *testing.T) { + short := RunLabel(testOrg, testRun) + if !strings.HasPrefix(short, "wfgrps-x-wfs-y-wfruns-z-") || len(short) != len("wfgrps-x-wfs-y-wfruns-z-")+8 { + t.Errorf("RunLabel = %q", short) + } + if RunLabel("/orgs/other", testRun) == short { + t.Error("the same run of two orgs got the same label") + } + long := RunLabel(testOrg, "/wfgrps/"+strings.Repeat("g", 80)+"/wfs/y/wfruns/z") + for _, v := range []string{short, long, RunLabel(testOrg, "/"), RunLabel(testOrg, "/a/"+strings.Repeat("-", 60))} { + if msgs := validation.IsValidLabelValue(v); len(msgs) > 0 || v == "" { + t.Errorf("RunLabel %q: %v", v, msgs) + } + } + if len(long) != 63 { + t.Errorf("long RunLabel is %d characters: %q", len(long), long) + } +} + +func TestPullPolicy(t *testing.T) { + for image, want := range map[string]corev1.PullPolicy{ + "ubuntu": corev1.PullAlways, + "ubuntu:latest": corev1.PullAlways, + "ubuntu:24.04": corev1.PullIfNotPresent, + "localhost:5000/team/img": corev1.PullAlways, + "localhost:5000/team/img:1": corev1.PullIfNotPresent, + "localhost:5000/team/img:latest": corev1.PullAlways, + "123.dkr.ecr.eu-central-1.amazonaws.com/x:v2": corev1.PullIfNotPresent, + "ubuntu@sha256:abcd": corev1.PullIfNotPresent, + "ubuntu:latest@sha256:abcd": corev1.PullIfNotPresent, + } { + if got := PullPolicy(image); got != want { + t.Errorf("PullPolicy(%q) = %s, want %s", image, got, want) + } + } +} + +func TestCPURequest(t *testing.T) { + for shares, want := range map[int64]string{0: "1m", 1: "1m", 2: "2m", 512: "500m", 1024: "1", 1536: "1500m", 100: "98m", 4096: "4"} { + q := CPURequest(shares) + if got := q.String(); got != want { + t.Errorf("CPURequest(%d) = %s, want %s", shares, got, want) + } + } +} + +func TestSecrets(t *testing.T) { + cfg := testConfig() + r := testRequest() + env := cfg.EnvSecret(testID, &r) + if env.Name != testID+"-env" || env.Namespace != "sg" || env.Type != corev1.SecretTypeOpaque || + string(env.Data["SG_WORKFLOW_RUN_PAYLOAD"]) != "{}" || string(env.Data["a.b-c_D"]) != "x" || len(env.Data) != 2 { + t.Errorf("env Secret %+v", env) + } + checkOwnedAndLabeled(t, env.ObjectMeta) + + if s, err := cfg.PullSecret(testID, &r); s != nil || err != nil { + t.Errorf("PullSecret without registryAuth = %v, %v", s, err) + } + r.RegistryAuth = &RegistryAuth{Registry: "https://index.docker.io/v1/", Username: "u", Password: "p:w"} + pull, err := cfg.PullSecret(testID, &r) + if err != nil { + t.Fatal(err) + } + if pull.Name != testID+"-pull" || pull.Type != corev1.SecretTypeDockerConfigJson { + t.Errorf("pull Secret %+v", pull) + } + checkOwnedAndLabeled(t, pull.ObjectMeta) + var dc struct { + Auths map[string]struct{ Username, Password, Auth string } + } + if err := json.Unmarshal(pull.Data[corev1.DockerConfigJsonKey], &dc); err != nil { + t.Fatal(err) + } + a := dc.Auths["https://index.docker.io/v1/"] + if a.Username != "u" || a.Password != "p:w" || a.Auth != base64.StdEncoding.EncodeToString([]byte("u:p:w")) || len(dc.Auths) != 1 { + t.Errorf("dockerconfigjson %s", pull.Data[corev1.DockerConfigJsonKey]) + } +} + +func checkOwnedAndLabeled(t *testing.T, m metav1.ObjectMeta) { + t.Helper() + want := []metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: "sg-runner-0", UID: "uid-0"}} + if !reflect.DeepEqual(m.OwnerReferences, want) { + t.Errorf("%s: ownerReferences %+v", m.Name, m.OwnerReferences) + } + if m.Labels[LabelComponent] != ComponentStep || m.Labels[LabelRunner] != "sg-runner-0" || + m.Labels[LabelRun] != RunLabel(testOrg, testRun) || m.Labels["app.kubernetes.io/name"] != "sg-runner" { + t.Errorf("%s: labels %v", m.Name, m.Labels) + } +} + +func TestPod(t *testing.T) { + cfg := testConfig() + cfg.WorkspaceSubPathPrefix = "shared/r0" + r := testRequest() + p := cfg.Pod(testID, &r, true, Wrap{Reason: "test"}) + + if p.Name != testID || p.Namespace != "sg" { + t.Errorf("name %s/%s", p.Namespace, p.Name) + } + checkOwnedAndLabeled(t, p.ObjectMeta) + if want := map[string]string{AnnotationOrg: testOrg, AnnotationRun: testRun, AnnotationStepIdx: "2", + AnnotationName: "example-org-plan-0", AnnotationWrapped: "false", AnnotationNotWrappedReason: "test"}; !reflect.DeepEqual(p.Annotations, want) { + t.Errorf("annotations %v", p.Annotations) + } + + s := p.Spec + if s.RestartPolicy != corev1.RestartPolicyNever || *s.TerminationGracePeriodSeconds != 30 || + *s.ActiveDeadlineSeconds != 2100+1800+300 || *s.AutomountServiceAccountToken || *s.EnableServiceLinks || + s.SecurityContext.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault || + !reflect.DeepEqual(s.NodeSelector, cfg.StepNodeSelector) || !reflect.DeepEqual(s.Tolerations, cfg.StepTolerations) || + !reflect.DeepEqual(s.ImagePullSecrets, []corev1.LocalObjectReference{{Name: testID + "-pull"}}) { + t.Errorf("pod spec %+v", s) + } + terms := s.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + if want := []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{ + Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{"node-a"}}}}}; !reflect.DeepEqual(terms, want) { + t.Errorf("node affinity %+v", terms) + } + if want := []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "sg-runner-0-workspace"}}}}; !reflect.DeepEqual(s.Volumes, want) { + t.Errorf("volumes %+v", s.Volumes) + } + + if len(s.Containers) != 1 { + t.Fatalf("%d containers", len(s.Containers)) + } + c := s.Containers[0] + if c.Name != "step" || c.Image != r.Image || c.ImagePullPolicy != corev1.PullIfNotPresent || c.Command != nil || + s.InitContainers != nil || c.TerminationMessagePath != "/dev/termination-log" || + !reflect.DeepEqual(c.Args, []string{"sh", "-c", "echo $$(HOME) $$HOME"}) || + *c.SecurityContext.Privileged || c.SecurityContext.Capabilities != nil { + t.Errorf("container %+v", c) + } + if want := []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: testID + "-env"}}}}; !reflect.DeepEqual(c.EnvFrom, want) { + t.Errorf("envFrom %+v", c.EnvFrom) + } + if req := c.Resources.Requests; req.Cpu().String() != "500m" || req.Memory().String() != "1Gi" || + c.Resources.Limits.Memory().String() != "1Gi" || c.Resources.Limits.Cpu().String() != "0" { + t.Errorf("resources %+v", c.Resources) + } + wantMounts := []corev1.VolumeMount{ + {Name: "workspace", MountPath: "/mnt/sg_workspace", SubPath: "shared/r0/orgs/example-org/wfgrps/x/wfs/y/wfruns/z/shared-workspace"}, + {Name: "workspace", MountPath: "/mnt/artifacts", SubPath: "shared/r0/orgs/example-org/wfgrps/x/wfs/y/wfruns/z/artifacts", ReadOnly: true}, + } + if !reflect.DeepEqual(c.VolumeMounts, wantMounts) { + t.Errorf("volume mounts %+v", c.VolumeMounts) + } +} + +func TestPodMinimal(t *testing.T) { + cfg := testConfig() + r := Request{Org: testOrg, Run: testRun, Image: "alpine", + Mounts: []Mount{{Source: testWS + "/w", Target: "/w"}}} + p := cfg.Pod(testID, &r, false, Wrap{Reason: "test"}) + c := p.Spec.Containers[0] + if p.Spec.ActiveDeadlineSeconds != nil || p.Spec.ImagePullSecrets != nil || c.Args != nil || + c.ImagePullPolicy != corev1.PullAlways || c.Resources.Requests.Cpu().String() != "1m" || + c.Resources.Limits != nil || c.Resources.Requests.Memory().Value() != 0 { + t.Errorf("minimal pod %+v", p.Spec) + } + if c.VolumeMounts[0].SubPath != "orgs/example-org/wfgrps/x/wfs/y/wfruns/z/w" { + t.Errorf("subPath without a prefix: %s", c.VolumeMounts[0].SubPath) + } + r.Mounts = nil + if p := cfg.Pod(testID, &r, false, Wrap{Reason: "test"}); p.Spec.Volumes != nil || p.Spec.Containers[0].VolumeMounts != nil { + t.Errorf("a step without mounts got the workspace: %+v", p.Spec.Volumes) + } +} + +func TestPodPullPolicyOverride(t *testing.T) { + cfg := testConfig() + for _, image := range []string{"alpine", "alpine:3.20"} { + r := Request{Org: testOrg, Run: testRun, Image: image} + for _, p := range []corev1.PullPolicy{corev1.PullNever, corev1.PullAlways, corev1.PullIfNotPresent} { + cfg.StepImagePullPolicy = p + if got := cfg.Pod(testID, &r, false, Wrap{Reason: "test"}).Spec.Containers[0].ImagePullPolicy; got != p { + t.Errorf("%s with %s: %s", image, p, got) + } + } + } +} + +func TestWrapCommand(t *testing.T) { + cfg := ImageConfig{Entrypoint: []string{"/entry", "--flag"}, Cmd: []string{"default"}} + for name, c := range map[string]struct { + cfg ImageConfig + args []string + want []string + }{ + "entrypoint and cmd": {cfg, nil, []string{WrapperPath, "--", "/entry", "--flag", "default"}}, + "args replace the cmd": {cfg, []string{"plan", "-x"}, []string{WrapperPath, "--", "/entry", "--flag", "plan", "-x"}}, + "empty args keep cmd": {cfg, []string{}, []string{WrapperPath, "--", "/entry", "--flag", "default"}}, + "cmd only": {ImageConfig{Cmd: []string{"sh"}}, nil, []string{WrapperPath, "--", "sh"}}, + "args only": {ImageConfig{}, []string{"run"}, []string{WrapperPath, "--", "run"}}, + "nothing": {ImageConfig{}, nil, nil}, + } { + if got := wrapCommand(c.cfg, c.args); !reflect.DeepEqual(got, c.want) { + t.Errorf("%s: %q, want %q", name, got, c.want) + } + } +} + +func TestWrappedPod(t *testing.T) { + cfg := testConfig() + r := testRequest() + r.RegistryAuth = &RegistryAuth{Registry: "registry.example.com"} + cmd := wrapCommand(ImageConfig{Entrypoint: []string{"/entry"}}, r.Args) + p := cfg.Pod(testID, &r, true, Wrap{Command: cmd}) + + if p.Annotations[AnnotationWrapped] != "true" || p.Annotations[AnnotationNotWrappedReason] != "" { + t.Errorf("annotations %v", p.Annotations) + } + c := p.Spec.Containers[0] + want := []string{WrapperPath, "--", "/entry", "sh", "-c", "echo $$(HOME) $$HOME"} + if !reflect.DeepEqual(c.Command, want) || c.Args != nil { + t.Errorf("command %q args %q", c.Command, c.Args) + } + if last := c.VolumeMounts[len(c.VolumeMounts)-1]; last != (corev1.VolumeMount{Name: "sg-wrapper", MountPath: "/sg-wrapper", ReadOnly: true}) { + t.Errorf("wrapper mount %+v", last) + } + if len(p.Spec.InitContainers) != 1 { + t.Fatalf("init containers %+v", p.Spec.InitContainers) + } + ic := p.Spec.InitContainers[0] + sc := ic.SecurityContext + if ic.Name != "sg-wrapper-install" || ic.Image != cfg.BrokerImage || ic.ImagePullPolicy != corev1.PullIfNotPresent || + !reflect.DeepEqual(ic.Command, []string{"/sg-step-wrapper", "install", "/sg-wrapper/sg-step-wrapper"}) || + !reflect.DeepEqual(ic.VolumeMounts, []corev1.VolumeMount{{Name: "sg-wrapper", MountPath: "/sg-wrapper"}}) || + ic.Resources.Requests.Cpu().String() != "10m" || ic.Resources.Requests.Memory().String() != "16Mi" || + !*sc.RunAsNonRoot || *sc.RunAsUser != 65532 || *sc.AllowPrivilegeEscalation || !*sc.ReadOnlyRootFilesystem || + !reflect.DeepEqual(sc.Capabilities.Drop, []corev1.Capability{"ALL"}) { + t.Errorf("init container %+v", ic) + } + if want := []corev1.LocalObjectReference{{Name: testID + "-pull"}, {Name: "regcred"}}; !reflect.DeepEqual(p.Spec.ImagePullSecrets, want) { + t.Errorf("pull secrets %+v", p.Spec.ImagePullSecrets) + } + var names []string + for _, v := range p.Spec.Volumes { + names = append(names, v.Name) + } + if strings.Join(names, ",") != "workspace,sg-wrapper" || p.Spec.Volumes[1].EmptyDir == nil { + t.Errorf("volumes %+v", p.Spec.Volumes) + } +} diff --git a/k8s/scaler/internal/steps/server.go b/k8s/scaler/internal/steps/server.go new file mode 100644 index 0000000..2ab6e62 --- /dev/null +++ b/k8s/scaler/internal/steps/server.go @@ -0,0 +1,623 @@ +package steps + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" +) + +// APIVersion is what GET /healthz reports. +const APIVersion = "v1" + +// maxBody caps a request body: the env's 1 MB, JSON-escaped, with room. +const maxBody = 8 << 20 + +// recordTTL is how long the broker remembers a step after its pod is gone. +const recordTTL = time.Hour + +// LogOpener streams a pod's logs. +type LogOpener func(ctx context.Context, namespace, pod string, opts *corev1.PodLogOptions) (io.ReadCloser, error) + +// Server is the broker's API on top of the cluster. +type Server struct { + Config Config + Client kubernetes.Interface + // Logs streams step logs (Client's when nil). Give it a client without + // a request timeout: a followed stream lasts as long as the step. + Logs LogOpener + Log *slog.Logger + // Random makes step ids (crypto/rand when nil). + Random io.Reader + // Now is the clock (time.Now when nil). + Now func() time.Time + // StopPollInterval is how often the broker reads a stopped step's pod + // until it's gone, for its exit code (250ms when 0). + StopPollInterval time.Duration + // PollInterval is how often a log request checks whether the container + // started (1s when 0). + PollInterval time.Duration + // Images looks up step images' entrypoint and cmd for the wrapper (steps + // aren't wrapped when nil). + Images ImageConfigs + + idRE *regexp.Regexp + + mu sync.Mutex + steps map[string]*stepRecord + streams map[string]int +} + +// stepRecord is what the broker remembers of a step besides its pod: a stop +// it was asked for, its final state, and what's left to clean up. +type stepRecord struct { + at time.Time + // stopped: a client asked for the stop; last is the latest state seen + // since. + stopped bool + last *State + // following: a goroutine reads the stopped pod until it's gone. + following bool + // final is the terminal state, once the broker saw it: the step's state + // for good, also once its pod is gone. terminalAt is when it saw it. + final *State + terminalAt time.Time + // finishing: one goroutine labels the pod with its outcome and deletes + // its Secrets; finished: done. + finishing bool + finished bool + // deleted: the pod and Secrets are gone (FINISHED_TTL, DELETE). + deleted bool + deletedAt time.Time + // secretsDeleted: the step's Secrets are gone. + secretsDeleted bool +} + +// recordLocked is a step's record, made when missing; s.mu is held. +func (s *Server) recordLocked(id string) *stepRecord { + rec := s.steps[id] + if rec == nil { + rec = &stepRecord{at: s.now()} + s.steps[id] = rec + } + return rec +} + +// see keeps a stopped step's latest state, but never trades an exit seen for +// a state without one. +func (rec *stepRecord) see(st State) { + if rec.last == nil || rec.last.State != StateExited || st.State == StateExited { + rec.last = &st + } +} + +// NewServer returns a broker for a configuration. +func NewServer(cfg Config, client kubernetes.Interface, log *slog.Logger) *Server { + return &Server{ + Config: cfg, + Client: client, + Log: log, + idRE: cfg.idRE(), + steps: map[string]*stepRecord{}, + streams: map[string]int{}, + } +} + +func (s *Server) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +// Handler is the HTTP API. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"api": APIVersion}) + }) + mux.HandleFunc("POST /v1/steps", s.create) + mux.HandleFunc("GET /v1/steps/{id}", s.get) + mux.HandleFunc("GET /v1/steps/{id}/logs", s.logs) + mux.HandleFunc("POST /v1/steps/{id}/stop", s.stop) + mux.HandleFunc("DELETE /v1/steps/{id}", s.delete) + // Every error is JSON, unknown routes too. + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + writeError(w, &Error{Status: http.StatusNotFound, Reason: ReasonNotFound, Message: "no route " + r.Method + " " + r.URL.Path}) + }) + return mux +} + +// Listen listens on the unix socket path, replacing a stale socket file, and +// lets anyone who reaches the file connect. +func Listen(socketPath string) (net.Listener, error) { + if err := os.MkdirAll(filepath.Dir(socketPath), 0o755); err != nil { + return nil, err + } + if err := os.Remove(socketPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + l, err := net.Listen("unix", socketPath) + if err != nil { + return nil, err + } + if err := os.Chmod(socketPath, 0o666); err != nil { + _ = l.Close() + return nil, err + } + return l, nil +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +// writeError writes an *Error, anything else as a KubernetesError. +func writeError(w http.ResponseWriter, err error) { + var e *Error + if !errors.As(err, &e) { + e = &Error{Status: http.StatusInternalServerError, Reason: ReasonKubernetesError, Message: err.Error()} + } + writeJSON(w, e.Status, e) +} + +func kubeError(what string, err error) *Error { + return &Error{Status: http.StatusInternalServerError, Reason: ReasonKubernetesError, Message: what + ": " + err.Error()} +} + +func notFound(id string) *Error { + return &Error{Status: http.StatusNotFound, Reason: ReasonNotFound, Message: "no step " + id} +} + +// stepID is the request's id when it names one of this runner's steps: the +// API never touches other pods. +func (s *Server) stepID(r *http.Request) (string, error) { + id := r.PathValue("id") + if !s.idRE.MatchString(id) { + return "", notFound(id) + } + return id, nil +} + +func ignoreNotFound(err error) error { + if apierrors.IsNotFound(err) { + return nil + } + return err +} + +func (s *Server) create(w http.ResponseWriter, r *http.Request) { + var req Request + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBody)).Decode(&req); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeError(w, badRequest(ReasonEnvTooLarge, "request body over %d bytes", maxBody)) + return + } + writeError(w, badRequest(ReasonInvalidRequest, "body: %v", err)) + return + } + if err := s.Config.Validate(&req); err != nil { + writeError(w, err) + return + } + random := s.Random + if random == nil { + random = rand.Reader + } + id, err := s.Config.NewID(random) + if err != nil { + writeError(w, fmt.Errorf("step id: %w", err)) + return + } + pull, err := s.Config.PullSecret(id, &req) + if err != nil { + writeError(w, badRequest(ReasonInvalidRequest, "registryAuth: %v", err)) + return + } + + ctx := r.Context() + wrap := s.wrap(ctx, &req) + secrets := s.Client.CoreV1().Secrets(s.Config.Namespace) + var created []string + // Secrets of a step that has no pod (else they go with the runner pod). + cleanup := func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + for _, name := range created { + if err := ignoreNotFound(secrets.Delete(ctx, name, metav1.DeleteOptions{})); err != nil { + s.Log.Warn("deleting a Secret of a step without a pod", "secret", name, "err", err) + } + } + } + if _, err := secrets.Create(ctx, s.Config.EnvSecret(id, &req), metav1.CreateOptions{}); err != nil { + writeError(w, kubeError("create Secret "+EnvSecretName(id), err)) + return + } + created = append(created, EnvSecretName(id)) + if pull != nil { + if _, err := secrets.Create(ctx, pull, metav1.CreateOptions{}); err != nil { + cleanup() + writeError(w, kubeError("create Secret "+PullSecretName(id), err)) + return + } + created = append(created, PullSecretName(id)) + } + pod := s.Config.Pod(id, &req, pull != nil, wrap) + if _, err := s.Client.CoreV1().Pods(s.Config.Namespace).Create(ctx, pod, metav1.CreateOptions{}); err != nil { + cleanup() + writeError(w, kubeError("create Pod "+id, err)) + return + } + s.Log.Info("step created", "id", id, "org", req.Org, "run", req.Run, "stepIdx", req.StepIdx, + "name", req.Name, "image", req.Image, "wrapped", wrap.Command != nil) + if wrap.Command == nil { + s.Log.Warn("step not wrapped: its exits 11 and 12 end the pod Failed", "id", id, "image", req.Image, "reason", wrap.Reason) + } + writeJSON(w, http.StatusCreated, map[string]string{"id": id}) +} + +// state is a step's state: its final one once the broker saw that, a +// stopped step's last once its pod is gone. Seeing it terminal the first time +// starts finishing it (outcome label, Secrets). +func (s *Server) state(ctx context.Context, id string) (State, error) { + if st, ok := s.final(id); ok { + return st, nil + } + p, err := s.Client.CoreV1().Pods(s.Config.Namespace).Get(ctx, id, metav1.GetOptions{}) + s.mu.Lock() + rec := s.steps[id] + if rec != nil && rec.final != nil { + st := *rec.final + s.mu.Unlock() + return st, nil + } + stopped := rec != nil && rec.stopped + var st State + switch { + case apierrors.IsNotFound(err) && stopped: + st, p = stoppedState(id, rec.last), nil + case apierrors.IsNotFound(err): + s.mu.Unlock() + return State{}, notFound(id) + case err != nil: + s.mu.Unlock() + return State{}, kubeError("get Pod "+id, err) + default: + st = s.Config.StepState(p, s.now(), stopped) + if stopped { + rec.see(st) + } + } + finish := st.Terminal() && s.terminalLocked(id, st) + s.mu.Unlock() + if finish { + go s.finishAsync(id, p) + } + return st, nil +} + +// final is a step's final state, if it has one. +func (s *Server) final(id string) (State, bool) { + s.mu.Lock() + defer s.mu.Unlock() + if rec := s.steps[id]; rec != nil && rec.final != nil { + return *rec.final, true + } + return State{}, false +} + +func (s *Server) get(w http.ResponseWriter, r *http.Request) { + id, err := s.stepID(r) + if err != nil { + writeError(w, err) + return + } + st, err := s.state(r.Context(), id) + if err != nil { + writeError(w, err) + return + } + writeJSON(w, http.StatusOK, st) +} + +func (s *Server) stop(w http.ResponseWriter, r *http.Request) { + id, err := s.stepID(r) + if err != nil { + writeError(w, err) + return + } + var body struct { + GraceSeconds *int64 `json:"graceSeconds"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + writeError(w, badRequest(ReasonInvalidRequest, "body: %v", err)) + return + } + grace := int64(30) + if body.GraceSeconds != nil { + grace = *body.GraceSeconds + } + if grace < 0 { + writeError(w, badRequest(ReasonInvalidRequest, "graceSeconds must not be negative")) + return + } + // 0 would remove the pod object before the kubelet killed the container; + // 1 still kills right away and keeps the pod (and the exit code) until then. + if grace == 0 { + grace = 1 + } + + ctx := r.Context() + st, err := s.state(ctx, id) + if err != nil { + writeError(w, err) + return + } + s.mu.Lock() + rec := s.recordLocked(id) + if !rec.stopped { + rec.stopped, rec.last = true, &st + } + follow := !rec.following + rec.following = true + s.mu.Unlock() + err = s.Client.CoreV1().Pods(s.Config.Namespace).Delete(ctx, id, metav1.DeleteOptions{GracePeriodSeconds: ptr.To(grace)}) + if err := ignoreNotFound(err); err != nil { + s.mu.Lock() + rec.following = false + s.mu.Unlock() + writeError(w, kubeError("delete Pod "+id, err)) + return + } + if follow { + go s.followStop(id, time.Duration(grace)*time.Second+30*time.Second) + } + s.Log.Info("step stopped", "id", id, "graceSeconds", grace, "state", st.State) + writeJSON(w, http.StatusAccepted, map[string]string{"id": id}) +} + +// followStop reads a stopped step's pod until it's gone (or for at most +// limit): the kubelet's final status, with the exit code, may last only a +// moment before the pod object goes. +func (s *Server) followStop(id string, limit time.Duration) { + every := s.StopPollInterval + if every <= 0 { + every = 250 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), limit) + defer cancel() + defer func() { + s.mu.Lock() + if rec := s.steps[id]; rec != nil { + rec.following = false + } + s.mu.Unlock() + }() + for { + p, err := s.Client.CoreV1().Pods(s.Config.Namespace).Get(ctx, id, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return + } + if err == nil { + st := s.Config.StepState(p, s.now(), true) + s.mu.Lock() + finish := false + if rec := s.steps[id]; rec != nil && rec.final == nil { + rec.see(st) + finish = st.Terminal() && s.terminalLocked(id, st) + } + s.mu.Unlock() + if finish { + s.finishAsync(id, p) + } + } + select { + case <-ctx.Done(): + return + case <-time.After(every): + } + } +} + +// deleteStep deletes a step's pod and Secrets; missing ones are fine. +func (s *Server) deleteStep(ctx context.Context, id string) error { + err := s.Client.CoreV1().Pods(s.Config.Namespace).Delete(ctx, id, metav1.DeleteOptions{ + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }) + if err := ignoreNotFound(err); err != nil { + return kubeError("delete Pod "+id, err) + } + return s.deleteSecrets(ctx, id) +} + +// deleteSecrets deletes a step's Secrets by name; missing ones are fine. +func (s *Server) deleteSecrets(ctx context.Context, id string) error { + for _, name := range []string{EnvSecretName(id), PullSecretName(id)} { + err := s.Client.CoreV1().Secrets(s.Config.Namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err := ignoreNotFound(err); err != nil { + return kubeError("delete Secret "+name, err) + } + } + return nil +} + +func (s *Server) delete(w http.ResponseWriter, r *http.Request) { + id, err := s.stepID(r) + if err != nil { + writeError(w, err) + return + } + if err := s.deleteStep(r.Context(), id); err != nil { + writeError(w, err) + return + } + s.mu.Lock() + if rec := s.steps[id]; rec != nil { + rec.deleted, rec.deletedAt, rec.secretsDeleted = true, s.now(), true + } + s.mu.Unlock() + s.Log.Info("step deleted", "id", id) + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) openStream(id string) { + s.mu.Lock() + s.streams[id]++ + s.mu.Unlock() +} + +func (s *Server) closeStream(id string) { + s.mu.Lock() + if s.streams[id]--; s.streams[id] <= 0 { + delete(s.streams, id) + } + s.mu.Unlock() +} + +// streaming reports an open log stream of a step. +func (s *Server) streaming(id string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.streams[id] > 0 +} + +func (s *Server) logs(w http.ResponseWriter, r *http.Request) { + id, err := s.stepID(r) + if err != nil { + writeError(w, err) + return + } + q := r.URL.Query() + follow := q.Get("follow") == "1" || q.Get("follow") == "true" + var since *time.Time + if raw := q.Get("sinceTime"); raw != "" { + t, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + writeError(w, badRequest(ReasonInvalidRequest, "sinceTime %q: want RFC3339Nano", raw)) + return + } + since = &t + } + ctx := r.Context() + if _, err := s.state(ctx, id); err != nil { + writeError(w, err) + return + } + + s.openStream(id) + defer s.closeStream(id) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + if flusher != nil { + flusher.Flush() + } + if follow && !s.waitStarted(ctx, id) { + return + } + + opts := &corev1.PodLogOptions{Container: ContainerName, Follow: follow, Timestamps: true} + if since != nil { + opts.SinceTime = &metav1.Time{Time: *since} + } + open := s.Logs + if open == nil { + open = func(ctx context.Context, ns, pod string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { + return s.Client.CoreV1().Pods(ns).GetLogs(pod, opts).Stream(ctx) + } + } + rc, err := open(ctx, s.Config.Namespace, id, opts) + if err != nil { + if ctx.Err() == nil { + s.Log.Warn("opening step logs", "id", id, "err", err) + } + return + } + defer rc.Close() + copyLines(w, flusher, rc, since) +} + +// waitStarted waits until a step's container started or ended; false when it +// never will (failed, gone) or the client left. +func (s *Server) waitStarted(ctx context.Context, id string) bool { + every := s.PollInterval + if every <= 0 { + every = time.Second + } + for { + st, err := s.state(ctx, id) + var e *Error + switch { + case errors.As(err, &e) && e.Status == http.StatusNotFound: + return false + case err != nil: + s.Log.Warn("waiting for a step to start", "id", id, "err", err) + case st.State == StateRunning || st.State == StateExited: + return true + case st.State == StateFailed: + return false + } + select { + case <-ctx.Done(): + return false + case <-time.After(every): + } + } +} + +// copyLines copies log lines, flushing whenever it has caught up. The API +// server passes sinceTime on in whole seconds: lines before since are +// dropped here. +func copyLines(w io.Writer, flusher http.Flusher, r io.Reader, since *time.Time) { + br := bufio.NewReaderSize(r, 64<<10) + for { + line, err := br.ReadBytes('\n') + if len(line) > 0 && !before(line, since) { + if _, werr := w.Write(line); werr != nil { + return + } + if flusher != nil && br.Buffered() == 0 { + flusher.Flush() + } + } + if err != nil { + if flusher != nil { + flusher.Flush() + } + return + } + } +} + +// before reports a line whose timestamp is before since. +func before(line []byte, since *time.Time) bool { + if since == nil { + return false + } + i := 0 + for i < len(line) && line[i] != ' ' { + i++ + } + t, err := time.Parse(time.RFC3339Nano, string(line[:i])) + return err == nil && t.Before(*since) +} diff --git a/k8s/scaler/internal/steps/server_test.go b/k8s/scaler/internal/steps/server_test.go new file mode 100644 index 0000000..7b7e908 --- /dev/null +++ b/k8s/scaler/internal/steps/server_test.go @@ -0,0 +1,505 @@ +package steps + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +type harness struct { + t *testing.T + client *fake.Clientset + s *Server + h http.Handler + now time.Time + logs *bytes.Buffer +} + +func newHarness(t *testing.T, objs ...runtime.Object) *harness { + hs := &harness{t: t, client: fake.NewClientset(objs...), now: t0.Add(time.Minute), logs: &bytes.Buffer{}} + hs.s = NewServer(testConfig(), hs.client, slog.New(slog.NewJSONHandler(hs.logs, nil))) + hs.s.Now = func() time.Time { return hs.now } + hs.s.Random = bytes.NewReader(bytes.Repeat([]byte{0, 1, 2, 3, 4, 5, 6, 7}, 64)) + hs.s.PollInterval = 5 * time.Millisecond + hs.h = hs.s.Handler() + return hs +} + +func (hs *harness) do(method, target, body string) *httptest.ResponseRecorder { + hs.t.Helper() + rec := httptest.NewRecorder() + hs.h.ServeHTTP(rec, httptest.NewRequest(method, target, strings.NewReader(body))) + return rec +} + +// wantError checks a JSON error response. +func wantError(t *testing.T, rec *httptest.ResponseRecorder, status int, reason string) { + t.Helper() + var e Error + if rec.Code != status || json.Unmarshal(rec.Body.Bytes(), &e) != nil || e.Reason != reason || e.Message == "" || + rec.Header().Get("Content-Type") != "application/json" { + t.Errorf("got %d %s, want %d %s", rec.Code, rec.Body, status, reason) + } +} + +func (hs *harness) pods() []corev1.Pod { + l, err := hs.client.CoreV1().Pods("sg").List(context.Background(), metav1.ListOptions{}) + if err != nil { + hs.t.Fatal(err) + } + return l.Items +} + +func (hs *harness) secretNames() []string { + l, err := hs.client.CoreV1().Secrets("sg").List(context.Background(), metav1.ListOptions{}) + if err != nil { + hs.t.Fatal(err) + } + var out []string + for _, s := range l.Items { + out = append(out, s.Name) + } + return out +} + +func requestJSON(t *testing.T, edit func(*Request)) string { + r := testRequest() + if edit != nil { + edit(&r) + } + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestHealthz(t *testing.T) { + rec := newHarness(t).do("GET", "/healthz", "") + if rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != `{"api":"v1"}` { + t.Errorf("healthz %d %s", rec.Code, rec.Body) + } +} + +func TestUnknownRouteIsJSON(t *testing.T) { + hs := newHarness(t) + wantError(t, hs.do("GET", "/v2/steps", ""), 404, ReasonNotFound) + wantError(t, hs.do("PUT", "/v1/steps", ""), 404, ReasonNotFound) +} + +func TestCreate(t *testing.T) { + hs := newHarness(t) + body := requestJSON(t, func(r *Request) { + r.RegistryAuth = &RegistryAuth{Registry: "registry.example.com", Username: "u", Password: "p"} + }) + rec := hs.do("POST", "/v1/steps", body) + var out map[string]string + if rec.Code != 201 || json.Unmarshal(rec.Body.Bytes(), &out) != nil { + t.Fatalf("create %d %s", rec.Code, rec.Body) + } + id := out["id"] + if id != "sg-runner-0-s-abcdefgh" { + t.Errorf("id %q", id) + } + pods := hs.pods() + if len(pods) != 1 || pods[0].Name != id || pods[0].Spec.ImagePullSecrets[0].Name != id+"-pull" { + t.Errorf("pods %+v", pods) + } + if got := strings.Join(hs.secretNames(), ","); got != id+"-env,"+id+"-pull" { + t.Errorf("secrets %s", got) + } + if strings.Contains(hs.logs.String(), "SG_WORKFLOW_RUN_PAYLOAD") || strings.Contains(hs.logs.String(), `"p"`) { + t.Errorf("the env or the password got logged: %s", hs.logs) + } + + // Unknown fields are fine: a newer controller may send more. + hs.s.Random = bytes.NewReader(bytes.Repeat([]byte{9}, 16)) + rec = hs.do("POST", "/v1/steps", strings.Replace(requestJSON(t, nil), "{", `{"future":1,`, 1)) + if rec.Code != 201 { + t.Errorf("unknown field: %d %s", rec.Code, rec.Body) + } + if got := hs.secretNames(); len(got) != 3 { + t.Errorf("no pull Secret without registryAuth: %v", got) + } +} + +func TestCreateRejects(t *testing.T) { + hs := newHarness(t) + wantError(t, hs.do("POST", "/v1/steps", "{"), 400, ReasonInvalidRequest) + wantError(t, hs.do("POST", "/v1/steps", `{"org":1}`), 400, ReasonInvalidRequest) + wantError(t, hs.do("POST", "/v1/steps", requestJSON(t, func(r *Request) { r.Image = "" })), 400, ReasonInvalidRequest) + wantError(t, hs.do("POST", "/v1/steps", requestJSON(t, func(r *Request) { r.Mounts[0].Source = "/var/tmp/workspace" })), 400, ReasonInvalidMount) + wantError(t, hs.do("POST", "/v1/steps", requestJSON(t, func(r *Request) { r.Env["A B"] = "" })), 400, ReasonInvalidEnv) + wantError(t, hs.do("POST", "/v1/steps", requestJSON(t, func(r *Request) { + r.Env = map[string]string{"A": strings.Repeat("x", MaxEnvBytes)} + })), 400, ReasonEnvTooLarge) + wantError(t, hs.do("POST", "/v1/steps", requestJSON(t, func(r *Request) { + r.Args = []string{strings.Repeat("x", maxBody)} + })), 400, ReasonEnvTooLarge) + if len(hs.pods()) != 0 || len(hs.secretNames()) != 0 { + t.Errorf("a rejected request created objects") + } +} + +func TestCreateCleansUpWhenThePodFails(t *testing.T) { + for _, failing := range []string{"pods", "secrets"} { + hs := newHarness(t) + calls := 0 + hs.client.PrependReactor("create", failing, func(a k8stesting.Action) (bool, runtime.Object, error) { + // The env Secret goes through; the pull Secret or the pod fails. + if calls++; failing == "secrets" && calls == 1 { + return false, nil, nil + } + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: failing}, "x", errors.New("quota")) + }) + body := requestJSON(t, func(r *Request) { r.RegistryAuth = &RegistryAuth{Registry: "r", Username: "u", Password: "p"} }) + rec := hs.do("POST", "/v1/steps", body) + wantError(t, rec, 500, ReasonKubernetesError) + if names := hs.secretNames(); len(names) != 0 || len(hs.pods()) != 0 { + t.Errorf("%s failing: left %v behind", failing, names) + } + } +} + +func TestGet(t *testing.T) { + started := t0.Add(10 * time.Second) + other := pod(corev1.PodRunning, running(started)) + other.Name = "sg-runner-1-s-aaaaaaaa" + unrelated := pod(corev1.PodRunning, running(started)) + unrelated.Name = "sg-runner-0" + hs := newHarness(t, pod(corev1.PodRunning, running(started)), other, unrelated) + + rec := hs.do("GET", "/v1/steps/"+testID, "") + want := `{"id":"sg-runner-0-s-7f3k9q2x","state":"running","exitCode":null,"reason":"","message":"",` + + `"startedAt":"2026-10-14T10:00:10Z","finishedAt":null}` + if rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != want { + t.Errorf("get %d %s", rec.Code, rec.Body) + } + for _, id := range []string{"sg-runner-0-s-zzzzzzzz", other.Name, unrelated.Name, testID + "-env"} { + wantError(t, hs.do("GET", "/v1/steps/"+id, ""), 404, ReasonNotFound) + } + + hs.client.PrependReactor("get", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("connection refused") + }) + wantError(t, hs.do("GET", "/v1/steps/"+testID, ""), 500, ReasonKubernetesError) +} + +// graces records the grace periods of pod deletes. +func graces(hs *harness) *[]int64 { + var out []int64 + hs.client.PrependReactor("delete", "pods", func(a k8stesting.Action) (bool, runtime.Object, error) { + // Stops set a grace period; deletions of finished steps don't. + if g := a.(k8stesting.DeleteActionImpl).DeleteOptions.GracePeriodSeconds; g != nil { + out = append(out, *g) + } + return false, nil, nil + }) + return &out +} + +func TestStop(t *testing.T) { + hs := newHarness(t, pod(corev1.PodRunning, running(t0))) + g := graces(hs) + wantError(t, hs.do("POST", "/v1/steps/sg-runner-0-s-zzzzzzzz/stop", `{"graceSeconds":5}`), 404, ReasonNotFound) + wantError(t, hs.do("POST", "/v1/steps/"+testID+"/stop", `{"graceSeconds":-1}`), 400, ReasonInvalidRequest) + if rec := hs.do("POST", "/v1/steps/"+testID+"/stop", `{"graceSeconds":5}`); rec.Code != 202 { + t.Fatalf("stop %d %s", rec.Code, rec.Body) + } + if len(hs.pods()) != 0 { + t.Error("stop left the pod") + } + // Its pod is gone: the step exited as killed. + rec := hs.do("GET", "/v1/steps/"+testID, "") + var st State + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &st) != nil || st.State != StateExited || + *st.ExitCode != 137 || st.Reason != ReasonStopped { + t.Errorf("after stop %d %s", rec.Code, rec.Body) + } + // Stopping again is fine, with the default grace of 30s; 0 kills (almost) + // right away. + if rec := hs.do("POST", "/v1/steps/"+testID+"/stop", ""); rec.Code != 202 { + t.Errorf("second stop %d %s", rec.Code, rec.Body) + } + hs0 := newHarness(t, pod(corev1.PodRunning, running(t0))) + g0 := graces(hs0) + hs0.do("POST", "/v1/steps/"+testID+"/stop", `{"graceSeconds":0}`) + if len(*g) != 2 || (*g)[0] != 5 || (*g)[1] != 30 || len(*g0) != 1 || (*g0)[0] != 1 { + t.Errorf("grace periods %v %v", *g, *g0) + } +} + +func TestStopKeepsTheExitCodeSeen(t *testing.T) { + hs := newHarness(t, pod(corev1.PodFailed, terminated(2, "Error", t0, t0.Add(time.Minute)))) + if rec := hs.do("POST", "/v1/steps/"+testID+"/stop", `{"graceSeconds":0}`); rec.Code != 202 { + t.Fatalf("stop %d %s", rec.Code, rec.Body) + } + var st State + rec := hs.do("GET", "/v1/steps/"+testID, "") + if json.Unmarshal(rec.Body.Bytes(), &st) != nil || st.State != StateExited || *st.ExitCode != 2 || st.Reason != "Error" { + t.Errorf("after stop %s", rec.Body) + } +} + +// TestStopCatchesTheFinalExitCode: the kubelet's terminated status lasts a +// moment before the pod goes; the broker keeps it. +func TestStopCatchesTheFinalExitCode(t *testing.T) { + hs := newHarness(t, pod(corev1.PodRunning, running(t0))) + hs.s.StopPollInterval = 5 * time.Millisecond + // A graceful delete keeps the pod until the kubelet is done. + hs.client.PrependReactor("delete", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + if rec := hs.do("POST", "/v1/steps/"+testID+"/stop", `{"graceSeconds":10}`); rec.Code != 202 { + t.Fatalf("stop %d %s", rec.Code, rec.Body) + } + p := pod(corev1.PodFailed, terminated(143, "Error", t0, t0.Add(time.Minute)), deleting) + if _, err := hs.client.CoreV1().Pods("sg").UpdateStatus(context.Background(), p, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + seen := func() bool { + hs.s.mu.Lock() + defer hs.s.mu.Unlock() + last := hs.s.steps[testID].last + return last.State == StateExited && *last.ExitCode == 143 + } + for deadline := time.Now().Add(5 * time.Second); !seen(); time.Sleep(5 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("the exit code was never caught") + } + } + gvr := corev1.SchemeGroupVersion.WithResource("pods") + if err := hs.client.Tracker().Delete(gvr, "sg", testID); err != nil { + t.Fatal(err) + } + following := func() bool { + hs.s.mu.Lock() + defer hs.s.mu.Unlock() + return hs.s.steps[testID].following + } + for deadline := time.Now().Add(5 * time.Second); following(); time.Sleep(5 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("still following a gone pod") + } + } + var st State + rec := hs.do("GET", "/v1/steps/"+testID, "") + if json.Unmarshal(rec.Body.Bytes(), &st) != nil || st.State != StateExited || *st.ExitCode != 143 || st.Reason != "Error" { + t.Errorf("after the pod went: %s", rec.Body) + } +} + +func TestStopRecordKeepsAnExit(t *testing.T) { + c := int32(143) + rec := &stepRecord{} + rec.see(State{State: StateRunning}) + rec.see(State{State: StateExited, ExitCode: &c}) + rec.see(State{State: StateFailed, Reason: FailedPodDeleted}) + if rec.last.State != StateExited || *rec.last.ExitCode != 143 { + t.Errorf("last %+v", rec.last) + } +} + +func TestDeletedWithoutStopIsNotFound(t *testing.T) { + hs := newHarness(t, pod(corev1.PodRunning, running(t0))) + if err := hs.client.CoreV1().Pods("sg").Delete(context.Background(), testID, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + wantError(t, hs.do("GET", "/v1/steps/"+testID, ""), 404, ReasonNotFound) +} + +func TestDelete(t *testing.T) { + sec := func(name string) *corev1.Secret { + return &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "sg"}} + } + hs := newHarness(t, pod(corev1.PodSucceeded, terminated(0, "Completed", t0, t0)), sec(testID+"-env"), sec(testID+"-pull"), sec("keep")) + hs.s.steps[testID] = &stepRecord{at: t0, stopped: true} + for i := 0; i < 2; i++ { + if rec := hs.do("DELETE", "/v1/steps/"+testID, ""); rec.Code != 204 { + t.Errorf("delete %d: %d %s", i, rec.Code, rec.Body) + } + } + if len(hs.pods()) != 0 || strings.Join(hs.secretNames(), ",") != "keep" || !hs.s.steps[testID].deleted { + t.Errorf("left pods %v, secrets %v, record %+v", hs.pods(), hs.secretNames(), hs.s.steps[testID]) + } + wantError(t, hs.do("DELETE", "/v1/steps/keep", ""), 404, ReasonNotFound) +} + +// fakeLogs serves canned logs and records the options asked for. +type fakeLogs struct { + mu sync.Mutex + opts []*corev1.PodLogOptions + body func() io.ReadCloser +} + +func (f *fakeLogs) open(ctx context.Context, ns, name string, opts *corev1.PodLogOptions) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.opts = append(f.opts, opts) + return f.body(), nil +} + +func (f *fakeLogs) calls() []*corev1.PodLogOptions { + f.mu.Lock() + defer f.mu.Unlock() + return f.opts +} + +const logLines = "2026-10-14T10:00:01.5Z one\n2026-10-14T10:00:02.25Z two\n2026-10-14T10:00:02.5Z three\n2026-10-14T10:00:03Z four" + +func TestLogs(t *testing.T) { + hs := newHarness(t, pod(corev1.PodRunning, running(t0))) + f := &fakeLogs{body: func() io.ReadCloser { return io.NopCloser(strings.NewReader(logLines)) }} + hs.s.Logs = f.open + + rec := hs.do("GET", "/v1/steps/"+testID+"/logs?follow=1", "") + if rec.Code != 200 || rec.Body.String() != logLines || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") { + t.Errorf("logs %d %q", rec.Code, rec.Body) + } + o := f.calls()[0] + if o.Container != "step" || !o.Follow || !o.Timestamps || o.SinceTime != nil { + t.Errorf("log options %+v", o) + } + + // The API server truncates sinceTime to seconds: earlier lines are + // dropped, lines at exactly sinceTime kept. + rec = hs.do("GET", "/v1/steps/"+testID+"/logs?follow=1&sinceTime=2026-10-14T10:00:02.25Z", "") + if want := "2026-10-14T10:00:02.25Z two\n2026-10-14T10:00:02.5Z three\n2026-10-14T10:00:03Z four"; rec.Body.String() != want { + t.Errorf("logs since %q", rec.Body) + } + if o := f.calls()[1]; o.SinceTime == nil || !o.SinceTime.Equal(&metav1.Time{Time: time.Date(2026, 10, 14, 10, 0, 2, 250e6, time.UTC)}) { + t.Errorf("since option %+v", o.SinceTime) + } + + wantError(t, hs.do("GET", "/v1/steps/"+testID+"/logs?sinceTime=yesterday", ""), 400, ReasonInvalidRequest) + wantError(t, hs.do("GET", "/v1/steps/sg-runner-0-s-zzzzzzzz/logs?follow=1", ""), 404, ReasonNotFound) +} + +func TestLogsWaitForTheContainer(t *testing.T) { + hs := newHarness(t, pod(corev1.PodPending, waiting("ContainerCreating", ""))) + f := &fakeLogs{body: func() io.ReadCloser { return io.NopCloser(strings.NewReader(logLines)) }} + hs.s.Logs = f.open + done := make(chan *httptest.ResponseRecorder) + go func() { done <- hs.do("GET", "/v1/steps/"+testID+"/logs?follow=1", "") }() + + time.Sleep(30 * time.Millisecond) + if len(f.calls()) != 0 { + t.Fatal("logs opened before the container started") + } + if _, err := hs.client.CoreV1().Pods("sg").UpdateStatus(context.Background(), + pod(corev1.PodRunning, running(t0)), metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + select { + case rec := <-done: + if rec.Body.String() != logLines { + t.Errorf("logs %q", rec.Body) + } + case <-time.After(5 * time.Second): + t.Fatal("logs never came") + } +} + +func TestLogsOfAFailedStepAreEmpty(t *testing.T) { + hs := newHarness(t, pod(corev1.PodPending, waiting("ImagePullBackOff", ""))) + f := &fakeLogs{body: func() io.ReadCloser { return io.NopCloser(strings.NewReader(logLines)) }} + hs.s.Logs = f.open + rec := hs.do("GET", "/v1/steps/"+testID+"/logs?follow=1", "") + if rec.Code != 200 || rec.Body.Len() != 0 || len(f.calls()) != 0 { + t.Errorf("logs of a failed step: %d %q, %d opens", rec.Code, rec.Body, len(f.calls())) + } + + // Deleted while waiting: the stream ends. + hs = newHarness(t, pod(corev1.PodPending, nil)) + hs.s.Logs = f.open + done := make(chan *httptest.ResponseRecorder) + go func() { done <- hs.do("GET", "/v1/steps/"+testID+"/logs?follow=1", "") }() + time.Sleep(20 * time.Millisecond) + _ = hs.client.CoreV1().Pods("sg").Delete(context.Background(), testID, metav1.DeleteOptions{}) + select { + case rec := <-done: + if rec.Code != 200 || rec.Body.Len() != 0 { + t.Errorf("logs of a deleted step: %d %q", rec.Code, rec.Body) + } + case <-time.After(5 * time.Second): + t.Fatal("the stream didn't end") + } +} + +func TestLogsCountOpenStreams(t *testing.T) { + hs := newHarness(t, pod(corev1.PodRunning, running(t0))) + pr, pw := io.Pipe() + hs.s.Logs = (&fakeLogs{body: func() io.ReadCloser { return pr }}).open + done := make(chan struct{}) + go func() { hs.do("GET", "/v1/steps/"+testID+"/logs?follow=1", ""); close(done) }() + _, _ = pw.Write([]byte("2026-10-14T10:00:01Z hi\n")) + if !hs.s.streaming(testID) { + t.Error("an open stream isn't counted") + } + _ = pw.Close() + <-done + if hs.s.streaming(testID) { + t.Error("a closed stream is still counted") + } +} + +// TestOverTheSocket runs the API on a unix socket like the controllers use it. +func TestOverTheSocket(t *testing.T) { + // macOS keeps unix socket paths under 104 bytes: a short temp dir. + dir, err := os.MkdirTemp("", "steps") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + sock := filepath.Join(dir, "sub", "steps.sock") + if err := os.MkdirAll(filepath.Dir(sock), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sock, []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + l, err := Listen(sock) + if err != nil { + t.Fatal(err) + } + fi, err := os.Stat(sock) + if err != nil || fi.Mode()&os.ModeSocket == 0 || fi.Mode().Perm() != 0o666 { + t.Errorf("socket %v %v", fi.Mode(), err) + } + srv := &http.Server{Handler: newHarness(t).s.Handler()} + go func() { _ = srv.Serve(l) }() + defer srv.Close() + + c := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", sock) + }}} + resp, err := c.Get("http://steps/healthz") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 || strings.TrimSpace(string(b)) != `{"api":"v1"}` { + t.Errorf("healthz over the socket %d %s", resp.StatusCode, b) + } +} diff --git a/k8s/scaler/internal/steps/state.go b/k8s/scaler/internal/steps/state.go new file mode 100644 index 0000000..efc065d --- /dev/null +++ b/k8s/scaler/internal/steps/state.go @@ -0,0 +1,199 @@ +package steps + +import ( + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" +) + +// Step states. +const ( + StatePending = "pending" + StateRunning = "running" + StateExited = "exited" + StateFailed = "failed" +) + +// Reasons of failed steps (and of stopped ones whose pod is gone). +const ( + FailedUnschedulable = "Unschedulable" + FailedImagePullBackOff = "ImagePullBackOff" + FailedErrImagePull = "ErrImagePull" + FailedInvalidImageName = "InvalidImageName" + FailedErrImageNeverPull = "ErrImageNeverPull" + FailedStartTimeout = "StartTimeout" + FailedEvicted = "Evicted" + FailedDeadlineExceeded = "DeadlineExceeded" + FailedPodDeleted = "PodDeleted" + // ReasonStopped is the reason of a stopped step the broker never saw exit. + ReasonStopped = "Stopped" +) + +// StoppedExitCode is a stopped step's exit code when the broker never saw +// its own (Docker's for a killed container). +const StoppedExitCode int32 = 137 + +// State is GET /v1/steps/{id}. +type State struct { + ID string `json:"id"` + State string `json:"state"` + ExitCode *int32 `json:"exitCode"` + Reason string `json:"reason"` + Message string `json:"message"` + StartedAt *time.Time `json:"startedAt"` + FinishedAt *time.Time `json:"finishedAt"` +} + +// Terminal reports a state that won't change. +func (s State) Terminal() bool { return s.State == StateExited || s.State == StateFailed } + +// stoppedState is a stopped step's state after its pod is gone. +func stoppedState(id string, last *State) State { + if last != nil && last.State == StateExited { + return *last + } + code := StoppedExitCode + return State{ID: id, State: StateExited, ExitCode: &code, Reason: ReasonStopped, Message: "stopped"} +} + +// ErrImagePull messages that won't get better with retries; any other pull +// error stays pending until the kubelet backs off. +var permanentPullErrors = []string{"not found", "manifest unknown", "unauthorized", "authentication required", "denied", "forbidden"} + +// StepState is what a step pod says about the step at now. stopped tells +// whether the broker was asked to stop it. +func (c *Config) StepState(p *corev1.Pod, now time.Time, stopped bool) State { + s := State{ID: p.Name} + var cs *corev1.ContainerStatus + for i := range p.Status.ContainerStatuses { + if p.Status.ContainerStatuses[i].Name == ContainerName { + cs = &p.Status.ContainerStatuses[i] + } + } + failed := func(reason, message string) State { + s.State, s.Reason, s.Message = StateFailed, reason, message + return s + } + + // The kubelet killed the pod: whatever the container says, the step was + // cut short. + if p.Status.Phase == corev1.PodFailed { + switch p.Status.Reason { + case FailedEvicted, FailedDeadlineExceeded: + s.FinishedAt = podFinishedAt(p, cs) + return failed(p.Status.Reason, p.Status.Message) + } + } + + if cs != nil { + if t := cs.State.Terminated; t != nil { + code, message := t.ExitCode, t.Message + if c, ok := wrappedExit(p, t); ok { + code, message = c, "" + } + s.State, s.ExitCode, s.Reason, s.Message = StateExited, &code, t.Reason, message + s.StartedAt, s.FinishedAt = timePtr(t.StartedAt.Time), timePtr(t.FinishedAt.Time) + return s + } + } + + if p.DeletionTimestamp != nil && !stopped { + return failed(FailedPodDeleted, "the step's pod was deleted") + } + + switch p.Status.Phase { + case corev1.PodFailed, corev1.PodSucceeded: + // Rejected or shut down by the node without a container status to + // tell: a container that never ran couldn't be placed there. + s.FinishedAt = podFinishedAt(p, cs) + msg := strings.TrimSpace(p.Status.Reason + ": " + p.Status.Message) + if cs == nil || cs.State.Running == nil && cs.LastTerminationState.Terminated == nil { + return failed(FailedUnschedulable, msg) + } + return failed(FailedEvicted, msg) + } + + if cs != nil && cs.State.Running != nil { + s.State = StateRunning + s.StartedAt = timePtr(cs.State.Running.StartedAt.Time) + return s + } + + created := p.CreationTimestamp.Time + if p.Spec.NodeName == "" { + for _, cond := range p.Status.Conditions { + if cond.Type == corev1.PodScheduled && cond.Status == corev1.ConditionFalse && + cond.Reason == corev1.PodReasonUnschedulable && now.Sub(created) > c.SchedulingTimeout { + s.FinishedAt = timePtr(created.Add(c.SchedulingTimeout)) + return failed(FailedUnschedulable, cond.Message) + } + } + } + + var waiting string + if cs != nil && cs.State.Waiting != nil { + w := cs.State.Waiting + waiting = strings.TrimSpace(w.Reason + ": " + w.Message) + switch w.Reason { + case FailedImagePullBackOff, FailedInvalidImageName, FailedErrImageNeverPull: + return failed(w.Reason, w.Message) + case FailedErrImagePull: + msg := strings.ToLower(w.Message) + for _, e := range permanentPullErrors { + if strings.Contains(msg, e) { + return failed(w.Reason, w.Message) + } + } + } + } + + if now.Sub(created) > c.StartTimeout { + s.FinishedAt = timePtr(created.Add(c.StartTimeout)) + msg := fmt.Sprintf("not started after %s", c.StartTimeout) + if waiting != "" { + msg += " (" + waiting + ")" + } + return failed(FailedStartTimeout, msg) + } + s.State, s.Message = StatePending, waiting + return s +} + +// wrappedExits are the termination messages of the wrapper's mapped exits. +var wrappedExits = map[string]int32{"sg-exit-code=11": 11, "sg-exit-code=12": 12} + +// wrappedExit is the step's own exit code when the wrapper mapped it to 0: +// only a wrapped pod's exit 0 with the wrapper's exact message counts. +func wrappedExit(p *corev1.Pod, t *corev1.ContainerStateTerminated) (int32, bool) { + if p.Annotations[AnnotationWrapped] != "true" || t.ExitCode != 0 { + return 0, false + } + code, ok := wrappedExits[t.Message] + return code, ok +} + +// podFinishedAt is when a pod the kubelet failed stopped: its container's +// end, else its conditions' last change. +func podFinishedAt(p *corev1.Pod, cs *corev1.ContainerStatus) *time.Time { + if cs != nil && cs.State.Terminated != nil && !cs.State.Terminated.FinishedAt.IsZero() { + return timePtr(cs.State.Terminated.FinishedAt.Time) + } + var last time.Time + for _, cond := range p.Status.Conditions { + if cond.LastTransitionTime.After(last) { + last = cond.LastTransitionTime.Time + } + } + return timePtr(last) +} + +// timePtr is t in UTC, nil when zero. +func timePtr(t time.Time) *time.Time { + if t.IsZero() { + return nil + } + t = t.UTC() + return &t +} diff --git a/k8s/scaler/internal/steps/state_test.go b/k8s/scaler/internal/steps/state_test.go new file mode 100644 index 0000000..2cdba23 --- /dev/null +++ b/k8s/scaler/internal/steps/state_test.go @@ -0,0 +1,179 @@ +package steps + +import ( + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var t0 = time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC) + +// pod is a step pod created at t0 in a phase, with the step container's state +// (nil for none). +func pod(phase corev1.PodPhase, cs *corev1.ContainerState, edit ...func(*corev1.Pod)) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: testID, Namespace: "sg", CreationTimestamp: metav1.NewTime(t0)}, + Spec: corev1.PodSpec{NodeName: "node-a"}, + Status: corev1.PodStatus{Phase: phase}, + } + if cs != nil { + p.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: ContainerName, State: *cs}} + } + for _, e := range edit { + e(p) + } + return p +} + +func waiting(reason, message string) *corev1.ContainerState { + return &corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: reason, Message: message}} +} + +func running(at time.Time) *corev1.ContainerState { + return &corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(at)}} +} + +func terminated(code int32, reason string, from, to time.Time) *corev1.ContainerState { + return &corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: code, Reason: reason, StartedAt: metav1.NewTime(from), FinishedAt: metav1.NewTime(to)}} +} + +func unscheduled(p *corev1.Pod) { + p.Spec.NodeName = "" + p.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, + Reason: corev1.PodReasonUnschedulable, Message: "0/3 nodes are available", LastTransitionTime: metav1.NewTime(t0)}} +} + +func deleting(p *corev1.Pod) { p.DeletionTimestamp = ptrTime(t0.Add(time.Hour)) } + +func podFailed(reason string) func(*corev1.Pod) { + return func(p *corev1.Pod) { + p.Status.Reason, p.Status.Message = reason, reason+" message" + p.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse, + LastTransitionTime: metav1.NewTime(t0.Add(7 * time.Minute))}} + } +} + +func wrapped(p *corev1.Pod) { p.Annotations = map[string]string{AnnotationWrapped: "true"} } + +func withMessage(cs *corev1.ContainerState, msg string) *corev1.ContainerState { + cs.Terminated.Message = msg + return cs +} + +func wrappedExit0(msg string) *corev1.ContainerState { + return withMessage(terminated(0, "Completed", t0.Add(time.Minute), t0.Add(6*time.Minute)), msg) +} + +func ptrTime(t time.Time) *metav1.Time { m := metav1.NewTime(t); return &m } + +func TestStepState(t *testing.T) { + cfg := testConfig() + started, ended := t0.Add(time.Minute), t0.Add(6*time.Minute) + type want struct { + state, reason string + code *int32 + start, finish *time.Time + } + code := func(c int32) *int32 { return &c } + at := func(t time.Time) *time.Time { return &t } + cases := map[string]struct { + pod *corev1.Pod + after time.Duration + stopped bool + want want + }{ + "just created": {pod(corev1.PodPending, nil), time.Second, false, want{state: StatePending}}, + "creating": {pod(corev1.PodPending, waiting("ContainerCreating", "")), time.Minute, false, want{state: StatePending}}, + "unscheduled": {pod(corev1.PodPending, nil, unscheduled), 4 * time.Minute, false, want{state: StatePending}}, + "unschedulable": {pod(corev1.PodPending, nil, unscheduled), 6 * time.Minute, false, want{state: StateFailed, reason: FailedUnschedulable, finish: at(t0.Add(5 * time.Minute))}}, + "pull backoff": {pod(corev1.PodPending, waiting("ImagePullBackOff", "Back-off pulling image")), time.Minute, false, want{state: StateFailed, reason: FailedImagePullBackOff}}, + "pull not found": {pod(corev1.PodPending, waiting("ErrImagePull", `rpc error: ... "x:1": not found`)), time.Minute, false, want{state: StateFailed, reason: FailedErrImagePull}}, + "pull unauthorized": {pod(corev1.PodPending, waiting("ErrImagePull", "failed to authorize: 401 Unauthorized")), time.Minute, false, want{state: StateFailed, reason: FailedErrImagePull}}, + "pull denied": {pod(corev1.PodPending, waiting("ErrImagePull", "pull access denied, repository does not exist")), time.Minute, false, want{state: StateFailed, reason: FailedErrImagePull}}, + "pull timeout": {pod(corev1.PodPending, waiting("ErrImagePull", "dial tcp: i/o timeout")), time.Minute, false, want{state: StatePending}}, + "invalid image": {pod(corev1.PodPending, waiting("InvalidImageName", "")), time.Minute, false, want{state: StateFailed, reason: FailedInvalidImageName}}, + "never pull": {pod(corev1.PodPending, waiting("ErrImageNeverPull", "")), time.Minute, false, want{state: StateFailed, reason: FailedErrImageNeverPull}}, + "config error": {pod(corev1.PodPending, waiting("CreateContainerConfigError", "secret not found")), 29 * time.Minute, false, want{state: StatePending}}, + "config error late": {pod(corev1.PodPending, waiting("CreateContainerConfigError", "secret not found")), 31 * time.Minute, false, want{state: StateFailed, reason: FailedStartTimeout, finish: at(t0.Add(30 * time.Minute))}}, + "start timeout": {pod(corev1.PodPending, waiting("ContainerCreating", "")), 31 * time.Minute, false, want{state: StateFailed, reason: FailedStartTimeout, finish: at(t0.Add(30 * time.Minute))}}, + "running": {pod(corev1.PodRunning, running(started)), time.Hour, false, want{state: StateRunning, start: &started}}, + "exited 0": {pod(corev1.PodSucceeded, terminated(0, "Completed", started, ended)), time.Hour, false, want{StateExited, "Completed", code(0), &started, &ended}}, + "exited 11": {pod(corev1.PodFailed, terminated(11, "Error", started, ended)), time.Hour, false, want{StateExited, "Error", code(11), &started, &ended}}, + "oom killed": {pod(corev1.PodFailed, terminated(137, "OOMKilled", started, ended)), time.Hour, false, want{StateExited, "OOMKilled", code(137), &started, &ended}}, + "exited, deleted": {pod(corev1.PodFailed, terminated(3, "Error", started, ended), deleting), time.Hour, false, want{StateExited, "Error", code(3), &started, &ended}}, + "evicted": {pod(corev1.PodFailed, terminated(137, "Error", started, ended), podFailed("Evicted")), time.Hour, false, want{state: StateFailed, reason: FailedEvicted, finish: &ended}}, + "evicted, no status": {pod(corev1.PodFailed, nil, podFailed("Evicted")), time.Hour, false, want{state: StateFailed, reason: FailedEvicted, finish: at(t0.Add(7 * time.Minute))}}, + "deadline": {pod(corev1.PodFailed, terminated(137, "Error", started, ended), podFailed("DeadlineExceeded")), time.Hour, false, want{state: StateFailed, reason: FailedDeadlineExceeded, finish: &ended}}, + "rejected by node": {pod(corev1.PodFailed, waiting("ContainerCreating", ""), podFailed("OutOfmemory")), time.Hour, false, want{state: StateFailed, reason: FailedUnschedulable, finish: at(t0.Add(7 * time.Minute))}}, + "shut down": {pod(corev1.PodFailed, running(started), podFailed("NodeShutdown")), time.Hour, false, want{state: StateFailed, reason: FailedEvicted, finish: at(t0.Add(7 * time.Minute))}}, + "deleted running": {pod(corev1.PodRunning, running(started), deleting), time.Hour, false, want{state: StateFailed, reason: FailedPodDeleted}}, + "deleted pending": {pod(corev1.PodPending, nil, deleting), time.Minute, false, want{state: StateFailed, reason: FailedPodDeleted}}, + "stopping running": {pod(corev1.PodRunning, running(started), deleting), time.Hour, true, want{state: StateRunning, start: &started}}, + "wrapped 11": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=11"), wrapped), time.Hour, false, want{StateExited, "Completed", code(11), &started, &ended}}, + "wrapped 12": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=12"), wrapped), time.Hour, false, want{StateExited, "Completed", code(12), &started, &ended}}, + "wrapped 0": {pod(corev1.PodSucceeded, wrappedExit0(""), wrapped), time.Hour, false, want{StateExited, "Completed", code(0), &started, &ended}}, + "wrapped other text": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=13"), wrapped), time.Hour, false, want{StateExited, "Completed", code(0), &started, &ended}}, + "wrapped near miss": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=11\n"), wrapped), time.Hour, false, want{StateExited, "Completed", code(0), &started, &ended}}, + "unwrapped message": {pod(corev1.PodSucceeded, wrappedExit0("sg-exit-code=11")), time.Hour, false, want{StateExited, "Completed", code(0), &started, &ended}}, + "wrapped, exit 1": {pod(corev1.PodFailed, withMessage(terminated(1, "Error", started, ended), "sg-exit-code=11"), wrapped), time.Hour, false, want{StateExited, "Error", code(1), &started, &ended}}, + "stopped exited": {pod(corev1.PodFailed, terminated(143, "Error", started, ended), deleting), time.Hour, true, want{StateExited, "Error", code(143), &started, &ended}}, + } + for name, c := range cases { + got := cfg.StepState(c.pod, t0.Add(c.after), c.stopped) + w := c.want + if got.ID != testID || got.State != w.state || got.Reason != w.reason || !eqCode(got.ExitCode, w.code) || + !eqTime(got.StartedAt, w.start) || !eqTime(got.FinishedAt, w.finish) { + t.Errorf("%s: got %+v (code %v, started %v, finished %v), want %+v", name, got, + deref(got.ExitCode), got.StartedAt, got.FinishedAt, w) + } + if got.Terminal() != (w.state == StateExited || w.state == StateFailed) { + t.Errorf("%s: Terminal = %v", name, got.Terminal()) + } + } +} + +func TestStepStateMessages(t *testing.T) { + cfg := testConfig() + st := cfg.StepState(pod(corev1.PodPending, waiting("ContainerCreating", "pulling")), t0.Add(time.Minute), false) + if st.Message != "ContainerCreating: pulling" { + t.Errorf("pending message %q", st.Message) + } + st = cfg.StepState(pod(corev1.PodPending, waiting("CreateContainerConfigError", "secret x not found")), t0.Add(time.Hour), false) + if st.Message != "not started after 30m0s (CreateContainerConfigError: secret x not found)" { + t.Errorf("start timeout message %q", st.Message) + } + st = cfg.StepState(pod(corev1.PodPending, nil, unscheduled), t0.Add(time.Hour), false) + if st.Message != "0/3 nodes are available" { + t.Errorf("unschedulable message %q", st.Message) + } +} + +func TestStoppedState(t *testing.T) { + st := stoppedState(testID, nil) + if st.State != StateExited || *st.ExitCode != 137 || st.Reason != ReasonStopped || st.ID != testID { + t.Errorf("stopped without a last state: %+v", st) + } + pending := State{ID: testID, State: StatePending} + if st := stoppedState(testID, &pending); st.State != StateExited || *st.ExitCode != 137 { + t.Errorf("stopped while pending: %+v", st) + } + c := int32(143) + exited := State{ID: testID, State: StateExited, ExitCode: &c, Reason: "Error"} + if st := stoppedState(testID, &exited); *st.ExitCode != 143 || st.Reason != "Error" { + t.Errorf("stopped after exiting: %+v", st) + } +} + +func eqCode(a, b *int32) bool { return a == nil && b == nil || a != nil && b != nil && *a == *b } + +func eqTime(a, b *time.Time) bool { return a == nil && b == nil || a != nil && b != nil && a.Equal(*b) } + +func deref(c *int32) any { + if c == nil { + return nil + } + return *c +} diff --git a/k8s/scaler/internal/teardown/teardown.go b/k8s/scaler/internal/teardown/teardown.go new file mode 100644 index 0000000..822b817 --- /dev/null +++ b/k8s/scaler/internal/teardown/teardown.go @@ -0,0 +1,213 @@ +// Package teardown removes every runner of a StatefulSet from StackGuardian +// before Helm deletes the release (the chart's pre-delete hook), so +// `helm uninstall` leaves no registered runners and no PVCs behind. +package teardown + +import ( + "context" + "fmt" + "log/slog" + "slices" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/decide" + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" +) + +// Teardown parks, deregisters and deletes every runner of one StatefulSet. +type Teardown struct { + Kube *kube.Kube + // ScalerDeployment is stopped first, so it can't wake, park or reap + // runners meanwhile. Empty = no scaler. + ScalerDeployment string + // ScalerSelector and RunnerSelector select the scaler's and the runners' + // pods, to wait for them to be gone. + ScalerSelector string + RunnerSelector string + // Poll is how often to check progress (5s when zero). + Poll time.Duration + Log *slog.Logger +} + +// Run stops the scaler, parks every runner (each one's preStop drain waits +// for its running workflows), deregisters each runner from its state PVC +// with a reap Job, and deletes the PVCs of every runner it deregistered. A +// runner that fails to deregister keeps its PVCs, and Run returns an error, +// so Helm stops before anything is orphaned. +func (t *Teardown) Run(ctx context.Context) error { + if t.Poll == 0 { + t.Poll = 5 * time.Second + } + if t.ScalerDeployment != "" { + t.Log.Info("stopping the scaler", "deployment", t.ScalerDeployment) + if err := t.Kube.ScaleDeployment(ctx, t.ScalerDeployment, 0); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("stop the scaler: %w", err) + } + if err := t.waitNoPods(ctx, t.ScalerSelector, "scaler"); err != nil { + return err + } + } + + t.Log.Info("parking every runner", "statefulset", t.Kube.StatefulSet) + if err := t.Kube.Scale(ctx, 0); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("park the runners: %w", err) + } + if err := t.waitNoPods(ctx, t.RunnerSelector, "runner"); err != nil { + return err + } + + claims, err := t.Kube.Claims(ctx) + if err != nil { + return fmt.Errorf("list runner PVCs: %w", err) + } + ordinals := make([]int, 0, len(claims)) + for n := range claims { + ordinals = append(ordinals, n) + } + slices.Sort(ordinals) + t.Log.Info("deregistering runners", "ordinals", ordinals) + + // A leftover reap Job (a failed reap by the scaler) would read as this + // run's result: start every reap afresh. + for _, n := range ordinals { + if err := t.deleteReapJob(ctx, n); err != nil { + return err + } + } + for _, n := range ordinals { + if err := t.Kube.CreateReapJob(ctx, n); err != nil { + return fmt.Errorf("start reap Job for runner %d: %w", n, err) + } + } + states, err := t.waitReaps(ctx, ordinals) + if err != nil { + return err + } + + var failed []int + for _, n := range ordinals { + if states[n] != decide.JobSucceeded { + failed = append(failed, n) + t.Log.Error("could not deregister runner; keeping its PVCs", "ordinal", n, "job", t.Kube.JobName(n)) + continue + } + for _, claim := range []string{t.Kube.ClaimName(n), t.Kube.DockerClaimName(n)} { + if err := t.Kube.DeleteClaim(ctx, claim); err != nil { + return fmt.Errorf("delete PVC %s: %w", claim, err) + } + } + if err := t.Kube.DeleteReapJob(ctx, n); err != nil { + return fmt.Errorf("delete reap Job %s: %w", t.Kube.JobName(n), err) + } + t.Log.Info("runner deregistered, PVCs deleted", "ordinal", n) + } + + // Placeholders asking the cluster autoscaler for nodes (the scaler is + // stopped, nothing needs them). + nudges, err := t.Kube.NudgeJobs(ctx) + if err != nil { + return fmt.Errorf("list placeholder Jobs: %w", err) + } + for n := range nudges { + if err := t.Kube.DeleteNudgeJob(ctx, n); err != nil { + return fmt.Errorf("delete placeholder Job %s: %w", t.Kube.NudgeJobName(n), err) + } + } + + // Docker caches whose runner is gone already (reaped earlier, or its state + // PVC deleted by hand). + docker, err := t.Kube.DockerClaims(ctx) + if err != nil { + return fmt.Errorf("list Docker cache PVCs: %w", err) + } + for _, n := range docker { + if _, ok := claims[n]; ok { + continue + } + if err := t.Kube.DeleteClaim(ctx, t.Kube.DockerClaimName(n)); err != nil { + return fmt.Errorf("delete PVC %s: %w", t.Kube.DockerClaimName(n), err) + } + } + + if len(failed) > 0 { + return fmt.Errorf("could not deregister runners %v: see the logs of their %s-reap- Jobs; "+ + "their PVCs are kept so nothing is orphaned", failed, t.Kube.StatefulSet) + } + t.Log.Info("teardown complete", "runners", len(ordinals)) + return nil +} + +// waitNoPods waits until no pod matches selector. Runners can take as long as +// their running workflows (their preStop drain waits for them). +func (t *Teardown) waitNoPods(ctx context.Context, selector, what string) error { + var last time.Time + for { + n, err := t.Kube.CountPods(ctx, selector) + if err != nil { + return fmt.Errorf("list %s pods: %w", what, err) + } + if n == 0 { + return nil + } + if time.Since(last) >= time.Minute { + t.Log.Info("waiting for pods to stop (runners wait for their running workflows)", "component", what, "pods", n) + last = time.Now() + } + if err := t.sleep(ctx); err != nil { + return err + } + } +} + +// deleteReapJob deletes the reap Job of an ordinal and waits until it's gone. +func (t *Teardown) deleteReapJob(ctx context.Context, n int) error { + if err := t.Kube.DeleteReapJob(ctx, n); err != nil { + return fmt.Errorf("delete old reap Job %s: %w", t.Kube.JobName(n), err) + } + for { + exists, err := t.Kube.ReapJobExists(ctx, n) + if err != nil { + return fmt.Errorf("check old reap Job %s: %w", t.Kube.JobName(n), err) + } + if !exists { + return nil + } + if err := t.sleep(ctx); err != nil { + return err + } + } +} + +// waitReaps waits until every ordinal's reap Job has finished. +func (t *Teardown) waitReaps(ctx context.Context, ordinals []int) (map[int]decide.JobState, error) { + for { + jobs, err := t.Kube.ReapJobs(ctx) + if err != nil { + return nil, fmt.Errorf("list reap Jobs: %w", err) + } + done := true + for _, n := range ordinals { + if s, ok := jobs[n]; !ok || s == decide.JobRunning { + done = false + break + } + } + if done { + return jobs, nil + } + if err := t.sleep(ctx); err != nil { + return nil, err + } + } +} + +func (t *Teardown) sleep(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(t.Poll): + return nil + } +} diff --git a/k8s/scaler/internal/teardown/teardown_test.go b/k8s/scaler/internal/teardown/teardown_test.go new file mode 100644 index 0000000..ed997bf --- /dev/null +++ b/k8s/scaler/internal/teardown/teardown_test.go @@ -0,0 +1,202 @@ +package teardown + +import ( + "bytes" + "context" + "log/slog" + "slices" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" + + "github.com/StackGuardian/sg-runner/k8s/scaler/internal/kube" +) + +const ns = "sg" + +func pod(name, component string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, + Labels: map[string]string{"app.kubernetes.io/component": component}}} +} + +func pvc(name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} +} + +// cluster stands in for the controllers and the kubelet: it removes pods +// once their workload is scaled to 0 and finishes reap Jobs, failing those +// of the ordinals in fail. +type cluster struct { + t *testing.T + k *kube.Kube + fail map[string]bool + logs *bytes.Buffer +} + +func newCluster(t *testing.T, fail []string, objs ...runtime.Object) *cluster { + c := &cluster{t: t, k: kube.New(fake.NewClientset(objs...), ns, "sg-runner", "runner:1", "creds", "cfg"), + fail: map[string]bool{}, logs: &bytes.Buffer{}} + for _, name := range fail { + c.fail[name] = true + } + return c +} + +func (c *cluster) step(ctx context.Context) { + cs := c.k.Client + if d, err := cs.AppsV1().Deployments(ns).Get(ctx, "sg-runner-scaler", metav1.GetOptions{}); err == nil && *d.Spec.Replicas == 0 { + c.deletePods(ctx, "scaler") + } + if s, err := c.k.GetStatefulSet(ctx); err == nil && *s.Spec.Replicas == 0 { + c.deletePods(ctx, "runner") + } + jobs, _ := cs.BatchV1().Jobs(ns).List(ctx, metav1.ListOptions{}) + for _, j := range jobs.Items { + if j.Status.Succeeded+j.Status.Failed > 0 { + continue + } + if c.fail[j.Name] { + j.Status.Failed = 1 + } else { + j.Status.Succeeded = 1 + } + if _, err := cs.BatchV1().Jobs(ns).UpdateStatus(ctx, &j, metav1.UpdateOptions{}); err != nil { + c.t.Error(err) + } + } +} + +func (c *cluster) deletePods(ctx context.Context, component string) { + pods, _ := c.k.Client.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/component=" + component}) + for _, p := range pods.Items { + _ = c.k.Client.CoreV1().Pods(ns).Delete(ctx, p.Name, metav1.DeleteOptions{}) + } +} + +func (c *cluster) run() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + go func() { + for ctx.Err() == nil { + c.step(ctx) + time.Sleep(time.Millisecond) + } + }() + td := &Teardown{ + Kube: c.k, + ScalerDeployment: "sg-runner-scaler", + ScalerSelector: "app.kubernetes.io/component=scaler", + RunnerSelector: "app.kubernetes.io/component=runner", + Poll: time.Millisecond, + Log: slog.New(slog.NewJSONHandler(c.logs, nil)), + } + return td.Run(ctx) +} + +func (c *cluster) claims() []string { + list, err := c.k.Client.CoreV1().PersistentVolumeClaims(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + c.t.Fatal(err) + } + var names []string + for _, p := range list.Items { + names = append(names, p.Name) + } + slices.Sort(names) + return names +} + +func workloads() []runtime.Object { + return []runtime.Object{ + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-scaler", Namespace: ns}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)}}, + &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner", Namespace: ns}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To[int32](2)}}, + pod("sg-runner-scaler-abc", "scaler"), + pod("sg-runner-0", "runner"), + pod("sg-runner-1", "runner"), + } +} + +func TestTeardownDeregistersEveryRunnerAndDeletesPVCs(t *testing.T) { + objs := append(workloads(), + // Awake runners 0 and 1, parked runner 3, an orphaned Docker cache 5, + // and a PVC of another release that must stay. + pvc("state-sg-runner-0"), pvc("docker-sg-runner-0"), + pvc("state-sg-runner-1"), pvc("docker-sg-runner-1"), + pvc("state-sg-runner-3"), pvc("docker-sg-runner-3"), + pvc("docker-sg-runner-5"), + pvc("state-other-0"), + // A failed reap the scaler left behind: must not count as this run's. + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "sg-runner-reap-3", Namespace: ns, + Labels: map[string]string{"app": "sg-runner-reaper"}}, Status: batchv1.JobStatus{Failed: 1}}, + ) + c := newCluster(t, nil, objs...) + if err := c.run(); err != nil { + t.Fatalf("Run: %v\n%s", err, c.logs) + } + if got, want := c.claims(), []string{"state-other-0"}; !slices.Equal(got, want) { + t.Errorf("PVCs left = %v, want %v", got, want) + } + jobs, _ := c.k.Client.BatchV1().Jobs(ns).List(context.Background(), metav1.ListOptions{}) + if len(jobs.Items) != 0 { + t.Errorf("%d reap Jobs left, want 0", len(jobs.Items)) + } + if !strings.Contains(c.logs.String(), `"teardown complete"`) { + t.Errorf("no completion log:\n%s", c.logs) + } +} + +func TestTeardownKeepsPVCsOfRunnersItCouldNotDeregister(t *testing.T) { + objs := append(workloads(), + pvc("state-sg-runner-0"), pvc("docker-sg-runner-0"), + pvc("state-sg-runner-1"), pvc("docker-sg-runner-1"), + ) + c := newCluster(t, []string{"sg-runner-reap-1"}, objs...) + err := c.run() + if err == nil || !strings.Contains(err.Error(), "could not deregister runners [1]") { + t.Fatalf("Run error = %v, want one naming runner 1", err) + } + if got, want := c.claims(), []string{"docker-sg-runner-1", "state-sg-runner-1"}; !slices.Equal(got, want) { + t.Errorf("PVCs left = %v, want %v", got, want) + } + // The failed Job stays for its logs. + if _, err := c.k.Client.BatchV1().Jobs(ns).Get(context.Background(), "sg-runner-reap-1", metav1.GetOptions{}); err != nil { + t.Errorf("failed reap Job deleted: %v", err) + } +} + +func TestTeardownWithoutScalerOrRunners(t *testing.T) { + // Scaler disabled and the StatefulSet already gone: nothing to do, no error. + c := newCluster(t, nil, pvc("docker-sg-runner-2")) + ctx := context.Background() + td := &Teardown{Kube: c.k, RunnerSelector: "app.kubernetes.io/component=runner", Poll: time.Millisecond, + Log: slog.New(slog.NewJSONHandler(c.logs, nil))} + if err := td.Run(ctx); err != nil { + t.Fatalf("Run: %v\n%s", err, c.logs) + } + if got := c.claims(); len(got) != 0 { + t.Errorf("PVCs left = %v, want none", got) + } +} + +func TestTeardownStopsOnCancel(t *testing.T) { + // A runner that never stops (a workflow still running) blocks until the + // context ends: the Job's deadline, not a hang. + c := newCluster(t, nil, pod("sg-runner-0", "runner")) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + td := &Teardown{Kube: c.k, RunnerSelector: "app.kubernetes.io/component=runner", Poll: time.Millisecond, + Log: slog.New(slog.NewJSONHandler(c.logs, nil))} + if err := td.Run(ctx); err == nil { + t.Fatal("Run returned nil with a runner pod that never stops") + } +} diff --git a/k8s/scaler/internal/wrapper/wrapper.go b/k8s/scaler/internal/wrapper/wrapper.go new file mode 100644 index 0000000..01e2091 --- /dev/null +++ b/k8s/scaler/internal/wrapper/wrapper.go @@ -0,0 +1,173 @@ +//go:build unix + +// Package wrapper is sg-step-wrapper: the init process of a step container. +// It runs the step's command as its child, forwards signals to it, reaps +// zombies, and exits 0 for the exit codes StackGuardian gives meaning to (11: +// approval required, 12: finished early), leaving the code in the +// container's termination message, so the pod ends Completed rather than +// Failed. See k8s/docs/steps-api.md. +package wrapper + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "syscall" + "time" +) + +// TerminationLog is where the kubelet reads a container's termination message. +const TerminationLog = "/dev/termination-log" + +// ExitCodePrefix starts the termination message of a mapped exit code. +const ExitCodePrefix = "sg-exit-code=" + +// Mapped are the exit codes reported in the termination message and turned +// into exit 0. +var Mapped = map[int]bool{11: true, 12: true} + +// forwarded are the signals passed on to the child. +var forwarded = []os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGUSR2} + +// Install copies the running executable to dest, executable by anyone. +func Install(dest string) error { + self, err := os.Executable() + if err != nil { + return err + } + in, err := os.Open(self) + if err != nil { + return err + } + defer in.Close() + tmp, err := os.CreateTemp(filepath.Dir(dest), ".sg-step-wrapper-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, in); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Chmod(0o755); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dest) +} + +// Options are a run's stdio and termination log (the process's own and +// TerminationLog when unset). +type Options struct { + Stdin, Stdout, Stderr *os.File + TerminationLog string +} + +// Run runs argv (argv[0] looked up in PATH) as a child until it exits and +// returns the exit code to exit with. +func Run(argv []string, o Options) int { + if o.Stdin == nil { + o.Stdin = os.Stdin + } + if o.Stdout == nil { + o.Stdout = os.Stdout + } + if o.Stderr == nil { + o.Stderr = os.Stderr + } + if o.TerminationLog == "" { + o.TerminationLog = TerminationLog + } + fail := func(code int, err error) int { + fmt.Fprintf(o.Stderr, "sg-step-wrapper: %v\n", err) + return code + } + if len(argv) == 0 { + return fail(2, errors.New("no command")) + } + + // Before the child starts, so neither a signal nor its exit is missed. + sigs := make(chan os.Signal, 32) + signal.Notify(sigs, append([]os.Signal{syscall.SIGCHLD}, forwarded...)...) + defer signal.Stop(sigs) + + path, err := exec.LookPath(argv[0]) + if err != nil { + // As shells do: 126 for a file that can't run, 127 for none. + if errors.Is(err, os.ErrPermission) { + return fail(126, err) + } + return fail(127, err) + } + // os.File stdio: the child gets the descriptors themselves, nothing is + // copied, and only reap() below waits for it. + cmd := &exec.Cmd{Path: path, Args: argv, Stdin: o.Stdin, Stdout: o.Stdout, Stderr: o.Stderr} + if err := cmd.Start(); err != nil { + if errors.Is(err, os.ErrPermission) { + return fail(126, err) + } + return fail(127, err) + } + child := cmd.Process.Pid + + // Signals can coalesce or, past the channel's buffer, drop: reap now and + // then anyway. + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + if ws, done := reap(child); done { + return exitCode(ws, o.TerminationLog) + } + select { + case sig := <-sigs: + if sig != syscall.SIGCHLD { + _ = cmd.Process.Signal(sig) + } + case <-tick.C: + } + } +} + +// reap waits for every exited child without blocking (as PID 1, orphans are +// ours too) and reports the main child's status once it exited. +func reap(child int) (syscall.WaitStatus, bool) { + var status syscall.WaitStatus + done := false + for { + var ws syscall.WaitStatus + pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil) + if errors.Is(err, syscall.EINTR) { + continue + } + if pid <= 0 || err != nil { + return status, done + } + if pid == child { + status, done = ws, true + } + } +} + +// exitCode is the code to exit with for the child's status; a mapped code +// goes to the termination log and becomes 0. +func exitCode(ws syscall.WaitStatus, terminationLog string) int { + if ws.Signaled() { + return 128 + int(ws.Signal()) + } + code := ws.ExitStatus() + if Mapped[code] { + // Best effort: without the message the step still ends; only its code + // is lost. + _ = os.WriteFile(terminationLog, []byte(ExitCodePrefix+strconv.Itoa(code)), 0o644) + return 0 + } + return code +} diff --git a/k8s/scaler/internal/wrapper/wrapper_test.go b/k8s/scaler/internal/wrapper/wrapper_test.go new file mode 100644 index 0000000..91e8fa9 --- /dev/null +++ b/k8s/scaler/internal/wrapper/wrapper_test.go @@ -0,0 +1,166 @@ +//go:build unix + +package wrapper + +import ( + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" +) + +// run runs a shell script under Run with its own termination log. +func run(t *testing.T, script string, args ...string) (code int, termLog string, stderr string) { + t.Helper() + dir := t.TempDir() + logPath := filepath.Join(dir, "termination-log") + errPath := filepath.Join(dir, "stderr") + errFile, err := os.Create(errPath) + if err != nil { + t.Fatal(err) + } + defer errFile.Close() + code = Run(append([]string{"sh", "-c", script}, args...), Options{Stderr: errFile, TerminationLog: logPath}) + b, _ := os.ReadFile(logPath) + e, _ := os.ReadFile(errPath) + return code, string(b), string(e) +} + +func TestExitCodes(t *testing.T) { + for _, c := range []struct { + script string + code int + message string + }{ + {"exit 0", 0, ""}, + {"exit 1", 1, ""}, + {"exit 10", 10, ""}, + {"exit 11", 0, "sg-exit-code=11"}, + {"exit 12", 0, "sg-exit-code=12"}, + {"exit 13", 13, ""}, + {"exit 137", 137, ""}, + {"kill -TERM $$", 128 + 15, ""}, + {"kill -KILL $$", 128 + 9, ""}, + } { + code, msg, _ := run(t, c.script) + if code != c.code || msg != c.message { + t.Errorf("%q: exit %d, message %q; want %d, %q", c.script, code, msg, c.code, c.message) + } + } +} + +func TestUnwritableTerminationLogStillExitsZero(t *testing.T) { + code := Run([]string{"sh", "-c", "exit 11"}, Options{TerminationLog: "/nonexistent/dir/termination-log"}) + if code != 0 { + t.Errorf("exit %d", code) + } +} + +func TestArgsAndPathLookup(t *testing.T) { + dir := t.TempDir() + out := filepath.Join(dir, "out") + // "sh" is found through PATH, the args pass as they are. + code, _, _ := run(t, `printf '%s|' "$@" > "$0"`, out, "a b", "$HOME", "") + b, _ := os.ReadFile(out) + if code != 0 || string(b) != "a b|$HOME||" { + t.Errorf("exit %d, args %q", code, b) + } +} + +func TestCommandNotFound(t *testing.T) { + dir := t.TempDir() + errFile, _ := os.Create(filepath.Join(dir, "stderr")) + defer errFile.Close() + if code := Run([]string{"sg-no-such-command"}, Options{Stderr: errFile}); code != 127 { + t.Errorf("not found: exit %d", code) + } + noExec := filepath.Join(dir, "script") + _ = os.WriteFile(noExec, []byte("#!/bin/sh\n"), 0o644) + if code := Run([]string{noExec}, Options{Stderr: errFile}); code != 126 { + t.Errorf("not executable: exit %d", code) + } + if code := Run(nil, Options{Stderr: errFile}); code != 2 { + t.Errorf("no command: exit %d", code) + } +} + +// TestForwardsSignals sends each signal to this process (as the kubelet would +// to the wrapper) and checks the child got it. +func TestForwardsSignals(t *testing.T) { + for name, sig := range map[string]syscall.Signal{ + "TERM": syscall.SIGTERM, "INT": syscall.SIGINT, "HUP": syscall.SIGHUP, + "QUIT": syscall.SIGQUIT, "USR1": syscall.SIGUSR1, "USR2": syscall.SIGUSR2, + } { + dir := t.TempDir() + ready, got := filepath.Join(dir, "ready"), filepath.Join(dir, "got") + script := `trap 'echo ` + name + ` > "$1"; exit 12' ` + name + `; touch "$0"; while :; do sleep 0.01; done` + done := make(chan struct{}) + var code int + var msg string + go func() { code, msg, _ = run(t, script, ready, got); close(done) }() + waitFile(t, ready) + if err := syscall.Kill(os.Getpid(), sig); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("%s: the child never ended", name) + } + b, _ := os.ReadFile(got) + if strings.TrimSpace(string(b)) != name || code != 0 || msg != "sg-exit-code=12" { + t.Errorf("%s: child got %q, exit %d, message %q", name, b, code, msg) + } + } +} + +// TestReapsZombies: an exited child that isn't the step's (as PID 1 inherits +// orphans) is reaped along the way. +func TestReapsZombies(t *testing.T) { + // This process's own child, never waited for: a zombie once it exits. + var attr os.ProcAttr + p, err := os.StartProcess("/bin/sh", []string{"sh", "-c", "exit 3"}, &attr) + if err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + if code, _, _ := run(t, "sleep 0.1; exit 4"); code != 4 { + t.Errorf("exit %d", code) + } + // Reaped already: nothing left to wait for. + var ws syscall.WaitStatus + if pid, err := syscall.Wait4(p.Pid, &ws, syscall.WNOHANG, nil); err == nil || pid != -1 { + t.Errorf("the zombie is still there: pid %d, err %v", pid, err) + } +} + +func TestInstall(t *testing.T) { + dest := filepath.Join(t.TempDir(), "sg-step-wrapper") + if err := Install(dest); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(dest) + self, _ := os.Executable() + sfi, _ := os.Stat(self) + if err != nil || fi.Mode().Perm() != 0o755 || fi.Size() != sfi.Size() { + t.Errorf("installed %v %v (want 0755, %d bytes)", fi.Mode(), err, sfi.Size()) + } + entries, _ := os.ReadDir(filepath.Dir(dest)) + if len(entries) != 1 { + t.Errorf("left temp files: %v", entries) + } +} + +func waitFile(t *testing.T, path string) { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(5 * time.Millisecond) { + if _, err := os.Stat(path); err == nil { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s never appeared", path) + } + } +} diff --git a/main.sh b/main.sh index ac8fadf..ee30a1e 100755 --- a/main.sh +++ b/main.sh @@ -239,7 +239,7 @@ print_details() { #{{{ echo details_frame "Host Information" details_item "Hostname" "$HOSTNAME" - details_item "Private IP Address" "$(ip route | grep default | cut -d" " -f9)" + details_item "Private IP Address" "$(detect_private_ip)" details_item "Public IP Address" "$(curl -fSs ifconfig.me)" details_item "HTTP PROXY" "$HTTP_PROXY" echo @@ -328,13 +328,19 @@ EOF #}}}: api_call update_runner_group() { #{{{ + # Best-effort report, sent at most once per error: api_call reports its own + # failures through err(), which would call back in here and retry until the + # stack overflows (one authenticated request per round) whenever the API is + # unreachable. The guard is dynamically scoped, so nested calls see it. + [[ -n "${REPORTING_RUNNER_GROUP_ERROR:-}" ]] && return 0 + local REPORTING_RUNNER_GROUP_ERROR=1 local err_msg ip_addr payload url url="${SG_BASE_API}/orgs/${ORGANIZATION_ID}/runnergroups/${RUNNER_GROUP_ID}/" err_msg=$(printf '%s' "$1" | tr -cd "[:print:]") debug "Error message ${err_msg}" - ip_addr=$(ip route | grep default | cut -d" " -f9) + ip_addr=$(detect_private_ip) payload=$(jq -n \ --arg ip "$ip_addr" \ @@ -344,11 +350,15 @@ update_runner_group() { #{{{ --arg command "${0} $*" \ '{RunnerRegistrationErrors: {($ip): {RunnerId: $runner_id, error: $error, timestamp: $timestamp, command: $command}}}') - if api_call "PATCH" "$url" "$payload"; then + # Subshell: api_call exits on an empty response, and a failed report must + # not end a caller that only meant to log a non-fatal error. + if (api_call "PATCH" "$url" "$payload"); then debug "updated runner group with error msg" else debug "failed to update runner group with error msg" fi + # debug() returns non-zero when --debug is off; a report never fails err(). + return 0 } #}}}: update_runner_group @@ -594,6 +604,27 @@ clean_local_setup() { #{{{ } #}}}: clean_local_setup +detect_private_ip() { #{{{ + local ip_addr + # Ask the kernel which source address it would use for outbound traffic. + # This is a local route lookup (no packet is sent) and works whether the + # default route is dhcp, static or a pod's (no "src" on the default route). + # Pick the token after "src" by name: its position varies between setups. + ip_addr=$(ip -4 route get 1.1.1.1 2>/dev/null | + awk '{for (i = 1; i < NF; i++) if ($i == "src") {print $(i + 1); exit}}') + # No usable route (e.g. no default gateway): take the first global IPv4. + if [[ -z "$ip_addr" ]]; then + ip_addr=$(ip -4 -o addr show scope global 2>/dev/null | + awk '{split($4, addr, "/"); print addr[1]; exit}') + fi + # Last resort for hosts where the ip queries fail; not every hostname has -I. + if [[ -z "$ip_addr" ]] && command -v hostname >/dev/null 2>&1; then + ip_addr=$(hostname -I 2>/dev/null | awk '{print $1}') + fi + printf '%s' "$ip_addr" +} +#}}}: detect_private_ip + check_variable_value() { #{{{ local variable_name=$1 [[ -z "${!variable_name}" ]] && diff --git a/test/README.md b/test/README.md index f27dca9..036c846 100644 --- a/test/README.md +++ b/test/README.md @@ -140,7 +140,8 @@ Set the override env vars **before** calling the function under test. Use | `curl` | `MOCK_CURL_OUTPUT` (empty) | `MOCK_CURL_EXIT` (`0`) | | `crontab` | `MOCK_CRONTAB_LIST` (empty; used by `crontab -l`) | `MOCK_CRONTAB_EXIT` (`0`) | | `iptables` | (no stdout); `MOCK_IPTABLES_CHECK_EXIT` (`1`) is the exit for `-C` rule checks | `MOCK_IPTABLES_EXIT` (`0`) | -| `ip` | `MOCK_IP_ROUTE_OUTPUT` (default route, field 9 = `10.0.0.1`) | `MOCK_IP_EXIT` (`0`) | +| `ip` | `MOCK_IP_ROUTE_GET_OUTPUT` (`route get` reply, src `10.0.0.1`), `MOCK_IP_ADDR_OUTPUT` (one global `10.0.0.1/24`), `MOCK_IP_ROUTE_OUTPUT` (dhcp default route) | `MOCK_IP_EXIT` (`0`); per query `MOCK_IP_ROUTE_GET_EXIT`, `MOCK_IP_ADDR_EXIT` | +| `hostname` | `MOCK_HOSTNAME_OUTPUT` (`10.0.0.1`; used by `hostname -I`) | `MOCK_HOSTNAME_EXIT` (`0`) | | `free` | `MOCK_FREE_OUTPUT` (a minimal `free -h` table) | `MOCK_FREE_EXIT` (`0`) | | `nproc` | `MOCK_NPROC_OUTPUT` (`4`) | `MOCK_NPROC_EXIT` (`0`) | | `uptime` | `MOCK_UPTIME_OUTPUT` (a typical `uptime` line with load average) | `MOCK_UPTIME_EXIT` (`0`) | diff --git a/test/helpers/k8s.bash b/test/helpers/k8s.bash new file mode 100644 index 0000000..804357a --- /dev/null +++ b/test/helpers/k8s.bash @@ -0,0 +1,33 @@ +# shellcheck shell=bash +# +# k8s.bash - extra setup for tests of k8s/runner.sh. Load after load.bash: +# load "${BATS_TEST_DIRNAME}/../helpers/load.bash" +# load "${BATS_TEST_DIRNAME}/../helpers/k8s.bash" + +SG_RUNNER_SH="${SG_REPO_ROOT}/k8s/runner.sh" +export SG_RUNNER_SH + +# Fake /proc files the agent sizing reads, so no test depends on the host. +export PROC_STATUS_FILE="${SG_TEST_TMPDIR}/proc-self-status" +export MEMINFO_FILE="${SG_TEST_TMPDIR}/proc-meminfo" + +# Source runner.sh (which sources main.sh from the repo root), exposing its +# functions. Its dispatcher only runs when executed, so nothing starts. +load_runner() { + # shellcheck source=/dev/null + if ! source "${SG_RUNNER_SH}"; then + echo "load_runner: failed to source ${SG_RUNNER_SH}" >&2 + return 1 + fi +} + +# Usage: fake_cpus_allowed , e.g. "0-3,8-11" +fake_cpus_allowed() { + printf 'Name:\tawk\nCpus_allowed:\tff\nCpus_allowed_list:\t%s\nMems_allowed_list:\t0\n' \ + "$1" >"${PROC_STATUS_FILE}" +} + +# Usage: fake_mem_total_mib +fake_mem_total_mib() { + printf 'MemTotal: %d kB\nMemFree: 1024 kB\n' "$(($1 * 1024))" >"${MEMINFO_FILE}" +} diff --git a/test/mocks/bin/hostname b/test/mocks/bin/hostname new file mode 100755 index 0000000..5574d0c --- /dev/null +++ b/test/mocks/bin/hostname @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Mock: hostname +# Records the call. detect_private_ip falls back to `hostname -I`. +# $MOCK_HOSTNAME_OUTPUT (default: 10.0.0.1) +# $MOCK_HOSTNAME_EXIT (default: 0) +printf '%s\n' "hostname $*" >>"${MOCK_CALL_LOG}" +echo "${MOCK_HOSTNAME_OUTPUT-10.0.0.1}" +exit "${MOCK_HOSTNAME_EXIT:-0}" diff --git a/test/mocks/bin/ip b/test/mocks/bin/ip index 9a44818..8ec1331 100755 --- a/test/mocks/bin/ip +++ b/test/mocks/bin/ip @@ -1,14 +1,26 @@ #!/usr/bin/env bash # Mock: ip -# Records the call. main.sh extracts the node IP with: -# ip route | grep default | cut -d" " -f9 -# so the default route line below is shaped to put 10.0.0.1 in field 9 -# (default via dev proto

src 10.0.0.1 metric ). -# $MOCK_IP_ROUTE_OUTPUT (default: a default route whose field 9 is 10.0.0.1) -# $MOCK_IP_EXIT (default: 0) +# Records the call. main.sh's detect_private_ip queries, in order: +# ip -4 route get 1.1.1.1 -> $MOCK_IP_ROUTE_GET_OUTPUT +# ip -4 -o addr show scope global -> $MOCK_IP_ADDR_OUTPUT +# Each query prints its output and exits with its own exit override, so a +# test can fail the route lookup and still exercise the addr fallback. +# $MOCK_IP_ROUTE_OUTPUT (default: a dhcp default route with src 10.0.0.1) +# $MOCK_IP_ROUTE_GET_OUTPUT (default: a route get reply with src 10.0.0.1) +# $MOCK_IP_ROUTE_GET_EXIT (default: $MOCK_IP_EXIT) +# $MOCK_IP_ADDR_OUTPUT (default: one global eth0 address, 10.0.0.1/24) +# $MOCK_IP_ADDR_EXIT (default: $MOCK_IP_EXIT) +# $MOCK_IP_EXIT (default: 0) printf '%s\n' "ip $*" >>"${MOCK_CALL_LOG}" -if [[ "$1" == "route" ]]; then +args=" $* " +if [[ "$args" == *" route get "* ]]; then + echo "${MOCK_IP_ROUTE_GET_OUTPUT-1.1.1.1 via 10.0.0.254 dev eth0 src 10.0.0.1 uid 0}" + exit "${MOCK_IP_ROUTE_GET_EXIT:-${MOCK_IP_EXIT:-0}}" +elif [[ "$args" == *" addr "* ]]; then + echo "${MOCK_IP_ADDR_OUTPUT-2: eth0 inet 10.0.0.1/24 brd 10.0.0.255 scope global eth0\ valid_lft forever preferred_lft forever}" + exit "${MOCK_IP_ADDR_EXIT:-${MOCK_IP_EXIT:-0}}" +elif [[ "$args" == *" route "* ]]; then echo "${MOCK_IP_ROUTE_OUTPUT:-default via 10.0.0.254 dev eth0 proto dhcp src 10.0.0.1 metric 100}" fi diff --git a/test/unit/detect_private_ip.bats b/test/unit/detect_private_ip.bats new file mode 100644 index 0000000..6464c80 --- /dev/null +++ b/test/unit/detect_private_ip.bats @@ -0,0 +1,65 @@ +#!/usr/bin/env bats +# +# Unit tests for detect_private_ip - reads the source address of the outbound +# route by name, then falls back to the first global IPv4, then hostname -I. + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" +} + +@test "detect_private_ip reads src from a dhcp-style route" { + export MOCK_IP_ROUTE_GET_OUTPUT="1.1.1.1 via 10.0.0.254 dev eth0 proto dhcp src 10.0.0.1 metric 100 uid 0" + load_main + run detect_private_ip + assert_success + assert_output "10.0.0.1" + assert_called ip "-4 route get 1.1.1.1" +} + +@test "detect_private_ip reads src behind a static default route" { + export MOCK_IP_ROUTE_GET_OUTPUT="1.1.1.1 via 10.0.0.1 dev eth0 src 10.0.0.5 uid 0" + load_main + run detect_private_ip + assert_success + assert_output "10.0.0.5" +} + +@test "detect_private_ip reads src in a Kubernetes pod" { + export MOCK_IP_ROUTE_GET_OUTPUT="1.1.1.1 via 192.168.194.1 dev eth0 src 192.168.194.8 uid 0" + load_main + run detect_private_ip + assert_success + assert_output "192.168.194.8" +} + +@test "detect_private_ip falls back to the first global address when route get fails" { + export MOCK_IP_ROUTE_GET_OUTPUT="" + export MOCK_IP_ROUTE_GET_EXIT=2 + export MOCK_IP_ADDR_OUTPUT="2: eth0 inet 172.16.4.20/16 brd 172.16.255.255 scope global eth0 +3: eth1 inet 172.17.0.1/16 scope global eth1" + load_main + run detect_private_ip + assert_success + assert_output "172.16.4.20" + assert_called ip "-4 -o addr show scope global" + refute_called hostname +} + +@test "detect_private_ip falls back to hostname -I when both ip queries fail" { + export MOCK_IP_EXIT=1 MOCK_IP_ROUTE_GET_OUTPUT="" MOCK_IP_ADDR_OUTPUT="" + export MOCK_HOSTNAME_OUTPUT="10.1.2.3 172.17.0.1" + load_main + run detect_private_ip + assert_success + assert_output "10.1.2.3" + assert_called hostname "-I" +} + +@test "detect_private_ip prints nothing when every source fails" { + export MOCK_IP_EXIT=1 MOCK_IP_ROUTE_GET_OUTPUT="" MOCK_IP_ADDR_OUTPUT="" + export MOCK_HOSTNAME_OUTPUT="" MOCK_HOSTNAME_EXIT=1 + load_main + run detect_private_ip + assert_success + assert_output "" +} diff --git a/test/unit/k8s_prune.bats b/test/unit/k8s_prune.bats new file mode 100644 index 0000000..04530d4 --- /dev/null +++ b/test/unit/k8s_prune.bats @@ -0,0 +1,135 @@ +#!/usr/bin/env bats +# +# Unit tests for the nested Docker cleanup in k8s/runner.sh (prune_loop in +# the dockerd container), which replaces the VM's `main.sh prune` cron. +# +# Same rule as test/unit/prune.bats: no sweep that can remove a container may +# run without the until= age guard, or it races the ECS agent between +# docker create and docker start (CannotStartContainerError). + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" + load "${BATS_TEST_DIRNAME}/../helpers/k8s.bash" + load_runner +} + +# Fake `df -P` output for the Docker data root at % used. +disk_at() { + export MOCK_DF_OUTPUT="Filesystem 1024-blocks Used Available Capacity Mounted on +/dev/vdb 20511312 0 0 $1% /var/lib/docker" +} + +prune_calls() { + mock_calls docker | grep -F "prune" || true +} + +#{{{ prune_enabled + +@test "prune_enabled defaults to on, every 4 hours" { + assert_equal "$SG_PRUNE_INTERVAL_SECONDS" "14400" + run prune_enabled + assert_success +} + +@test "prune_enabled is off at interval 0 or a non-number" { + SG_PRUNE_INTERVAL_SECONDS=0 + run prune_enabled + assert_failure + + SG_PRUNE_INTERVAL_SECONDS=4h + run prune_enabled + assert_failure +} + +#}}}: prune_enabled + +#{{{ should_prune_images + +@test "should_prune_images keeps the image cache below the threshold" { + SG_PRUNE_IMAGES_ABOVE_PCT=70 + run should_prune_images 69 + assert_failure + run should_prune_images 70 + assert_success +} + +@test "should_prune_images always prunes at threshold 0" { + SG_PRUNE_IMAGES_ABOVE_PCT=0 + run should_prune_images 1 + assert_success +} + +@test "should_prune_images prunes when disk usage is unknown" { + run should_prune_images "" + assert_success +} + +@test "docker_disk_used_pct reads Capacity from df" { + disk_at 42 + run docker_disk_used_pct + assert_output "42" + assert_called df "-P /var/lib/docker" +} + +#}}}: should_prune_images + +#{{{ prune_docker + +@test "prune_docker age-guards every sweep that can remove a container" { + run prune_docker + assert_success + + assert_called docker "container prune -f --filter until=24h" + refute [ -n "$(prune_calls | grep -F "system prune")" ] + refute [ -n "$(prune_calls | grep -E "(container|image|builder) prune" | grep -vF -- "--filter until=")" ] +} + +@test "prune_docker reclaims volumes with a volume-only prune" { + run prune_docker + assert_success + assert_called docker "volume prune -f" +} + +@test "prune_docker honours SG_PRUNE_UNTIL" { + SG_PRUNE_UNTIL=6h + run prune_docker + assert_called docker "container prune -f --filter until=6h" + assert_called docker "image prune -f --filter until=6h" + assert_called docker "builder prune -f --filter until=6h" +} + +@test "prune_docker keeps unused images while the disk has room" { + disk_at 30 + run prune_docker + assert_called docker "image prune -f --filter until=24h" + refute [ -n "$(prune_calls | grep -F "image prune -f -a")" ] +} + +@test "prune_docker prunes unused images once the disk fills up" { + disk_at 85 + run prune_docker + assert_called docker "image prune -f -a --filter until=24h" + assert_output --partial "unused images too" +} + +@test "prune_docker logs what each sweep reclaimed" { + export MOCK_DOCKER_VOLUME_OUTPUT="Deleted Volumes: +abc + +Total reclaimed space: 1.5GB" + run prune_docker + assert_success + assert_output --partial "volumes 1.5GB" + assert_output --partial "containers 0B" +} + +@test "prune_docker carries on and reports a failing sweep" { + export MOCK_DOCKER_EXIT=1 + run prune_docker + assert_success + assert_output --partial "containers failed" + assert_output --partial "volumes failed" + assert_called docker "volume prune -f" +} + +#}}}: prune_docker diff --git a/test/unit/k8s_runner.bats b/test/unit/k8s_runner.bats new file mode 100644 index 0000000..95ae838 --- /dev/null +++ b/test/unit/k8s_runner.bats @@ -0,0 +1,388 @@ +#!/usr/bin/env bats +# +# Unit tests for the pure helpers of k8s/runner.sh: agent sizing (CPU +# affinity, reserved memory), re-registering on a size change, reading the +# registration in ecs.config, and dockerd's authorization plugin. + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" + load "${BATS_TEST_DIRNAME}/../helpers/k8s.bash" + load_runner + mkdir -p "$ECS_CONFIG_DIR" +} + +#{{{ cpu_affinity_list + +@test "cpu_affinity_list takes the first n CPUs of a range" { + fake_cpus_allowed "0-7" + run cpu_affinity_list 2 + assert_success + assert_output "0,1" +} + +@test "cpu_affinity_list walks split ranges and single CPUs" { + fake_cpus_allowed "2-3,6,9-11" + run cpu_affinity_list 5 + assert_success + assert_output "2,3,6,9,10" +} + +@test "cpu_affinity_list prints nothing when n covers every allowed CPU" { + fake_cpus_allowed "0-3" + run cpu_affinity_list 4 + assert_success + assert_output "" + + run cpu_affinity_list 16 + assert_success + assert_output "" +} + +@test "cpu_affinity_list pins to the advertised CPUs out of a whole node" { + # Native steps: ecs-agent has no CPU limit, so it sees the node's CPUs. + fake_cpus_allowed "0-15" + run cpu_affinity_list 4 + assert_success + assert_output "0,1,2,3" +} + +#}}}: cpu_affinity_list + +#{{{ size_agent_memory + +@test "size_agent_memory leaves the agent alone without a memory limit" { + fake_mem_total_mib 16384 + unset SG_TASK_MEMORY_LIMIT_MIB ECS_RESERVED_MEMORY + size_agent_memory + assert [ -z "${ECS_RESERVED_MEMORY+set}" ] +} + +@test "size_agent_memory ignores a non-numeric limit" { + fake_mem_total_mib 16384 + unset ECS_RESERVED_MEMORY + SG_TASK_MEMORY_LIMIT_MIB="4Gi" size_agent_memory + assert [ -z "${ECS_RESERVED_MEMORY+set}" ] +} + +@test "size_agent_memory reserves everything but the limit minus dockerd overhead" { + fake_mem_total_mib 16384 + SG_TASK_MEMORY_LIMIT_MIB=4096 + size_agent_memory >/dev/null + # 16384 - (4096 - 256) + assert_equal "$ECS_RESERVED_MEMORY" "12544" + + run size_agent_memory + assert_output --partial "3840 of 16384" +} + +@test "size_agent_memory reserves nothing when the limit exceeds the node" { + fake_mem_total_mib 2048 + SG_TASK_MEMORY_LIMIT_MIB=4096 + size_agent_memory >/dev/null + assert_equal "$ECS_RESERVED_MEMORY" "0" +} + +@test "size_agent_memory refuses a reservation the agent would read as 0" { + # 128 GiB node, 4 GiB limit: 127232 MiB doesn't fit the agent's uint16. + fake_mem_total_mib 131072 + SG_TASK_MEMORY_LIMIT_MIB=4096 + run size_agent_memory + assert_failure + assert_output --partial "exceeds 65535MiB" +} + +@test "size_agent_memory advertises the native steps capacity in full" { + # Native steps: advertise 6Gi on a 16 GiB node, nothing kept for dockerd. + fake_mem_total_mib 16384 + SG_TASK_MEMORY_LIMIT_MIB=6144 SG_DOCKERD_OVERHEAD_MIB=0 size_agent_memory >/dev/null + assert_equal "$ECS_RESERVED_MEMORY" "10240" + + SG_TASK_MEMORY_LIMIT_MIB=6144 SG_DOCKERD_OVERHEAD_MIB=0 run size_agent_memory + assert_output --partial "6144 of 16384" +} + +@test "size_agent_memory advertises the whole node when native steps ask for more" { + fake_mem_total_mib 8192 + SG_TASK_MEMORY_LIMIT_MIB=12288 SG_DOCKERD_OVERHEAD_MIB=0 size_agent_memory >/dev/null + assert_equal "$ECS_RESERVED_MEMORY" "0" +} + +@test "size_agent_memory accepts a native steps reservation right at the uint16 limit" { + # 65535 MiB reserved still fits; one more doesn't. + fake_mem_total_mib $((65535 + 4096)) + SG_TASK_MEMORY_LIMIT_MIB=4096 SG_DOCKERD_OVERHEAD_MIB=0 size_agent_memory >/dev/null + assert_equal "$ECS_RESERVED_MEMORY" "65535" + + SG_TASK_MEMORY_LIMIT_MIB=4095 SG_DOCKERD_OVERHEAD_MIB=0 run size_agent_memory + assert_failure + assert_output --partial "exceeds 65535MiB" +} + +#}}}: size_agent_memory + +#{{{ advertised_resources + +@test "advertised_resources matches the dockerd limits minus dockerd overhead" { + # small: 2 CPUs / 4Gi on an 8-CPU 16 GiB node. + fake_cpus_allowed "0-7" + fake_mem_total_mib 16384 + SG_TASK_CPU_LIMIT=2 SG_TASK_MEMORY_LIMIT_MIB=4096 run advertised_resources + assert_success + assert_output "2048 3840" +} + +@test "advertised_resources takes native steps capacity in full" { + fake_cpus_allowed "0-15" + fake_mem_total_mib 16384 + SG_TASK_CPU_LIMIT=4 SG_TASK_MEMORY_LIMIT_MIB=6144 SG_DOCKERD_OVERHEAD_MIB=0 run advertised_resources + assert_success + assert_output "4096 6144" +} + +@test "advertised_resources caps at the node" { + fake_cpus_allowed "0-1" + fake_mem_total_mib 2048 + SG_TASK_CPU_LIMIT=4 SG_TASK_MEMORY_LIMIT_MIB=4096 run advertised_resources + assert_success + assert_output "2048 2048" +} + +@test "advertised_resources without limits is the whole node" { + fake_cpus_allowed "0-3" + fake_mem_total_mib 8192 + unset SG_TASK_CPU_LIMIT SG_TASK_MEMORY_LIMIT_MIB + run advertised_resources + assert_success + assert_output "4096 8192" +} + +@test "advertised_resources fails where the agent would refuse the reservation" { + fake_cpus_allowed "0-7" + fake_mem_total_mib 131072 + SG_TASK_CPU_LIMIT=2 SG_TASK_MEMORY_LIMIT_MIB=4096 run advertised_resources + assert_failure +} + +#}}}: advertised_resources + +#{{{ resized_while_idle / cmd_register + +# Usage: fake_runner_group [runner id] +# The runner group as SG returns it: this runner's container instance among +# others, and a secret that must never be printed. +fake_runner_group() { + local instance + instance=$(jq -nc --arg r "${5:-abc123-runner}" --argjson cpu "$2" --argjson mem "$3" --argjson run "$4" \ + '{runnerId: $r, status: "ACTIVE", runningTasksCount: $run, pendingTasksCount: 0, + registeredResources: [{name: "CPU", type: "INTEGER", integerValue: $cpu}, + {name: "MEMORY", type: "INTEGER", integerValue: $mem}, + {name: "PORTS", type: "STRINGSET", stringSetValue: ["22"]}]}') + export MOCK_CURL_OUTPUT="HTTP/1.1 $1 OK +Content-Type: application/json + +{\"msg\":{\"RunnerToken\":\"runner-group-secret-6f1d\",\"ContainerInstances\":[{\"runnerId\":\"other-runner\"},$instance]}}" +} + +setup_registered_runner() { + echo 'ECS_INSTANCE_ATTRIBUTES={"sg_runner_id":"abc123-runner"}' >"$ECS_CONFIG_DIR/ecs.config" + export SG_NODE_TOKEN=token SG_ORGANIZATION=org SG_RUNNER_GROUP=group + # Advertises 2048 / 3840 (small on an 8-CPU 16 GiB node). + fake_cpus_allowed "0-7" + fake_mem_total_mib 16384 + export SG_TASK_CPU_LIMIT=2 SG_TASK_MEMORY_LIMIT_MIB=4096 + is_registered() { return 0; } + deregister_runner() { echo "deregister called"; rm -f "$ECS_CONFIG_DIR/ecs.config"; } + # Stands in for the fresh registration that follows. + load_sg_credentials() { echo "fresh registration"; exit 0; } +} + +@test "cmd_register keeps a registration of the same size" { + setup_registered_runner + fake_runner_group 200 2048 3840 0 + run cmd_register + assert_success + assert_output --partial "Already registered" + refute_output --partial "deregister called" + refute_output --partial "fresh registration" + refute_output --partial "runner-group-secret" +} + +@test "cmd_register registers an idle runner of another size again" { + setup_registered_runner + fake_runner_group 200 1024 1792 0 + run cmd_register + assert_success + assert_output --partial "Registered CPU/memory 1024/1792, now 2048/3840:" + assert_output --partial "re-registering" + assert_output --partial "deregister called" + assert_output --partial "fresh registration" + refute_output --partial "runner-group-secret" +} + +@test "cmd_register keeps the old size while tasks run" { + setup_registered_runner + fake_runner_group 200 1024 1792 2 + run cmd_register + assert_success + assert_output --partial "2 task(s) running or pending, keeping the old size" + refute_output --partial "deregister called" + refute_output --partial "fresh registration" +} + +@test "cmd_register keeps the registration when SG can't be read" { + setup_registered_runner + fake_runner_group 500 1024 1792 0 + run cmd_register + assert_success + assert_output --partial "failed (500)" + assert_output --partial "Can't read the registered size from SG" + refute_output --partial "deregister called" + refute_output --partial "runner-group-secret" + + export MOCK_CURL_OUTPUT="" + run cmd_register + assert_success + assert_output --partial "Can't read the registered size from SG" + refute_output --partial "deregister called" +} + +@test "cmd_register keeps the registration when its instance isn't listed" { + setup_registered_runner + fake_runner_group 200 1024 1792 0 someone-else + run cmd_register + assert_success + assert_output --partial "Can't read the registered size from SG" + refute_output --partial "deregister called" +} + +@test "cmd_register keeps the registration without registered resources" { + setup_registered_runner + export MOCK_CURL_OUTPUT='HTTP/1.1 200 OK + +{"msg":{"ContainerInstances":[{"runnerId":"abc123-runner","runningTasksCount":0}]}}' + run cmd_register + assert_success + assert_output --partial "Can't read the registered size from SG" + refute_output --partial "deregister called" +} + +@test "cmd_register keeps the registration when deregistering fails" { + setup_registered_runner + fake_runner_group 200 1024 1792 0 + deregister_runner() { return 1; } + run cmd_register + assert_success + assert_output --partial "Could not deregister" + refute_output --partial "fresh registration" +} + +@test "cmd_register skips the size check when opted out" { + setup_registered_runner + fake_runner_group 200 1024 1792 0 + SG_REREGISTER_ON_RESIZE=false run cmd_register + assert_success + assert_output --partial "Already registered" + refute_output --partial "deregister called" + assert [ -z "$(mock_calls curl)" ] +} + +#}}}: resized_while_idle / cmd_register + +#{{{ load_ecs_config / configured_runner_id + +@test "load_ecs_config exports values with spaces, quotes and = intact" { + cat >"$ECS_CONFIG_DIR/ecs.config" <<'CFG' +# written by configure_local_data +ECS_CLUSTER=sg-cluster + +ECS_INSTANCE_ATTRIBUTES={"sg_runner_id":"runner 1","note":"say \"hi\" a=b"} +CFG + # No trailing newline on the last line. + printf 'ECS_LOGLEVEL=info' >>"$ECS_CONFIG_DIR/ecs.config" + + load_ecs_config + + assert_equal "$ECS_CLUSTER" "sg-cluster" + assert_equal "$ECS_INSTANCE_ATTRIBUTES" '{"sg_runner_id":"runner 1","note":"say \"hi\" a=b"}' + assert_equal "$(jq -r .note <<<"$ECS_INSTANCE_ATTRIBUTES")" 'say "hi" a=b' + assert_equal "$ECS_LOGLEVEL" "info" + assert_equal "$(bash -c 'printf %s "$ECS_CLUSTER"')" "sg-cluster" +} + +@test "configured_runner_id reads sg_runner_id from the instance attributes" { + cat >"$ECS_CONFIG_DIR/ecs.config" <<'CFG' +ECS_CLUSTER=sg-cluster +ECS_INSTANCE_ATTRIBUTES={"sg_organization":"org","sg_runner_id":"abc123-runner"} +CFG + run configured_runner_id + assert_success + assert_output "abc123-runner" +} + +@test "configured_runner_id dies when the runner isn't registered" { + run configured_runner_id + assert_failure + assert_output --partial "Not registered" +} + +#}}}: load_ecs_config / configured_runner_id + +#{{{ cmd_reap + +@test "cmd_reap succeeds without deregistering a runner that never registered" { + deregister_runner() { echo "deregister called"; } + run cmd_reap + assert_success + assert_output --partial "nothing to deregister" + refute_output --partial "deregister called" +} + +@test "cmd_reap deregisters a registered runner" { + echo 'ECS_INSTANCE_ATTRIBUTES={"sg_runner_id":"abc123-runner"}' >"$ECS_CONFIG_DIR/ecs.config" + deregister_runner() { echo "deregister called"; } + run cmd_reap + assert_success + assert_output "deregister called" +} + +#}}}: cmd_reap + +#{{{ dockerd authorization plugin + +# A unix socket at a short path (sun_path is ~104 bytes on macOS). +make_socket() { + python3 -c 'import socket, sys; socket.socket(socket.AF_UNIX).bind(sys.argv[1])' "$1" +} + +@test "dockerd_args turns on the steps broker's authorization plugin" { + run dockerd_args + assert_success + assert_line "--host=unix:///var/run/docker.sock" + assert_line "--authorization-plugin=sg-steps-guard" +} + +@test "wait_for_authz_plugin returns once the plugin socket exists" { + local dir + dir=$(mktemp -d /tmp/sg-authz.XXXXXX) + make_socket "$dir/sg-steps-guard.sock" + SG_AUTHZ_PLUGIN_SOCKET="$dir/sg-steps-guard.sock" run wait_for_authz_plugin + rm -rf "$dir" + assert_success + assert_output --partial "Authorization plugin" +} + +@test "wait_for_authz_plugin dies without the socket: dockerd never starts unguarded" { + SG_AUTHZ_PLUGIN_SOCKET="$SG_TEST_TMPDIR/missing.sock" SG_AUTHZ_PLUGIN_WAIT_SECONDS=1 \ + SG_AUTHZ_PLUGIN_POLL_SECONDS=0.2 run wait_for_authz_plugin + assert_failure + assert_output --partial "not starting dockerd unguarded" +} + +@test "wait_for_authz_plugin doesn't take a plain file for the socket" { + touch "$SG_TEST_TMPDIR/plain.sock" + SG_AUTHZ_PLUGIN_SOCKET="$SG_TEST_TMPDIR/plain.sock" SG_AUTHZ_PLUGIN_WAIT_SECONDS=1 \ + SG_AUTHZ_PLUGIN_POLL_SECONDS=0.2 run wait_for_authz_plugin + assert_failure +} + +#}}}: dockerd authorization plugin diff --git a/test/unit/update_runner_group.bats b/test/unit/update_runner_group.bats new file mode 100644 index 0000000..30d6405 --- /dev/null +++ b/test/unit/update_runner_group.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats +# +# Unit tests for update_runner_group: error reports are best effort and sent +# once, even when the API is unreachable (api_call reports its own failure +# through err(), which calls update_runner_group again). + +setup() { + load "${BATS_TEST_DIRNAME}/../helpers/load.bash" + export SG_NODE_TOKEN="test-token" + export ORGANIZATION_ID="demo-org" + export RUNNER_GROUP_ID="demo-group" +} + +curl_calls() { + grep -c '^curl ' "${MOCK_CALL_LOG}" || true +} + +@test "err with an unreachable API reports once and returns" { + export MOCK_CURL_OUTPUT="" + load_main + + run err "Registration failed" "boom" + assert_success + assert_output --partial "Registration failed" + assert_equal "$(curl_calls)" "1" +} + +@test "die with an unreachable API exits 1 after a single report" { + export MOCK_CURL_OUTPUT="" + load_main + + run die "Could not fetch data from API." "000" + assert_failure 1 + assert_equal "$(curl_calls)" "1" +} + +@test "err reports to the runner group endpoint when the API answers" { + export MOCK_CURL_OUTPUT="$(cat "${BATS_TEST_DIRNAME}/../fixtures/api/200_ok.http")" + load_main + + run err "Registration failed" "boom" + assert_success + assert_equal "$(curl_calls)" "1" + # The JSON payload spans lines in the call log, so check method and URL apart. + run grep -E '^curl .*-X PATCH' "${MOCK_CALL_LOG}" + assert_success + run grep -F '/orgs/demo-org/runnergroups/demo-group/' "${MOCK_CALL_LOG}" + assert_success +}