diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38e6a9a..a948ad1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,11 @@ on: branches: [main, master] pull_request: +# A newer push to the same branch cancels the in-flight run. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + jobs: build-test: runs-on: ubuntu-latest @@ -34,8 +39,28 @@ jobs: # Redis is provided above; tests that need Postgres skip automatically # unless FORGEQUEUE_PG_DSN is set. - - name: Test - run: go test ./... -count=1 + - name: Test (race) + run: go test ./... -race -count=1 + + - name: Vulnerability scan + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + # only-new-issues keeps the gate useful without failing on pre-existing + # style debt; tighten by removing it once the tree is clean. + - uses: golangci/golangci-lint-action@v6 + with: + version: latest + only-new-issues: true images: runs-on: ubuntu-latest @@ -62,5 +87,70 @@ jobs: context: . file: ${{ matrix.dockerfile }} push: false + load: true + tags: forgequeue/${{ matrix.name }}:ci cache-from: type=gha cache-to: type=gha,mode=max + + - name: Scan ${{ matrix.name }} image + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: forgequeue/${{ matrix.name }}:ci + format: table + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: "1" + + e2e: + runs-on: ubuntu-latest + needs: images + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build images + run: | + for svc in apiserver gateway worker web; do + docker build -f "dockerfiles/${svc}.Dockerfile" -t "forgequeue/${svc}:latest" . + done + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: forgequeue + config: deploy/k8s/kind-config.yaml + + - name: Load images + run: | + kind load docker-image --name forgequeue \ + forgequeue/apiserver:latest forgequeue/gateway:latest \ + forgequeue/worker:latest forgequeue/web:latest + + - name: Deploy + run: kubectl apply -k deploy/k8s/overlays/kind + + - name: Wait for rollouts + run: | + kubectl -n forgequeue rollout status statefulset/redis --timeout=180s + kubectl -n forgequeue rollout status statefulset/postgres --timeout=180s + kubectl -n forgequeue rollout status deployment/gateway --timeout=180s + kubectl -n forgequeue rollout status deployment/apiserver --timeout=180s + kubectl -n forgequeue rollout status deployment/worker --timeout=180s + kubectl -n forgequeue rollout status deployment/web --timeout=180s + + - name: Smoke test the API + run: | + kubectl -n forgequeue port-forward svc/apiserver 8080:8080 & + PF_PID=$! + trap 'kill $PF_PID' EXIT + for i in $(seq 1 30); do + if curl -fsS http://localhost:8080/healthz; then exit 0; fi + sleep 2 + done + echo "apiserver never became healthy" >&2 + kubectl -n forgequeue get pods + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..17a3d66 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,90 @@ +name: release + +# Tag a commit (e.g. v1.0.0) to cut a release. Builds the four images, pushes +# them to GHCR, attaches SBOM + provenance attestations, and signs each image +# with cosign (keyless, via the workflow's OIDC identity). +on: + push: + tags: + - "v*" + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository_owner }}/forgequeue + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write # cosign keyless signing + strategy: + fail-fast: false + matrix: + include: + - name: apiserver + dockerfile: dockerfiles/apiserver.Dockerfile + - name: gateway + dockerfile: dockerfiles/gateway.Dockerfile + - name: worker + dockerfile: dockerfiles/worker.Dockerfile + - name: web + dockerfile: dockerfiles/web.Dockerfile + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.name }} + tags: | + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ github.ref_name }} + COMMIT=${{ github.sha }} + BUILD_DATE=${{ github.event.head_commit.timestamp }} + VITE_APP_VERSION=${{ github.ref_name }} + # SLSA provenance + an SPDX SBOM, stored as in-registry attestations. + provenance: true + sbom: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Sign the image (keyless) + env: + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.name }} + run: cosign sign --yes "${IMAGE}@${DIGEST}" + + - name: Record digest + run: | + echo "### ${{ matrix.name }}" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.name }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 4c1cd38..6767fc7 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,6 @@ ForgeQueue is a Redis-backed task queue written in Go. It implements durable, at-least-once task execution with leases, heartbeats, crash recovery, and ownership fencing from first principles. -> **What this is.** A learning / portfolio project that implements a task queue -> the way a real one works: atomic Lua state transitions, lease ownership, -> heartbeats, recovery, retries and a dead-letter queue. It is deliberately -> small and readable rather than production-hardened. It is **not** a drop-in -> replacement for Redis-backed job systems like Asynq, Celery, BullMQ or -> Sidekiq — see [Limitations](#limitations) for the honest list of what it does -> not do. - Key features: - At-least-once execution with lease-based ownership and fencing - Automatic retries with exponential backoff @@ -137,6 +129,27 @@ be the accurate signal. Metrics are scraped via the `ServiceMonitor` in kube-prometheus-stack (the `monitoring.coreos.com` CRDs must exist). The Ingress targets the `forgequeue.local` host and needs an ingress-nginx controller. +Launch hardening baked into the manifests: + +- **TLS at the edge.** The Ingress terminates TLS (cert-manager + `letsencrypt-prod` by default) and redirects HTTP to HTTPS; the console and + `/api` share one origin, so the API needs no CORS grant in production. +- **Redis AUTH.** The Redis StatefulSet runs with `requirepass` sourced from + `forgequeue-secrets` (`FORGEQUEUE_REDIS_PASSWORD`); every service reads it + through `redisutil`. `maxmemory-policy noeviction` bounds memory without + silently dropping queued tasks. +- **Alerting.** `observability/prometheusrule.yaml` ships alerts for backlog + growth, dead-lettering, lease losses, a down scrape target, failed backups + and filling persistent volumes. +- **Queue-depth autoscaling.** `observability/keda-scaledobject.yaml` scales + workers on the ready-list length (KEDA); it supersedes the CPU HPA. +- **Off-cluster backups.** `base/backup.yaml` adds a daily `pg_dump` to object + storage in addition to the on-PVC copy, and no-ops when unconfigured. + +Images are built and signed (SBOM + provenance + cosign) by +`.github/workflows/release.yml` on a `v*` tag. Pin the printed digests in +`overlays/prod` before deploying. + ## Architecture ForgeQueue consists of two main components that communicate through Redis: @@ -279,6 +292,16 @@ The batch's heartbeats are only stopped **after** the ACK/requeue is committed. | Setting | Location | Default | | --- | --- | --- | | Redis address | `FORGEQUEUE_REDIS_ADDR` | `localhost:6379` | +| Redis password (AUTH) | `FORGEQUEUE_REDIS_PASSWORD` | none | +| Redis database index | `FORGEQUEUE_REDIS_DB` | `0` | +| Redis TLS | `FORGEQUEUE_REDIS_TLS` | off | +| Redis TLS verify skip (dev) | `FORGEQUEUE_REDIS_TLS_INSECURE` | off | +| Redis TLS CA bundle | `FORGEQUEUE_REDIS_TLS_CA` | system roots | +| gRPC TLS cert / key | `FORGEQUEUE_GRPC_TLS_CERT` / `_KEY` | off (plaintext) | +| gRPC TLS client CA (mTLS) | `FORGEQUEUE_GRPC_TLS_CLIENT_CA` | off | +| API CORS origins (comma list) | `FORGEQUEUE_CORS_ORIGINS` | same-origin only | +| API max request body | `FORGEQUEUE_MAX_BODY_BYTES` | 1 MiB | +| Postgres sslmode (DSN) | `FORGEQUEUE_PG_SSLMODE` | `disable` | | Gateway gRPC listen | `gateway.Start()` (hardcoded) | `:50051` | | Gateway metrics address | `FORGEQUEUE_METRICS_ADDR` | disabled | | HTTP API listen | `FORGEQUEUE_API_ADDR` | `:8080` | @@ -310,6 +333,52 @@ The batch's heartbeats are only stopped **after** the ACK/requeue is committed. | Backoff base | `wp.backoffBase` (unexported) | 1s | | Backoff cap | `wp.backoffCap` (unexported) | 60s | +## Observability + +The gateway and worker each expose Prometheus metrics at `/metrics` on +`FORGEQUEUE_METRICS_ADDR` (disabled by default). They are scraped as two +separate jobs so "accepted" (gateway) and "executed" (worker) stay +distinguishable: + +- `forgequeue_tasks_enqueued_total` — tasks the gateway accepted +- `forgequeue_tasks_processed_total` — tasks the worker ACKed +- `forgequeue_tasks_retried_total` / `forgequeue_tasks_dead_lettered_total` +- `forgequeue_queue_ready / inflight / scheduled / dead` — queue depth by state, sampled from Redis every 5s (O(1) per scrape) +- `forgequeue_task_duration_seconds` — handler duration histogram (p50/p95/p99) +- `forgequeue_redis_claim/ack/heartbeat_seconds` — Redis round-trip histograms +- `forgequeue_lease_losses_total` — fenced lease losses + +Run the whole telemetry stack locally with `deploy/observability.sh` (needs +`prometheus` on PATH; Grafana is optional and only started if installed): + +```bash +./deploy/observability.sh +# in another terminal, generate some load: +go run ./cmd/demoenqueue +``` + +That prints the URLs: gateway `:9090/metrics`, worker `:9091/metrics`, +Prometheus UI on `:9095`, and Grafana on `:3000` (no login) when available. +Logs go to `/tmp/fq-*.log`. + +### Kubernetes + +Under `deploy/k8s/observability` (applied separately after kube-prometheus-stack, since its CRDs must exist): + +- `servicemonitor.yaml` — scrapes the `worker` and `gateway` `metrics` ports every 15s +- `prometheusrule.yaml` — alerts for backlog growth, dead-lettering, lease losses, a down scrape target, failed backups and filling persistent volumes +- `keda-scaledobject.yaml` — scales workers on ready-list length; supersedes the CPU HPA in `base/` + +### Grafana + +Dashboards live under `deploy/grafana`: + +- `dashboards/forgequeue.json` — "ForgeQueue Overview": enqueued/processed/retried/dead-lettered stats, queue depth by state, in-flight vs ready, handler duration p50/p95/p99, Redis round trips p95, lease losses, and enqueued vs processed throughput +- `provisioning/datasources/prometheus.yml` — provisions the Prometheus datasource (`uid: forgequeue`, `http://localhost:9095`) so the dashboard wires itself up on import +- `provisioning/dashboards/forgequeue.yml` — file provider that loads every dashboard JSON in `dashboards/` + +The dashboard assumes the `forgequeue` Prometheus datasource UID matches what the provisioning file registers. + ## Running the whole stack ```bash @@ -426,7 +495,13 @@ Real handlers dominate throughput in practice, so treat these as an upper bound. - **No exactly-once.** Delivery is at-least-once: a lease can expire (worker crash, Redis blip, GC pause) and a task can run twice. Handlers must be idempotent. - **Single Redis.** No replication, sharding, or failover logic — durability is whatever the Redis deployment provides. The bundled Kubernetes manifest runs one AOF-backed Redis on a PersistentVolumeClaim, but that is still a single point of failure. -- **Partial auth.** The HTTP API has JWT + RBAC, with optional OIDC/SSO login. The **gRPC gateway and Redis have no authentication or TLS**, and the HTTP API itself is plain HTTP. Do not expose any of it to a public network. +- **Partial auth.** The HTTP API and the gRPC gateway authenticate with JWT + bearer tokens (RBAC on the API, per-method permissions on gRPC), with + optional OIDC/SSO login. Transport TLS is available but **off by default**: + enable it on gRPC with the `FORGEQUEUE_GRPC_TLS_*` vars and terminate HTTP + TLS at the ingress (see the Kubernetes section). Redis supports + `requirepass` and TLS through its env vars. On a trusted pod network the + defaults are plaintext; do not expose any port publicly without enabling TLS. - **No multi-tenancy / queue isolation.** "Queues" are a single logical namespace, not separate Redis keyspaces or broker connections. - **Completed bodies are deleted**, so a finished job cannot be looked up by ID afterwards (see the HTTP API note above). - **Idle polling** uses a short sleep interval per worker; blocking variants (e.g. `BLMOVE`) are not implemented. diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 4b63419..b0bab22 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -5,6 +5,5 @@ import ( ) func main() { - // Call the execution function inside your internal package gateway.Start() } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 048ee80..86dcb28 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -38,7 +38,6 @@ func main() { log.Printf("worker starting: redis=%s concurrency=%d batch_size=%d", redisAddr, concurrency, batchSize) - // fail fast bootCtx, cancelBoot := context.WithTimeout(context.Background(), 5*time.Second) defer cancelBoot() @@ -47,13 +46,11 @@ func main() { log.Fatalf("failed to create worker pool: %v", err) } - // Batching collapses the claim and ACK round trips for a whole batch - // into one each. It is opt-in because a batch is handled sequentially, - // so a large value trades per-task latency for throughput. + // A batch is claimed and ACKed in one round trip but processed one task + // at a time. wp.WithBatchSize(batchSize) - // Telemetry is opt-in: with no scrape address configured the pool keeps - // no-op instruments and opens no port. + // No scrape address means metrics stay off. var ( telemetry *metrics.Metrics metricsSrv *metrics.Server @@ -69,9 +66,8 @@ func main() { metricsSrv = metrics.NewServer(metricsAddr, m) } - // The durable journal is optional and opt-in, matching the gateway. When - // attached, the reconciler heals the window where a task is in Postgres - // but not yet in Redis. + // Optional. With a DSN set, the reconciler heals tasks that are in + // Postgres but not yet in Redis. if dsn := os.Getenv(envPostgresDSN); dsn != "" { store, err := durable.NewPostgres(bootCtx, dsn) if err != nil { @@ -84,24 +80,20 @@ func main() { log.Println("durable journal disabled (set FORGEQUEUE_PG_DSN to enable)") } - // Register the built-in handlers, and say which ones, so a task type - // nobody handles is obvious from the startup log rather than from a - // stream of timeouts later. + // Log the handler list so a task type nobody handles shows up at startup + // instead of as timeouts later. registered := handlers.Default() for taskType, h := range registered { wp.RegisterHandler(taskType, h) } log.Printf("handlers registered: %s", sortedKeys(registered)) - // Fail before serving if there is nothing to serve. A pool that only - // polls Redis with no handler registered looks healthy while silently - // leaking every task it claims. + // A pool with no handlers would claim tasks it cannot process. if len(registered) == 0 { log.Fatal("no handlers registered: the worker would claim tasks it cannot process") } - // The scrape endpoint goes up before the workers so telemetry is - // reachable even if the pool is saturated. + // Start the scrape endpoint before the pool. if metricsSrv != nil { if err := metricsSrv.Start(); err != nil { log.Fatalf("failed to start metrics server: %v", err) @@ -109,29 +101,21 @@ func main() { log.Printf("metrics endpoint online on %s/metrics", metricsAddr) } - // One signal channel for the whole process. SIGINT is Ctrl-C, SIGTERM is - // what a container runtime sends on `docker stop` and on Kubernetes - // pod deletion, so both need to shut down cleanly rather than kill the - // process mid-task. + // SIGINT is Ctrl-C, SIGTERM is what the container runtime sends. shutdown := make(chan os.Signal, 1) signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM) wp.Start() - // Block until a signal arrives. sig := <-shutdown log.Printf("received shutdown signal: %v", sig) - // Stop accepting new claims and wait for in-flight handlers to return. - // Stop cancels the pool context, which also stops the recovery, - // promotion, and queue-depth loops. Tasks still holding a lease when - // this returns are not lost: recovery reclaims them once the lease - // expires. + // Cancels the pool context and waits for in-flight handlers. Tasks still + // holding a lease are reclaimed by recovery after it expires. wp.Stop() log.Println("worker pool stopped") - // Flush buffered metrics after the pool is idle, so a scrape during - // shutdown still sees the final counts. + // Flush buffered metrics after the pool is idle. if metricsSrv != nil { stopCtx, stopCancel := context.WithTimeout(context.Background(), 3*time.Second) defer stopCancel() @@ -155,13 +139,8 @@ func envString(key, def string) string { return def } -// envInt returns the value of key parsed as an int, or def when it is unset, -// empty, or not a valid number. -// -// A malformed value falls back to the default rather than exiting, on the -// grounds that a typo in an optional tuning knob should not stop the worker -// from processing tasks. The effective value is logged at startup either way, -// so a silently-ignored typo is still visible. +// envInt returns key parsed as an int, or def when it is unset, empty, or not +// a valid number. A bad value falls back to def and is logged. func envInt(key string, def int) int { raw := os.Getenv(key) if raw == "" { @@ -177,8 +156,7 @@ func envInt(key string, def int) int { return v } -// sortedKeys returns the map keys in a stable order so startup logs are -// comparable between runs and between replicas. +// sortedKeys returns map keys in a stable order for logging. func sortedKeys(m map[string]handlers.WorkerHandler) string { keys := make([]string, 0, len(m)) for k := range m { diff --git a/deploy/k8s/base/apiserver.yaml b/deploy/k8s/base/apiserver.yaml index 0c26187..cba4339 100644 --- a/deploy/k8s/base/apiserver.yaml +++ b/deploy/k8s/base/apiserver.yaml @@ -17,6 +17,15 @@ spec: app.kubernetes.io/part-of: forgequeue spec: terminationGracePeriodSeconds: 30 + # Spread replicas across nodes so one node loss cannot take out every + # copy. ScheduleAnyway keeps scheduling working on single-node kind. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: apiserver securityContext: runAsNonRoot: true runAsUser: 65532 @@ -32,6 +41,15 @@ spec: runAsUser: 65532 capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi containers: - name: apiserver image: forgequeue/apiserver:latest @@ -71,6 +89,8 @@ spec: readOnlyRootFilesystem: true capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault --- apiVersion: v1 kind: Service diff --git a/deploy/k8s/base/backup.yaml b/deploy/k8s/base/backup.yaml new file mode 100644 index 0000000..811bd92 --- /dev/null +++ b/deploy/k8s/base/backup.yaml @@ -0,0 +1,251 @@ +# Logical backup of the durable journal. This is a pattern, not full DR: a +# PVC-backed dump protects against operator error and logical corruption, but +# it lives on the same cluster as the data. For real disaster recovery, point +# the dump destination at object storage (S3/GCS) or ship the base volume +# snapshots off-cluster. WAL archiving (pgBackRest/WAL-G) is the step beyond +# this that gives point-in-time recovery. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-backup + labels: + app.kubernetes.io/name: postgres-backup + app.kubernetes.io/part-of: forgequeue +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: postgres-backup + labels: + app.kubernetes.io/name: postgres-backup + app.kubernetes.io/part-of: forgequeue +spec: + schedule: "0 3 * * *" # 03:00 daily + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app.kubernetes.io/name: postgres-backup + app.kubernetes.io/part-of: forgequeue + spec: + restartPolicy: OnFailure + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + containers: + - name: pg-dump + image: postgres:16.15 + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: POSTGRES_PASSWORD + command: + - sh + - -c + - | + set -eu + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + out="/backups/forgequeue-${stamp}.dump" + pg_dump -h postgres -U forgequeue -d forgequeue \ + --format=custom --no-owner --no-privileges -f "$out" + echo "wrote $out" + # Retain the 7 most recent dumps. + ls -1t /backups/forgequeue-*.dump | tail -n +8 | xargs -r rm -f + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: backups + mountPath: /backups + - name: tmp + mountPath: /tmp + volumes: + - name: backups + persistentVolumeClaim: + claimName: postgres-backup + - name: tmp + emptyDir: {} +--- +# Off-cluster copy. The PVC dump above lives on the same cluster as the data, +# so it protects against operator error but not against losing the cluster. +# This job streams the same logical dump to object storage, which is the copy +# that actually restores you after a region/cluster loss. +# +# Self-guarding: if FORGEQUEUE_BACKUP_BUCKET (and credentials) are absent from +# the forgequeue-secrets Secret, the job logs and exits 0 instead of failing, +# so it is harmless in clusters that only want the on-PVC backup. +# +# To enable, add to forgequeue-secrets: +# FORGEQUEUE_BACKUP_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, +# AWS_DEFAULT_REGION (optional FORGEQUEUE_BACKUP_PREFIX) +# Prefer a scoped IAM user, or swap the static keys for IRSA/Workload Identity. +# +# This is still logical DR, not point-in-time recovery: for PITR add WAL +# archiving (pgBackRest or WAL-G) to the object store as well. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: postgres-backup-offsite + labels: + app.kubernetes.io/name: postgres-backup-offsite + app.kubernetes.io/part-of: forgequeue +spec: + schedule: "30 3 * * *" # 03:30 daily, after the on-cluster dump + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app.kubernetes.io/name: postgres-backup-offsite + app.kubernetes.io/part-of: forgequeue + spec: + restartPolicy: OnFailure + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + initContainers: + - name: dump + image: postgres:16.15 + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: POSTGRES_PASSWORD + command: + - sh + - -c + - | + set -eu + pg_dump -h postgres -U forgequeue -d forgequeue \ + --format=custom --no-owner --no-privileges \ + -f /dump/forgequeue.dump + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: dump + mountPath: /dump + - name: tmp + mountPath: /tmp + containers: + - name: upload + image: amazon/aws-cli:2.17.0 + # aws-cli wants a writable HOME; the root filesystem is read-only. + env: + - name: HOME + value: /tmp + - name: FORGEQUEUE_BACKUP_BUCKET + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: FORGEQUEUE_BACKUP_BUCKET + optional: true + - name: FORGEQUEUE_BACKUP_PREFIX + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: FORGEQUEUE_BACKUP_PREFIX + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: AWS_SECRET_ACCESS_KEY + optional: true + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: AWS_DEFAULT_REGION + optional: true + command: + - sh + - -c + - | + set -eu + if [ -z "${FORGEQUEUE_BACKUP_BUCKET:-}" ]; then + echo "FORGEQUEUE_BACKUP_BUCKET unset; skipping offsite upload" + exit 0 + fi + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + prefix="${FORGEQUEUE_BACKUP_PREFIX:-forgequeue}" + aws s3 cp /dump/forgequeue.dump \ + "s3://${FORGEQUEUE_BACKUP_BUCKET}/${prefix}/forgequeue-${stamp}.dump" + echo "uploaded forgequeue-${stamp}.dump" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: dump + mountPath: /dump + - name: tmp + mountPath: /tmp + volumes: + - name: dump + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/deploy/k8s/base/gateway.yaml b/deploy/k8s/base/gateway.yaml index cd2b262..84c772d 100644 --- a/deploy/k8s/base/gateway.yaml +++ b/deploy/k8s/base/gateway.yaml @@ -17,6 +17,15 @@ spec: app.kubernetes.io/part-of: forgequeue spec: terminationGracePeriodSeconds: 30 + # Spread replicas across nodes so one node loss cannot take out every + # copy. ScheduleAnyway keeps scheduling working on single-node kind. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: gateway securityContext: runAsNonRoot: true runAsUser: 65532 @@ -32,6 +41,15 @@ spec: runAsUser: 65532 capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi - name: wait-for-postgres image: busybox:1.36 command: ["sh", "-c", "until nc -z postgres 5432; do echo waiting for postgres; sleep 1; done"] @@ -42,6 +60,15 @@ spec: runAsUser: 65532 capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi containers: - name: gateway image: forgequeue/gateway:latest @@ -59,10 +86,10 @@ spec: secretKeyRef: name: forgequeue-secrets key: POSTGRES_PASSWORD - # The journal DSN is derived from the shared Postgres password so - # the credential lives in exactly one place. + # Derive the journal DSN from the shared Postgres password so the + # credential lives in exactly one place. - name: FORGEQUEUE_PG_DSN - value: "postgres://forgequeue:$(POSTGRES_PASSWORD)@postgres:5432/forgequeue?sslmode=disable" + value: "postgres://forgequeue:$(POSTGRES_PASSWORD)@postgres:5432/forgequeue?sslmode=$(FORGEQUEUE_PG_SSLMODE)" ports: - name: grpc containerPort: 50051 @@ -92,6 +119,8 @@ spec: readOnlyRootFilesystem: true capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault --- apiVersion: v1 kind: Service diff --git a/deploy/k8s/base/ingress.yaml b/deploy/k8s/base/ingress.yaml index 75a93f2..382ee04 100644 --- a/deploy/k8s/base/ingress.yaml +++ b/deploy/k8s/base/ingress.yaml @@ -7,8 +7,19 @@ metadata: annotations: nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + # Terminate TLS at the edge and force HTTPS. cert-manager issues the + # certificate into the secret referenced under spec.tls below. + nginx.ingress.kubernetes.io/ssl-redirect: "true" + cert-manager.io/cluster-issuer: letsencrypt-prod spec: ingressClassName: nginx + # TLS terminates here, so credentials and bearer tokens never cross the + # public network in clear text. Everything behind the ingress is the + # in-cluster pod network. + tls: + - hosts: + - forgequeue.local + secretName: forgequeue-tls rules: - host: forgequeue.local http: diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index e94c6cc..41f0bbc 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -7,6 +7,7 @@ resources: - namespace.yaml - redis.yaml - postgres.yaml + - backup.yaml - worker.yaml - gateway.yaml - apiserver.yaml @@ -26,3 +27,12 @@ configMapGenerator: - FORGEQUEUE_BATCH_SIZE=1 - FORGEQUEUE_METRICS_ADDR=:9100 - FORGEQUEUE_API_ADDR=:8080 + # TLS mode for the Postgres connection. "disable" fits the bundled, + # in-cluster Postgres (loopback pod network). Override to "require" or + # "verify-full" in an overlay that points at a managed, TLS-terminating + # Postgres (close FORGEQUEUE_PG_DSN accordingly). + - FORGEQUEUE_PG_SSLMODE=disable + # Browser origins allowed to call the API with credentials. Empty means + # same-origin only (the ingress serves the console and /api together), + # which is the production shape. Dev sets the web origin explicitly. + - FORGEQUEUE_CORS_ORIGINS= diff --git a/deploy/k8s/base/networkpolicy.yaml b/deploy/k8s/base/networkpolicy.yaml index 9ce7160..fcc59b0 100644 --- a/deploy/k8s/base/networkpolicy.yaml +++ b/deploy/k8s/base/networkpolicy.yaml @@ -1,4 +1,4 @@ -# Redis holds live queue state, so only the queue services may reach it. This +# Redis holds live queue state, so only the queue services can reach it. This # is the network half of the boundary; the gRPC gateway adds token auth on top. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -41,7 +41,7 @@ spec: matchExpressions: - key: app.kubernetes.io/name operator: In - values: ["gateway", "worker"] + values: ["gateway", "worker", "postgres-backup"] ports: - protocol: TCP port: 5432 diff --git a/deploy/k8s/base/pdb.yaml b/deploy/k8s/base/pdb.yaml index bd707e9..63b22a0 100644 --- a/deploy/k8s/base/pdb.yaml +++ b/deploy/k8s/base/pdb.yaml @@ -1,5 +1,19 @@ apiVersion: policy/v1 kind: PodDisruptionBudget +metadata: + name: postgres + labels: + app.kubernetes.io/part-of: forgequeue +spec: + # A single-replica datastore: block voluntary disruptions rather than let a + # node drain take the journal down without a human deciding to. + maxUnavailable: 0 + selector: + matchLabels: + app.kubernetes.io/name: postgres +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget metadata: name: worker labels: @@ -33,3 +47,15 @@ spec: selector: matchLabels: app.kubernetes.io/name: apiserver +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: web + labels: + app.kubernetes.io/part-of: forgequeue +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: web diff --git a/deploy/k8s/base/postgres.yaml b/deploy/k8s/base/postgres.yaml index 96a8951..bd4d04a 100644 --- a/deploy/k8s/base/postgres.yaml +++ b/deploy/k8s/base/postgres.yaml @@ -8,6 +8,11 @@ metadata: spec: serviceName: postgres replicas: 1 + # Retain the data volume on delete and on scale-down. This is the only + # durable copy, so we don't want a StatefulSet change dropping it. + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain selector: matchLabels: app.kubernetes.io/name: postgres @@ -22,9 +27,10 @@ spec: runAsUser: 999 runAsGroup: 999 fsGroup: 999 + automountServiceAccountToken: false containers: - name: postgres - image: postgres:16 + image: postgres:16.15 env: - name: POSTGRES_DB value: forgequeue @@ -35,33 +41,57 @@ spec: secretKeyRef: name: forgequeue-secrets key: POSTGRES_PASSWORD - # Keep the cluster data in a subdir so the volume root can hold the - # lost+found directory some CSI drivers create. + # Data goes in a subdir so the volume root can hold lost+found, + # which some CSI drivers create. - name: PGDATA value: /var/lib/postgresql/data/pgdata ports: - name: postgres containerPort: 5432 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + # Generous startupProbe for first-boot initdb and crash recovery. + # Liveness only starts after the DB accepts connections, so a slow + # disk can't restart-loop us into a half-initialized PGDATA. + startupProbe: + exec: + command: ["pg_isready", "-U", "forgequeue", "-d", "forgequeue"] + periodSeconds: 10 + failureThreshold: 30 readinessProbe: exec: command: ["pg_isready", "-U", "forgequeue", "-d", "forgequeue"] - initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: exec: command: ["pg_isready", "-U", "forgequeue", "-d", "forgequeue"] - initialDelaySeconds: 15 periodSeconds: 20 resources: + # requests == limits gives Postgres Guaranteed QoS, so node memory + # pressure evicts best-effort pods instead of the journal. requests: - cpu: 100m - memory: 128Mi + cpu: 500m + memory: 512Mi limits: cpu: 500m memory: 512Mi volumeMounts: - name: data mountPath: /var/lib/postgresql/data + - name: run + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + volumes: + - name: run + emptyDir: {} + - name: tmp + emptyDir: {} volumeClaimTemplates: - metadata: name: data diff --git a/deploy/k8s/base/redis.yaml b/deploy/k8s/base/redis.yaml index 3cc82bf..3d2c15f 100644 --- a/deploy/k8s/base/redis.yaml +++ b/deploy/k8s/base/redis.yaml @@ -8,6 +8,11 @@ metadata: spec: serviceName: redis replicas: 1 + # Keep the AOF volume on delete and scale-down: it is the only copy of the + # queue until the Postgres journal reconstitutes it. + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain selector: matchLabels: app.kubernetes.io/name: redis @@ -22,20 +27,40 @@ spec: runAsUser: 999 runAsGroup: 999 fsGroup: 999 + seccompProfile: + type: RuntimeDefault + automountServiceAccountToken: false containers: - name: redis image: redis:7.4-alpine + # REDISCLI_AUTH lets the readiness/liveness probes (and any in-pod + # redis-cli) authenticate without putting the password on the command + # line, where it would leak via `ps`. + env: + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: FORGEQUEUE_REDIS_PASSWORD # AOF is the durability mechanism: appendonly + everysec fsync, plus # an RDB snapshot every 60s as a fast restart path. The appendonly # data lives on a PVC so it outlives the pod. - args: - - "--appendonly" - - "yes" - - "--appendfsync" - - "everysec" - - "--save" - - "60" - - "1000" + # + # requirepass closes the gap the NetworkPolicy cannot: any pod that + # reaches the port must still authenticate. maxmemory-policy is + # noeviction on purpose — evicting a queued task silently loses work, + # so instead we bound memory and let the alert fire on noeviction. + command: + - sh + - -c + - > + exec redis-server + --appendonly yes + --appendfsync everysec + --save 60 1000 + --requirepass "$FORGEQUEUE_REDIS_PASSWORD" + --maxmemory 192mb + --maxmemory-policy noeviction ports: - name: redis containerPort: 6379 @@ -50,15 +75,30 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 resources: + # requests == limits keeps Redis out of the eviction path; the + # maxmemory above is deliberately below the 256Mi limit so Redis + # hits its own ceiling before the kernel OOM-kills the pod. requests: - cpu: 50m - memory: 64Mi + cpu: 100m + memory: 256Mi limits: - cpu: 250m + cpu: 500m memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault volumeMounts: - name: data mountPath: /data + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} volumeClaimTemplates: - metadata: name: data @@ -69,7 +109,7 @@ spec: accessModes: ["ReadWriteOnce"] resources: requests: - storage: 1Gi + storage: 2Gi --- apiVersion: v1 kind: Service @@ -80,7 +120,8 @@ metadata: app.kubernetes.io/part-of: forgequeue spec: selector: - app.kubernetes.io/name: redis + matchLabels: + app.kubernetes.io/name: redis ports: - name: redis port: 6379 diff --git a/deploy/k8s/base/web.yaml b/deploy/k8s/base/web.yaml index 0738755..c7fa74c 100644 --- a/deploy/k8s/base/web.yaml +++ b/deploy/k8s/base/web.yaml @@ -17,6 +17,15 @@ spec: app.kubernetes.io/part-of: forgequeue spec: terminationGracePeriodSeconds: 15 + # Spread replicas across nodes so one node loss cannot take out every + # copy. ScheduleAnyway keeps scheduling working on single-node kind. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: web securityContext: runAsNonRoot: true runAsUser: 1000 @@ -52,6 +61,8 @@ spec: readOnlyRootFilesystem: true capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault volumeMounts: - name: tmp mountPath: /tmp diff --git a/deploy/k8s/base/worker.yaml b/deploy/k8s/base/worker.yaml index 1c11126..0810690 100644 --- a/deploy/k8s/base/worker.yaml +++ b/deploy/k8s/base/worker.yaml @@ -17,6 +17,15 @@ spec: app.kubernetes.io/part-of: forgequeue spec: terminationGracePeriodSeconds: 30 + # Spread replicas across nodes so one node loss cannot take out every + # copy. ScheduleAnyway keeps scheduling working on single-node kind. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: worker securityContext: runAsNonRoot: true runAsUser: 65532 @@ -32,6 +41,15 @@ spec: runAsUser: 65532 capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi - name: wait-for-postgres image: busybox:1.36 command: ["sh", "-c", "until nc -z postgres 5432; do echo waiting for postgres; sleep 1; done"] @@ -42,6 +60,15 @@ spec: runAsUser: 65532 capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi containers: - name: worker image: forgequeue/worker:latest @@ -60,9 +87,9 @@ spec: name: forgequeue-secrets key: POSTGRES_PASSWORD # Attaching the journal turns on the worker's reconciler, which - # rebuilds any non-terminal task Redis is missing. + # rebuilds any non-terminal task that Redis is missing. - name: FORGEQUEUE_PG_DSN - value: "postgres://forgequeue:$(POSTGRES_PASSWORD)@postgres:5432/forgequeue?sslmode=disable" + value: "postgres://forgequeue:$(POSTGRES_PASSWORD)@postgres:5432/forgequeue?sslmode=$(FORGEQUEUE_PG_SSLMODE)" ports: - name: metrics containerPort: 9100 @@ -90,6 +117,8 @@ spec: readOnlyRootFilesystem: true capabilities: drop: ["ALL"] + seccompProfile: + type: RuntimeDefault --- apiVersion: v1 kind: Service diff --git a/deploy/k8s/kind-config.yaml b/deploy/k8s/kind-config.yaml index f68c9b4..838e9e1 100644 --- a/deploy/k8s/kind-config.yaml +++ b/deploy/k8s/kind-config.yaml @@ -1,8 +1,8 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +# A control-plane plus one worker. No host mounts: manifests are applied with +# `kubectl apply -k`, so nothing needs to be bind-mounted into the node, and +# the file stays portable across machines and CI. nodes: -- role: control-plane - extraMounts: - - hostPath: /Users/yashbahuguna/forgequeue/deploy/k8s - containerPath: /etc/k8s-manifests -- role: worker + - role: control-plane + - role: worker diff --git a/deploy/k8s/observability/keda-scaledobject.yaml b/deploy/k8s/observability/keda-scaledobject.yaml new file mode 100644 index 0000000..96fef5b --- /dev/null +++ b/deploy/k8s/observability/keda-scaledobject.yaml @@ -0,0 +1,48 @@ +# Queue-depth autoscaling for workers. The CPU HPA in base/ scales on polling +# overhead, which is the wrong signal for an idle-polling worker; this scales +# on the ready-list length instead. +# +# Requires KEDA (both the keda.sh CRDs and its operator): +# helm repo add kedacore https://kedacore.github.io/charts +# helm upgrade --install keda kedacore/keda -n keda --create-namespace +# +# IMPORTANT: KEDA and the CPU HorizontalPodAutoscaler both target the worker +# Deployment. Applying this file starts a fight over replica count. When you +# enable KEDA, remove the worker HPA from base/hpa.yaml (drop worker from the +# document, or delete it from the overlay). +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: worker + labels: + app.kubernetes.io/part-of: forgequeue +spec: + scaleTargetRef: + name: worker + minReplicaCount: 1 + maxReplicaCount: 10 + # Wait out a burst without thrashing; queues tolerate brief over-provision. + cooldownPeriod: 120 + pollingInterval: 15 + triggers: + - type: redis + metadata: + address: redis:6379 + listName: queue:tasks:immediate + # One worker replica per 100 ready tasks. + listLength: "100" + enableTLS: "false" + authenticationRef: + name: keda-redis-auth +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-redis-auth + labels: + app.kubernetes.io/part-of: forgequeue +spec: + secretTargetRef: + - parameter: password + name: forgequeue-secrets + key: FORGEQUEUE_REDIS_PASSWORD diff --git a/deploy/k8s/observability/kustomization.yaml b/deploy/k8s/observability/kustomization.yaml index fe2f8b2..5a83365 100644 --- a/deploy/k8s/observability/kustomization.yaml +++ b/deploy/k8s/observability/kustomization.yaml @@ -12,3 +12,7 @@ namespace: forgequeue resources: - servicemonitor.yaml + - prometheusrule.yaml + # Requires the KEDA CRDs. Do not include this alongside the CPU HPA for the + # worker (both target the same Deployment); see the file's header. + # - keda-scaledobject.yaml diff --git a/deploy/k8s/observability/prometheusrule.yaml b/deploy/k8s/observability/prometheusrule.yaml new file mode 100644 index 0000000..792c70e --- /dev/null +++ b/deploy/k8s/observability/prometheusrule.yaml @@ -0,0 +1,80 @@ +# Alerting rules for the queue internals and the platform around it. Requires +# kube-prometheus-stack's PrometheusRule CRD (monitoring.coreos.com/v1), the +# same dependency the ServiceMonitor has. +# +# OTel translates dotted instrument names to underscores in Prometheus, so +# forgequeue.queue.dead becomes forgequeue_queue_dead. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: forgequeue + labels: + app.kubernetes.io/part-of: forgequeue + release: kube-prometheus-stack +spec: + groups: + - name: forgequeue.queue + rules: + - alert: ForgeQueueBacklogGrowing + # A readily-visible backlog that is not draining. Tune to your + # expected steady-state depth. + expr: forgequeue_queue_ready > 1000 + for: 15m + labels: + severity: warning + annotations: + summary: "Queue ready backlog above 1000 for 15m" + description: "{{ $value }} tasks are ready and the queue is not draining; scale workers or investigate a slow handler." + + - alert: ForgeQueueDeadLettering + expr: increase(forgequeue_tasks_dead_lettered_total[10m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Tasks are being dead-lettered" + description: "{{ $value }} tasks moved to the DLQ in the last 10m; check handler errors and the DLQ in the console." + + - alert: ForgeQueueLeaseLosses + # Lease loss means a worker could not renew ownership and the handler + # context was cancelled — often Redis latency or a crash. + expr: increase(forgequeue_lease_losses_total[5m]) > 5 + for: 5m + labels: + severity: warning + annotations: + summary: "Elevated lease losses" + description: "{{ $value }} lease losses in 5m; check Redis latency/availability and worker restarts." + + - name: forgequeue.platform + rules: + - alert: ForgeQueueTargetDown + expr: up{job="forgequeue"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "ForgeQueue metrics target down" + description: "{{ $labels.instance }} has been unscrapeable for 5m." + + - alert: PostgresBackupFailed + expr: kube_job_status_failed{job_name=~"postgres-backup.*"} > 0 + for: 10m + labels: + severity: critical + annotations: + summary: "Postgres backup job failed" + description: "A postgres-backup CronJob failed; the durable journal has no recent logical backup." + + - alert: PersistentVolumeFillingUp + expr: > + kubelet_volume_stats_used_bytes{persistentvolumeclaim=~"redis-data-.*|data-postgres-0"} + / + kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"redis-data-.*|data-postgres-0"} + > 0.85 + for: 15m + labels: + severity: warning + annotations: + summary: "Volume {{ $labels.persistentvolumeclaim }} is over 85% full" + description: "Grow the PVC or add retention; a full Redis/Postgres volume stops accepting writes." diff --git a/deploy/k8s/overlays/kind/kustomization.yaml b/deploy/k8s/overlays/kind/kustomization.yaml index aa443ee..31d4688 100644 --- a/deploy/k8s/overlays/kind/kustomization.yaml +++ b/deploy/k8s/overlays/kind/kustomization.yaml @@ -16,6 +16,8 @@ secretGenerator: - FORGEQUEUE_AUTH_SECRET=devsecret-change-me - FORGEQUEUE_ADMIN_USER=admin - FORGEQUEUE_ADMIN_PASSWORD=devpassword + # AUTH for the Redis StatefulSet and every service that connects to it. + - FORGEQUEUE_REDIS_PASSWORD=devredis # Shared by the Postgres StatefulSet (POSTGRES_PASSWORD) and the derived # FORGEQUEUE_PG_DSN that gateway/worker build in the base manifests. - POSTGRES_PASSWORD=devpostgres diff --git a/deploy/k8s/overlays/prod/imagepullsecret-patch.yaml b/deploy/k8s/overlays/prod/imagepullsecret-patch.yaml new file mode 100644 index 0000000..a31e619 --- /dev/null +++ b/deploy/k8s/overlays/prod/imagepullsecret-patch.yaml @@ -0,0 +1,49 @@ +# Optional. Add this file to overlays/prod/kustomization.yaml's `patches:` list +# only if your GHCR packages are private. Create the pull secret first: +# +# kubectl -n forgequeue create secret docker-registry ghcr-cred \ +# --docker-server=ghcr.io \ +# --docker-username= \ +# --docker-password= +# +# A referenced-but-missing imagePullSecret makes every pod ErrImagePull, so +# this is opt-in rather than on by default. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gateway +spec: + template: + spec: + imagePullSecrets: + - name: ghcr-cred +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker +spec: + template: + spec: + imagePullSecrets: + - name: ghcr-cred +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: apiserver +spec: + template: + spec: + imagePullSecrets: + - name: ghcr-cred +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web +spec: + template: + spec: + imagePullSecrets: + - name: ghcr-cred diff --git a/deploy/k8s/overlays/prod/kustomization.yaml b/deploy/k8s/overlays/prod/kustomization.yaml index 5a70e13..4181105 100644 --- a/deploy/k8s/overlays/prod/kustomization.yaml +++ b/deploy/k8s/overlays/prod/kustomization.yaml @@ -7,9 +7,21 @@ resources: - ../../base # forgequeue-secrets is applied out of band (SOPS + age), not generated here. -# It MUST include FORGEQUEUE_AUTH_SECRET, FORGEQUEUE_ADMIN_USER/PASSWORD, and -# POSTGRES_PASSWORD (the base derives FORGEQUEUE_PG_DSN from it). For an -# external managed Postgres, override FORGEQUEUE_PG_DSN in the secret too. +# It MUST include FORGEQUEUE_AUTH_SECRET, FORGEQUEUE_ADMIN_USER/PASSWORD, +# FORGEQUEUE_REDIS_PASSWORD, and POSTGRES_PASSWORD (the base derives +# FORGEQUEUE_PG_DSN from it). For an external managed Postgres, override +# FORGEQUEUE_PG_DSN in the secret too. +# +# Recommended next steps for a real launch: +# * Pin images by digest, not the mutable :v1.0.0 tag. The release workflow +# prints the digest for each image; copy it here, e.g. +# newName: ghcr.io/...-apiserver@sha256:... +# * If your packages are private, enable the imagePullSecret patch: +# patches: +# - path: imagepullsecret-patch.yaml +# * When pointing at a TLS-terminating managed Postgres, also set +# FORGEQUEUE_PG_SSLMODE=verify-full via a configMapGenerator override so +# the derived DSN requires TLS. images: - name: forgequeue/worker diff --git a/dockerfiles/apiserver.Dockerfile b/dockerfiles/apiserver.Dockerfile index ffd4125..0797434 100644 --- a/dockerfiles/apiserver.Dockerfile +++ b/dockerfiles/apiserver.Dockerfile @@ -7,8 +7,15 @@ RUN go mod download COPY cmd ./cmd COPY internal ./internal COPY proto ./proto +ARG VERSION=v1.0.0 +ARG COMMIT=unknown +ARG BUILD_DATE=unknown RUN CGO_ENABLED=0 GOOS=linux go build \ - -trimpath -ldflags="-s -w" \ + -trimpath \ + -ldflags="-s -w \ + -X forgequeue/internal/version.Version=${VERSION} \ + -X forgequeue/internal/version.Commit=${COMMIT} \ + -X forgequeue/internal/version.Date=${BUILD_DATE}" \ -o /out/apiserver ./cmd/apiserver FROM gcr.io/distroless/static-debian12:nonroot diff --git a/dockerfiles/gateway.Dockerfile b/dockerfiles/gateway.Dockerfile index e5d5c9d..970a70a 100644 --- a/dockerfiles/gateway.Dockerfile +++ b/dockerfiles/gateway.Dockerfile @@ -7,8 +7,15 @@ RUN go mod download COPY cmd ./cmd COPY internal ./internal COPY proto ./proto +ARG VERSION=v1.0.0 +ARG COMMIT=unknown +ARG BUILD_DATE=unknown RUN CGO_ENABLED=0 GOOS=linux go build \ - -trimpath -ldflags="-s -w" \ + -trimpath \ + -ldflags="-s -w \ + -X forgequeue/internal/version.Version=${VERSION} \ + -X forgequeue/internal/version.Commit=${COMMIT} \ + -X forgequeue/internal/version.Date=${BUILD_DATE}" \ -o /out/gateway ./cmd/gateway FROM gcr.io/distroless/static-debian12:nonroot diff --git a/dockerfiles/web.Dockerfile b/dockerfiles/web.Dockerfile index c0aefb0..bd8919f 100644 --- a/dockerfiles/web.Dockerfile +++ b/dockerfiles/web.Dockerfile @@ -12,6 +12,8 @@ COPY web/ ./ # web origin and the ingress can route /api to the apiserver service. ARG VITE_API_BASE_URL=/api ENV VITE_API_BASE_URL=$VITE_API_BASE_URL +ARG VITE_APP_VERSION=v1.0.0 +ENV VITE_APP_VERSION=$VITE_APP_VERSION ENV NITRO_PRESET=node-server RUN bun run build diff --git a/dockerfiles/worker.Dockerfile b/dockerfiles/worker.Dockerfile index a17fdfa..9a90e3a 100644 --- a/dockerfiles/worker.Dockerfile +++ b/dockerfiles/worker.Dockerfile @@ -7,8 +7,15 @@ RUN go mod download COPY cmd ./cmd COPY internal ./internal COPY proto ./proto +ARG VERSION=v1.0.0 +ARG COMMIT=unknown +ARG BUILD_DATE=unknown RUN CGO_ENABLED=0 GOOS=linux go build \ - -trimpath -ldflags="-s -w" \ + -trimpath \ + -ldflags="-s -w \ + -X forgequeue/internal/version.Version=${VERSION} \ + -X forgequeue/internal/version.Commit=${COMMIT} \ + -X forgequeue/internal/version.Date=${BUILD_DATE}" \ -o /out/worker ./cmd/worker FROM gcr.io/distroless/static-debian12:nonroot diff --git a/internal/durable/migrations_test.go b/internal/durable/migrations_test.go new file mode 100644 index 0000000..a07346d --- /dev/null +++ b/internal/durable/migrations_test.go @@ -0,0 +1,27 @@ +package durable + +import "testing" + +// TestMigrationsAreOrdered guards the append-only contract: versions must be +// strictly increasing and unique so the ledger applies them exactly once and +// in a deterministic order. +func TestMigrationsAreOrdered(t *testing.T) { + if len(migrations) == 0 { + t.Fatal("no migrations defined") + } + for i, m := range migrations { + if m.Version <= 0 { + t.Errorf("migration %d has non-positive version %d", i, m.Version) + } + if m.Name == "" { + t.Errorf("migration %d (version %d) has empty name", i, m.Version) + } + if m.SQL == "" { + t.Errorf("migration %d (%s) has empty SQL", i, m.Name) + } + if i > 0 && m.Version <= migrations[i-1].Version { + t.Errorf("migration %q version %d is not greater than previous %d", + m.Name, m.Version, migrations[i-1].Version) + } + } +} diff --git a/internal/durable/postgres.go b/internal/durable/postgres.go index ac55bf8..e8d9cde 100644 --- a/internal/durable/postgres.go +++ b/internal/durable/postgres.go @@ -12,10 +12,13 @@ import ( // Override with FORGEQUEUE_PG_DSN. const DefaultDSN = "postgres://localhost:5432/forgequeue" -// One table, one row per enqueued task, following it until succeeded or dead. -// payload is BYTEA not JSONB: it is the exact body Redis stores, so the -// reconciler can rebuild Redis without re-encoding. -const schema = ` +// Migration 1: one table, one row per enqueued task, following it until +// succeeded or dead. payload is BYTEA not JSONB: it is the exact body Redis +// stores, so the reconciler can rebuild Redis without re-encoding. +// +// Every statement is idempotent (IF NOT EXISTS) so a database created by an +// older binary, before this file tracked versions, adopts cleanly. +const migration1QueueTasks = ` CREATE TABLE IF NOT EXISTS queue_tasks ( id TEXT PRIMARY KEY, task_type TEXT NOT NULL, @@ -38,18 +41,42 @@ CREATE INDEX IF NOT EXISTS queue_tasks_active_idx DROP INDEX IF EXISTS queue_tasks_pending_idx; ` +// migrations is the ordered, append-only list of schema changes. To evolve the +// schema, add the next version here; never edit an applied one. Each runs +// exactly once and is recorded in schema_migrations. +var migrations = []migration{ + {Version: 1, Name: "queue_tasks", SQL: migration1QueueTasks}, +} + +type migration struct { + Version int + Name string + SQL string +} + +// migrationsTable records which migrations have run. It is created before any +// migration so the very first one is tracked too. +const migrationsTable = ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +` + // Postgres is a Store backed by PostgreSQL. type Postgres struct { pool *pgxpool.Pool } -// NewPostgres connects to Postgres and makes sure the schema exists. Returns -// an error if Postgres is unreachable. -// advisoryLock serializes one-time schema creation across pods. Multiple -// replicas (gateway + workers) all call NewPostgres on boot, and executing the -// same CREATE/INDEX DDL concurrently raises transient catalog races (e.g. -// duplicate key on pg_type_typname_nsp_index). The lock is transaction-scoped, -// so it releases when the commit ends. +// NewPostgres connects to Postgres, applies any pending migrations, and returns +// the store. Returns an error if Postgres is unreachable. +// +// advisoryLock serializes migrations across pods. Multiple replicas (gateway + +// workers) all call NewPostgres on boot, and executing the same DDL +// concurrently raises transient catalog races (e.g. duplicate key on +// pg_type_typname_nsp_index). The lock is transaction-scoped, so it releases +// when the commit ends. const advisoryLock int64 = 884422 func NewPostgres(ctx context.Context, dsn string) (*Postgres, error) { @@ -61,20 +88,57 @@ func NewPostgres(ctx context.Context, dsn string) (*Postgres, error) { pool.Close() return nil, fmt.Errorf("durable: ping postgres: %w", err) } - if _, err := pool.Exec(ctx, schema); err != nil { - // A concurrent peer may be mid-migration (gateway and workers all - // initialize on boot). Retry a few times before giving up. - if err := ensureSchema(ctx, pool); err != nil { - pool.Close() - return nil, fmt.Errorf("durable: ensure schema: %w", err) - } + if err := migrate(ctx, pool); err != nil { + pool.Close() + return nil, fmt.Errorf("durable: migrate: %w", err) } return &Postgres{pool: pool}, nil } -func ensureSchema(ctx context.Context, pool *pgxpool.Pool) error { +// migrate creates the ledger, then applies every migration that has not run, +// in order, each guarded by an advisory lock. A transient failure (a peer +// mid-migration) is retried a few times before giving up. +func migrate(ctx context.Context, pool *pgxpool.Pool) error { var lastErr error for attempt := 0; attempt < 5; attempt++ { + if err := migrateOnce(ctx, pool); err != nil { + lastErr = err + time.Sleep(200 * time.Millisecond) + continue + } + return nil + } + return lastErr +} + +func migrateOnce(ctx context.Context, pool *pgxpool.Pool) error { + if _, err := pool.Exec(ctx, migrationsTable); err != nil { + return err + } + + // Which migrations are already applied? + applied := map[int]bool{} + rows, err := pool.Query(ctx, `SELECT version FROM schema_migrations`) + if err != nil { + return err + } + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + rows.Close() + return err + } + applied[v] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + + for _, m := range migrations { + if applied[m.Version] { + continue + } tx, err := pool.Begin(ctx) if err != nil { return err @@ -83,15 +147,33 @@ func ensureSchema(ctx context.Context, pool *pgxpool.Pool) error { tx.Rollback(ctx) return err } - if _, err := tx.Exec(ctx, schema); err != nil { + // Re-check under the lock: a peer may have applied it while we waited. + var exists bool + if err := tx.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1)`, + m.Version).Scan(&exists); err != nil { + tx.Rollback(ctx) + return err + } + if exists { tx.Rollback(ctx) - lastErr = err - time.Sleep(200 * time.Millisecond) continue } - return tx.Commit(ctx) + if _, err := tx.Exec(ctx, m.SQL); err != nil { + tx.Rollback(ctx) + return fmt.Errorf("migration %d (%s): %w", m.Version, m.Name, err) + } + if _, err := tx.Exec(ctx, + `INSERT INTO schema_migrations (version, name) VALUES ($1, $2)`, + m.Version, m.Name); err != nil { + tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } } - return lastErr + return nil } func (p *Postgres) Close() error { diff --git a/internal/gateway/main.go b/internal/gateway/main.go index 5ec1f92..7f541bc 100644 --- a/internal/gateway/main.go +++ b/internal/gateway/main.go @@ -2,6 +2,9 @@ package gateway import ( "context" + "crypto/tls" + "crypto/x509" + "fmt" "log" "net" "os" @@ -12,6 +15,7 @@ import ( "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" "forgequeue/internal/auth" @@ -58,6 +62,49 @@ func (s *server) EnqueueTask( }, nil } +// serverTLSOption builds gRPC transport credentials from the environment. +// Returns (nil, nil) when TLS is not configured, so the gateway still runs +// plaintext on a trusted pod network for local development. +// +// FORGEQUEUE_GRPC_TLS_CERT server certificate (PEM) +// FORGEQUEUE_GRPC_TLS_KEY server private key (PEM) +// FORGEQUEUE_GRPC_TLS_CLIENT_CA optional CA bundle; when set, clients must +// present a certificate signed by it (mTLS) +func serverTLSOption() (grpc.ServerOption, error) { + certFile := os.Getenv("FORGEQUEUE_GRPC_TLS_CERT") + keyFile := os.Getenv("FORGEQUEUE_GRPC_TLS_KEY") + if certFile == "" && keyFile == "" { + return nil, nil + } + if certFile == "" || keyFile == "" { + return nil, fmt.Errorf("both FORGEQUEUE_GRPC_TLS_CERT and FORGEQUEUE_GRPC_TLS_KEY must be set") + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("load key pair: %w", err) + } + cfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, + } + + if caFile := os.Getenv("FORGEQUEUE_GRPC_TLS_CLIENT_CA"); caFile != "" { + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read client CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no certificates parsed from %s", caFile) + } + cfg.ClientCAs = pool + cfg.ClientAuth = tls.RequireAndVerifyClientCert + } + + return grpc.Creds(credentials.NewTLS(cfg)), nil +} + // raw netowrk listener socket opening func Start() { // Initialize your custom Redis connection pool pointer @@ -126,9 +173,16 @@ func Start() { } //Initialize the core gRPC microservice server container - grpcServer := grpc.NewServer( + opts := []grpc.ServerOption{ grpc.UnaryInterceptor(newAuthInterceptor(authSvc)), - ) + } + if tlsOpt, err := serverTLSOption(); err != nil { + log.Fatalf("failed to configure gRPC TLS: %v", err) + } else if tlsOpt != nil { + opts = append(opts, tlsOpt) + log.Println("gRPC TLS enabled on port :50051") + } + grpcServer := grpc.NewServer(opts...) //Register our server implementation with the gRPC engine queuev1.RegisterQueueServiceServer( diff --git a/internal/httpapi/handlers.go b/internal/httpapi/handlers.go index 377b8b1..58f79ed 100644 --- a/internal/httpapi/handlers.go +++ b/internal/httpapi/handlers.go @@ -11,6 +11,7 @@ import ( "github.com/redis/go-redis/v9" "forgequeue/internal/auth" + "forgequeue/internal/version" ) type Handler struct { @@ -713,7 +714,11 @@ func (h *Handler) Throughput(w http.ResponseWriter, r *http.Request) { h.writeJSON(w, http.StatusOK, points) } func (h *Handler) Version(w http.ResponseWriter, r *http.Request) { - h.writeJSON(w, http.StatusOK, map[string]string{"version": "v0.9.1"}) + h.writeJSON(w, http.StatusOK, map[string]string{ + "version": version.Version, + "commit": version.Commit, + "date": version.Date, + }) } func genSeries(rangeQ string, rdb *redis.Client, ctx context.Context) []ThroughputPoint { diff --git a/internal/httpapi/middleware.go b/internal/httpapi/middleware.go new file mode 100644 index 0000000..1df742f --- /dev/null +++ b/internal/httpapi/middleware.go @@ -0,0 +1,115 @@ +package httpapi + +import ( + "net/http" + "strconv" + "strings" +) + +// securityHeaders sets conservative response headers on every API response. +// The API only serves JSON, so a locked-down CSP is cheap insurance against a +// browser being tricked into treating a JSON body as a document. +func securityHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("X-Content-Type-Options", "nosniff") + h.Set("X-Frame-Options", "DENY") + h.Set("Referrer-Policy", "no-referrer") + h.Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'") + // Only advertise HSTS when the request already arrived over TLS + // (the ingress terminates TLS and sets this header upstream). + if r.TLS != nil || strings.EqualFold(r.Header.Get("X-Forwarded-Proto"), "https") { + h.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") + } + next.ServeHTTP(w, r) + }) +} + +// corsMiddleware allows only the configured browser origins. An empty +// allowlist disables cross-origin access entirely, which is correct for the +// production shape where the console and the API share one origin behind the +// ingress. A wildcard is deliberately not supported: with a wildcard any site +// could make authenticated calls with a stolen token. +func corsMiddleware(allowed []string) func(http.Handler) http.Handler { + set := make(map[string]struct{}, len(allowed)) + for _, o := range allowed { + if o = strings.TrimSpace(o); o != "" { + set[o] = struct{}{} + } + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + origin := r.Header.Get("Origin") + if origin != "" { + if _, ok := set[origin]; ok { + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") + w.Header().Set("Access-Control-Max-Age", "600") + // The response varies by Origin, so caches must not + // serve one origin's CORS headers to another. + w.Header().Add("Vary", "Origin") + } + } + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// limitBody caps the request body so a client cannot make the process allocate +// unbounded memory by streaming an oversized body. Over the limit, the read +// fails and decoding returns an error the handler turns into a 400. +func limitBody(maxBytes int64) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + r.Body = http.MaxBytesReader(w, r.Body, maxBytes) + } + next.ServeHTTP(w, r) + }) + } +} + +// clientIP extracts the caller's address. Behind the ingress the connection +// peer is the proxy, so X-Forwarded-For carries the real client; the leftmost +// entry is the original. This is used only for rate limiting, never for +// authorization. +func clientIP(r *http.Request) string { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + if first := strings.TrimSpace(strings.Split(xff, ",")[0]); first != "" { + return first + } + } + host := r.RemoteAddr + if i := strings.LastIndex(host, ":"); i >= 0 { + host = host[:i] + } + return host +} + +// parseOrigins turns a comma-separated env value into a list. +func parseOrigins(raw string) []string { + var out []string + for _, p := range strings.Split(raw, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +// envInt64 reads an int64 env var, falling back to def on empty/invalid input. +func envInt64(raw string, def int64) int64 { + if raw == "" { + return def + } + v, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64) + if err != nil { + return def + } + return v +} diff --git a/internal/httpapi/middleware_test.go b/internal/httpapi/middleware_test.go new file mode 100644 index 0000000..b6ab16f --- /dev/null +++ b/internal/httpapi/middleware_test.go @@ -0,0 +1,94 @@ +package httpapi + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestCORSAllowlist(t *testing.T) { + h := corsMiddleware([]string{"https://console.example.com"})(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) + + // Allowed origin is echoed. + req := httptest.NewRequest(http.MethodGet, "/api/queues", nil) + req.Header.Set("Origin", "https://console.example.com") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://console.example.com" { + t.Errorf("ACAO = %q, want echoed origin", got) + } + + // Unknown origin gets no CORS grant (no wildcard). + req = httptest.NewRequest(http.MethodGet, "/api/queues", nil) + req.Header.Set("Origin", "https://evil.example.com") + rec = httptest.NewRecorder() + h.ServeHTTP(rec, req) + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" { + t.Errorf("ACAO = %q, want empty for unlisted origin", got) + } +} + +func TestCORSDisabledWhenEmpty(t *testing.T) { + h := corsMiddleware(nil)(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) + + req := httptest.NewRequest(http.MethodGet, "/api/queues", nil) + req.Header.Set("Origin", "https://console.example.com") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" { + t.Errorf("ACAO = %q, want empty when no origins configured", got) + } +} + +func TestLimitBodyRejectsOversize(t *testing.T) { + handler := limitBody(4)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, err := io.ReadAll(r.Body); err != nil { + http.Error(w, "too big", http.StatusRequestEntityTooLarge) + return + } + w.WriteHeader(http.StatusOK) + })) + + req := httptest.NewRequest(http.MethodPost, "/api/auth/login", strings.NewReader("this body is far too long")) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Errorf("status = %d, want 413", rec.Code) + } +} + +func TestLimitBodyAllowsUnderCap(t *testing.T) { + handler := limitBody(64)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, err := io.ReadAll(r.Body); err != nil { + http.Error(w, "too big", http.StatusRequestEntityTooLarge) + return + } + w.WriteHeader(http.StatusOK) + })) + + req := httptest.NewRequest(http.MethodPost, "/api/auth/login", strings.NewReader("small")) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want 200", rec.Code) + } +} + +func TestSecurityHeaders(t *testing.T) { + h := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if got := rec.Header().Get("X-Content-Type-Options"); got != "nosniff" { + t.Errorf("X-Content-Type-Options = %q, want nosniff", got) + } + if rec.Header().Get("Content-Security-Policy") == "" { + t.Error("missing Content-Security-Policy") + } +} diff --git a/internal/httpapi/ratelimit.go b/internal/httpapi/ratelimit.go new file mode 100644 index 0000000..0dcc76a --- /dev/null +++ b/internal/httpapi/ratelimit.go @@ -0,0 +1,96 @@ +package httpapi + +import ( + "net/http" + "sync" + "time" +) + +// rateLimiter is a per-key token bucket, used to slow down credential +// stuffing against the login endpoint. It is in-process: with several +// apiserver replicas each enforces its own budget, so the effective limit is +// (replicas x rate). That is fine for blunting brute force; a shared store +// (Redis) would be needed for an exact global limit. +type rateLimiter struct { + mu sync.Mutex + buckets map[string]*bucket + rate float64 // tokens replenished per second + burst float64 // bucket capacity + now func() time.Time +} + +type bucket struct { + tokens float64 + last time.Time +} + +func newRateLimiter(ratePerSecond, burst float64) *rateLimiter { + if ratePerSecond <= 0 { + ratePerSecond = 0.2 // 12 per minute + } + if burst <= 0 { + burst = 10 + } + return &rateLimiter{ + buckets: make(map[string]*bucket), + rate: ratePerSecond, + burst: burst, + now: time.Now, + } +} + +// allow consumes one token for key, returning false when the bucket is empty. +func (l *rateLimiter) allow(key string) bool { + now := l.now() + + l.mu.Lock() + defer l.mu.Unlock() + + b, ok := l.buckets[key] + if !ok { + b = &bucket{tokens: l.burst, last: now} + l.buckets[key] = b + } + + // Refill for elapsed time, capped at burst. + elapsed := now.Sub(b.last).Seconds() + if elapsed > 0 { + b.tokens += elapsed * l.rate + if b.tokens > l.burst { + b.tokens = l.burst + } + b.last = now + } + + if b.tokens < 1 { + return false + } + b.tokens-- + // Opportunistic sweep so a flood of distinct keys cannot grow the map + // without bound. Full buckets are indistinguishable from idle ones. + if len(l.buckets) > 4096 { + l.sweepLocked(now) + } + return true +} + +func (l *rateLimiter) sweepLocked(now time.Time) { + for k, b := range l.buckets { + if now.Sub(b.last) > 10*time.Minute { + delete(l.buckets, k) + } + } +} + +// middleware returns a handler that rejects callers over budget with 429. +// It keys on client IP; the login handler adds a second, per-username limiter. +func (l *rateLimiter) middleware(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !l.allow(clientIP(r)) { + w.Header().Set("Retry-After", "60") + writeError(w, http.StatusTooManyRequests, "too many requests") + return + } + next(w, r) + } +} diff --git a/internal/httpapi/ratelimit_test.go b/internal/httpapi/ratelimit_test.go new file mode 100644 index 0000000..fc598d3 --- /dev/null +++ b/internal/httpapi/ratelimit_test.go @@ -0,0 +1,47 @@ +package httpapi + +import ( + "testing" + "time" +) + +func TestRateLimiterBurstThenDeny(t *testing.T) { + now := time.Unix(0, 0) + l := newRateLimiter(1, 3) // 1 token/sec, burst 3 + l.now = func() time.Time { return now } + + for i := 0; i < 3; i++ { + if !l.allow("1.2.3.4") { + t.Fatalf("burst request %d denied, want allowed", i+1) + } + } + if l.allow("1.2.3.4") { + t.Fatal("request over burst allowed, want denied") + } + + // Two seconds later two tokens have refilled. + now = now.Add(2 * time.Second) + if !l.allow("1.2.3.4") || !l.allow("1.2.3.4") { + t.Fatal("refilled tokens denied, want allowed") + } + if l.allow("1.2.3.4") { + t.Fatal("bucket not emptied after consuming refill") + } +} + +func TestRateLimiterIsolatesKeys(t *testing.T) { + l := newRateLimiter(1, 1) + now := time.Unix(0, 0) + l.now = func() time.Time { return now } + + if !l.allow("a") { + t.Fatal("first key denied") + } + if l.allow("a") { + t.Fatal("second call for same key allowed") + } + // A different key has its own bucket. + if !l.allow("b") { + t.Fatal("independent key denied") + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 6f37aea..b1ac23c 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "os" "strings" "time" @@ -17,6 +18,10 @@ type Server struct { auth *auth.Service oidc *auth.OIDC h *Handler + + // Login brute-force guards: one global backstop and one per client IP. + loginGlobal *rateLimiter + loginIP *rateLimiter } func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { @@ -24,7 +29,13 @@ func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { addr = ":8080" } - s := &Server{auth: authSvc} + s := &Server{ + auth: authSvc, + // Burst generous enough to absorb a human retyping a password; the + // sustained rate is what stops automated guessing. + loginGlobal: newRateLimiter(20, 40), + loginIP: newRateLimiter(0.2, 10), + } h := NewHandler(rdb, authSvc) mux := http.NewServeMux() @@ -41,7 +52,7 @@ func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { writeJSON(w, http.StatusOK, map[string]string{"status": "ready"}) }) - mux.HandleFunc("POST /api/auth/login", h.Login) + mux.HandleFunc("POST /api/auth/login", s.loginLimiter(h.Login)) mux.HandleFunc("GET /api/auth/config", h.AuthConfig) mux.HandleFunc("GET /api/auth/oidc/login", h.OIDCLogin) mux.HandleFunc("GET /api/auth/oidc/callback", h.OIDCCallback) @@ -100,15 +111,32 @@ func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { mux.HandleFunc("GET /api/metrics/throughput", s.require(auth.PermMetricsRead, h.Throughput)) srv := &http.Server{ - Addr: addr, - Handler: corsMiddleware(mux), + Addr: addr, + // Outer-most middleware first: headers on every response, then CORS, + // then a body cap, then the router. + Handler: securityHeaders(corsMiddleware(parseOrigins(os.Getenv("FORGEQUEUE_CORS_ORIGINS")))(limitBody(envInt64(os.Getenv("FORGEQUEUE_MAX_BODY_BYTES"), 1<<20))(mux))), ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } s.srv = srv s.h = h return s } +// loginLimiter rejects credential attempts over budget, per IP and globally. +func (s *Server) loginLimiter(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !s.loginGlobal.allow("global") || !s.loginIP.allow(clientIP(r)) { + w.Header().Set("Retry-After", "60") + writeError(w, http.StatusTooManyRequests, "too many login attempts, try again later") + return + } + next(w, r) + } +} + // SetOIDC enables single sign-on on the server. Call before Start. func (s *Server) SetOIDC(o *auth.OIDC) { s.oidc = o @@ -155,19 +183,6 @@ func (s *Server) authenticate(r *http.Request) (auth.Identity, bool) { return id, true } -func corsMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") - if r.Method == http.MethodOptions { - w.WriteHeader(http.StatusNoContent) - return - } - next.ServeHTTP(w, r) - }) -} - func (s *Server) Start() error { go func() { _ = s.srv.ListenAndServe() diff --git a/internal/redisutil/client.go b/internal/redisutil/client.go index 3d05c50..161b286 100644 --- a/internal/redisutil/client.go +++ b/internal/redisutil/client.go @@ -2,21 +2,94 @@ package redisutil import ( "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "strconv" + "strings" "github.com/redis/go-redis/v9" ) -// NewClient creates a redis client and checks the connection -func NewClient(addr string, poolSize int, ctx context.Context) (*redis.Client, error) { - // creating a redis client instance - client := redis.NewClient(&redis.Options{ - // redis.NewClient already returns a pointer +// OptionsFromEnv builds redis.Options from the environment. Every service that +// talks to Redis (gateway, worker, apiserver) funnels through here so auth and +// TLS are configured in exactly one place. +// +// FORGEQUEUE_REDIS_PASSWORD AUTH password (empty disables AUTH) +// FORGEQUEUE_REDIS_DB database index, default 0 +// FORGEQUEUE_REDIS_TLS "true" enables TLS +// FORGEQUEUE_REDIS_TLS_INSECURE "true" skips certificate verification (dev only) +// FORGEQUEUE_REDIS_TLS_CA path to a PEM CA bundle to trust +// +// A password with TLS turned off still authenticates; Redis just carries the +// credential in cleartext, which is acceptable only on a trusted pod network. +func OptionsFromEnv(addr string, poolSize int) (*redis.Options, error) { + if poolSize <= 0 { + poolSize = 10 + } + opts := &redis.Options{ Addr: addr, PoolSize: poolSize, - }) - // ping to check the server is up + } + + if pw := os.Getenv("FORGEQUEUE_REDIS_PASSWORD"); pw != "" { + opts.Password = pw + } + + if raw := strings.TrimSpace(os.Getenv("FORGEQUEUE_REDIS_DB")); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n < 0 { + return nil, fmt.Errorf("redisutil: invalid FORGEQUEUE_REDIS_DB %q", raw) + } + opts.DB = n + } + + if isTruthy(os.Getenv("FORGEQUEUE_REDIS_TLS")) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12} + if isTruthy(os.Getenv("FORGEQUEUE_REDIS_TLS_INSECURE")) { + // VerifyPeerCertificate stays set, but the chain is not checked. + cfg.InsecureSkipVerify = true + } + if caPath := os.Getenv("FORGEQUEUE_REDIS_TLS_CA"); caPath != "" { + pem, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("redisutil: read FORGEQUEUE_REDIS_TLS_CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("redisutil: no certificates parsed from %s", caPath) + } + cfg.RootCAs = pool + } + opts.TLSConfig = cfg + } + + return opts, nil +} + +// NewClient builds a client from the environment and verifies the connection +// before returning. A Redis that needs AUTH will fail fast here rather than at +// the first claim. +func NewClient(addr string, poolSize int, ctx context.Context) (*redis.Client, error) { + opts, err := OptionsFromEnv(addr, poolSize) + if err != nil { + return nil, err + } + + client := redis.NewClient(opts) if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() return nil, err } return client, nil } + +func isTruthy(v string) bool { + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} diff --git a/internal/redisutil/client_test.go b/internal/redisutil/client_test.go new file mode 100644 index 0000000..647879c --- /dev/null +++ b/internal/redisutil/client_test.go @@ -0,0 +1,54 @@ +package redisutil + +import ( + "crypto/tls" + "testing" +) + +func TestOptionsFromEnvPassword(t *testing.T) { + t.Setenv("FORGEQUEUE_REDIS_PASSWORD", "s3cret") + t.Setenv("FORGEQUEUE_REDIS_DB", "3") + + opts, err := OptionsFromEnv("redis:6379", 0) + if err != nil { + t.Fatalf("OptionsFromEnv: %v", err) + } + if opts.Password != "s3cret" { + t.Errorf("password = %q, want s3cret", opts.Password) + } + if opts.DB != 3 { + t.Errorf("db = %d, want 3", opts.DB) + } + if opts.PoolSize <= 0 { + t.Errorf("pool size = %d, want > 0 default", opts.PoolSize) + } + if opts.TLSConfig != nil { + t.Errorf("TLS unexpectedly enabled") + } +} + +func TestOptionsFromEnvTLS(t *testing.T) { + t.Setenv("FORGEQUEUE_REDIS_TLS", "true") + t.Setenv("FORGEQUEUE_REDIS_TLS_INSECURE", "true") + + opts, err := OptionsFromEnv("redis:6379", 10) + if err != nil { + t.Fatalf("OptionsFromEnv: %v", err) + } + if opts.TLSConfig == nil { + t.Fatal("expected TLS config") + } + if !opts.TLSConfig.InsecureSkipVerify { + t.Errorf("expected InsecureSkipVerify") + } + if opts.TLSConfig.MinVersion != tls.VersionTLS12 { + t.Errorf("min TLS version = %d, want TLS 1.2", opts.TLSConfig.MinVersion) + } +} + +func TestOptionsFromEnvBadDB(t *testing.T) { + t.Setenv("FORGEQUEUE_REDIS_DB", "not-a-number") + if _, err := OptionsFromEnv("redis:6379", 1); err == nil { + t.Fatal("expected error for non-numeric FORGEQUEUE_REDIS_DB") + } +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..b005253 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,21 @@ +// Package version holds build identity for every binary. The values are +// injected at build time with -ldflags so a running process can report exactly +// which commit it came from; the defaults keep local builds working. +package version + +var ( + // Version is the semantic release tag, e.g. "v1.0.0". + Version = "v1.0.0" + // Commit is the git SHA the binary was built from. + Commit = "unknown" + // Date is the build timestamp (RFC3339). + Date = "unknown" +) + +// Full returns "version (commit)" or just the version when no commit is known. +func Full() string { + if Commit == "" || Commit == "unknown" { + return Version + } + return Version + " (" + Commit + ")" +} diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 62bb53b..c05f57a 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -16,6 +16,8 @@ import ( "forgequeue/internal/durable" "forgequeue/internal/metrics" "forgequeue/internal/model" + "forgequeue/internal/redisutil" + "forgequeue/internal/version" ) // worker registry keys. a pool publishes its identity on Start and keeps it @@ -138,29 +140,22 @@ type TaskClaim struct { } func NewWorkerPool(addr string, concurrencyLimit int) (*WorkerPool, error) { - client := redis.NewClient(&redis.Options{ - Addr: addr, - - // enough connections for the workers and their concurrent Redis ops. - PoolSize: concurrencyLimit * 2, - }) - - // one context for the whole pool, passed to every Redis operation so - // Stop() can interrupt them. + // One context for the whole pool, passed to every Redis operation so + // Stop() can interrupt them. It also bounds the connection check below. ctx, cancel := context.WithCancel(context.Background()) - // separate context for the background loops, so a graceful drain can stop - // them without cancelling handlers still finishing under ctx. - bgCtx, bgCancel := context.WithCancel(context.Background()) - - // make sure Redis is reachable before returning the pool. - if err := client.Ping(context.Background()).Err(); err != nil { - // pool won't start, so drop the contexts we just made. + // enough connections for the workers and their concurrent Redis ops. + // redisutil applies auth/TLS from the environment in one place. + client, err := redisutil.NewClient(addr, concurrencyLimit*2, ctx) + if err != nil { cancel() - bgCancel() return nil, err } + // separate context for the background loops, so a graceful drain can stop + // them without cancelling handlers still finishing under ctx. + bgCtx, bgCancel := context.WithCancel(context.Background()) + hostname, _ := os.Hostname() return &WorkerPool{ @@ -170,7 +165,7 @@ func NewWorkerPool(addr string, concurrencyLimit int) (*WorkerPool, error) { id: uuid.New().String(), hostname: hostname, startedAt: time.Now(), - version: "v0.9.1", + version: version.Version, wg: sync.WaitGroup{}, diff --git a/web/.env.development b/web/.env.development index 9608399..86d6aeb 100644 --- a/web/.env.development +++ b/web/.env.development @@ -1,2 +1,2 @@ VITE_API_BASE_URL=http://localhost:8080/api -VITE_APP_VERSION=v0.9.1 +VITE_APP_VERSION=v1.0.0 diff --git a/web/.env.example b/web/.env.example index fa593f0..e997cc8 100644 --- a/web/.env.example +++ b/web/.env.example @@ -1,3 +1,3 @@ # Base URL for the ForgeQueue gRPC Gateway (HTTP/JSON). Relative paths work when served under the Go binary. VITE_API_BASE_URL=/api -VITE_APP_VERSION=v0.9.1 +VITE_APP_VERSION=v1.0.0 diff --git a/web/src/components/app-shell.tsx b/web/src/components/app-shell.tsx index 6587255..0872656 100644 --- a/web/src/components/app-shell.tsx +++ b/web/src/components/app-shell.tsx @@ -258,7 +258,7 @@ function Sidebar({ collapsed }: { collapsed: boolean }) { > {!collapsed && (
- {import.meta.env["VITE_APP_VERSION"] || "v0.9.1"} + {import.meta.env["VITE_APP_VERSION"] || "dev"}
)} diff --git a/web/src/routes/login.tsx b/web/src/routes/login.tsx index 773f065..f5a3702 100644 --- a/web/src/routes/login.tsx +++ b/web/src/routes/login.tsx @@ -91,7 +91,7 @@ function LoginPage() { const apiBase = getPrefs().apiBase; const env = apiBase.startsWith("/") ? "local-dev" : apiBase.replace(/^https?:\/\//, ""); - const version = import.meta.env["VITE_APP_VERSION"] || "v0.9.1"; + const version = import.meta.env["VITE_APP_VERSION"] || "dev"; useEffect(() => { if (session) navigate({ to: "/", replace: true });