deploy.yml

name: Test & Deploy

# One workflow, one app, one image. Command Center and the Sentinel AI
# agent both ship from here: the image carries both, and fly.toml's
# [processes] block runs them as two process groups on separate machines.
# There is no separate agent workflow or agent app any more.
#
# Path filtering is applied to `push` ONLY, never to `pull_request`, and
# that asymmetry is load-bearing. `master` requires these status checks:
#
#   Backend tests (sqlite) · Backend tests (postgres) ·
#   Frontend audit + build
#
# GitHub reports NO status for a workflow that a path filter skipped —
# not "skipped", nothing at all — so a required check on a path-filtered
# PR trigger blocks the PR forever. Hence: PRs always run everything,
# pushes stay filtered. Deploys are the thing filtering exists to avoid,
# and they only happen on push.
#
# If you add a path filter to the pull_request trigger below, you will
# hang every PR that misses it, and the failure looks like a stuck check
# rather than a config error.
on:
  push:
    branches:
      - master
    paths-ignore:
      - 'docs/**'
      - '**/*.md'
  # Manual re-deploy. Needed because Dependabot auto-merges land on
  # master WITHOUT triggering this workflow — see the GITHUB_TOKEN note
  # in dependabot-auto-merge.yml. Without a manual trigger the only way
  # to ship those commits is to push an unrelated one on top.
  workflow_dispatch: {}
  # Validate PRs (incl. Dependabot bumps) BEFORE merge. The deploy job
  # below is gated to push-only, so a PR runs tests/audits/build but
  # never deploys. This is what makes auto-merging dependency PRs safe —
  # see .github/workflows/dependabot-auto-merge.yml.
  #
  # DELIBERATELY UNFILTERED, unlike the push trigger above. These jobs are
  # required status checks on master, and GitHub reports NO status for a
  # workflow skipped by a path filter — so a filtered PR trigger would
  # leave every docs-only or agent-only PR waiting forever on a check
  # that is never going to arrive.
  #
  # The cost is a few wasted CI-minutes on PRs that touch nothing this
  # workflow builds. The thing path filtering actually exists to prevent
  # is redundant *deploys*, and deploys only happen on push, where the
  # filter is still in force.
  pull_request:
    branches:
      - master

# Least-privilege token. Without this block a job gets whatever the
# repository's default GITHUB_TOKEN scope is, which is broader than
# anything here needs — CodeQL's actions/missing-workflow-permissions
# flagged every job in this file. Nothing here writes through the API:
# no `gh` calls, no git push, no package publish. Artifact upload does
# not need a contents scope either (it uses the runtime token).
#
# A job that genuinely needs more should declare it at the JOB level
# rather than widening this.
permissions:
  contents: read

jobs:
  test:
    # Explicit name so the branch-protection check name is STABLE.
    # Without it GitHub derives the name from the whole matrix entry —
    # "Backend tests (postgres, postgresql+psycopg://sentinel:sentinel@
    # localhost:5432/sentinel_test)" — and editing that URL would rename
    # the check, silently detaching it from branch protection.
    name: Backend tests (${{ matrix.dialect }})
    runs-on: ubuntu-latest
    # One codebase, two databases: hosted Command Center runs Postgres,
    # self-hosted runs SQLite. Testing only one of them means shipping
    # the other unverified, so the suite runs against both and a failure
    # in either blocks the deploy.
    #
    # fail-fast is off deliberately — when a dialect-specific bug lands,
    # seeing which engine broke (and whether the other survived) is the
    # whole diagnostic.
    strategy:
      fail-fast: false
      matrix:
        include:
          - dialect: sqlite
            test_database_url: ""
          - dialect: postgres
            test_database_url: postgresql+psycopg://sentinel:sentinel@localhost:5432/sentinel_test
    services:
      # Always started, used only by the postgres leg. GitHub has no
      # per-matrix-entry `services`, and an idle container costs a few
      # seconds of startup — cheaper than splitting this into two jobs
      # that would then duplicate every step below.
      postgres:
        image: postgres:16-alpine
        env:
          POSTGRES_USER: sentinel
          POSTGRES_PASSWORD: sentinel
          POSTGRES_DB: sentinel_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v7

      - name: Install uv
        # astral-sh/setup-uv doesn't publish sliding major-version
        # tags (v7, v8 etc.) the way actions/* and docker/* do — only
        # specific versions like v8.1.0.  Pin to the exact tag and
        # bump manually when there's a reason to.  v8.x is the first
        # Node.js 24-ready major.
        uses: astral-sh/setup-uv@v10.0.1

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        working-directory: backend
        run: uv sync --extra dev

      # Lint before tests — ruff is sub-second and catches the kind of
      # issue (unused imports, import-sort drift, missing `from err` on
      # re-raises) that masks real bugs and clutters review diffs.  Ruleset
      # is conservative (F + E9 + W6 + I + B + UP) so the bar is "real
      # problems only" rather than "every style nit"; tighten in pyproject
      # when the team agrees on each new rule.
      - name: Lint with ruff
        working-directory: backend
        run: uv run ruff check

      # Scan our Python dep tree against the PyPA Advisory DB.
      # ``--strict`` makes any vulnerability a non-zero exit so a
      # known-bad transitive dep blocks the deploy.  Mirrors the
      # frontend's npm-audit gate (--audit-level=high --omit=dev)
      # in policy: this also blocks at high+ tiers — pip-audit
      # doesn't currently expose a severity gate, so anything in
      # the advisory DB counts.  When a CVE shows up with no fix
      # yet, add ``--ignore-vuln <PYSEC-ID>`` here with a comment
      # citing the upstream issue and the date we plan to revisit.
      - name: Dependency scan (pip-audit)
        working-directory: backend
        run: uv run pip-audit --strict

      # TEST_DATABASE_URL is empty for the sqlite leg, and conftest.py
      # falls back to in-memory SQLite when it's unset — so the two
      # matrix entries share one command.
      - name: Run tests (${{ matrix.dialect }})
        working-directory: backend
        env:
          TEST_DATABASE_URL: ${{ matrix.test_database_url }}
        run: uv run pytest -v

  frontend:
    name: Frontend audit + build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v7
        with:
          node-version: "24"
          cache: "npm"
          cache-dependency-path: frontend/package-lock.json

      - name: Install dependencies
        working-directory: frontend
        run: npm ci

      # `npm audit` flags vulnerabilities in production dependencies.
      # Threshold = high so low/moderate findings (transitive dev-only
      # CVEs that don't reach prod, advisories with no fix yet, etc.)
      # don't block legitimate deploys.  High + critical findings DO
      # block — those are real and need a fix or a documented
      # `--omit=optional` / override / waiver.
      #
      # `--omit=dev` skips devDependencies because they don't ship
      # to production; the prod bundle is what reaches a user.
      #
      # Requires npm 11+ (hence Node 24 above, not 20). npm registry
      # retired the legacy `/security/audits/quick` endpoint that npm 10
      # uses for `--omit=dev`, and it now returns 400 Bad Request —
      # which failed this step and blocked every deploy, with nothing in
      # the repo having changed. npm 11 uses the bulk advisory endpoint
      # instead. Verified directly: npm@10 fails here, npm@11 passes.
      - name: npm audit (production deps only, high+critical)
        working-directory: frontend
        run: npm audit --audit-level=high --omit=dev

      # ESLint. This step did not exist until 2026-09-11, which made
      # `eslint.config.js` decorative: the config was added specifically
      # to fix `npm run lint` silently erroring with "couldn't find a
      # config", and then nothing ever ran it. Thirteen errors
      # accumulated in master unnoticed, and the eslint 9 -> 10 bump was
      # reviewed by running the linter by hand because CI could not say
      # anything about it.
      #
      # eslint exits non-zero on errors only, so the ~34 deliberate
      # `warn`-level findings (React Compiler advisories, unused-vars)
      # stay visible in the log without blocking a deploy. That split is
      # the whole point — see the rule notes in eslint.config.js.
      - name: Lint (eslint)
        working-directory: frontend
        run: npm run lint

      # Vitest component tests — run BEFORE the build so a regression
      # caught by tests doesn't get the chance to ship via a successful
      # build.  We have ~50 tests today (HelpTooltip, InstallCameraNodeCard,
      # UpgradeModal, EmptyState, the API service helpers, the docs
      # page, and a sanity smoke test); the suite runs in <10s on CI
      # so the speed cost is negligible.
      #
      # ``npm test`` is the package.json alias for ``vitest run``
      # (one-shot, exits with status, no watch).  Vitest auto-discovers
      # files matching the ``include: ["tests/**/*.test.{js,jsx}"]``
      # pattern in vite.config.js.
      - name: Run frontend tests (vitest)
        working-directory: frontend
        run: npm test

      # Build now so a syntax or type error fails CI here rather than
      # mid-deploy.  Catches the same class of bug as backend pytest.
      - name: Build production bundle
        working-directory: frontend
        run: npm run build

  deploy:
    name: Deploy to Fly.io
    runs-on: ubuntu-latest
    needs: [test, frontend]
    # Push-to-master only. PRs run test+frontend for validation but must
    # never deploy (no secrets exposure, no half-reviewed code shipped).
    if: github.event_name == 'push'
    concurrency:
      group: deploy
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v7

      - uses: superfly/flyctl-actions/setup-flyctl@master

      # Why two-step (build → machine update) instead of plain `fly deploy`:
      #
      # We run a single Fly Machine with a single persistent volume
      # (`sentinel_data` at /data).  Since 2026-09-07 that volume holds
      # only HLS segment working files and /data/backups — the database
      # is Postgres on the sentinel-postgres cluster — but the volume is
      # still single-attachment, so the deploy topology below is
      # unchanged.  `fly deploy`
      # for this topology is non-deterministic: sometimes it sees the
      # existing machine and updates it in place, sometimes it decides
      # the image config has "drifted enough" and tries to provision a
      # NEW machine alongside the old.  The new-machine path errors
      # immediately because the volume only has one attachment slot:
      #     "creating a new machine in group 'app' requires an
      #      unattached 'sentinel_data' volume."
      # We hit this on consecutive runs 2026-04-28 with strategy=rolling
      # AND strategy=immediate, and `max_unavailable` is rolling-only so
      # it didn't help either.
      #
      # `fly machine update --image …` is the explicit in-place API.
      # It targets a specific machine ID, restarts it on the new image,
      # and the volume stays attached throughout.  It cannot try to
      # create a new machine.  ~30-60s of downtime per deploy (same as
      # `strategy = "immediate"` on a good day) but reliably works.
      #
      # Builder choice has flipped THREE times now:
      #   - Original: --depot=true (depot.dev managed builder).
      #   - 2026-04-28: depot.dev timed out 5 min × 2 in a row
      #     (~10 min wasted per deploy).  Switched to --depot=false
      #     (Fly's standard remote builder).  ~100 deploys worked.
      #   - 2026-05-04 morning: Fly's standard remote builder started
      #     returning `unauthorized` on the WireGuard heartbeat for
      #     valid deploy tokens (Request ID 01KQTE4AHWKB2PAAS8A372EKNP).
      #     Swapped tokens — same failure.  Server-side scope change
      #     or platform incident; either way, CI was wedged.  Tried
      #     --depot=true again briefly: depot built fine but tagged
      #     the manifest under its own internal namespace
      #     (vo4x1o84n7ozql5y), so the subsequent `fly machine update`
      #     got MANIFEST_UNKNOWN looking for the image at the
      #     sentinel-command path.  Mismatch between depot's push
      #     and the two-step pattern we use.
      #   - 2026-05-04 afternoon (current): build locally on the
      #     GitHub runner with docker/build-push-action and push
      #     directly to registry.fly.io.  No third-party builders.
      #     No WireGuard.  Same FLY_API_TOKEN works for the registry
      #     push (proven on the failed depot run — depot's push to
      #     registry.fly.io itself succeeded; the namespace mismatch
      #     was on its side, not the registry's).  Dockerfile is a
      #     standard multi-stage build (node:20-alpine for frontend,
      #     uv:python3.12-bookworm-slim for backend) — no special
      #     build hardware needed.
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Log in to Fly registry
        uses: docker/login-action@v4
        with:
          registry: registry.fly.io
          # Fly's registry ignores the username; only the token matters.
          username: x
          password: ${{ secrets.FLY_API_TOKEN }}

      - name: Build + push image to Fly registry
        id: build
        uses: docker/build-push-action@v7
        with:
          context: .
          push: true
          # Tag with the commit SHA so each deploy is uniquely
          # addressable + grep-friendly.  The previous flyctl-managed
          # format was deployment-<ULID> — a SHA is more useful for
          # cross-referencing the image to the source revision when
          # debugging.
          tags: registry.fly.io/sentinel-command:deployment-${{ github.sha }}
          # Clerk's publishable key. `VITE_*` is baked in at BUILD time, so
          # this cannot be a Fly secret — see the ARG in the Dockerfile.
          # Empty until a CLERK_PUBLISHABLE_KEY repo secret exists, and the
          # Dockerfile falls back to the committed .env.production when it
          # is empty, so this line changes nothing on its own. Setting the
          # secret is what flips production from the test key to the live
          # one; no code change needed at that point.
          # Note the name has NO `VITE_` prefix, on purpose — see the long
          # comment on the ARG in the Dockerfile. A VITE_-prefixed build arg
          # lands in the build environment and overrides .env.production
          # with an empty value, shipping a bundle with no Clerk key.
          build-args: |
            CLERK_PUBLISHABLE_KEY=${{ secrets.CLERK_PUBLISHABLE_KEY }}
          # GitHub Actions cache for layers — ~30s saved on warm
          # builds vs. cold.  scope=deploy keeps it isolated from
          # any future workflows that might also use buildx.
          cache-from: type=gha,scope=deploy
          cache-to: type=gha,scope=deploy,mode=max

      - name: Export image tag for next step
        id: image
        run: |
          IMAGE="registry.fly.io/sentinel-command:deployment-${{ github.sha }}"
          echo "Captured image: $IMAGE"
          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"

      - name: Release the prebuilt image to both process groups
        run: |
          set -e
          # Was: `flyctl machine update` on `machines list | .[0].id`.
          # That took the FIRST machine and updated only it, which the
          # step's own comment flagged as needing a loop once we ran more
          # than one. Two things now break it outright:
          #
          #   1. There are two process groups (app, agent), so ".[0]" is
          #      a coin flip over which one gets the new image.
          #   2. `machine update` can only touch machines that already
          #      EXIST. It has no way to create the agent machine for a
          #      newly-declared process group, so the group would simply
          #      never come up.
          #
          # `flyctl deploy --image` keeps our own build-and-push (and its
          # layer cache) while letting Fly reconcile fly.toml's
          # [processes] against reality — creating what's missing and
          # updating what isn't.
          #
          # `--strategy immediate` because the `app` group still mounts a
          # single-attachment volume: rolling tries to stand up a parallel
          # machine and errors on the attachment slot. The agent group has
          # no volume, but strategy is app-wide.
          # --ha=false because Fly provisions TWO machines by default for
          # a newly-declared process group. For `agent` that is actively
          # wrong: two workers drain the same run queue, and POST /start
          # is idempotent rather than exclusive, so both would claim the
          # same run and we would pay for the LLM call twice. The first
          # deploy of the agent group did exactly this.
          echo "Releasing $IMAGE to sentinel-command"
          flyctl deploy --image "$IMAGE" --strategy immediate --ha=false --yes -a sentinel-command

          echo "--- process groups after release ---"
          flyctl machines list -a sentinel-command --json \
            | jq -r '.[] | "  \(.config.metadata.fly_process_group // "?")  \(.id)  \(.state)"'
        env:
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
          # Image tag comes from the `image` step (the export step
          # after docker/build-push-action), not the build step.
          IMAGE: ${{ steps.image.outputs.image }}