deploy-backend.yml

# Dev CD pipeline: build once in CI, deploy that exact artifact, verify it
# with real API tests, and roll the image back automatically if verification
# fails.
#
#   build -> deploy -> smoke-tests -> { rollback-on-smoke-failure | finalize }
#
# Deploys are pinned to an immutable digest, never a tag: `dev` is a moving
# pointer, so "roll back to the previous dev tag" is not a thing you can
# express. The host records the digest it was running before each deploy in
# .deploy/previous_image, which is what makes cross-job rollback possible at
# all (rollback runs on a different runner, so no shell state survives).
#
# MIGRATIONS ARE NOT ROLLED BACK. A rollback restores the previous image and
# says so loudly; the schema stays forward. Migrations must therefore be
# written additively / backward-compatibly, so the previous image can still
# run against the newer schema. This is a policy constraint on how you write
# migrations, not something this pipeline can enforce for you.
#
# The target host is not a standalone checkout of this repo -- it's the
# DataExBackend submodule inside the separate CivicDataLab/DataExchange
# superproject, already running its own self-contained compose project
# there (confirmed via `docker ps` compose labels: project=dataexbackend,
# workdir=~/DataExchange/DataExBackend). This pipeline only ever touches
# that directory; it never touches the DataExchange repo or its top-level
# compose file.
#
# Several structural choices here mirror ParakhAPI's proven dev CD pipeline
# (deploy-parakh-api-dev.yml in CivicDataLab/ParakhAI-Backend) -- see the
# notes at each site before "simplifying" them. In particular:
# appleboy/ssh-action's inline multi-line `script:` input was found there to
# reproducibly fail with a spurious "syntax error near unexpected token ';'"
# for reasons never fully root-caused (confirmed the script text itself was
# valid bash both locally and on the real target host every time -- the
# corruption happened somewhere in the action's own transport). scp-action
# never had that problem. So no SSH step here ever carries more than a
# single trivial invocation line; all real logic lives in scripts/ci-*.sh,
# shipped as files.

name: Deploy Backend to Dev EC2

on:
  push:
    branches: ['dev']
  workflow_dispatch:
    inputs:
      force_smoke_failure:
        description: "Deliberately fail the smoke gate, to exercise the rollback path. Testing only."
        type: boolean
        required: false
        default: false

# Queue overlapping deploys rather than cancelling: a cancelled run mid-deploy
# could leave .deploy/ state and the running containers disagreeing.
concurrency:
  group: dataspace-backend-dev-deploy
  cancel-in-progress: false

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: civicdatalab/dataspacebackend
  DEPLOY_PATH: ${{ vars.DEPLOY_PATH || 'DataExchange/DataExBackend' }}

jobs:
  build:
    name: Build and push image
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: read
      packages: write
    outputs:
      image_ref: ${{ steps.ref.outputs.image_ref }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      # driver: docker-container explicitly, not the ambient default --
      # this runner's default buildx context reports driver "docker",
      # which does not support cache export (cache-to: type=gha below
      # fails outright without this).
      - name: Set up Buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push
        id: build
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          build-args: |
            GIT_COMMIT_SHA=${{ github.sha }}
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }}
          # mode=min (the default), not max: this Dockerfile is single-stage
          # (no multi-stage FROM ... AS builder), so mode=max's extra
          # intermediate-stage caching buys nothing here -- it only added a
          # slow cache-export step that got stuck writing one large layer
          # (chromium + apt packages) and blew the job's timeout on the
          # first (cold-cache) run, even though the actual image build and
          # push had already completed successfully by that point.
          cache-from: type=gha
          cache-to: type=gha

      - name: Pin image reference by digest
        id: ref
        run: |
          echo "image_ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" >> "$GITHUB_OUTPUT"

      # A size ceiling, because image size is a deploy-time failure mode here
      # and nothing else surfaces it. The image reached 14.1GB - 6.6GB of it
      # CUDA runtime on a GPU-less box - and `docker pull` then ran past the
      # deploy step's 40 minute command_timeout, so deploys failed with
      # "Run Command Timeout" and no indication of why. Builds stayed green
      # throughout; the cost only appeared on the host.
      #
      # Fails the build rather than the deploy, so the feedback lands on the
      # PR that caused it instead of an hour later on a broken environment.
      # Raise MAX_IMAGE_GB deliberately if the image legitimately grows.
      - name: Enforce image size ceiling
        env:
          MAX_IMAGE_GB: 8
        run: |
          set -euo pipefail
          docker pull -q "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}"
          BYTES=$(docker image inspect \
            "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" \
            --format "{{.Size}}")
          GB=$(awk -v b="$BYTES" 'BEGIN{printf "%.2f", b/1024/1024/1024}')
          echo "Image size: ${GB} GB (ceiling ${MAX_IMAGE_GB} GB)"
          echo "- image size: **${GB} GB** (ceiling ${MAX_IMAGE_GB} GB)" >> "$GITHUB_STEP_SUMMARY"
          if awk -v g="$GB" -v m="$MAX_IMAGE_GB" 'BEGIN{exit !(g > m)}'; then
            echo "::error::Image is ${GB} GB, over the ${MAX_IMAGE_GB} GB ceiling. Pulling this on the deploy host will run past the SSH command_timeout and the deploy will fail. Check for CUDA/GPU wheels (nvidia/*, torch, triton) being pulled in place of CPU builds."
            exit 1
          fi

      - name: Sanity-check the built image
        # Cheap, real gate: catches import errors and bad settings before
        # anything touches the host. Live testing showed every layer
        # downloading successfully every single retry, with toomanyrequests
        # firing right after the LAST layer completes (the final
        # manifest/config fetch) -- consistently, even across 10 attempts
        # 30s apart. That pattern looks like GHCR hasn't finished settling
        # a just-pushed manifest yet, not a generic quota, so this waits
        # up front before the first attempt rather than only reacting
        # after failures.
        run: |
          sleep 90
          for attempt in 1 2 3 4 5 6 7 8 9 10; do
            if docker run --rm \
              -e SECRET_KEY=ci-sanity-check-not-a-real-key \
              -e URL_WHITELIST=http://localhost \
              -e DB_ENGINE=django.db.backends.sqlite3 \
              --entrypoint python \
              "${{ steps.ref.outputs.image_ref }}" \
              manage.py check; then
              exit 0
            fi
            echo "attempt $attempt failed, retrying in 30s..."
            sleep 30
          done
          echo "::error::Sanity check failed after 10 attempts."
          exit 1

  deploy:
    name: Deploy to EC2
    needs: build
    runs-on: ubuntu-latest
    environment: development
    # 50m: must comfortably exceed the Deploy step's own 40m
    # command_timeout (see that step's comment for why it's 40m).
    timeout-minutes: 50
    # packages: read -- GITHUB_TOKEN needs this explicitly granted to pull
    # from GHCR; it isn't covered by the repo's default token permissions.
    permissions:
      contents: read
      packages: read
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Write GHCR token file
        run: printf '%s' "${{ secrets.GITHUB_TOKEN }}" > .ghcr_token

      # Ship the compose files rather than relying on the DataExBackend
      # submodule pointer inside the separate DataExchange repo -- bumping
      # that pointer is a change to a different, shared repo and out of
      # scope here. Shipping the file directly keeps this pipeline
      # self-contained.
      - name: Ship deploy files to host
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.EC2_HOST }}
          username: ${{ secrets.EC2_USERNAME }}
          key: ${{ secrets.EC2_PRIVATE_KEY }}
          source: docker-compose.yml,docker-compose.hotreload.yml,scripts/ci-deploy.sh,.ghcr_token
          target: ${{ env.DEPLOY_PATH }}

      - name: Deploy
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ vars.EC2_HOST }}
          username: ${{ secrets.EC2_USERNAME }}
          key: ${{ secrets.EC2_PRIVATE_KEY }}
          script_stop: true
          # 40m: live testing showed one large layer (chromium + its X11
          # libs, almost certainly) take ~15 minutes just downloading via
          # Docker's own internal per-layer retry (not our retry loop --
          # this is a single docker pull invocation struggling), then hang
          # with zero output for several more minutes afterward (most
          # likely extraction stalling under memory pressure -- this host
          # runs Postgres, two separate Elasticsearch instances, Redis,
          # Keycloak, and telemetry tooling alongside the app, confirmed
          # via `free -h`: ~169Mi truly free, 828Mi already swapped). The
          # previous 20m ceiling cut it off mid-extraction, right after
          # the layer had already finished downloading.
          command_timeout: 40m
          # sudo: docker/docker compose require it on this host (confirmed
          # passwordless -- sudo -n succeeds non-interactively).
          script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-deploy.sh "${{ needs.build.outputs.image_ref }}" "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}"

  smoke-tests:
    name: Smoke Tests
    needs: deploy
    # No `environment:` here -- GitHub rejects the entire workflow file at
    # parse time if a `uses:` job declares one. Consequence: this job also
    # cannot see environment-scoped vars, which is why api_base_url comes
    # from a repo-level var.
    uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI
    with:
      api_base_url: ${{ vars.DEV_API_BASE_URL }}
      # An obviously-wrong sentinel SHA fails the reusable workflow's own
      # deployed-SHA-vs-live-/health/ assertion on purpose, for the
      # force_smoke_failure test path.
      deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }}
      min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }}
    secrets:
      HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }}
      TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }}
      TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
      TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
      TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
      # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a
      # confidential client, so without this the token request returns 401
      # and the job fails its preflight.
      KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }}

  rollback-on-smoke-failure:
    name: Rollback (smoke tests failed)
    # `deploy` must be in needs: for needs.deploy.result to resolve here.
    needs: [deploy, smoke-tests]
    # failure()/success() builtins rather than needs.smoke-tests.result --
    # both are false on cancellation, which is the behaviour we want;
    # if: always() would ignore cancellation entirely.
    if: failure() && needs.deploy.result == 'success'
    runs-on: ubuntu-latest
    environment: development
    # 50m: must comfortably exceed the Restore previous image step's own
    # 40m command_timeout (see the Deploy job's equivalent step for why).
    timeout-minutes: 50
    permissions:
      contents: read
      packages: read
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Write GHCR token file
        run: printf '%s' "${{ secrets.GITHUB_TOKEN }}" > .ghcr_token

      - name: Ship rollback files to host
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.EC2_HOST }}
          username: ${{ secrets.EC2_USERNAME }}
          key: ${{ secrets.EC2_PRIVATE_KEY }}
          source: scripts/ci-rollback.sh,.ghcr_token
          target: ${{ env.DEPLOY_PATH }}

      - name: Restore previous image
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ vars.EC2_HOST }}
          username: ${{ secrets.EC2_USERNAME }}
          key: ${{ secrets.EC2_PRIVATE_KEY }}
          script_stop: true
          # 40m -- see the Deploy job's equivalent step for why (one large
          # layer took ~15 min to download plus several more to extract
          # under this host's memory pressure in live testing).
          command_timeout: 40m
          script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}"

      - name: Mark this run as failed
        # The mitigation succeeded, but the run must still read RED -- a bad
        # deploy that silently self-heals is a bad deploy nobody investigates.
        run: |
          echo "::error::Smoke tests failed after deploy; the image was rolled back. Migrations were NOT reverted -- see the rollback step's log."
          exit 1

  finalize-deploy:
    name: Finalize Deploy
    needs: [deploy, smoke-tests]
    if: success()
    runs-on: ubuntu-latest
    environment: development
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Ship finalize script to host
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ vars.EC2_HOST }}
          username: ${{ secrets.EC2_USERNAME }}
          key: ${{ secrets.EC2_PRIVATE_KEY }}
          source: scripts/ci-finalize.sh
          target: ${{ env.DEPLOY_PATH }}

      - name: Prune to current + previous image
        uses: appleboy/ssh-action@v1.0.3
        with:
          host: ${{ vars.EC2_HOST }}
          username: ${{ secrets.EC2_USERNAME }}
          key: ${{ secrets.EC2_PRIVATE_KEY }}
          script_stop: true
          script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-finalize.sh