diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d0df3a5..a4b6d69 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,271 +1,305 @@ -name: CloudMonitor CD — Deploy to AWS EC2 +name: CloudMonitor CD — Deploy to AWS EC2 via SSM +# Manual-only during migration. Re-enable push-to-main only after SSM is verified. on: - push: - branches: - - main workflow_dispatch: inputs: - target_env: - description: "Target environment to deploy" + operation: + description: "Choose OIDC check, SSM preflight, deploy, or rollback" required: true - default: "production" type: choice - options: - - production - diagnostic_only: - description: "Run diagnostic connectivity check only (reports runner egress IP, skips deployment)" + options: [oidc-check, ssm-preflight, deploy, rollback] + default: oidc-check + confirm_production: + description: "Required for production deploy and rollback" required: false - default: false type: boolean - -concurrency: - group: production-deployment - cancel-in-progress: false + default: false permissions: contents: read +concurrency: + group: cloudmonitor-production-deployment + cancel-in-progress: false + jobs: - diagnostics: - name: Runner Network & IP Diagnostics - if: github.event_name == 'workflow_dispatch' && inputs.diagnostic_only == true + ssm: + name: CloudMonitor Production — SSM runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Discover Runner Public Egress IP - run: | - echo "=== GitHub-hosted Runner Egress IP Discovery ===" - RUNNER_IP=$(curl -s -m 5 https://api.ipify.org || curl -s -m 5 https://ifconfig.me || curl -s -m 5 https://icanhazip.com) - echo "Runner Public Egress IP: $RUNNER_IP" - echo "=================================================" - echo "Note: If inbound SSH traffic is restricted, verify that network access control permits traffic from this runner's IP address." - - - name: Probe TCP Port 22 Reachability - env: - EC2_HOST: ${{ secrets.EC2_HOST }} - run: | - if [ -z "$EC2_HOST" ]; then - echo "::warning::Secret EC2_HOST is not configured; skipping port 22 probe." - exit 0 - fi - echo "Probing TCP port 22 reachability to EC2 host..." - if nc -z -w 5 "$EC2_HOST" 22 >/dev/null 2>&1; then - echo "Result: Port 22 is OPEN and REACHABLE from this runner." - else - echo "::error::Result: Port 22 is unreachable from this runner (connection timed out or refused). This may be due to an AWS Security Group, Network ACL, routing, host firewall, or other network restriction." - exit 1 - fi - - deploy: - name: Deploy to AWS EC2 - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.diagnostic_only) environment: production - runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 35 + permissions: + contents: read + id-token: write + env: + AWS_REGION: ap-southeast-2 + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} + AWS_DEPLOY_ROLE_ARN: ${{ vars.AWS_DEPLOY_ROLE_ARN }} + EC2_INSTANCE_ID: ${{ vars.EC2_INSTANCE_ID }} + OPERATION: ${{ inputs.operation }} + CONFIRM_PRODUCTION: ${{ inputs.confirm_production }} steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Validate Deployment Secrets - env: - EC2_HOST: ${{ secrets.EC2_HOST }} - EC2_USER: ${{ secrets.EC2_USER }} - EC2_KEY: ${{ secrets.EC2_SSH_PRIVATE_KEY }} - EC2_KNOWN_HOSTS: ${{ secrets.EC2_SSH_KNOWN_HOSTS }} + - name: Validate requested operation and configuration + shell: bash run: | - MISSING=0 - if [ -z "$EC2_HOST" ]; then - echo "::error::Secret EC2_HOST is not configured in repository or environment secrets." - MISSING=1 - fi - if [ -z "$EC2_USER" ]; then - echo "::error::Secret EC2_USER is not configured in repository or environment secrets." - MISSING=1 - fi - if [ -z "$EC2_KEY" ]; then - echo "::error::Secret EC2_SSH_PRIVATE_KEY is not configured in repository or environment secrets." - MISSING=1 - fi - if [ -z "$EC2_KNOWN_HOSTS" ]; then - echo "::error::Secret EC2_SSH_KNOWN_HOSTS is not configured in repository or environment secrets. Strict SSH host-key verification requires a verified known_hosts entry." - MISSING=1 - fi - - if [ "$MISSING" -ne 0 ]; then - echo "::error::Deployment prerequisites missing. Configure the required secrets in GitHub: Settings -> Secrets and variables -> Actions (Repository secrets or Environment 'production' secrets)." + set -euo pipefail + case "$OPERATION" in + oidc-check|ssm-preflight|deploy|rollback) ;; + *) echo "::error::Unsupported operation."; exit 1 ;; + esac + if [[ "$OPERATION" == "deploy" || "$OPERATION" == "rollback" ]] && [[ "$CONFIRM_PRODUCTION" != "true" ]]; then + echo "::error::Set confirm_production=true for production deploy/rollback." exit 1 fi - echo "All required deployment secrets are present." - - - name: Set up SSH Private Key and Known Hosts - env: - EC2_SSH_PRIVATE_KEY: ${{ secrets.EC2_SSH_PRIVATE_KEY }} - EC2_SSH_KNOWN_HOSTS: ${{ secrets.EC2_SSH_KNOWN_HOSTS }} - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - - # Write private key with strict permissions, removing any Windows CRLF - printf "%s\n" "$EC2_SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/deploy_key - chmod 600 ~/.ssh/deploy_key - - # Write independently verified known_hosts entry, removing any Windows CRLF - printf "%s\n" "$EC2_SSH_KNOWN_HOSTS" | tr -d '\r' > ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - - # Configure SSH client with strict host key checking enabled and keep-alive - cat << 'EOF' > ~/.ssh/config - Host ec2-target - HostName ${{ secrets.EC2_HOST }} - User ${{ secrets.EC2_USER }} - IdentityFile ~/.ssh/deploy_key - StrictHostKeyChecking yes - UserKnownHostsFile ~/.ssh/known_hosts - ConnectTimeout 15 - ServerAliveInterval 15 - ServerAliveCountMax 4 - BatchMode yes - EOF - chmod 600 ~/.ssh/config + if [[ "$OPERATION" != "oidc-check" ]]; then + for name in AWS_ACCOUNT_ID AWS_DEPLOY_ROLE_ARN EC2_INSTANCE_ID; do + if [[ -z "${!name}" ]]; then + echo "::error::Missing GitHub Environment variable: $name (environment: production)." + exit 1 + fi + done + [[ "$AWS_ACCOUNT_ID" =~ ^[0-9]{12}$ ]] || { echo "::error::AWS_ACCOUNT_ID must be 12 digits."; exit 1; } + [[ "$EC2_INSTANCE_ID" =~ ^i-[0-9a-f]{8,17}$ ]] || { echo "::error::Invalid EC2_INSTANCE_ID."; exit 1; } + case "$AWS_DEPLOY_ROLE_ARN" in + "arn:aws:iam::${AWS_ACCOUNT_ID}:role/"*) ;; + *) echo "::error::AWS_DEPLOY_ROLE_ARN does not match AWS_ACCOUNT_ID."; exit 1 ;; + esac + fi - - name: Verify SSH Connectivity with Host Verification + - name: Inspect OIDC claims (token value is never logged) + shell: bash run: | - echo "Verifying secure SSH connectivity with strict host-key verification..." - ssh ec2-target "uptime && docker --version" - - - name: Deploy Application Stack via Docker Compose + set -euo pipefail + python3 - <<'PY' + import base64, json, os, urllib.request + url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"] + "&audience=sts.amazonaws.com" + request = urllib.request.Request(url, headers={ + "Authorization": f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}" + }) + with urllib.request.urlopen(request, timeout=15) as response: + token = json.load(response)["value"] + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + claims = json.loads(base64.urlsafe_b64decode(payload)) + print("OIDC issuer:", claims.get("iss")) + print("OIDC audience:", claims.get("aud")) + print("OIDC subject:", claims.get("sub")) + if claims.get("iss") != "https://token.actions.githubusercontent.com": + raise SystemExit("Unexpected OIDC issuer.") + aud = claims.get("aud") + if aud != "sts.amazonaws.com" and not (isinstance(aud, list) and "sts.amazonaws.com" in aud): + raise SystemExit("Unexpected OIDC audience.") + PY + + - name: Configure AWS credentials using GitHub OIDC + uses: aws-actions/configure-aws-credentials@v6.3.0 + with: + role-to-assume: ${{ env.AWS_DEPLOY_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: CloudMonitor-SSM-${{ github.run_id }} + allowed-account-ids: ${{ env.AWS_ACCOUNT_ID }} + + - name: Verify AWS role assumption and optional SSM managed-node status + shell: bash run: | - echo "Syncing latest configuration and deploying services to EC2..." - ssh ec2-target "bash -s" << 'EOF' - set -euo pipefail - DEPLOY_DIR="/home/ubuntu/cloud_monitor" - - if [ ! -d "$DEPLOY_DIR" ]; then - echo "::error::Target directory $DEPLOY_DIR does not exist on host." - exit 1 - fi - cd "$DEPLOY_DIR" - - # 1. Protect production configuration (never automatically create or overwrite) - if [ ! -f .env ]; then - echo "::error::Production environment file ($DEPLOY_DIR/.env) does not exist! Deployment stopped to protect database and secrets." + set -euo pipefail + ACTUAL_ACCOUNT="$(aws sts get-caller-identity --query Account --output text)" + [[ "$ACTUAL_ACCOUNT" == "$AWS_ACCOUNT_ID" ]] || { echo "::error::Unexpected AWS account."; exit 1; } + ACTUAL_ARN="$(aws sts get-caller-identity --query Arn --output text)" + echo "GitHub OIDC successfully assumed the configured AWS role: $ACTUAL_ARN" + + if [[ "$OPERATION" != "oidc-check" ]]; then + PING_STATUS="$(aws ssm describe-instance-information \ + --filters "Key=InstanceIds,Values=${EC2_INSTANCE_ID}" \ + --query 'InstanceInformationList[0].PingStatus' --output text)" + [[ "$PING_STATUS" == "Online" ]] || { + echo "::error::EC2 is not Online in SSM (status: $PING_STATUS). Fix instance role, SSM Agent, and outbound HTTPS before deploying." exit 1 - fi - - # 2. Update code from main branch if git repository is initialized - if [ -d .git ]; then - git config --global --add safe.directory "$DEPLOY_DIR" || true - git fetch origin main - git checkout -f main - git reset --hard origin/main - fi - - # 3. Pull required external images - sudo docker compose -f docker-compose.prod.yml pull prometheus node-exporter alertmanager postgres || true - - # 4. Safely build/update backend and frontend without disturbing PostgreSQL volumes - sudo docker compose -f docker-compose.prod.yml build backend frontend - - # 5. Bring up services idempotently (preserves postgres_data volume) - sudo docker compose -f docker-compose.prod.yml up -d --remove-orphans - - # 6. Verify container status on host - echo "Verifying running container states on host..." - sudo docker compose -f docker-compose.prod.yml ps - - echo "Stack deployment commands completed successfully." - EOF + } + echo "EC2 is Online in Systems Manager." + fi - - name: Verify Production Health and Telemetry Endpoints - env: - EC2_HOST: ${{ secrets.EC2_HOST }} + - name: Run SSM preflight, deployment, or rollback + if: inputs.operation != 'oidc-check' + shell: bash run: | - echo "Performing post-deployment health check against EC2 entrypoint..." - FRONTEND_URL="http://${EC2_HOST}/" - BACKEND_HEALTH_URL="http://${EC2_HOST}/health" - TARGETS_URL="http://${EC2_HOST}/prom/api/v1/targets" - - # 1. Bounded health check for Frontend SPA public entrypoint - echo "Checking frontend public endpoint at $FRONTEND_URL..." - FRONTEND_SUCCESS=false - for i in {1..12}; do - STATUS=$(curl -s -m 5 -o /dev/null -w "%{http_code}" "$FRONTEND_URL" || true) - if [ "$STATUS" = "200" ]; then - echo "Frontend public endpoint responded with HTTP 200 on attempt $i." - FRONTEND_SUCCESS=true - break - fi - echo "Waiting for frontend entrypoint (attempt $i/12, status: $STATUS)..." - sleep 5 - done - - if [ "$FRONTEND_SUCCESS" != "true" ]; then - echo "::error::Frontend public endpoint check failed at $FRONTEND_URL after 12 attempts." + set -euo pipefail + case "$OPERATION" in + ssm-preflight) + REMOTE_SCRIPT="$(cat <<'REMOTE' + set -Eeuo pipefail + DEPLOY_DIR="/home/ubuntu/cloud_monitor" + test -d "$DEPLOY_DIR" + test -f "$DEPLOY_DIR/.env" + if sudo -u ubuntu -- git -C "$DEPLOY_DIR" ls-files --error-unmatch .env >/dev/null 2>&1; then + echo "ERROR: .env is tracked by Git. Refusing to continue." exit 1 fi - - # 2. Bounded health check for FastAPI backend & Nginx reverse proxy - echo "Checking backend health at $BACKEND_HEALTH_URL..." - BACKEND_SUCCESS=false - for i in {1..12}; do - BACKEND_RES=$(curl -s -m 5 "$BACKEND_HEALTH_URL" || true) - if echo "$BACKEND_RES" | grep -q '"status":"healthy"'; then - echo "Backend health check succeeded on attempt $i: $BACKEND_RES" - BACKEND_SUCCESS=true - break - fi - echo "Waiting for backend health probe (attempt $i/12)..." - sleep 5 - done - - if [ "$BACKEND_SUCCESS" != "true" ]; then - echo "::error::Backend health check failed at $BACKEND_HEALTH_URL after 12 attempts." + cd "$DEPLOY_DIR" + command -v git >/dev/null + command -v curl >/dev/null + sudo -n docker --version + sudo -n docker compose version + sudo -n docker compose -f docker-compose.prod.yml config --quiet + sudo -n docker compose -f docker-compose.prod.yml ps + echo "SSM preflight passed. No application or database changes were made." + REMOTE + )" + ;; + deploy) + REMOTE_SCRIPT="$(cat <<'REMOTE' + set -Eeuo pipefail + DEPLOY_DIR="/home/ubuntu/cloud_monitor" + test -d "$DEPLOY_DIR" + test -f "$DEPLOY_DIR/.env" + if sudo -u ubuntu -- git -C "$DEPLOY_DIR" ls-files --error-unmatch .env >/dev/null 2>&1; then + echo "ERROR: .env is tracked by Git. Refusing to deploy." exit 1 fi - - # 3 & 4. Bounded check for Prometheus scrape targets API and all required targets UP - echo "Validating Prometheus scrape targets proxy at $TARGETS_URL..." - TARGETS_SUCCESS=false - EXPECTED_TARGETS=("alertmanager" "cloudmonitor-backend" "node-exporter" "prometheus") - - for i in {1..10}; do - TARGETS_RES=$(curl -s -m 5 "$TARGETS_URL" || true) - if echo "$TARGETS_RES" | jq -e '.status == "success"' >/dev/null 2>&1; then - DOWN_TARGETS=$(echo "$TARGETS_RES" | jq -r '[.data.activeTargets[] | select(.health != "up") | .scrapePool] | join(", ")' 2>/dev/null || true) - ACTIVE_COUNT=$(echo "$TARGETS_RES" | jq -r '.data.activeTargets | length' 2>/dev/null || echo "0") - - MISSING_TARGETS=() - for target in "${EXPECTED_TARGETS[@]}"; do - FOUND=$(echo "$TARGETS_RES" | jq -r --arg t "$target" '[.data.activeTargets[] | select(.scrapePool == $t)] | length' 2>/dev/null || echo "0") - if [ "$FOUND" -eq 0 ]; then - MISSING_TARGETS+=("$target") - fi - done - - if [ "$ACTIVE_COUNT" -ge 4 ] && [ -z "$DOWN_TARGETS" ] && [ ${#MISSING_TARGETS[@]} -eq 0 ]; then - echo "Prometheus targets check succeeded on attempt $i: $ACTIVE_COUNT targets active and all UP (${EXPECTED_TARGETS[*]})." - TARGETS_SUCCESS=true - break - else - [ -n "$DOWN_TARGETS" ] && echo "::warning::Unhealthy targets detected: $DOWN_TARGETS (attempt $i/10)..." - [ ${#MISSING_TARGETS[@]} -gt 0 ] && echo "::warning::Missing expected targets: ${MISSING_TARGETS[*]} (attempt $i/10)..." - fi - fi - echo "Waiting for Prometheus target discovery (attempt $i/10)..." - sleep 5 - done - - if [ "$TARGETS_SUCCESS" != "true" ]; then - echo "::error::Prometheus scrape targets verification failed at $TARGETS_URL after 10 attempts." + cd "$DEPLOY_DIR" + + PREVIOUS_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)" + rollback_on_error() { + local rc=$? + trap - ERR + echo "Deployment failed; attempting rollback to ${PREVIOUS_SHA}." + sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main "$PREVIOUS_SHA" || true + sudo -n docker compose -f docker-compose.prod.yml build backend frontend || true + sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans || true + exit "$rc" + } + trap rollback_on_error ERR + + sudo -n install -d -m 0750 /var/lib/cloudmonitor + printf '%s\n' "$PREVIOUS_SHA" | sudo -n tee /var/lib/cloudmonitor/previous-deploy-sha >/dev/null + sudo -u ubuntu -- git -C "$DEPLOY_DIR" fetch --prune origin +refs/heads/main:refs/remotes/origin/main + sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main origin/main + DEPLOY_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)" + + sudo -n docker compose -f docker-compose.prod.yml config --quiet + sudo -n docker compose -f docker-compose.prod.yml pull prometheus node-exporter alertmanager + sudo -n docker compose -f docker-compose.prod.yml build backend frontend + sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans + + wait_for_http() { + local url="$1" attempt status + for attempt in $(seq 1 12); do + status="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "$url" || true)" + [[ "$status" == "200" ]] && return 0 + sleep 5 + done + echo "HTTP health check failed: $url" + return 1 + } + wait_for_backend() { + local attempt body + for attempt in $(seq 1 12); do + body="$(curl -fsS --max-time 5 http://127.0.0.1/health 2>/dev/null || true)" + if printf '%s' "$body" | python3 -c 'import json,sys; sys.exit(0 if json.load(sys.stdin).get("status") == "healthy" else 1)' 2>/dev/null; then return 0; fi + sleep 5 + done + echo "Backend did not report status=healthy." + return 1 + } + wait_for_prometheus() { + local attempt body + for attempt in $(seq 1 12); do + body="$(curl -fsS --max-time 5 http://127.0.0.1/prom/api/v1/targets 2>/dev/null || true)" + if printf '%s' "$body" | python3 -c 'import json,sys; p=json.load(sys.stdin); required={"alertmanager","cloudmonitor-backend","node-exporter","prometheus"}; ts=p.get("data",{}).get("activeTargets",[]); bad=[n for n in required if not any(t.get("scrapePool")==n and t.get("health")=="up" for t in ts)]; sys.exit(0 if p.get("status")=="success" and not bad else 1)' 2>/dev/null; then return 0; fi + sleep 5 + done + echo "Prometheus targets are not all healthy." + return 1 + } + + wait_for_http "http://127.0.0.1/" + wait_for_backend + wait_for_prometheus + printf '%s\n' "$DEPLOY_SHA" | sudo -n tee /var/lib/cloudmonitor/last-successful-deploy-sha >/dev/null + trap - ERR + echo "Deployment and health checks succeeded for ${DEPLOY_SHA}." + echo "No PostgreSQL volume was deleted or recreated." + REMOTE + )" + ;; + rollback) + REMOTE_SCRIPT="$(cat <<'REMOTE' + set -Eeuo pipefail + DEPLOY_DIR="/home/ubuntu/cloud_monitor" + test -d "$DEPLOY_DIR" + test -f "$DEPLOY_DIR/.env" + if sudo -u ubuntu -- git -C "$DEPLOY_DIR" ls-files --error-unmatch .env >/dev/null 2>&1; then + echo "ERROR: .env is tracked by Git. Refusing to roll back." exit 1 fi + cd "$DEPLOY_DIR" + CURRENT_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)" + ROLLBACK_SHA="$(sudo -n cat /var/lib/cloudmonitor/previous-deploy-sha)" + sudo -u ubuntu -- git -C "$DEPLOY_DIR" cat-file -e "${ROLLBACK_SHA}^{commit}" + + restore_current_on_error() { + local rc=$? + trap - ERR + sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main "$CURRENT_SHA" || true + sudo -n docker compose -f docker-compose.prod.yml build backend frontend || true + sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans || true + exit "$rc" + } + trap restore_current_on_error ERR + + sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main "$ROLLBACK_SHA" + sudo -n docker compose -f docker-compose.prod.yml config --quiet + sudo -n docker compose -f docker-compose.prod.yml build backend frontend + sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans + curl -fsS --max-time 5 http://127.0.0.1/ >/dev/null + BACKEND_BODY="$(curl -fsS --max-time 5 http://127.0.0.1/health)" + printf '%s' "$BACKEND_BODY" | python3 -c 'import json,sys; sys.exit(0 if json.load(sys.stdin).get("status") == "healthy" else 1)' + printf '%s\n' "$CURRENT_SHA" | sudo -n tee /var/lib/cloudmonitor/previous-deploy-sha >/dev/null + printf '%s\n' "$ROLLBACK_SHA" | sudo -n tee /var/lib/cloudmonitor/last-successful-deploy-sha >/dev/null + trap - ERR + echo "Rollback to ${ROLLBACK_SHA} succeeded; PostgreSQL volumes were preserved." + REMOTE + )" + ;; + *) + echo "::error::Unsupported SSM operation." + exit 1 + ;; + esac + + PARAMETERS="$(jq -n --arg script "$REMOTE_SCRIPT" '{commands: [("bash -lc " + ($script | @sh))]}')" + COMMAND_ID="$(aws ssm send-command \ + --instance-ids "$EC2_INSTANCE_ID" \ + --document-name "AWS-RunShellScript" \ + --comment "CloudMonitor ${OPERATION} via GitHub Actions" \ + --timeout-seconds 900 \ + --parameters "$PARAMETERS" \ + --query 'Command.CommandId' --output text)" + echo "SSM Command ID: $COMMAND_ID" + + for attempt in $(seq 1 360); do + INVOCATION="$(aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" --instance-id "$EC2_INSTANCE_ID" \ + --output json 2>/dev/null || true)" + STATUS="$(printf '%s' "$INVOCATION" | jq -r '.Status // "Pending"' 2>/dev/null || echo "Pending")" + case "$STATUS" in + Success) + printf '%s\n' "$INVOCATION" | jq -r '.StandardOutputContent // ""' + exit 0 + ;; + Failed|TimedOut|ExecutionTimedOut|DeliveryTimedOut|Cancelled|Cancelling|Undeliverable|Terminated|InvalidPlatform|AccessDenied|NoInstancesInTag|InvalidDocument|Incomplete|RateExceeded) + echo "::error::SSM command ended with status: $STATUS" + printf '%s\n' "$INVOCATION" | jq -r '"Status details: \(.StatusDetails // "unknown")\nResponse code: \(.ResponseCode // "unknown")\nStandard output:\n\(.StandardOutputContent // "")\nStandard error:\n\(.StandardErrorContent // "")"' + exit 1 + ;; + esac + sleep 5 + done + echo "::error::Timed out waiting for SSM command completion. Command ID: $COMMAND_ID" + exit 1 - echo "All production health, frontend, backend, and monitoring checks passed." - - - name: Clean up SSH Temporary Credentials - if: always() - run: | - rm -f ~/.ssh/deploy_key ~/.ssh/config ~/.ssh/known_hosts - echo "SSH temporary credentials and host keys removed from runner environment." + - name: Confirm deployment mode + if: inputs.operation == 'deploy' + run: echo "SSM production deployment completed. Review output before re-enabling push-based deployment." diff --git a/docs/SSM_DEPLOYMENT_MIGRATION.md b/docs/SSM_DEPLOYMENT_MIGRATION.md new file mode 100644 index 0000000..c3e32ce --- /dev/null +++ b/docs/SSM_DEPLOYMENT_MIGRATION.md @@ -0,0 +1,156 @@ +# Issue #9 — Migrate EC2 deployment from SSH to AWS Systems Manager + +**Status:** SSM workflow prepared on a feature branch. The existing EC2 instance role is attached, SSM reports the instance Online, the GitHub OIDC provider and deployment role exist, and the repository production environment has the three AWS variables plus a main-only deployment rule. GitHub OIDC role assumption and Run Command execution still require workflow verification. + +## Why the SSH deployment is failing + +The current CD run fails at the SSH connectivity step with a TCP port 22 connection timeout. This is a network reachability failure before the deployment command starts; it is not evidence of a bad SSH key. + +The replacement workflow uses GitHub Actions OIDC to assume a narrowly scoped AWS role and Systems Manager Run Command to contact one EC2 instance. The workflow wraps each remote script with `bash -lc` so Bash-specific constructs are not accidentally interpreted by the default POSIX shell. The EC2 SSM Agent initiates outbound HTTPS connections. GitHub-hosted runners do not need inbound SSH access to the instance. AWS documents outbound HTTPS on port 443 to regional SSM endpoints; no inbound SSM port is required: https://docs.aws.amazon.com/systems-manager/latest/userguide/troubleshooting-ssm-agent.html + +## Important rollout guard + +The updated `.github/workflows/cd.yml` is intentionally **workflow_dispatch-only** during migration. It does not deploy on a push or PR merge. This prevents merging the workflow from unexpectedly initiating production deployment before AWS prerequisites are in place. + +Available operations: + +1. `oidc-check` — requests a GitHub OIDC token, prints only issuer/audience/subject claims (never the token), assumes the configured AWS role with STS, and verifies the AWS account/caller identity. It does not send an SSM command or change EC2. +2. `ssm-preflight` — authenticates to AWS, verifies the configured EC2 instance is Online in SSM, and sends a read-only preflight command. +3. `deploy` — requires `confirm_production=true`; deploys `origin/main` through SSM and runs local frontend/backend/Prometheus health checks on EC2. +4. `rollback` — requires `confirm_production=true`; restores the previous Git revision recorded on EC2. + +After one successful controlled SSM deployment and operational review, a follow-up change can re-enable push-to-main for automatic CD. Do not add the push trigger before SSM is verified. + +## AWS setup checklist + +### 1. Confirm the existing EC2 SSM registration + +The CloudMonitor EC2 instance already has CloudMonitorEC2SSMRole with AmazonSSMManagedInstanceCore, and Fleet Manager reports the instance as Online. The GitHub role policy is restricted to the exact EC2 instance ARN, so an EC2 tag is not required for this implementation. The workflow polls for up to 30 minutes and sets a 35-minute GitHub job timeout to allow image build and health-check time. + +### 2. EC2 instance profile — completed + +The existing CloudMonitor EC2 instance has CloudMonitorEC2SSMRole attached with the AWS-managed AmazonSSMManagedInstanceCore policy. SSM Agent diagnostics passed, and the instance is listed as Online in Fleet Manager. No further EC2 role change is needed. + +### 3. SSM Agent and network connectivity — completed + +The Ubuntu 24.04 host has SSM Agent 3.3.4793.0 running and enabled. ssm-cli get-diagnostics --output table reported success for IMDS, SSM, EC2 Messages, SSM Messages, S3, KMS, Logs, Monitoring, AWS role credentials, and agent service checks. Fleet Manager subsequently showed the CloudMonitor instance as Online. No inbound SSH change is required for GitHub-hosted runners to use SSM. Keep administrator SSH access as a break-glass path until the new deployment route is fully tested. + +### 4. GitHub OIDC identity provider — already exists + +The AWS account already has the OpenID Connect provider token.actions.githubusercontent.com with audience sts.amazonaws.com. Reuse it; do not create a duplicate or delete the existing provider. + +### 5. GitHub deployment role trust policy — already created + +The role CloudMonitorGitHubDeployRole exists and trusts the GitHub OIDC provider for the immutable CloudMonitor repository identity and GitHub Environment production. The expected subject is repo:ayushcmd@252997256/cloud_monitor@1419373799:environment:production. The workflow's oidc-check now verifies the emitted subject and performs an actual AWS STS role assumption. If the STS step fails, compare the emitted subject with the role trust policy before changing either. + +Depending on the repository's OIDC subject-claim setting, the subject may use one of these forms: + +- Legacy: `repo:ayushcmd/cloud_monitor:environment:production` +- Immutable: `repo:ayushcmd@252997256/cloud_monitor@1419373799:environment:production` + +The environment's deployment branch rule allows main only. Do not allow repository-wide or branch-wide wildcards in the IAM trust policy. + +Example trust policy (replace the account ID and subject with the actual values): + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": "" + } + } + } + ] +} +``` + +### 6. Attach the deployment role permissions + +The GitHub role is limited to AWS-RunShellScript on the exact CloudMonitor EC2 instance, plus the read operations needed to verify the command. The policy attached in AWS uses the actual account and instance identifiers. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DescribeManagedInstances", + "Effect": "Allow", + "Action": "ssm:DescribeInstanceInformation", + "Resource": "*" + }, + { + "Sid": "SendOnlyAwsRunShellScriptDocument", + "Effect": "Allow", + "Action": "ssm:SendCommand", + "Resource": "arn:aws:ssm:ap-southeast-2::document/AWS-RunShellScript" + }, + { + "Sid": "SendOnlyToCloudMonitorInstance", + "Effect": "Allow", + "Action": "ssm:SendCommand", + "Resource": "arn:aws:ec2:ap-southeast-2::instance/" + }, + { + "Sid": "ReadCommandInvocationResult", + "Effect": "Allow", + "Action": "ssm:GetCommandInvocation", + "Resource": "*" + } + ] +} +``` + +The read actions above use `Resource: "*"`; verify resource-level support and condition keys against the current Systems Manager service authorization reference: https://docs.aws.amazon.com/service-authorization/latest/reference/list_ssm.html + +### 7. Add GitHub Environment variables + +In GitHub → Settings → Environments → `production` → Environment variables, configure: + +| Variable | Value | +|---|---| +| `AWS_ACCOUNT_ID` | Your 12-digit AWS account ID | +| `AWS_DEPLOY_ROLE_ARN` | ARN of the GitHub OIDC deployment role | +| `EC2_INSTANCE_ID` | Existing CloudMonitor EC2 instance ID | + +The region is set in the workflow to `ap-southeast-2`. Consider adding required reviewers to the `production` GitHub Environment so deploy and rollback operations receive an approval gate. + +## Runbook after this PR is merged + +1. Open Actions → **CloudMonitor CD — Deploy to AWS EC2 via SSM** → Run workflow. +2. Select `oidc-check`. Confirm the emitted OIDC subject matches the IAM trust policy and that STS returns the expected AWS account and assumed role. +3. Run `ssm-preflight`. It must confirm the EC2 managed node is Online and complete the harmless read-only command. +5. Run `deploy` with `confirm_production=true` and complete any GitHub Environment approval. Verify the public application from outside EC2 and verify PostgreSQL data/persistence using a known non-sensitive application-level check. The workflow's health probes run against localhost on EC2. +6. Use `rollback` with explicit confirmation if rollback is required. +7. Only after the first successful SSM deployment and rollback procedure are reviewed should a follow-up PR restore automatic deployment on pushes to `main`. + +## Data and access safety + +- No SSH key or known-host secret is used by the SSM workflow. +- The workflow does not require inbound SSH from GitHub-hosted runners. +- It checks that `/home/ubuntu/cloud_monitor/.env` exists and is not Git-tracked; it never prints the file contents. +- It uses `docker compose config --quiet`, `build`, and `up -d --remove-orphans`. +- It does **not** use `docker compose down -v`, delete volumes, or recreate PostgreSQL storage. +- The deploy operation records the previous Git revision under `/var/lib/cloudmonitor/` and attempts a source/build rollback if deployment or local health checks fail. This is a **code/config rollback only**: it does not restore PostgreSQL data or reverse database schema migrations. Keep schema changes backward-compatible across rollback, and take/verify a separate database backup before any release that changes schema or data. +- Keep SSH access available for trusted administration until SSM is Online, the read-only preflight succeeds, and the required break-glass access method has been tested. +- No AWS resources are changed by merging this workflow/documentation PR. Actual AWS configuration and production deployment remain separate, deliberate steps. + +## Current acceptance status + +| Acceptance criterion | Status | +|---|---| +| EC2 registered and Online in SSM | Complete — Fleet Manager reports Online | +| GitHub OIDC provider, role, and trust policy | Configured — live STS assumption pending `oidc-check` | +| GitHub invokes the approved SSM command | Pending `ssm-preflight` | +| Runner no longer requires inbound SSH | Implemented in the prepared workflow; pending live preflight | +| Production deployment and health checks succeed | Pending controlled deployment | +| PostgreSQL persistent storage is preserved | Workflow avoids volume removal/recreation; live verification pending | +| Least-privilege IAM and rollback procedure | Policy and runbook prepared; AWS review/verification pending |