From 303cb667990aa07d55a6ab269c00f99018760f1f Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sat, 31 Oct 2026 20:28:54 +0530
Subject: [PATCH 1/8] ci(cd): migrate EC2 deployment workflow to SSM Run
 Command

---
 .github/workflows/cd.yml | 506 +++++++++++++++++++++------------------
 1 file changed, 268 insertions(+), 238 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index d0df3a5..b1ec026 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -1,271 +1,301 @@
-name: CloudMonitor CD — Deploy to AWS EC2
+name: CloudMonitor CD — Deploy to AWS EC2 via SSM
 
+# Manual-only during migration. Re-enable push-to-main only after SSM is verified.
 on:
-  push:
-    branches:
-      - main
   workflow_dispatch:
     inputs:
-      target_env:
-        description: "Target environment to deploy"
+      operation:
+        description: "Choose OIDC check, SSM preflight, deploy, or rollback"
         required: true
-        default: "production"
         type: choice
-        options:
-          - production
-      diagnostic_only:
-        description: "Run diagnostic connectivity check only (reports runner egress IP, skips deployment)"
+        options: [oidc-check, ssm-preflight, deploy, rollback]
+        default: oidc-check
+      confirm_production:
+        description: "Required for production deploy and rollback"
         required: false
-        default: false
         type: boolean
-
-concurrency:
-  group: production-deployment
-  cancel-in-progress: false
+        default: false
 
 permissions:
   contents: read
 
+concurrency:
+  group: cloudmonitor-production-deployment
+  cancel-in-progress: false
+
 jobs:
-  diagnostics:
-    name: Runner Network & IP Diagnostics
-    if: github.event_name == 'workflow_dispatch' && inputs.diagnostic_only == true
+  ssm:
+    name: CloudMonitor Production — SSM
     runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Discover Runner Public Egress IP
-        run: |
-          echo "=== GitHub-hosted Runner Egress IP Discovery ==="
-          RUNNER_IP=$(curl -s -m 5 https://api.ipify.org || curl -s -m 5 https://ifconfig.me || curl -s -m 5 https://icanhazip.com)
-          echo "Runner Public Egress IP: $RUNNER_IP"
-          echo "================================================="
-          echo "Note: If inbound SSH traffic is restricted, verify that network access control permits traffic from this runner's IP address."
-
-      - name: Probe TCP Port 22 Reachability
-        env:
-          EC2_HOST: ${{ secrets.EC2_HOST }}
-        run: |
-          if [ -z "$EC2_HOST" ]; then
-            echo "::warning::Secret EC2_HOST is not configured; skipping port 22 probe."
-            exit 0
-          fi
-          echo "Probing TCP port 22 reachability to EC2 host..."
-          if nc -z -w 5 "$EC2_HOST" 22 >/dev/null 2>&1; then
-            echo "Result: Port 22 is OPEN and REACHABLE from this runner."
-          else
-            echo "::error::Result: Port 22 is unreachable from this runner (connection timed out or refused). This may be due to an AWS Security Group, Network ACL, routing, host firewall, or other network restriction."
-            exit 1
-          fi
-
-  deploy:
-    name: Deploy to AWS EC2
-    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.diagnostic_only)
     environment: production
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 25
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      AWS_REGION: ap-southeast-2
+      AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }}
+      AWS_DEPLOY_ROLE_ARN: ${{ vars.AWS_DEPLOY_ROLE_ARN }}
+      EC2_INSTANCE_ID: ${{ vars.EC2_INSTANCE_ID }}
+      OPERATION: ${{ inputs.operation }}
+      CONFIRM_PRODUCTION: ${{ inputs.confirm_production }}
 
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Validate Deployment Secrets
-        env:
-          EC2_HOST: ${{ secrets.EC2_HOST }}
-          EC2_USER: ${{ secrets.EC2_USER }}
-          EC2_KEY: ${{ secrets.EC2_SSH_PRIVATE_KEY }}
-          EC2_KNOWN_HOSTS: ${{ secrets.EC2_SSH_KNOWN_HOSTS }}
+      - name: Validate requested operation and configuration
+        shell: bash
         run: |
-          MISSING=0
-          if [ -z "$EC2_HOST" ]; then
-            echo "::error::Secret EC2_HOST is not configured in repository or environment secrets."
-            MISSING=1
-          fi
-          if [ -z "$EC2_USER" ]; then
-            echo "::error::Secret EC2_USER is not configured in repository or environment secrets."
-            MISSING=1
-          fi
-          if [ -z "$EC2_KEY" ]; then
-            echo "::error::Secret EC2_SSH_PRIVATE_KEY is not configured in repository or environment secrets."
-            MISSING=1
-          fi
-          if [ -z "$EC2_KNOWN_HOSTS" ]; then
-            echo "::error::Secret EC2_SSH_KNOWN_HOSTS is not configured in repository or environment secrets. Strict SSH host-key verification requires a verified known_hosts entry."
-            MISSING=1
-          fi
-
-          if [ "$MISSING" -ne 0 ]; then
-            echo "::error::Deployment prerequisites missing. Configure the required secrets in GitHub: Settings -> Secrets and variables -> Actions (Repository secrets or Environment 'production' secrets)."
+          set -euo pipefail
+          case "$OPERATION" in
+            oidc-check|ssm-preflight|deploy|rollback) ;;
+            *) echo "::error::Unsupported operation."; exit 1 ;;
+          esac
+          if [[ "$OPERATION" == "deploy" || "$OPERATION" == "rollback" ]] && [[ "$CONFIRM_PRODUCTION" != "true" ]]; then
+            echo "::error::Set confirm_production=true for production deploy/rollback."
             exit 1
           fi
-          echo "All required deployment secrets are present."
-
-      - name: Set up SSH Private Key and Known Hosts
-        env:
-          EC2_SSH_PRIVATE_KEY: ${{ secrets.EC2_SSH_PRIVATE_KEY }}
-          EC2_SSH_KNOWN_HOSTS: ${{ secrets.EC2_SSH_KNOWN_HOSTS }}
-        run: |
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-
-          # Write private key with strict permissions, removing any Windows CRLF
-          printf "%s\n" "$EC2_SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-
-          # Write independently verified known_hosts entry, removing any Windows CRLF
-          printf "%s\n" "$EC2_SSH_KNOWN_HOSTS" | tr -d '\r' > ~/.ssh/known_hosts
-          chmod 644 ~/.ssh/known_hosts
-
-          # Configure SSH client with strict host key checking enabled and keep-alive
-          cat << 'EOF' > ~/.ssh/config
-          Host ec2-target
-            HostName ${{ secrets.EC2_HOST }}
-            User ${{ secrets.EC2_USER }}
-            IdentityFile ~/.ssh/deploy_key
-            StrictHostKeyChecking yes
-            UserKnownHostsFile ~/.ssh/known_hosts
-            ConnectTimeout 15
-            ServerAliveInterval 15
-            ServerAliveCountMax 4
-            BatchMode yes
-          EOF
-          chmod 600 ~/.ssh/config
+          if [[ "$OPERATION" != "oidc-check" ]]; then
+            for name in AWS_ACCOUNT_ID AWS_DEPLOY_ROLE_ARN EC2_INSTANCE_ID; do
+              if [[ -z "${!name}" ]]; then
+                echo "::error::Missing GitHub Environment variable: $name (environment: production)."
+                exit 1
+              fi
+            done
+            [[ "$AWS_ACCOUNT_ID" =~ ^[0-9]{12}$ ]] || { echo "::error::AWS_ACCOUNT_ID must be 12 digits."; exit 1; }
+            [[ "$EC2_INSTANCE_ID" =~ ^i-[0-9a-f]{8,17}$ ]] || { echo "::error::Invalid EC2_INSTANCE_ID."; exit 1; }
+            case "$AWS_DEPLOY_ROLE_ARN" in
+              "arn:aws:iam::${AWS_ACCOUNT_ID}:role/"*) ;;
+              *) echo "::error::AWS_DEPLOY_ROLE_ARN does not match AWS_ACCOUNT_ID."; exit 1 ;;
+            esac
+          fi
 
-      - name: Verify SSH Connectivity with Host Verification
+      - name: Inspect OIDC claims (token value is never logged)
+        shell: bash
         run: |
-          echo "Verifying secure SSH connectivity with strict host-key verification..."
-          ssh ec2-target "uptime && docker --version"
-
-      - name: Deploy Application Stack via Docker Compose
+          set -euo pipefail
+          python3 - <<'PY'
+          import base64, json, os, urllib.request
+          url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"] + "&audience=sts.amazonaws.com"
+          request = urllib.request.Request(url, headers={
+              "Authorization": f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"
+          })
+          with urllib.request.urlopen(request, timeout=15) as response:
+              token = json.load(response)["value"]
+          payload = token.split(".")[1]
+          payload += "=" * (-len(payload) % 4)
+          claims = json.loads(base64.urlsafe_b64decode(payload))
+          print("OIDC issuer:", claims.get("iss"))
+          print("OIDC audience:", claims.get("aud"))
+          print("OIDC subject:", claims.get("sub"))
+          aud = claims.get("aud")
+          if aud != "sts.amazonaws.com" and not (isinstance(aud, list) and "sts.amazonaws.com" in aud):
+              raise SystemExit("Unexpected OIDC audience.")
+          PY
+
+      - name: Configure AWS credentials using GitHub OIDC
+        if: inputs.operation != 'oidc-check'
+        uses: aws-actions/configure-aws-credentials@v6.3.0
+        with:
+          role-to-assume: ${{ env.AWS_DEPLOY_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+          role-session-name: CloudMonitor-SSM-${{ github.run_id }}
+          allowed-account-ids: ${{ env.AWS_ACCOUNT_ID }}
+
+      - name: Verify AWS identity and SSM managed-node status
+        if: inputs.operation != 'oidc-check'
+        shell: bash
         run: |
-          echo "Syncing latest configuration and deploying services to EC2..."
-          ssh ec2-target "bash -s" << 'EOF'
-            set -euo pipefail
-            DEPLOY_DIR="/home/ubuntu/cloud_monitor"
-
-            if [ ! -d "$DEPLOY_DIR" ]; then
-              echo "::error::Target directory $DEPLOY_DIR does not exist on host."
-              exit 1
-            fi
-            cd "$DEPLOY_DIR"
-
-            # 1. Protect production configuration (never automatically create or overwrite)
-            if [ ! -f .env ]; then
-              echo "::error::Production environment file ($DEPLOY_DIR/.env) does not exist! Deployment stopped to protect database and secrets."
-              exit 1
-            fi
-
-            # 2. Update code from main branch if git repository is initialized
-            if [ -d .git ]; then
-              git config --global --add safe.directory "$DEPLOY_DIR" || true
-              git fetch origin main
-              git checkout -f main
-              git reset --hard origin/main
-            fi
-
-            # 3. Pull required external images
-            sudo docker compose -f docker-compose.prod.yml pull prometheus node-exporter alertmanager postgres || true
-
-            # 4. Safely build/update backend and frontend without disturbing PostgreSQL volumes
-            sudo docker compose -f docker-compose.prod.yml build backend frontend
-
-            # 5. Bring up services idempotently (preserves postgres_data volume)
-            sudo docker compose -f docker-compose.prod.yml up -d --remove-orphans
-
-            # 6. Verify container status on host
-            echo "Verifying running container states on host..."
-            sudo docker compose -f docker-compose.prod.yml ps
-
-            echo "Stack deployment commands completed successfully."
-          EOF
+          set -euo pipefail
+          ACTUAL_ACCOUNT="$(aws sts get-caller-identity --query Account --output text)"
+          [[ "$ACTUAL_ACCOUNT" == "$AWS_ACCOUNT_ID" ]] || { echo "::error::Unexpected AWS account."; exit 1; }
+          PING_STATUS="$(aws ssm describe-instance-information \
+            --filters "Key=InstanceIds,Values=${EC2_INSTANCE_ID}" \
+            --query 'InstanceInformationList[0].PingStatus' --output text)"
+          [[ "$PING_STATUS" == "Online" ]] || {
+            echo "::error::EC2 is not Online in SSM (status: $PING_STATUS). Fix instance role, SSM Agent, and outbound HTTPS before deploying."
+            exit 1
+          }
+          echo "Expected AWS account verified; EC2 is Online in Systems Manager."
 
-      - name: Verify Production Health and Telemetry Endpoints
-        env:
-          EC2_HOST: ${{ secrets.EC2_HOST }}
+      - name: Run SSM preflight, deployment, or rollback
+        if: inputs.operation != 'oidc-check'
+        shell: bash
         run: |
-          echo "Performing post-deployment health check against EC2 entrypoint..."
-          FRONTEND_URL="http://${EC2_HOST}/"
-          BACKEND_HEALTH_URL="http://${EC2_HOST}/health"
-          TARGETS_URL="http://${EC2_HOST}/prom/api/v1/targets"
-
-          # 1. Bounded health check for Frontend SPA public entrypoint
-          echo "Checking frontend public endpoint at $FRONTEND_URL..."
-          FRONTEND_SUCCESS=false
-          for i in {1..12}; do
-            STATUS=$(curl -s -m 5 -o /dev/null -w "%{http_code}" "$FRONTEND_URL" || true)
-            if [ "$STATUS" = "200" ]; then
-              echo "Frontend public endpoint responded with HTTP 200 on attempt $i."
-              FRONTEND_SUCCESS=true
-              break
-            fi
-            echo "Waiting for frontend entrypoint (attempt $i/12, status: $STATUS)..."
-            sleep 5
-          done
-
-          if [ "$FRONTEND_SUCCESS" != "true" ]; then
-            echo "::error::Frontend public endpoint check failed at $FRONTEND_URL after 12 attempts."
+          set -euo pipefail
+          case "$OPERATION" in
+            ssm-preflight)
+              REMOTE_SCRIPT="$(cat <<'REMOTE'
+          set -Eeuo pipefail
+          DEPLOY_DIR="/home/ubuntu/cloud_monitor"
+          git config --global --add safe.directory "$DEPLOY_DIR" || true
+          test -d "$DEPLOY_DIR"
+          test -f "$DEPLOY_DIR/.env"
+          cd "$DEPLOY_DIR"
+          if git ls-files --error-unmatch .env >/dev/null 2>&1; then
+            echo "ERROR: .env is tracked by Git. Refusing to continue."
             exit 1
           fi
-
-          # 2. Bounded health check for FastAPI backend & Nginx reverse proxy
-          echo "Checking backend health at $BACKEND_HEALTH_URL..."
-          BACKEND_SUCCESS=false
-          for i in {1..12}; do
-            BACKEND_RES=$(curl -s -m 5 "$BACKEND_HEALTH_URL" || true)
-            if echo "$BACKEND_RES" | grep -q '"status":"healthy"'; then
-              echo "Backend health check succeeded on attempt $i: $BACKEND_RES"
-              BACKEND_SUCCESS=true
-              break
-            fi
-            echo "Waiting for backend health probe (attempt $i/12)..."
-            sleep 5
-          done
-
-          if [ "$BACKEND_SUCCESS" != "true" ]; then
-            echo "::error::Backend health check failed at $BACKEND_HEALTH_URL after 12 attempts."
+          command -v git >/dev/null
+          command -v curl >/dev/null
+          sudo -n docker --version
+          sudo -n docker compose version
+          sudo -n docker compose -f docker-compose.prod.yml config --quiet
+          sudo -n docker compose -f docker-compose.prod.yml ps
+          echo "SSM preflight passed. No application or database changes were made."
+          REMOTE
+          )"
+              ;;
+            deploy)
+              REMOTE_SCRIPT="$(cat <<'REMOTE'
+          set -Eeuo pipefail
+          DEPLOY_DIR="/home/ubuntu/cloud_monitor"
+          git config --global --add safe.directory "$DEPLOY_DIR" || true
+          cd "$DEPLOY_DIR"
+          test -f .env
+          if git ls-files --error-unmatch .env >/dev/null 2>&1; then
+            echo "ERROR: .env is tracked by Git. Refusing to deploy."
             exit 1
           fi
 
-          # 3 & 4. Bounded check for Prometheus scrape targets API and all required targets UP
-          echo "Validating Prometheus scrape targets proxy at $TARGETS_URL..."
-          TARGETS_SUCCESS=false
-          EXPECTED_TARGETS=("alertmanager" "cloudmonitor-backend" "node-exporter" "prometheus")
-
-          for i in {1..10}; do
-            TARGETS_RES=$(curl -s -m 5 "$TARGETS_URL" || true)
-            if echo "$TARGETS_RES" | jq -e '.status == "success"' >/dev/null 2>&1; then
-              DOWN_TARGETS=$(echo "$TARGETS_RES" | jq -r '[.data.activeTargets[] | select(.health != "up") | .scrapePool] | join(", ")' 2>/dev/null || true)
-              ACTIVE_COUNT=$(echo "$TARGETS_RES" | jq -r '.data.activeTargets | length' 2>/dev/null || echo "0")
-
-              MISSING_TARGETS=()
-              for target in "${EXPECTED_TARGETS[@]}"; do
-                FOUND=$(echo "$TARGETS_RES" | jq -r --arg t "$target" '[.data.activeTargets[] | select(.scrapePool == $t)] | length' 2>/dev/null || echo "0")
-                if [ "$FOUND" -eq 0 ]; then
-                  MISSING_TARGETS+=("$target")
-                fi
-              done
-
-              if [ "$ACTIVE_COUNT" -ge 4 ] && [ -z "$DOWN_TARGETS" ] && [ ${#MISSING_TARGETS[@]} -eq 0 ]; then
-                echo "Prometheus targets check succeeded on attempt $i: $ACTIVE_COUNT targets active and all UP (${EXPECTED_TARGETS[*]})."
-                TARGETS_SUCCESS=true
-                break
-              else
-                [ -n "$DOWN_TARGETS" ] && echo "::warning::Unhealthy targets detected: $DOWN_TARGETS (attempt $i/10)..."
-                [ ${#MISSING_TARGETS[@]} -gt 0 ] && echo "::warning::Missing expected targets: ${MISSING_TARGETS[*]} (attempt $i/10)..."
-              fi
-            fi
-            echo "Waiting for Prometheus target discovery (attempt $i/10)..."
-            sleep 5
-          done
-
-          if [ "$TARGETS_SUCCESS" != "true" ]; then
-            echo "::error::Prometheus scrape targets verification failed at $TARGETS_URL after 10 attempts."
+          PREVIOUS_SHA="$(git rev-parse HEAD)"
+          rollback_on_error() {
+            local rc=$?
+            trap - ERR
+            echo "Deployment failed; attempting rollback to ${PREVIOUS_SHA}."
+            git checkout -B main "$PREVIOUS_SHA" || true
+            sudo -n docker compose -f docker-compose.prod.yml build backend frontend || true
+            sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans || true
+            exit "$rc"
+          }
+          trap rollback_on_error ERR
+
+          sudo -n install -d -m 0750 /var/lib/cloudmonitor
+          printf '%s\n' "$PREVIOUS_SHA" | sudo -n tee /var/lib/cloudmonitor/previous-deploy-sha >/dev/null
+          git fetch --prune origin main
+          git checkout -B main origin/main
+          DEPLOY_SHA="$(git rev-parse HEAD)"
+
+          sudo -n docker compose -f docker-compose.prod.yml config --quiet
+          sudo -n docker compose -f docker-compose.prod.yml pull prometheus node-exporter alertmanager
+          sudo -n docker compose -f docker-compose.prod.yml build backend frontend
+          sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans
+
+          wait_for_http() {
+            local url="$1" attempt status
+            for attempt in $(seq 1 12); do
+              status="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "$url" || true)"
+              [[ "$status" == "200" ]] && return 0
+              sleep 5
+            done
+            echo "HTTP health check failed: $url"
+            return 1
+          }
+          wait_for_backend() {
+            local attempt body
+            for attempt in $(seq 1 12); do
+              body="$(curl -fsS --max-time 5 http://127.0.0.1/health 2>/dev/null || true)"
+              if printf '%s' "$body" | python3 -c 'import json,sys; sys.exit(0 if json.load(sys.stdin).get("status") == "healthy" else 1)' 2>/dev/null; then return 0; fi
+              sleep 5
+            done
+            echo "Backend did not report status=healthy."
+            return 1
+          }
+          wait_for_prometheus() {
+            local attempt body
+            for attempt in $(seq 1 12); do
+              body="$(curl -fsS --max-time 5 http://127.0.0.1/prom/api/v1/targets 2>/dev/null || true)"
+              if printf '%s' "$body" | python3 -c 'import json,sys; p=json.load(sys.stdin); required={"alertmanager","cloudmonitor-backend","node-exporter","prometheus"}; ts=p.get("data",{}).get("activeTargets",[]); bad=[n for n in required if not any(t.get("scrapePool")==n and t.get("health")=="up" for t in ts)]; sys.exit(0 if p.get("status")=="success" and not bad else 1)' 2>/dev/null; then return 0; fi
+              sleep 5
+            done
+            echo "Prometheus targets are not all healthy."
+            return 1
+          }
+
+          wait_for_http "http://127.0.0.1/"
+          wait_for_backend
+          wait_for_prometheus
+          printf '%s\n' "$DEPLOY_SHA" | sudo -n tee /var/lib/cloudmonitor/last-successful-deploy-sha >/dev/null
+          trap - ERR
+          echo "Deployment and health checks succeeded for ${DEPLOY_SHA}."
+          echo "No PostgreSQL volume was deleted or recreated."
+          REMOTE
+          )"
+              ;;
+            rollback)
+              REMOTE_SCRIPT="$(cat <<'REMOTE'
+          set -Eeuo pipefail
+          DEPLOY_DIR="/home/ubuntu/cloud_monitor"
+          git config --global --add safe.directory "$DEPLOY_DIR" || true
+          cd "$DEPLOY_DIR"
+          test -f .env
+          if git ls-files --error-unmatch .env >/dev/null 2>&1; then
+            echo "ERROR: .env is tracked by Git. Refusing to roll back."
             exit 1
           fi
+          CURRENT_SHA="$(git rev-parse HEAD)"
+          ROLLBACK_SHA="$(sudo -n cat /var/lib/cloudmonitor/previous-deploy-sha)"
+          git cat-file -e "${ROLLBACK_SHA}^{commit}"
+
+          restore_current_on_error() {
+            local rc=$?
+            trap - ERR
+            git checkout -B main "$CURRENT_SHA" || true
+            sudo -n docker compose -f docker-compose.prod.yml build backend frontend || true
+            sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans || true
+            exit "$rc"
+          }
+          trap restore_current_on_error ERR
+
+          git checkout -B main "$ROLLBACK_SHA"
+          sudo -n docker compose -f docker-compose.prod.yml config --quiet
+          sudo -n docker compose -f docker-compose.prod.yml build backend frontend
+          sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans
+          curl -fsS --max-time 5 http://127.0.0.1/ >/dev/null
+          BACKEND_BODY="$(curl -fsS --max-time 5 http://127.0.0.1/health)"
+          printf '%s' "$BACKEND_BODY" | python3 -c 'import json,sys; sys.exit(0 if json.load(sys.stdin).get("status") == "healthy" else 1)'
+          printf '%s\n' "$CURRENT_SHA" | sudo -n tee /var/lib/cloudmonitor/previous-deploy-sha >/dev/null
+          printf '%s\n' "$ROLLBACK_SHA" | sudo -n tee /var/lib/cloudmonitor/last-successful-deploy-sha >/dev/null
+          trap - ERR
+          echo "Rollback to ${ROLLBACK_SHA} succeeded; PostgreSQL volumes were preserved."
+          REMOTE
+          )"
+              ;;
+            *)
+              echo "::error::Unsupported SSM operation."
+              exit 1
+              ;;
+          esac
+
+          PARAMETERS="$(jq -n --arg script "$REMOTE_SCRIPT" '{commands: [$script]}')"
+          COMMAND_ID="$(aws ssm send-command \
+            --instance-ids "$EC2_INSTANCE_ID" \
+            --document-name "AWS-RunShellScript" \
+            --comment "CloudMonitor ${OPERATION} via GitHub Actions" \
+            --timeout-seconds 900 \
+            --parameters "$PARAMETERS" \
+            --query 'Command.CommandId' --output text)"
+          echo "SSM Command ID: $COMMAND_ID"
+
+          for attempt in $(seq 1 120); do
+            INVOCATION="$(aws ssm get-command-invocation \
+              --command-id "$COMMAND_ID" --instance-id "$EC2_INSTANCE_ID" \
+              --output json 2>/dev/null || true)"
+            STATUS="$(printf '%s' "$INVOCATION" | jq -r '.Status // "Pending"' 2>/dev/null || echo "Pending")"
+            case "$STATUS" in
+              Success)
+                printf '%s\n' "$INVOCATION" | jq -r '.StandardOutputContent // ""'
+                exit 0
+                ;;
+              Failed|TimedOut|Cancelled|Undeliverable|Terminated|InvalidPlatform|AccessDenied)
+                echo "::error::SSM command ended with status: $STATUS"
+                printf '%s\n' "$INVOCATION" | jq -r '"Status details: \(.StatusDetails // "unknown")\nResponse code: \(.ResponseCode // "unknown")\nStandard output:\n\(.StandardOutputContent // "")\nStandard error:\n\(.StandardErrorContent // "")"'
+                exit 1
+                ;;
+            esac
+            sleep 5
+          done
+          echo "::error::Timed out waiting for SSM command completion. Command ID: $COMMAND_ID"
+          exit 1
 
-          echo "All production health, frontend, backend, and monitoring checks passed."
-
-      - name: Clean up SSH Temporary Credentials
-        if: always()
-        run: |
-          rm -f ~/.ssh/deploy_key ~/.ssh/config ~/.ssh/known_hosts
-          echo "SSH temporary credentials and host keys removed from runner environment."
+      - name: Confirm deployment mode
+        if: inputs.operation == 'deploy'
+        run: echo "SSM production deployment completed. Review output before re-enabling push-based deployment."

From 7fca445b336803abc4eaf712147e679c1d8e511e Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sat, 31 Oct 2026 20:29:28 +0530
Subject: [PATCH 2/8] docs: add SSM deployment migration setup and runbook

---
 docs/SSM_DEPLOYMENT_MIGRATION.md | 192 +++++++++++++++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 docs/SSM_DEPLOYMENT_MIGRATION.md

diff --git a/docs/SSM_DEPLOYMENT_MIGRATION.md b/docs/SSM_DEPLOYMENT_MIGRATION.md
new file mode 100644
index 0000000..9bea726
--- /dev/null
+++ b/docs/SSM_DEPLOYMENT_MIGRATION.md
@@ -0,0 +1,192 @@
+# Issue #9 — Migrate EC2 deployment from SSH to AWS Systems Manager
+
+**Status:** SSM workflow prepared on a feature branch. AWS account configuration and live EC2 verification are still required before production deployment.
+
+## Why the SSH deployment is failing
+
+The current CD run fails at the SSH connectivity step with a TCP port 22 connection timeout. This is a network reachability failure before the deployment command starts; it is not evidence of a bad SSH key.
+
+The replacement workflow uses GitHub Actions OIDC to assume a narrowly scoped AWS role and Systems Manager Run Command to contact one EC2 instance. The EC2 SSM Agent initiates outbound HTTPS connections. GitHub-hosted runners do not need inbound SSH access to the instance. AWS documents outbound HTTPS on port 443 to regional SSM endpoints; no inbound SSM port is required: https://docs.aws.amazon.com/systems-manager/latest/userguide/troubleshooting-ssm-agent.html
+
+## Important rollout guard
+
+The updated `.github/workflows/cd.yml` is intentionally **workflow_dispatch-only** during migration. It does not deploy on a push or PR merge. This prevents merging the workflow from unexpectedly initiating production deployment before AWS prerequisites are in place.
+
+Available operations:
+
+1. `oidc-check` — prints the workflow's OIDC issuer, audience, and subject only; it does not call AWS.
+2. `ssm-preflight` — authenticates to AWS, verifies the configured EC2 instance is Online in SSM, and sends a read-only preflight command.
+3. `deploy` — requires `confirm_production=true`; deploys `origin/main` through SSM and runs local frontend/backend/Prometheus health checks on EC2.
+4. `rollback` — requires `confirm_production=true`; restores the previous Git revision recorded on EC2.
+
+After one successful controlled SSM deployment and operational review, a follow-up change can re-enable push-to-main for automatic CD. Do not add the push trigger before SSM is verified.
+
+## AWS setup checklist
+
+### 1. Tag the production EC2 instance
+
+In EC2 → Instances → select the CloudMonitor instance → Tags, add:
+
+| Key | Value |
+|---|---|
+| `CloudMonitor` | `production` |
+
+The GitHub deployment role's `ssm:SendCommand` permission should be restricted to the intended EC2 instance and this tag.
+
+### 2. Give the EC2 instance permission to use SSM
+
+Create an IAM role trusted by the **EC2** service, for example `CloudMonitorEC2SSMRole`. Attach the AWS-managed policy:
+
+`AmazonSSMManagedInstanceCore`
+
+Then attach this role to the existing EC2 instance:
+
+EC2 → Instances → select instance → Actions → Security → Modify IAM role → choose `CloudMonitorEC2SSMRole` → Update IAM role.
+
+Do not replace or remove other required instance permissions without auditing them first. This operation must not recreate the EC2 instance or its EBS volume.
+
+AWS documents the instance-profile path and policy attachment steps here: https://docs.aws.amazon.com/systems-manager/latest/userguide/setup-instance-permissions.html
+
+### 3. Verify SSM Agent and network connectivity
+
+On the existing Ubuntu 24.04 host, verify the SSM Agent is installed and running before installing or replacing anything. Depending on the package source, the service may be named `amazon-ssm-agent` or `snap.amazon-ssm-agent.amazon-ssm-agent`.
+
+The instance needs outbound TCP 443 connectivity to the regional Systems Manager endpoints, including:
+
+- `ssm.ap-southeast-2.amazonaws.com`
+- `ssmmessages.ap-southeast-2.amazonaws.com`
+
+This can be provided by its existing outbound internet route/NAT or suitable Systems Manager VPC endpoints. Do not open inbound SSH to GitHub-hosted runner IP ranges for SSM. Keep existing administrator SSH access as a break-glass path until SSM and any required Session Manager access are verified.
+
+AWS endpoint requirements: https://docs.aws.amazon.com/systems-manager/latest/userguide/troubleshooting-ssm-agent.html
+
+### 4. Create the GitHub OIDC identity provider
+
+In AWS IAM → Identity providers, create an OpenID Connect provider:
+
+- Provider URL: `https://token.actions.githubusercontent.com`
+- Audience: `sts.amazonaws.com`
+
+Do not create or store long-lived AWS access keys in GitHub.
+
+### 5. Create the GitHub deployment role trust policy
+
+Create an IAM role trusted by the GitHub OIDC provider. The workflow runs in GitHub Environment `production`, so the `sub` claim must be scoped to that environment. Use the **exact subject printed by the workflow's `oidc-check` operation**.
+
+Depending on the repository's OIDC subject-claim setting, the subject may use one of these forms:
+
+- Legacy: `repo:ayushcmd/cloud_monitor:environment:production`
+- Immutable: `repo:ayushcmd@252997256/cloud_monitor@1419373799:environment:production`
+
+The workflow's `oidc-check` output is authoritative. Do not allow repository-wide or branch-wide wildcards.
+
+Example trust policy (replace the account ID and subject with the actual values):
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::<AWS_ACCOUNT_ID>:oidc-provider/token.actions.githubusercontent.com"
+      },
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "token.actions.githubusercontent.com:aud": "sts.amazonaws.com",
+          "token.actions.githubusercontent.com:sub": "<EXACT_SUBJECT_FROM_OIDC_CHECK>"
+        }
+      }
+    }
+  ]
+}
+```
+
+### 6. Attach the deployment role permissions
+
+The GitHub role should be limited to `AWS-RunShellScript` on the CloudMonitor EC2 instance, plus the read operations needed to verify the command. Replace account and instance placeholders. Review the policy against the current AWS service authorization reference before attaching.
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "DescribeManagedInstances",
+      "Effect": "Allow",
+      "Action": "ssm:DescribeInstanceInformation",
+      "Resource": "*"
+    },
+    {
+      "Sid": "SendOnlyAwsRunShellScriptDocument",
+      "Effect": "Allow",
+      "Action": "ssm:SendCommand",
+      "Resource": "arn:aws:ssm:ap-southeast-2::document/AWS-RunShellScript"
+    },
+    {
+      "Sid": "SendOnlyToCloudMonitorInstance",
+      "Effect": "Allow",
+      "Action": "ssm:SendCommand",
+      "Resource": "arn:aws:ec2:ap-southeast-2:<AWS_ACCOUNT_ID>:instance/<EC2_INSTANCE_ID>",
+      "Condition": {
+        "StringEquals": {
+          "ssm:resourceTag/CloudMonitor": "production"
+        }
+      }
+    },
+    {
+      "Sid": "ReadCommandInvocationResult",
+      "Effect": "Allow",
+      "Action": "ssm:GetCommandInvocation",
+      "Resource": "*"
+    }
+  ]
+}
+```
+
+The read actions above use `Resource: "*"`; verify resource-level support and condition keys against the current Systems Manager service authorization reference: https://docs.aws.amazon.com/service-authorization/latest/reference/list_ssm.html
+
+### 7. Add GitHub Environment variables
+
+In GitHub → Settings → Environments → `production` → Environment variables, configure:
+
+| Variable | Value |
+|---|---|
+| `AWS_ACCOUNT_ID` | Your 12-digit AWS account ID |
+| `AWS_DEPLOY_ROLE_ARN` | ARN of the GitHub OIDC deployment role |
+| `EC2_INSTANCE_ID` | Existing CloudMonitor EC2 instance ID |
+
+The region is set in the workflow to `ap-southeast-2`. Consider adding required reviewers to the `production` GitHub Environment so deploy and rollback operations receive an approval gate.
+
+## Runbook after this PR is merged
+
+1. Open Actions → **CloudMonitor CD — Deploy to AWS EC2 via SSM** → Run workflow.
+2. Select `oidc-check` first. Record the printed OIDC subject and create the exact IAM trust policy.
+3. Configure the EC2 instance role, SSM Agent connectivity, EC2 tag, GitHub deployment role, and GitHub Environment variables.
+4. Run `ssm-preflight`. It must show the EC2 managed node as Online and complete the harmless read-only command.
+5. Run `deploy` with `confirm_production=true` and complete any GitHub Environment approval. Verify the public application and PostgreSQL data after the run.
+6. Use `rollback` with explicit confirmation if rollback is required.
+7. Only after the first successful SSM deployment and rollback procedure are reviewed should a follow-up PR restore automatic deployment on pushes to `main`.
+
+## Data and access safety
+
+- No SSH key or known-host secret is used by the SSM workflow.
+- The workflow does not require inbound SSH from GitHub-hosted runners.
+- It checks that `/home/ubuntu/cloud_monitor/.env` exists and is not Git-tracked; it never prints the file contents.
+- It uses `docker compose config --quiet`, `build`, and `up -d --remove-orphans`.
+- It does **not** use `docker compose down -v`, delete volumes, or recreate PostgreSQL storage.
+- The deploy operation records the previous Git revision under `/var/lib/cloudmonitor/` and attempts a source/build rollback if deployment or local health checks fail.
+- Keep SSH access available for trusted administration until SSM is Online, the read-only preflight succeeds, and the required break-glass access method has been tested.
+- No AWS resources are changed by merging this workflow/documentation PR. Actual AWS configuration and production deployment remain separate, deliberate steps.
+
+## Current acceptance status
+
+| Acceptance criterion | Status |
+|---|---|
+| EC2 registered and Online in SSM | Pending AWS-side setup/verification |
+| GitHub OIDC can assume the deployment role | Pending AWS trust policy and `oidc-check` |
+| GitHub invokes the approved SSM command | Pending `ssm-preflight` |
+| Runner no longer requires inbound SSH | Implemented in the prepared workflow; pending live preflight |
+| Production deployment and health checks succeed | Pending controlled deployment |
+| PostgreSQL persistent storage is preserved | Workflow avoids volume removal/recreation; live verification pending |
+| Least-privilege IAM and rollback procedure | Policy and runbook prepared; AWS review/verification pending |

From 10115d2bdd360dc007b39ba13055df02ae738eb3 Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sat, 31 Oct 2026 20:32:34 +0530
Subject: [PATCH 3/8] ci(cd): harden SSM shell execution and command polling

---
 .github/workflows/cd.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index b1ec026..c22f80e 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -28,7 +28,7 @@ jobs:
     name: CloudMonitor Production — SSM
     runs-on: ubuntu-latest
     environment: production
-    timeout-minutes: 25
+    timeout-minutes: 35
     permissions:
       contents: read
       id-token: write
@@ -265,7 +265,7 @@ jobs:
               ;;
           esac
 
-          PARAMETERS="$(jq -n --arg script "$REMOTE_SCRIPT" '{commands: [$script]}')"
+          PARAMETERS="$(jq -n --arg script "$REMOTE_SCRIPT" '{commands: [("bash -lc " + ($script | @sh))]}')"
           COMMAND_ID="$(aws ssm send-command \
             --instance-ids "$EC2_INSTANCE_ID" \
             --document-name "AWS-RunShellScript" \
@@ -275,7 +275,7 @@ jobs:
             --query 'Command.CommandId' --output text)"
           echo "SSM Command ID: $COMMAND_ID"
 
-          for attempt in $(seq 1 120); do
+          for attempt in $(seq 1 360); do
             INVOCATION="$(aws ssm get-command-invocation \
               --command-id "$COMMAND_ID" --instance-id "$EC2_INSTANCE_ID" \
               --output json 2>/dev/null || true)"
@@ -285,7 +285,7 @@ jobs:
                 printf '%s\n' "$INVOCATION" | jq -r '.StandardOutputContent // ""'
                 exit 0
                 ;;
-              Failed|TimedOut|Cancelled|Undeliverable|Terminated|InvalidPlatform|AccessDenied)
+              Failed|TimedOut|ExecutionTimedOut|DeliveryTimedOut|Cancelled|Cancelling|Undeliverable|Terminated|InvalidPlatform|AccessDenied|NoInstancesInTag|InvalidDocument|Incomplete|RateExceeded)
                 echo "::error::SSM command ended with status: $STATUS"
                 printf '%s\n' "$INVOCATION" | jq -r '"Status details: \(.StatusDetails // "unknown")\nResponse code: \(.ResponseCode // "unknown")\nStandard output:\n\(.StandardOutputContent // "")\nStandard error:\n\(.StandardErrorContent // "")"'
                 exit 1

From bc765ea9020604146595e8edbfe46fe6222c0b80 Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sat, 31 Oct 2026 20:33:03 +0530
Subject: [PATCH 4/8] docs: clarify SSM shell execution and rollback limits

---
 docs/SSM_DEPLOYMENT_MIGRATION.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/SSM_DEPLOYMENT_MIGRATION.md b/docs/SSM_DEPLOYMENT_MIGRATION.md
index 9bea726..5698abc 100644
--- a/docs/SSM_DEPLOYMENT_MIGRATION.md
+++ b/docs/SSM_DEPLOYMENT_MIGRATION.md
@@ -6,7 +6,7 @@
 
 The current CD run fails at the SSH connectivity step with a TCP port 22 connection timeout. This is a network reachability failure before the deployment command starts; it is not evidence of a bad SSH key.
 
-The replacement workflow uses GitHub Actions OIDC to assume a narrowly scoped AWS role and Systems Manager Run Command to contact one EC2 instance. The EC2 SSM Agent initiates outbound HTTPS connections. GitHub-hosted runners do not need inbound SSH access to the instance. AWS documents outbound HTTPS on port 443 to regional SSM endpoints; no inbound SSM port is required: https://docs.aws.amazon.com/systems-manager/latest/userguide/troubleshooting-ssm-agent.html
+The replacement workflow uses GitHub Actions OIDC to assume a narrowly scoped AWS role and Systems Manager Run Command to contact one EC2 instance. The workflow wraps each remote script with `bash -lc` so Bash-specific constructs are not accidentally interpreted by the default POSIX shell. The EC2 SSM Agent initiates outbound HTTPS connections. GitHub-hosted runners do not need inbound SSH access to the instance. AWS documents outbound HTTPS on port 443 to regional SSM endpoints; no inbound SSM port is required: https://docs.aws.amazon.com/systems-manager/latest/userguide/troubleshooting-ssm-agent.html
 
 ## Important rollout guard
 
@@ -31,7 +31,7 @@ In EC2 → Instances → select the CloudMonitor instance → Tags, add:
 |---|---|
 | `CloudMonitor` | `production` |
 
-The GitHub deployment role's `ssm:SendCommand` permission should be restricted to the intended EC2 instance and this tag.
+The GitHub deployment role's `ssm:SendCommand` permission should be restricted to the intended EC2 instance and this tag. The workflow polls for up to 30 minutes and sets a 35-minute GitHub job timeout to allow image build and health-check time.
 
 ### 2. Give the EC2 instance permission to use SSM
 
@@ -164,7 +164,7 @@ The region is set in the workflow to `ap-southeast-2`. Consider adding required
 2. Select `oidc-check` first. Record the printed OIDC subject and create the exact IAM trust policy.
 3. Configure the EC2 instance role, SSM Agent connectivity, EC2 tag, GitHub deployment role, and GitHub Environment variables.
 4. Run `ssm-preflight`. It must show the EC2 managed node as Online and complete the harmless read-only command.
-5. Run `deploy` with `confirm_production=true` and complete any GitHub Environment approval. Verify the public application and PostgreSQL data after the run.
+5. Run `deploy` with `confirm_production=true` and complete any GitHub Environment approval. Verify the public application from outside EC2 and verify PostgreSQL data/persistence using a known non-sensitive application-level check. The workflow's health probes run against localhost on EC2.
 6. Use `rollback` with explicit confirmation if rollback is required.
 7. Only after the first successful SSM deployment and rollback procedure are reviewed should a follow-up PR restore automatic deployment on pushes to `main`.
 
@@ -175,7 +175,7 @@ The region is set in the workflow to `ap-southeast-2`. Consider adding required
 - It checks that `/home/ubuntu/cloud_monitor/.env` exists and is not Git-tracked; it never prints the file contents.
 - It uses `docker compose config --quiet`, `build`, and `up -d --remove-orphans`.
 - It does **not** use `docker compose down -v`, delete volumes, or recreate PostgreSQL storage.
-- The deploy operation records the previous Git revision under `/var/lib/cloudmonitor/` and attempts a source/build rollback if deployment or local health checks fail.
+- The deploy operation records the previous Git revision under `/var/lib/cloudmonitor/` and attempts a source/build rollback if deployment or local health checks fail. This is a **code/config rollback only**: it does not restore PostgreSQL data or reverse database schema migrations. Keep schema changes backward-compatible across rollback, and take/verify a separate database backup before any release that changes schema or data.
 - Keep SSH access available for trusted administration until SSM is Online, the read-only preflight succeeds, and the required break-glass access method has been tested.
 - No AWS resources are changed by merging this workflow/documentation PR. Actual AWS configuration and production deployment remain separate, deliberate steps.
 

From f4bc5ebb0c01c7b699833781c090175e5efdb302 Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sat, 31 Oct 2026 20:34:10 +0530
Subject: [PATCH 5/8] ci(cd): validate GitHub OIDC issuer

---
 .github/workflows/cd.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index c22f80e..48507b1 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -86,6 +86,8 @@ jobs:
           print("OIDC issuer:", claims.get("iss"))
           print("OIDC audience:", claims.get("aud"))
           print("OIDC subject:", claims.get("sub"))
+          if claims.get("iss") != "https://token.actions.githubusercontent.com":
+              raise SystemExit("Unexpected OIDC issuer.")
           aud = claims.get("aud")
           if aud != "sts.amazonaws.com" and not (isinstance(aud, list) and "sts.amazonaws.com" in aud):
               raise SystemExit("Unexpected OIDC audience.")

From 684d77df9c30e30e51035e11b014559ce8b13ca0 Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sun, 1 Nov 2026 10:39:48 +0530
Subject: [PATCH 6/8] ci(cd): verify OIDC role assumption and preserve deploy
 ownership

---
 .github/workflows/cd.yml | 62 ++++++++++++++++++++--------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 48507b1..bac36ee 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -94,7 +94,6 @@ jobs:
           PY
 
       - name: Configure AWS credentials using GitHub OIDC
-        if: inputs.operation != 'oidc-check'
         uses: aws-actions/configure-aws-credentials@v6.3.0
         with:
           role-to-assume: ${{ env.AWS_DEPLOY_ROLE_ARN }}
@@ -102,21 +101,26 @@ jobs:
           role-session-name: CloudMonitor-SSM-${{ github.run_id }}
           allowed-account-ids: ${{ env.AWS_ACCOUNT_ID }}
 
-      - name: Verify AWS identity and SSM managed-node status
-        if: inputs.operation != 'oidc-check'
+      - name: Verify AWS role assumption and optional SSM managed-node status
+        if: inputs.operation != 'oidc-check' || success()
         shell: bash
         run: |
           set -euo pipefail
           ACTUAL_ACCOUNT="$(aws sts get-caller-identity --query Account --output text)"
           [[ "$ACTUAL_ACCOUNT" == "$AWS_ACCOUNT_ID" ]] || { echo "::error::Unexpected AWS account."; exit 1; }
-          PING_STATUS="$(aws ssm describe-instance-information \
-            --filters "Key=InstanceIds,Values=${EC2_INSTANCE_ID}" \
-            --query 'InstanceInformationList[0].PingStatus' --output text)"
-          [[ "$PING_STATUS" == "Online" ]] || {
-            echo "::error::EC2 is not Online in SSM (status: $PING_STATUS). Fix instance role, SSM Agent, and outbound HTTPS before deploying."
-            exit 1
-          }
-          echo "Expected AWS account verified; EC2 is Online in Systems Manager."
+          ACTUAL_ARN="$(aws sts get-caller-identity --query Arn --output text)"
+          echo "GitHub OIDC successfully assumed the configured AWS role: $ACTUAL_ARN"
+
+          if [[ "$OPERATION" != "oidc-check" ]]; then
+            PING_STATUS="$(aws ssm describe-instance-information \
+              --filters "Key=InstanceIds,Values=${EC2_INSTANCE_ID}" \
+              --query 'InstanceInformationList[0].PingStatus' --output text)"
+            [[ "$PING_STATUS" == "Online" ]] || {
+              echo "::error::EC2 is not Online in SSM (status: $PING_STATUS). Fix instance role, SSM Agent, and outbound HTTPS before deploying."
+              exit 1
+            }
+            echo "EC2 is Online in Systems Manager."
+          fi
 
       - name: Run SSM preflight, deployment, or rollback
         if: inputs.operation != 'oidc-check'
@@ -128,11 +132,9 @@ jobs:
               REMOTE_SCRIPT="$(cat <<'REMOTE'
           set -Eeuo pipefail
           DEPLOY_DIR="/home/ubuntu/cloud_monitor"
-          git config --global --add safe.directory "$DEPLOY_DIR" || true
           test -d "$DEPLOY_DIR"
           test -f "$DEPLOY_DIR/.env"
-          cd "$DEPLOY_DIR"
-          if git ls-files --error-unmatch .env >/dev/null 2>&1; then
+          if sudo -u ubuntu -- git -C "$DEPLOY_DIR" ls-files --error-unmatch .env >/dev/null 2>&1; then
             echo "ERROR: .env is tracked by Git. Refusing to continue."
             exit 1
           fi
@@ -150,20 +152,19 @@ jobs:
               REMOTE_SCRIPT="$(cat <<'REMOTE'
           set -Eeuo pipefail
           DEPLOY_DIR="/home/ubuntu/cloud_monitor"
-          git config --global --add safe.directory "$DEPLOY_DIR" || true
-          cd "$DEPLOY_DIR"
-          test -f .env
-          if git ls-files --error-unmatch .env >/dev/null 2>&1; then
+          test -d "$DEPLOY_DIR"
+          test -f "$DEPLOY_DIR/.env"
+          if sudo -u ubuntu -- git -C "$DEPLOY_DIR" ls-files --error-unmatch .env >/dev/null 2>&1; then
             echo "ERROR: .env is tracked by Git. Refusing to deploy."
             exit 1
           fi
 
-          PREVIOUS_SHA="$(git rev-parse HEAD)"
+          PREVIOUS_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)"
           rollback_on_error() {
             local rc=$?
             trap - ERR
             echo "Deployment failed; attempting rollback to ${PREVIOUS_SHA}."
-            git checkout -B main "$PREVIOUS_SHA" || true
+            sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main "$PREVIOUS_SHA" || true
             sudo -n docker compose -f docker-compose.prod.yml build backend frontend || true
             sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans || true
             exit "$rc"
@@ -172,9 +173,9 @@ jobs:
 
           sudo -n install -d -m 0750 /var/lib/cloudmonitor
           printf '%s\n' "$PREVIOUS_SHA" | sudo -n tee /var/lib/cloudmonitor/previous-deploy-sha >/dev/null
-          git fetch --prune origin main
-          git checkout -B main origin/main
-          DEPLOY_SHA="$(git rev-parse HEAD)"
+          sudo -u ubuntu -- git -C "$DEPLOY_DIR" fetch --prune origin +refs/heads/main:refs/remotes/origin/main
+          sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main origin/main
+          DEPLOY_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)"
 
           sudo -n docker compose -f docker-compose.prod.yml config --quiet
           sudo -n docker compose -f docker-compose.prod.yml pull prometheus node-exporter alertmanager
@@ -226,28 +227,27 @@ jobs:
               REMOTE_SCRIPT="$(cat <<'REMOTE'
           set -Eeuo pipefail
           DEPLOY_DIR="/home/ubuntu/cloud_monitor"
-          git config --global --add safe.directory "$DEPLOY_DIR" || true
-          cd "$DEPLOY_DIR"
-          test -f .env
-          if git ls-files --error-unmatch .env >/dev/null 2>&1; then
+          test -d "$DEPLOY_DIR"
+          test -f "$DEPLOY_DIR/.env"
+          if sudo -u ubuntu -- git -C "$DEPLOY_DIR" ls-files --error-unmatch .env >/dev/null 2>&1; then
             echo "ERROR: .env is tracked by Git. Refusing to roll back."
             exit 1
           fi
-          CURRENT_SHA="$(git rev-parse HEAD)"
+          CURRENT_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)"
           ROLLBACK_SHA="$(sudo -n cat /var/lib/cloudmonitor/previous-deploy-sha)"
-          git cat-file -e "${ROLLBACK_SHA}^{commit}"
+          sudo -u ubuntu -- git -C "$DEPLOY_DIR" cat-file -e "${ROLLBACK_SHA}^{commit}"
 
           restore_current_on_error() {
             local rc=$?
             trap - ERR
-            git checkout -B main "$CURRENT_SHA" || true
+            sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main "$CURRENT_SHA" || true
             sudo -n docker compose -f docker-compose.prod.yml build backend frontend || true
             sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans || true
             exit "$rc"
           }
           trap restore_current_on_error ERR
 
-          git checkout -B main "$ROLLBACK_SHA"
+          sudo -u ubuntu -- git -C "$DEPLOY_DIR" checkout -B main "$ROLLBACK_SHA"
           sudo -n docker compose -f docker-compose.prod.yml config --quiet
           sudo -n docker compose -f docker-compose.prod.yml build backend frontend
           sudo -n docker compose -f docker-compose.prod.yml up -d --remove-orphans

From 69e113505003d01710a96d14d4b66eaaab4a3396 Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sun, 1 Nov 2026 10:41:21 +0530
Subject: [PATCH 7/8] docs: align SSM runbook with completed AWS setup

---
 docs/SSM_DEPLOYMENT_MIGRATION.md | 74 ++++++++------------------------
 1 file changed, 19 insertions(+), 55 deletions(-)

diff --git a/docs/SSM_DEPLOYMENT_MIGRATION.md b/docs/SSM_DEPLOYMENT_MIGRATION.md
index 5698abc..c3e32ce 100644
--- a/docs/SSM_DEPLOYMENT_MIGRATION.md
+++ b/docs/SSM_DEPLOYMENT_MIGRATION.md
@@ -1,6 +1,6 @@
 # Issue #9 — Migrate EC2 deployment from SSH to AWS Systems Manager
 
-**Status:** SSM workflow prepared on a feature branch. AWS account configuration and live EC2 verification are still required before production deployment.
+**Status:** SSM workflow prepared on a feature branch. The existing EC2 instance role is attached, SSM reports the instance Online, the GitHub OIDC provider and deployment role exist, and the repository production environment has the three AWS variables plus a main-only deployment rule. GitHub OIDC role assumption and Run Command execution still require workflow verification.
 
 ## Why the SSH deployment is failing
 
@@ -14,7 +14,7 @@ The updated `.github/workflows/cd.yml` is intentionally **workflow_dispatch-only
 
 Available operations:
 
-1. `oidc-check` — prints the workflow's OIDC issuer, audience, and subject only; it does not call AWS.
+1. `oidc-check` — requests a GitHub OIDC token, prints only issuer/audience/subject claims (never the token), assumes the configured AWS role with STS, and verifies the AWS account/caller identity. It does not send an SSM command or change EC2.
 2. `ssm-preflight` — authenticates to AWS, verifies the configured EC2 instance is Online in SSM, and sends a read-only preflight command.
 3. `deploy` — requires `confirm_production=true`; deploys `origin/main` through SSM and runs local frontend/backend/Prometheus health checks on EC2.
 4. `rollback` — requires `confirm_production=true`; restores the previous Git revision recorded on EC2.
@@ -23,62 +23,32 @@ After one successful controlled SSM deployment and operational review, a follow-
 
 ## AWS setup checklist
 
-### 1. Tag the production EC2 instance
+### 1. Confirm the existing EC2 SSM registration
 
-In EC2 → Instances → select the CloudMonitor instance → Tags, add:
+The CloudMonitor EC2 instance already has CloudMonitorEC2SSMRole with AmazonSSMManagedInstanceCore, and Fleet Manager reports the instance as Online. The GitHub role policy is restricted to the exact EC2 instance ARN, so an EC2 tag is not required for this implementation. The workflow polls for up to 30 minutes and sets a 35-minute GitHub job timeout to allow image build and health-check time.
 
-| Key | Value |
-|---|---|
-| `CloudMonitor` | `production` |
-
-The GitHub deployment role's `ssm:SendCommand` permission should be restricted to the intended EC2 instance and this tag. The workflow polls for up to 30 minutes and sets a 35-minute GitHub job timeout to allow image build and health-check time.
-
-### 2. Give the EC2 instance permission to use SSM
-
-Create an IAM role trusted by the **EC2** service, for example `CloudMonitorEC2SSMRole`. Attach the AWS-managed policy:
-
-`AmazonSSMManagedInstanceCore`
-
-Then attach this role to the existing EC2 instance:
-
-EC2 → Instances → select instance → Actions → Security → Modify IAM role → choose `CloudMonitorEC2SSMRole` → Update IAM role.
-
-Do not replace or remove other required instance permissions without auditing them first. This operation must not recreate the EC2 instance or its EBS volume.
-
-AWS documents the instance-profile path and policy attachment steps here: https://docs.aws.amazon.com/systems-manager/latest/userguide/setup-instance-permissions.html
+### 2. EC2 instance profile — completed
 
-### 3. Verify SSM Agent and network connectivity
+The existing CloudMonitor EC2 instance has CloudMonitorEC2SSMRole attached with the AWS-managed AmazonSSMManagedInstanceCore policy. SSM Agent diagnostics passed, and the instance is listed as Online in Fleet Manager. No further EC2 role change is needed.
 
-On the existing Ubuntu 24.04 host, verify the SSM Agent is installed and running before installing or replacing anything. Depending on the package source, the service may be named `amazon-ssm-agent` or `snap.amazon-ssm-agent.amazon-ssm-agent`.
+### 3. SSM Agent and network connectivity — completed
 
-The instance needs outbound TCP 443 connectivity to the regional Systems Manager endpoints, including:
+The Ubuntu 24.04 host has SSM Agent 3.3.4793.0 running and enabled. ssm-cli get-diagnostics --output table reported success for IMDS, SSM, EC2 Messages, SSM Messages, S3, KMS, Logs, Monitoring, AWS role credentials, and agent service checks. Fleet Manager subsequently showed the CloudMonitor instance as Online. No inbound SSH change is required for GitHub-hosted runners to use SSM. Keep administrator SSH access as a break-glass path until the new deployment route is fully tested.
 
-- `ssm.ap-southeast-2.amazonaws.com`
-- `ssmmessages.ap-southeast-2.amazonaws.com`
+### 4. GitHub OIDC identity provider — already exists
 
-This can be provided by its existing outbound internet route/NAT or suitable Systems Manager VPC endpoints. Do not open inbound SSH to GitHub-hosted runner IP ranges for SSM. Keep existing administrator SSH access as a break-glass path until SSM and any required Session Manager access are verified.
+The AWS account already has the OpenID Connect provider token.actions.githubusercontent.com with audience sts.amazonaws.com. Reuse it; do not create a duplicate or delete the existing provider.
 
-AWS endpoint requirements: https://docs.aws.amazon.com/systems-manager/latest/userguide/troubleshooting-ssm-agent.html
+### 5. GitHub deployment role trust policy — already created
 
-### 4. Create the GitHub OIDC identity provider
-
-In AWS IAM → Identity providers, create an OpenID Connect provider:
-
-- Provider URL: `https://token.actions.githubusercontent.com`
-- Audience: `sts.amazonaws.com`
-
-Do not create or store long-lived AWS access keys in GitHub.
-
-### 5. Create the GitHub deployment role trust policy
-
-Create an IAM role trusted by the GitHub OIDC provider. The workflow runs in GitHub Environment `production`, so the `sub` claim must be scoped to that environment. Use the **exact subject printed by the workflow's `oidc-check` operation**.
+The role CloudMonitorGitHubDeployRole exists and trusts the GitHub OIDC provider for the immutable CloudMonitor repository identity and GitHub Environment production. The expected subject is repo:ayushcmd@252997256/cloud_monitor@1419373799:environment:production. The workflow's oidc-check now verifies the emitted subject and performs an actual AWS STS role assumption. If the STS step fails, compare the emitted subject with the role trust policy before changing either.
 
 Depending on the repository's OIDC subject-claim setting, the subject may use one of these forms:
 
 - Legacy: `repo:ayushcmd/cloud_monitor:environment:production`
 - Immutable: `repo:ayushcmd@252997256/cloud_monitor@1419373799:environment:production`
 
-The workflow's `oidc-check` output is authoritative. Do not allow repository-wide or branch-wide wildcards.
+The environment's deployment branch rule allows main only. Do not allow repository-wide or branch-wide wildcards in the IAM trust policy.
 
 Example trust policy (replace the account ID and subject with the actual values):
 
@@ -105,7 +75,7 @@ Example trust policy (replace the account ID and subject with the actual values)
 
 ### 6. Attach the deployment role permissions
 
-The GitHub role should be limited to `AWS-RunShellScript` on the CloudMonitor EC2 instance, plus the read operations needed to verify the command. Replace account and instance placeholders. Review the policy against the current AWS service authorization reference before attaching.
+The GitHub role is limited to AWS-RunShellScript on the exact CloudMonitor EC2 instance, plus the read operations needed to verify the command. The policy attached in AWS uses the actual account and instance identifiers.
 
 ```json
 {
@@ -127,12 +97,7 @@ The GitHub role should be limited to `AWS-RunShellScript` on the CloudMonitor EC
       "Sid": "SendOnlyToCloudMonitorInstance",
       "Effect": "Allow",
       "Action": "ssm:SendCommand",
-      "Resource": "arn:aws:ec2:ap-southeast-2:<AWS_ACCOUNT_ID>:instance/<EC2_INSTANCE_ID>",
-      "Condition": {
-        "StringEquals": {
-          "ssm:resourceTag/CloudMonitor": "production"
-        }
-      }
+      "Resource": "arn:aws:ec2:ap-southeast-2:<AWS_ACCOUNT_ID>:instance/<EC2_INSTANCE_ID>"
     },
     {
       "Sid": "ReadCommandInvocationResult",
@@ -161,9 +126,8 @@ The region is set in the workflow to `ap-southeast-2`. Consider adding required
 ## Runbook after this PR is merged
 
 1. Open Actions → **CloudMonitor CD — Deploy to AWS EC2 via SSM** → Run workflow.
-2. Select `oidc-check` first. Record the printed OIDC subject and create the exact IAM trust policy.
-3. Configure the EC2 instance role, SSM Agent connectivity, EC2 tag, GitHub deployment role, and GitHub Environment variables.
-4. Run `ssm-preflight`. It must show the EC2 managed node as Online and complete the harmless read-only command.
+2. Select `oidc-check`. Confirm the emitted OIDC subject matches the IAM trust policy and that STS returns the expected AWS account and assumed role.
+3. Run `ssm-preflight`. It must confirm the EC2 managed node is Online and complete the harmless read-only command.
 5. Run `deploy` with `confirm_production=true` and complete any GitHub Environment approval. Verify the public application from outside EC2 and verify PostgreSQL data/persistence using a known non-sensitive application-level check. The workflow's health probes run against localhost on EC2.
 6. Use `rollback` with explicit confirmation if rollback is required.
 7. Only after the first successful SSM deployment and rollback procedure are reviewed should a follow-up PR restore automatic deployment on pushes to `main`.
@@ -183,8 +147,8 @@ The region is set in the workflow to `ap-southeast-2`. Consider adding required
 
 | Acceptance criterion | Status |
 |---|---|
-| EC2 registered and Online in SSM | Pending AWS-side setup/verification |
-| GitHub OIDC can assume the deployment role | Pending AWS trust policy and `oidc-check` |
+| EC2 registered and Online in SSM | Complete — Fleet Manager reports Online |
+| GitHub OIDC provider, role, and trust policy | Configured — live STS assumption pending `oidc-check` |
 | GitHub invokes the approved SSM command | Pending `ssm-preflight` |
 | Runner no longer requires inbound SSH | Implemented in the prepared workflow; pending live preflight |
 | Production deployment and health checks succeed | Pending controlled deployment |

From 957bb4b414b3d1b0a58397211057ef705dab5326 Mon Sep 17 00:00:00 2001
From: Ayush Raj <ayush_24a12res897@iitp.ac.in>
Date: Sun, 1 Nov 2026 10:42:04 +0530
Subject: [PATCH 8/8] ci(cd): set SSM working directory for compose operations

---
 .github/workflows/cd.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index bac36ee..a4b6d69 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -102,7 +102,6 @@ jobs:
           allowed-account-ids: ${{ env.AWS_ACCOUNT_ID }}
 
       - name: Verify AWS role assumption and optional SSM managed-node status
-        if: inputs.operation != 'oidc-check' || success()
         shell: bash
         run: |
           set -euo pipefail
@@ -138,6 +137,7 @@ jobs:
             echo "ERROR: .env is tracked by Git. Refusing to continue."
             exit 1
           fi
+          cd "$DEPLOY_DIR"
           command -v git >/dev/null
           command -v curl >/dev/null
           sudo -n docker --version
@@ -158,6 +158,7 @@ jobs:
             echo "ERROR: .env is tracked by Git. Refusing to deploy."
             exit 1
           fi
+          cd "$DEPLOY_DIR"
 
           PREVIOUS_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)"
           rollback_on_error() {
@@ -233,6 +234,7 @@ jobs:
             echo "ERROR: .env is tracked by Git. Refusing to roll back."
             exit 1
           fi
+          cd "$DEPLOY_DIR"
           CURRENT_SHA="$(sudo -u ubuntu -- git -C "$DEPLOY_DIR" rev-parse HEAD)"
           ROLLBACK_SHA="$(sudo -n cat /var/lib/cloudmonitor/previous-deploy-sha)"
           sudo -u ubuntu -- git -C "$DEPLOY_DIR" cat-file -e "${ROLLBACK_SHA}^{commit}"