VED-1103: Refactor Batch Processor (recordprocessor) deployment to a dedicated pipeline stage

.github/workflows/continuous-deployment.yml

@@ -15,11 +15,31 @@ jobs:
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
+  detect-recordprocessor-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      has_changes: ${{ steps.detect.outputs.has_changes }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          fetch-depth: 0
+
+      - name: Detect recordprocessor changes compared to master
+        id: detect
+        run: |
+          if git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" | grep -q '^lambdas/recordprocessor/'; then
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          fi
+
   deploy-internal-dev-backend:
-    needs: [run-quality-checks]
+    needs: [run-quality-checks, detect-recordprocessor-changes]
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
+      build_recordprocessor_image: ${{needs.detect-recordprocessor-changes.outputs.has_changes == 'true'}}
       create_mns_subscription: true
       environment: dev
       sub_environment: internal-dev
@@ -75,13 +95,14 @@ jobs:
       STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}
 
   deploy-higher-dev-envs:
-    needs: [run-internal-dev-tests]
+    needs: [run-internal-dev-tests, detect-recordprocessor-changes]
     strategy:
       matrix:
         sub_environment_name: [ref, internal-qa]
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: ${{ matrix.sub_environment_name }}
+      build_recordprocessor_image: ${{needs.detect-recordprocessor-changes.outputs.has_changes == 'true'}}
       create_mns_subscription: true
       environment: dev
       sub_environment: ${{ matrix.sub_environment_name }}

.github/workflows/deploy-backend.yml

@@ -6,6 +6,10 @@ on:
       apigee_environment:
         required: true
         type: string
+      build_recordprocessor_image:
+        required: false
+        type: boolean
+        default: false
       create_mns_subscription:
         required: false
         type: boolean
@@ -39,6 +43,11 @@ on:
           - dev
           - preprod
           - prod
+      build_recordprocessor_image:
+        description: Build and push a new recordprocessor image for this deployment
+        required: false
+        type: boolean
+        default: false
       sub_environment:
         type: string
         description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments
@@ -51,11 +60,113 @@ env: # Sonarcloud - do not allow direct usage of untrusted data
 run-name: Deploy Backend - ${{ inputs.environment }} ${{ inputs.sub_environment }}
 
 jobs:
+  build-and-push-recordprocessor:
+    if: ${{ inputs.build_recordprocessor_image }}
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      recordprocessor_image_tag: ${{ steps.build-image.outputs.recordprocessor_image_tag }}
+    name: Build and push recordprocessor image
+    runs-on: ubuntu-latest
+
+    environment:
+      name: ${{ inputs.environment }}
+
+    env:
+      AWS_REGION: eu-west-2
+      SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076
+
+      - name: Build and push Docker image
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        working-directory: lambdas
+        run: |
+          IMAGE_TAG="${SUB_ENVIRONMENT}-${GITHUB_SHA}"
+          REPOSITORY_NAME="imms-recordprocessor-repo"
+          IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}"
+
+          docker build -f recordprocessor/Dockerfile -t "${IMAGE_URI}" .
+          docker push "${IMAGE_URI}"
+          echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+
+  resolve-recordprocessor-image-tag:
+    if: ${{ !inputs.build_recordprocessor_image }}
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      recordprocessor_image_tag: ${{ steps.resolve-image-tag.outputs.recordprocessor_image_tag }}
+    name: Resolve existing recordprocessor image tag
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
+    env:
+      AWS_REGION: eu-west-2
+    steps:
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - name: Resolve latest matching recordprocessor image tag
+        id: resolve-image-tag
+        env:
+          REPOSITORY_NAME: imms-recordprocessor-repo
+          TAG_PREFIX: ${{ inputs.sub_environment }}-
+        run: |
+          IMAGE_TAG=$(
+            aws ecr describe-images \
+              --repository-name "${REPOSITORY_NAME}" \
+              --region "${AWS_REGION}" \
+              --filter tagStatus=TAGGED \
+              --query 'sort_by(imageDetails,&imagePushedAt)[*].imageTags[*]' \
+              --output text \
+            | tr '\t' '\n' \
+            | grep "^${TAG_PREFIX}" \
+            | tail -n1 || true
+          )
+
+          if [ -z "${IMAGE_TAG}" ]; then
+            echo "No existing recordprocessor image found for prefix '${TAG_PREFIX}'."
+            echo "Trigger a run with build_recordprocessor_image=true to build one."
+            exit 1
+          fi
+
+          echo "Using existing recordprocessor image tag: ${IMAGE_TAG}"
+          echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+
   terraform-plan:
     permissions:
       id-token: write
       contents: read
+    needs:
+      - build-and-push-recordprocessor
+      - resolve-recordprocessor-image-tag
+    if: ${{ !cancelled() && (needs.build-and-push-recordprocessor.result == 'success' || needs.resolve-recordprocessor-image-tag.result == 'success') }}
+    outputs:
+      recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }}
     runs-on: ubuntu-latest
+    env:
+      TF_VAR_recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }}
     environment:
       name: ${{ inputs.environment }}
     steps:
@@ -96,6 +207,8 @@ jobs:
       contents: read
     needs: terraform-plan
     runs-on: ubuntu-latest
+    env:
+      TF_VAR_recordprocessor_image_tag: ${{ needs.terraform-plan.outputs.recordprocessor_image_tag }}
     environment:
       name: ${{ inputs.environment }}
     steps:

.github/workflows/pr-deploy-and-test.yml

@@ -9,16 +9,40 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  detect-recordprocessor-changes:
+    runs-on: ubuntu-latest
+    if: github.event.action == 'synchronize'
+    outputs:
+      has_changes: ${{ steps.detect.outputs.has_changes }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98
+        with:
+          fetch-depth: 0
+
+      - name: Detect recordprocessor changes in PR
+        id: detect
+        run: |
+          git fetch origin "${{ github.event.pull_request.base.ref }}"
+
+          if git diff --name-only "origin/${{ github.event.pull_request.base.ref }}" "${{ github.sha }}" | grep -q '^lambdas/recordprocessor/'; then
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          fi
+
   run-quality-checks:
     uses: ./.github/workflows/quality-checks.yml
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
   deploy-pr-backend:
-    needs: [run-quality-checks]
+    needs: [run-quality-checks, detect-recordprocessor-changes]
+    if: ${{ always() && !failure() && !cancelled() }}
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
+      build_recordprocessor_image: ${{ github.event.action == 'opened' || github.event.action == 'reopened' || needs.detect-recordprocessor-changes.outputs.has_changes == 'true' }}
       create_mns_subscription: true
       environment: dev
       sub_environment: pr-${{github.event.pull_request.number}}

infrastructure/account/recordprocessor_ecr_repo.tf

@@ -0,0 +1,9 @@
+resource "aws_ecr_repository" "processing_repository" {
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+  image_tag_mutability = "IMMUTABLE"
+  name                 = "imms-recordprocessor-repo"
+}
+
+#TODO add lifecycle policy to manage images

infrastructure/instance/ecs_batch_processor_config.tf

@@ -1,4 +1,3 @@
-# Define the ECS Cluster
 resource "aws_ecs_cluster" "ecs_cluster" {
   name = "${local.short_prefix}-ecs-cluster"
 
@@ -11,59 +10,8 @@ resource "aws_ecs_cluster" "ecs_cluster" {
   }
 }
 
-# Locals for Lambda processing paths and hash
-locals {
-  processing_lambda_dir     = abspath("${path.root}/../../lambdas/recordprocessor")
-  processing_path_include   = ["**"]
-  processing_path_exclude   = ["**/__pycache__/**"]
-  processing_files_include  = setunion([for f in local.processing_path_include : fileset(local.processing_lambda_dir, f)]...)
-  processing_files_exclude  = setunion([for f in local.processing_path_exclude : fileset(local.processing_lambda_dir, f)]...)
-  processing_lambda_files   = sort(setsubtract(local.processing_files_include, local.processing_files_exclude))
-  processing_lambda_dir_sha = sha1(join("", [for f in local.processing_lambda_files : filesha1("${local.processing_lambda_dir}/${f}")]))
-  image_tag                 = "latest"
-}
-
-# Create ECR Repository for processing.
-resource "aws_ecr_repository" "processing_repository" {
-  image_scanning_configuration {
-    scan_on_push = true
-  }
-  name         = "${local.short_prefix}-processing-repo"
-  force_delete = local.is_temp
-}
-
-# Build and Push Docker Image to ECR (Reusing the existing module)
-module "processing_docker_image" {
-  source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
-  version = "8.7.0"
-
-  create_ecr_repo  = false
-  docker_file_path = "./recordprocessor/Dockerfile"
-  ecr_repo         = aws_ecr_repository.processing_repository.name
-  ecr_repo_lifecycle_policy = jsonencode({
-    "rules" : [
-      {
-        "rulePriority" : 1,
-        "description" : "Keep only the last 2 images",
-        "selection" : {
-          "tagStatus" : "any",
-          "countType" : "imageCountMoreThan",
-          "countNumber" : 2
-        },
-        "action" : {
-          "type" : "expire"
-        }
-      }
-    ]
-  })
-
-  platform      = "linux/amd64"
-  use_image_tag = false
-  source_path   = abspath("${path.root}/../../lambdas")
-  triggers = {
-    dir_sha        = local.processing_lambda_dir_sha
-    shared_dir_sha = local.shared_dir_sha
-  }
+data "aws_ecr_repository" "recordprocessor_repository" {
+  name = "${var.project_short_name}-recordprocessor-repo"
 }
 
 # Define the IAM Role for ECS Task Execution
@@ -157,7 +105,7 @@ resource "aws_iam_policy" "ecs_task_exec_policy" {
         Action = [
           "ecr:GetAuthorizationToken"
         ],
-        Resource = "arn:aws:ecr:${var.aws_region}:${var.immunisation_account_id}:repository/${local.short_prefix}-processing-repo"
+        Resource = "arn:aws:ecr:${var.aws_region}:${var.immunisation_account_id}:repository/${data.aws_ecr_repository.recordprocessor_repository.name}"
       },
       {
         "Effect" : "Allow",
@@ -197,7 +145,7 @@ resource "aws_ecs_task_definition" "ecs_task" {
 
   container_definitions = jsonencode([{
     name      = "${local.short_prefix}-process-records-container"
-    image     = "${aws_ecr_repository.processing_repository.repository_url}:${local.image_tag}"
+    image     = "${data.aws_ecr_repository.recordprocessor_repository.repository_url}:${var.recordprocessor_image_tag}"
     essential = true
     environment = [
       {

infrastructure/instance/variables.tf

@@ -99,6 +99,12 @@ variable "dynamodb_point_in_time_recovery_enabled" {
   default     = false
 }
 
+variable "recordprocessor_image_tag" {
+  description = "Tag of the recordprocessor (batch processor) container image in ECR"
+  type        = string
+  default     = "latest"
+}
+
 locals {
   prefix              = "${var.project_name}-${var.service}-${var.sub_environment}"
   short_prefix        = "${var.project_short_name}-${var.sub_environment}"