diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml index 1e2bebdfb..0cd93a630 100644 --- a/.github/workflows/continuous-deployment.yml +++ b/.github/workflows/continuous-deployment.yml @@ -15,11 +15,31 @@ jobs: secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + detect-recordprocessor-changes: + runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.detect.outputs.has_changes }} + steps: + - name: Checkout + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + fetch-depth: 0 + + - name: Detect recordprocessor changes compared to master + id: detect + run: | + if git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" | grep -q '^lambdas/recordprocessor/'; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else + echo "has_changes=false" >> "$GITHUB_OUTPUT" + fi + deploy-internal-dev-backend: - needs: [run-quality-checks] + needs: [run-quality-checks, detect-recordprocessor-changes] uses: ./.github/workflows/deploy-backend.yml with: apigee_environment: internal-dev + build_recordprocessor_image: ${{needs.detect-recordprocessor-changes.outputs.has_changes == 'true'}} create_mns_subscription: true environment: dev sub_environment: internal-dev @@ -75,13 +95,14 @@ jobs: STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }} deploy-higher-dev-envs: - needs: [run-internal-dev-tests] + needs: [run-internal-dev-tests, detect-recordprocessor-changes] strategy: matrix: sub_environment_name: [ref, internal-qa] uses: ./.github/workflows/deploy-backend.yml with: apigee_environment: ${{ matrix.sub_environment_name }} + build_recordprocessor_image: ${{needs.detect-recordprocessor-changes.outputs.has_changes == 'true'}} create_mns_subscription: true environment: dev sub_environment: ${{ matrix.sub_environment_name }} diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 9c742cdc1..5ad490df9 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -6,6 +6,10 @@ on: apigee_environment: required: true type: string + build_recordprocessor_image: + required: false + type: boolean + default: false create_mns_subscription: required: false type: boolean @@ -39,6 +43,11 @@ on: - dev - preprod - prod + build_recordprocessor_image: + description: Build and push a new recordprocessor image for this deployment + required: false + type: boolean + default: false sub_environment: type: string description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments @@ -51,11 +60,113 @@ env: # Sonarcloud - do not allow direct usage of untrusted data run-name: Deploy Backend - ${{ inputs.environment }} ${{ inputs.sub_environment }} jobs: + build-and-push-recordprocessor: + if: ${{ inputs.build_recordprocessor_image }} + permissions: + id-token: write + contents: read + outputs: + recordprocessor_image_tag: ${{ steps.build-image.outputs.recordprocessor_image_tag }} + name: Build and push recordprocessor image + runs-on: ubuntu-latest + + environment: + name: ${{ inputs.environment }} + + env: + AWS_REGION: eu-west-2 + SUB_ENVIRONMENT: ${{ inputs.sub_environment }} + + steps: + - name: Checkout + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + + - name: Connect to AWS + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 + with: + aws-region: eu-west-2 + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops + role-session-name: github-actions + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 + + - name: Build and push Docker image + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + working-directory: lambdas + run: | + IMAGE_TAG="${SUB_ENVIRONMENT}-${GITHUB_SHA}" + REPOSITORY_NAME="imms-recordprocessor-repo" + IMAGE_URI="${ECR_REGISTRY}/${REPOSITORY_NAME}:${IMAGE_TAG}" + + docker build -f recordprocessor/Dockerfile -t "${IMAGE_URI}" . + docker push "${IMAGE_URI}" + echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + + resolve-recordprocessor-image-tag: + if: ${{ !inputs.build_recordprocessor_image }} + permissions: + id-token: write + contents: read + outputs: + recordprocessor_image_tag: ${{ steps.resolve-image-tag.outputs.recordprocessor_image_tag }} + name: Resolve existing recordprocessor image tag + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + env: + AWS_REGION: eu-west-2 + steps: + - name: Connect to AWS + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 + with: + aws-region: eu-west-2 + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops + role-session-name: github-actions + + - name: Resolve latest matching recordprocessor image tag + id: resolve-image-tag + env: + REPOSITORY_NAME: imms-recordprocessor-repo + TAG_PREFIX: ${{ inputs.sub_environment }}- + run: | + IMAGE_TAG=$( + aws ecr describe-images \ + --repository-name "${REPOSITORY_NAME}" \ + --region "${AWS_REGION}" \ + --filter tagStatus=TAGGED \ + --query 'sort_by(imageDetails,&imagePushedAt)[*].imageTags[*]' \ + --output text \ + | tr '\t' '\n' \ + | grep "^${TAG_PREFIX}" \ + | tail -n1 || true + ) + + if [ -z "${IMAGE_TAG}" ]; then + echo "No existing recordprocessor image found for prefix '${TAG_PREFIX}'." + echo "Trigger a run with build_recordprocessor_image=true to build one." + exit 1 + fi + + echo "Using existing recordprocessor image tag: ${IMAGE_TAG}" + echo "recordprocessor_image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + terraform-plan: permissions: id-token: write contents: read + needs: + - build-and-push-recordprocessor + - resolve-recordprocessor-image-tag + if: ${{ !cancelled() && (needs.build-and-push-recordprocessor.result == 'success' || needs.resolve-recordprocessor-image-tag.result == 'success') }} + outputs: + recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }} runs-on: ubuntu-latest + env: + TF_VAR_recordprocessor_image_tag: ${{ needs.build-and-push-recordprocessor.outputs.recordprocessor_image_tag || needs.resolve-recordprocessor-image-tag.outputs.recordprocessor_image_tag }} environment: name: ${{ inputs.environment }} steps: @@ -95,7 +206,10 @@ jobs: id-token: write contents: read needs: terraform-plan + if: ${{ !cancelled() && needs.terraform-plan.result == 'success' }} runs-on: ubuntu-latest + env: + TF_VAR_recordprocessor_image_tag: ${{ needs.terraform-plan.outputs.recordprocessor_image_tag }} environment: name: ${{ inputs.environment }} steps: diff --git a/.github/workflows/pr-deploy-and-test.yml b/.github/workflows/pr-deploy-and-test.yml index 76db9846d..3c325f37a 100644 --- a/.github/workflows/pr-deploy-and-test.yml +++ b/.github/workflows/pr-deploy-and-test.yml @@ -9,16 +9,40 @@ concurrency: cancel-in-progress: true jobs: + detect-recordprocessor-changes: + runs-on: ubuntu-latest + if: github.event.action == 'synchronize' + outputs: + has_changes: ${{ steps.detect.outputs.has_changes }} + steps: + - name: Checkout + uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 + with: + fetch-depth: 0 + + - name: Detect recordprocessor changes in PR + id: detect + run: | + git fetch origin "${{ github.event.pull_request.base.ref }}" + + if git diff --name-only "origin/${{ github.event.pull_request.base.ref }}" "${{ github.sha }}" | grep -q '^lambdas/recordprocessor/'; then + echo "has_changes=true" >> "$GITHUB_OUTPUT" + else + echo "has_changes=false" >> "$GITHUB_OUTPUT" + fi + run-quality-checks: uses: ./.github/workflows/quality-checks.yml secrets: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} deploy-pr-backend: - needs: [run-quality-checks] + needs: [run-quality-checks, detect-recordprocessor-changes] + if: ${{ always() && !failure() && !cancelled() }} uses: ./.github/workflows/deploy-backend.yml with: apigee_environment: internal-dev + build_recordprocessor_image: ${{ github.event.action == 'opened' || github.event.action == 'reopened' || needs.detect-recordprocessor-changes.outputs.has_changes == 'true' }} create_mns_subscription: true environment: dev sub_environment: pr-${{github.event.pull_request.number}} diff --git a/infrastructure/account/recordprocessor_ecr_repo.tf b/infrastructure/account/recordprocessor_ecr_repo.tf new file mode 100644 index 000000000..3c890ff11 --- /dev/null +++ b/infrastructure/account/recordprocessor_ecr_repo.tf @@ -0,0 +1,9 @@ +resource "aws_ecr_repository" "processing_repository" { + image_scanning_configuration { + scan_on_push = true + } + image_tag_mutability = "IMMUTABLE" + name = "imms-recordprocessor-repo" +} + +#TODO add lifecycle policy to manage images \ No newline at end of file diff --git a/infrastructure/instance/ecs_batch_processor_config.tf b/infrastructure/instance/ecs_batch_processor_config.tf index 653b220a9..c3ef792a0 100644 --- a/infrastructure/instance/ecs_batch_processor_config.tf +++ b/infrastructure/instance/ecs_batch_processor_config.tf @@ -1,4 +1,3 @@ -# Define the ECS Cluster resource "aws_ecs_cluster" "ecs_cluster" { name = "${local.short_prefix}-ecs-cluster" @@ -11,59 +10,8 @@ resource "aws_ecs_cluster" "ecs_cluster" { } } -# Locals for Lambda processing paths and hash -locals { - processing_lambda_dir = abspath("${path.root}/../../lambdas/recordprocessor") - processing_path_include = ["**"] - processing_path_exclude = ["**/__pycache__/**"] - processing_files_include = setunion([for f in local.processing_path_include : fileset(local.processing_lambda_dir, f)]...) - processing_files_exclude = setunion([for f in local.processing_path_exclude : fileset(local.processing_lambda_dir, f)]...) - processing_lambda_files = sort(setsubtract(local.processing_files_include, local.processing_files_exclude)) - processing_lambda_dir_sha = sha1(join("", [for f in local.processing_lambda_files : filesha1("${local.processing_lambda_dir}/${f}")])) - image_tag = "latest" -} - -# Create ECR Repository for processing. -resource "aws_ecr_repository" "processing_repository" { - image_scanning_configuration { - scan_on_push = true - } - name = "${local.short_prefix}-processing-repo" - force_delete = local.is_temp -} - -# Build and Push Docker Image to ECR (Reusing the existing module) -module "processing_docker_image" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "8.7.0" - - create_ecr_repo = false - docker_file_path = "./recordprocessor/Dockerfile" - ecr_repo = aws_ecr_repository.processing_repository.name - ecr_repo_lifecycle_policy = jsonencode({ - "rules" : [ - { - "rulePriority" : 1, - "description" : "Keep only the last 2 images", - "selection" : { - "tagStatus" : "any", - "countType" : "imageCountMoreThan", - "countNumber" : 2 - }, - "action" : { - "type" : "expire" - } - } - ] - }) - - platform = "linux/amd64" - use_image_tag = false - source_path = abspath("${path.root}/../../lambdas") - triggers = { - dir_sha = local.processing_lambda_dir_sha - shared_dir_sha = local.shared_dir_sha - } +data "aws_ecr_repository" "recordprocessor_repository" { + name = "${var.project_short_name}-recordprocessor-repo" } # Define the IAM Role for ECS Task Execution @@ -157,7 +105,7 @@ resource "aws_iam_policy" "ecs_task_exec_policy" { Action = [ "ecr:GetAuthorizationToken" ], - Resource = "arn:aws:ecr:${var.aws_region}:${var.immunisation_account_id}:repository/${local.short_prefix}-processing-repo" + Resource = "arn:aws:ecr:${var.aws_region}:${var.immunisation_account_id}:repository/${data.aws_ecr_repository.recordprocessor_repository.name}" }, { "Effect" : "Allow", @@ -197,7 +145,7 @@ resource "aws_ecs_task_definition" "ecs_task" { container_definitions = jsonencode([{ name = "${local.short_prefix}-process-records-container" - image = "${aws_ecr_repository.processing_repository.repository_url}:${local.image_tag}" + image = "${data.aws_ecr_repository.recordprocessor_repository.repository_url}:${var.recordprocessor_image_tag}" essential = true environment = [ { diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf index af8863069..5d6d5932e 100644 --- a/infrastructure/instance/variables.tf +++ b/infrastructure/instance/variables.tf @@ -99,6 +99,12 @@ variable "dynamodb_point_in_time_recovery_enabled" { default = false } +variable "recordprocessor_image_tag" { + description = "Tag of the recordprocessor (batch processor) container image in ECR" + type = string + default = "latest" +} + locals { prefix = "${var.project_name}-${var.service}-${var.sub_environment}" short_prefix = "${var.project_short_name}-${var.sub_environment}"