CCM-14203 - Callback Delivery Replay and Recovery

.github/actions/acceptance-tests/action.yaml

@@ -45,6 +45,14 @@ runs:
           echo "PR_NUMBER=${{ inputs.targetEnvironment }}" >> $GITHUB_ENV
           echo "ENVIRONMENT=${{ inputs.targetEnvironment }}" >> $GITHUB_ENV
 
+    - name: "Set TEST_MOCK_WEBHOOK_URL environment variable"
+      shell: bash
+      run: |
+        mock_webhook_url=$(jq -r '.mock_webhook_url.value // empty' terraform_output.json)
+        if [ -n "$mock_webhook_url" ]; then
+          echo "TEST_MOCK_WEBHOOK_URL=${mock_webhook_url}" >> $GITHUB_ENV
+        fi
+
     - name: Run test - ${{ inputs.testType }}
       shell: bash
       env:

.github/workflows/cicd-1-pull-request.yaml

@@ -171,7 +171,7 @@ jobs:
             --terraformAction "apply" \
             --overrideProjectName "nhs" \
             --overrideRoleName "nhs-main-acct-client-callbacks-github-deploy" \
-            --overrides "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+            --overrides "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}},deploy_mock_webhook=true"
   acceptance-stage: # Recommended maximum execution time is 10 minutes
     name: "Acceptance stage"
     needs: [metadata, build-stage, pr-create-dynamic-environment]

infrastructure/terraform/components/callbacks/README.md

@@ -42,13 +42,15 @@
 | <a name="module_client_config_bucket"></a> [client\_config\_bucket](#module\_client\_config\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.28/terraform-s3bucket.zip | n/a |
 | <a name="module_client_destination"></a> [client\_destination](#module\_client\_destination) | ../../modules/client-destination | n/a |
 | <a name="module_client_transform_filter_lambda"></a> [client\_transform\_filter\_lambda](#module\_client\_transform\_filter\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
+| <a name="module_debug_log_bucket"></a> [debug\_log\_bucket](#module\_debug\_log\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.28/terraform-s3bucket.zip | n/a |
 | <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-kms.zip | n/a |
 | <a name="module_mock_webhook_lambda"></a> [mock\_webhook\_lambda](#module\_mock\_webhook\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_sqs_inbound_event"></a> [sqs\_inbound\_event](#module\_sqs\_inbound\_event) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-sqs.zip | n/a |
 ## Outputs
 
 | Name | Description |
 |------|-------------|
+| <a name="output_debug_log_bucket_name"></a> [debug\_log\_bucket\_name](#output\_debug\_log\_bucket\_name) | S3 bucket name for debug logs (integration testing, deploy\_mock\_webhook=true only) |
 | <a name="output_deployment"></a> [deployment](#output\_deployment) | Deployment details used for post-deployment scripts |
 | <a name="output_mock_webhook_lambda_log_group_name"></a> [mock\_webhook\_lambda\_log\_group\_name](#output\_mock\_webhook\_lambda\_log\_group\_name) | CloudWatch log group name for mock webhook lambda (for integration test queries) |
 | <a name="output_mock_webhook_url"></a> [mock\_webhook\_url](#output\_mock\_webhook\_url) | URL endpoint for mock webhook (for TEST\_WEBHOOK\_URL environment variable) |

infrastructure/terraform/components/callbacks/cloudwatch_dashboard_dlq.tf

@@ -0,0 +1,100 @@
+locals {
+  dlq_overview_metrics = [
+    for client_key in sort(keys(local.all_clients)) : [
+      "AWS/SQS",
+      "ApproximateNumberOfMessagesVisible",
+      "QueueName",
+      "${local.csi}-${client_key}-dlq-queue",
+      { "label" : "${client_key} DLQ" }
+    ]
+  ]
+
+  dlq_timeseries_widgets = [
+    for idx, client_key in sort(keys(local.all_clients)) : {
+      height = 6
+      width  = 12
+      y      = 5 + (floor(idx / 2) * 6)
+      x      = (idx % 2) * 12
+      type   = "metric"
+      properties = {
+        title   = "${client_key} DLQ Depth"
+        view    = "timeSeries"
+        stacked = false
+        region  = var.region
+        stat    = "Maximum"
+        period  = 300
+        metrics = [
+          [
+            "AWS/SQS",
+            "ApproximateNumberOfMessagesVisible",
+            "QueueName",
+            "${local.csi}-${client_key}-dlq-queue",
+            { "label" : "Messages awaiting redrive", "color" : "#d62728" }
+          ],
+          [
+            "AWS/SQS",
+            "NumberOfMessagesMoved",
+            "QueueName",
+            "${local.csi}-${client_key}-dlq-queue",
+            { "label" : "Messages being redriven", "color" : "#2ca02c" }
+          ]
+        ]
+        annotations = {
+          horizontal = [
+            {
+              label = "Alarm threshold"
+              value = 0
+              fill  = "above"
+            }
+          ]
+        }
+        yAxis = {
+          left = {
+            min       = 0
+            showUnits = false
+            label     = "Message count"
+          }
+        }
+      }
+    }
+  ]
+}
+
+resource "aws_cloudwatch_dashboard" "dlq" {
+  dashboard_name = "${local.csi}-dlq"
+
+  dashboard_body = jsonencode({
+    widgets = concat(
+      [
+        {
+          height = 1
+          width  = 24
+          y      = 0
+          x      = 0
+          type   = "text"
+          properties = {
+            markdown = "## DLQ Monitoring\nMessages in a DLQ indicate failed callback deliveries."
+          }
+        },
+        {
+          height = 4
+          width  = 24
+          y      = 1
+          x      = 0
+          type   = "metric"
+          properties = {
+            title                    = "DLQ Depth Overview"
+            view                     = "singleValue"
+            sparkline                = true
+            region                   = var.region
+            stat                     = "Maximum"
+            period                   = 300
+            singleValueFullPrecision = false
+            metrics                  = local.dlq_overview_metrics
+          }
+        }
+      ],
+      local.dlq_timeseries_widgets,
+    )
+  })
+}

infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_dlq_depth.tf

@@ -0,0 +1,31 @@
+resource "aws_cloudwatch_metric_alarm" "client_dlq_depth" {
+  for_each = toset(keys(local.all_clients))
+
+  alarm_name = "${local.csi}-${each.key}-dlq-depth"
+  alarm_description = join(" ", [
+    "RELIABILITY: Messages are in DLQ for ${each.key}.",
+    "Failed callback deliveries require operator attention.",
+  ])
+
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ApproximateNumberOfMessagesVisible"
+  namespace           = "AWS/SQS"
+  period              = 300
+  statistic           = "Sum"
+  threshold           = 0
+  actions_enabled     = true
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    QueueName = "${local.csi}-${each.key}-dlq-queue"
+  }
+
+  tags = merge(
+    local.default_tags,
+    {
+      Name   = "${local.csi}-${each.key}-dlq-depth"
+      Client = each.key
+    },
+  )
+}

infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf

@@ -36,8 +36,9 @@ module "mock_webhook_lambda" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = {
-    LOG_LEVEL = var.log_level
-    API_KEY   = random_password.mock_webhook_api_key[0].result
+    LOG_LEVEL         = var.log_level
+    API_KEY           = random_password.mock_webhook_api_key[0].result
+    DEBUG_BUCKET_NAME = module.debug_log_bucket[0].id
   }
 }
 
@@ -63,6 +64,19 @@ data "aws_iam_policy_document" "mock_webhook_lambda" {
       module.kms.key_arn,
     ]
   }
+
+  statement {
+    sid    = "DebugLogBucketWrite"
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+    ]
+
+    resources = [
+      "${module.debug_log_bucket[0].arn}/*",
+    ]
+  }
 }
 
 # Lambda Function URL for mock webhook (test/dev only)

infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf

@@ -36,7 +36,8 @@ module "client_transform_filter_lambda" {
 
   lambda_env_vars = {
     ENVIRONMENT                           = var.environment
-    METRICS_NAMESPACE                     = "nhs-notify-client-callbacks-metrics"
+    METRICS_NAMESPACE                     = "nhs-notify-client-callbacks"
+    DEBUG_BUCKET_NAME                     = var.deploy_mock_webhook ? module.debug_log_bucket[0].id : ""
     CLIENT_SUBSCRIPTION_CONFIG_BUCKET     = module.client_config_bucket.id
     CLIENT_SUBSCRIPTION_CONFIG_PREFIX     = "client_subscriptions/"
     CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS = "60"
@@ -83,4 +84,21 @@ data "aws_iam_policy_document" "client_transform_filter_lambda" {
       "*",
     ]
   }
+
+  dynamic "statement" {
+    for_each = var.deploy_mock_webhook ? toset(["enabled"]) : toset([])
+
+    content {
+      sid    = "DebugLogBucketWrite"
+      effect = "Allow"
+
+      actions = [
+        "s3:PutObject",
+      ]
+
+      resources = [
+        "${module.debug_log_bucket[0].arn}/*",
+      ]
+    }
+  }
 }

infrastructure/terraform/components/callbacks/outputs.tf

@@ -15,7 +15,7 @@ output "deployment" {
 }
 
 ##
-# Mock Webhook Lambda Outputs (test/dev environments only)
+# Mock Webhook Lambda Outputs (test/dev environments only).
 ##
 
 output "mock_webhook_lambda_log_group_name" {
@@ -27,3 +27,8 @@ output "mock_webhook_url" {
   description = "URL endpoint for mock webhook (for TEST_WEBHOOK_URL environment variable)"
   value       = var.deploy_mock_webhook ? aws_lambda_function_url.mock_webhook[0].function_url : null
 }
+
+output "debug_log_bucket_name" {
+  description = "S3 bucket name for debug logs (integration testing, deploy_mock_webhook=true only)"
+  value       = var.deploy_mock_webhook ? module.debug_log_bucket[0].id : null
+}

infrastructure/terraform/components/callbacks/s3_bucket_debug_log.tf

@@ -0,0 +1,93 @@
+module "debug_log_bucket" {
+  count  = var.deploy_mock_webhook ? 1 : 0
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.28/terraform-s3bucket.zip"
+
+  name = "debug-log"
+
+  aws_account_id = var.aws_account_id
+  component      = var.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+
+  default_tags = merge(
+    local.default_tags,
+    {
+      Description = "Debug log storage for integration testing"
+    }
+  )
+
+  kms_key_arn        = module.kms.key_arn
+  force_destroy      = true
+  versioning         = false
+  object_ownership   = "BucketOwnerPreferred"
+  bucket_key_enabled = true
+
+  policy_documents = [
+    data.aws_iam_policy_document.debug_log_bucket[0].json,
+  ]
+}
+
+data "aws_iam_policy_document" "debug_log_bucket" {
+  count = var.deploy_mock_webhook ? 1 : 0
+
+  statement {
+    sid    = "AllowLambdaWriteAccess"
+    effect = "Allow"
+
+    principals {
+      type = "AWS"
+      identifiers = [
+        module.mock_webhook_lambda[0].iam_role_arn,
+        module.client_transform_filter_lambda.iam_role_arn,
+      ]
+    }
+
+    actions = [
+      "s3:PutObject",
+    ]
+
+    resources = [
+      "${module.debug_log_bucket[0].arn}/*",
+    ]
+  }
+
+  statement {
+    sid    = "DenyInsecureTransport"
+    effect = "Deny"
+
+    principals {
+      type        = "*"
+      identifiers = ["*"]
+    }
+
+    actions = [
+      "s3:*",
+    ]
+
+    resources = [
+      module.debug_log_bucket[0].arn,
+      "${module.debug_log_bucket[0].arn}/*",
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "debug_log" {
+  count  = var.deploy_mock_webhook ? 1 : 0
+  bucket = module.debug_log_bucket[0].id
+
+  rule {
+    id     = "expire-debug-logs"
+    status = "Enabled"
+
+    expiration {
+      days = 1
+    }
+  }
+}

jest.config.base.ts

@@ -5,7 +5,7 @@ export const baseJestConfig: Config = {
   clearMocks: true,
   collectCoverage: true,
   coverageDirectory: "./.reports/unit/coverage",
-  coverageProvider: "v8",
+  coverageProvider: "babel",
   coveragePathIgnorePatterns: ["/__tests__/", "/node_modules/"],
   transform: { "^.+\\.ts$": "ts-jest" },
   testPathIgnorePatterns: [".build"],

lambdas/client-transform-filter-lambda/package.json

@@ -1,12 +1,12 @@
 {
   "dependencies": {
     "@aws-sdk/client-s3": "^3.821.0",
+    "@nhs-notify-client-callbacks/logger": "*",
     "@nhs-notify-client-callbacks/models": "*",
     "aws-embedded-metrics": "^4.2.1",
     "cloudevents": "^8.0.2",
     "esbuild": "^0.25.0",
     "p-map": "^4.0.0",
-    "pino": "^9.6.0",
     "zod": "^4.1.13"
   },
   "devDependencies": {

lambdas/client-transform-filter-lambda/src/__tests__/index.test.ts

@@ -489,6 +489,8 @@ describe("createHandler default wiring", () => {
 
       jest.doMock("services/logger", () => ({
         Logger: state.LoggerCtor,
+
+        flushLogs: jest.fn().mockResolvedValue(undefined),
       }));
 
       jest.doMock("services/observability", () => ({