CCM-14200 - event driven callback delivery

.tool-versions

@@ -1,18 +1,16 @@
-act 0.2.64
-gitleaks 8.24.0
-jq 1.6
-nodejs 22.11.0
-pre-commit 3.6.0
-ruby 3.3.6
-terraform 1.10.1
+act            0.2.64
+gitleaks       8.24.0
+jq             1.6
+nodejs         22.13
+pre-commit     3.6.0
+ruby           3.3.6
+terraform      1.10.1
 terraform-docs 0.19.0
-trivy 0.61.0
-vale 3.6.0
-python 3.13.2
-
+trivy          0.61.0
+vale           3.6.0
+python         3.13.2
 # ==============================================================================
 # The section below is reserved for Docker image versions.
-
 # TODO: Move this section - consider using a different file for the repository template dependencies.
 # docker/ghcr.io/anchore/grype v0.104.3@sha256:d340f4f8b3b7e6e72a6c9c0152f25402ed8a2d7375dba1dfce4e53115242feb6  # SEE: https://github.com/anchore/grype/pkgs/container/grype
 # docker/ghcr.io/anchore/syft v1.39.0@sha256:6f13bb010923c33fb197047c8f88888e77071bd32596b3f605d62a133e493ce4 # SEE: https://github.com/anchore/syft/pkgs/container/syft

.vscode/settings.json

@@ -1,5 +1,4 @@
 {
-  "autoOpenWorkspace.enableAutoOpenIfSingleWorkspace": true,
   "files.exclude": {
     "**/.DS_Store": true,
     "**/.git": true,
@@ -11,5 +10,10 @@
     ".devcontainer": true,
     ".github": false,
     ".vscode": false
+  },
+  "jest.jestCommandLine": "npm run test:unit --workspaces --",
+  "sonarlint.connectedMode.project": {
+    "connectionId": "nhsdigital",
+    "projectKey": "NHSDigital_nhs-notify-client-callbacks"
   }
 }

AGENTS.md

@@ -81,7 +81,7 @@ When proposing a change, agents should:
 
   to catch formatting and basic lint issues. Domain specific checks will be defined in appropriate nested AGENTS.md files.
 
-- Suggest at least one extra validation step (for example `npm test` in a lambda, or triggering a specific workflow).
+- Suggest at least one extra validation step (for example `npm test:unit` in a lambda, or triggering a specific workflow).
 - Any required follow up activites which fall outside of the current task's scope should be clearly marked with a 'TODO: CCM-12345' comment. The human user should be prompted to create and provide a JIRA ticket ID to be added to the comment.
 
 ## Security & Safety
@@ -93,3 +93,11 @@ When proposing a change, agents should:
 ## Escalation / Blockers
 
 If you are blocked by an unavailable secret, unclear architectural constraint, missing upstream module, or failing tooling you cannot safely fix, stop and ask a single clear clarifying question rather than guessing.
+
+## Comment Policy
+
+- No JSDoc unless it's a public API with non-obvious behavior
+- No inline comments that just describe what the next line does
+- Only comment when explaining WHY, not WHAT
+- Prefer better naming over comments
+- Trust developers can read TypeScript

eslint.config.mjs

@@ -223,6 +223,13 @@ export default defineConfig([
       },
     },
   },
+  {
+    files: ["**/jest.config.ts"],
+    rules: {
+      "no-relative-import-paths/no-relative-import-paths": 0,
+      "import-x/no-relative-packages": 0,
+    },
+  },
   {
     files: ["scripts/**"],
     rules: {

infrastructure/terraform/components/callbacks/README.md

@@ -8,6 +8,7 @@
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.10.1 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | 6.13 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | ~> 3.0 |
 ## Inputs
 
 | Name | Description | Type | Default | Required |
@@ -16,6 +17,7 @@
 | <a name="input_clients"></a> [clients](#input\_clients) | n/a | <pre>list(object({<br/>    connection_name                  = string<br/>    destination_name                 = string<br/>    invocation_endpoint              = string<br/>    invocation_rate_limit_per_second = optional(number, 10)<br/>    http_method                      = optional(string, "POST")<br/>    header_name                      = optional(string, "x-api-key")<br/>    header_value                     = string<br/>    client_detail                    = list(string)<br/>  }))</pre> | `[]` | no |
 | <a name="input_component"></a> [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"callbacks"` | no |
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
+| <a name="input_deploy_mock_webhook"></a> [deploy\_mock\_webhook](#input\_deploy\_mock\_webhook) | Flag to deploy mock webhook lambda for integration testing (test/dev environments only) | `bool` | `false` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_force_lambda_code_deploy"></a> [force\_lambda\_code\_deploy](#input\_force\_lambda\_code\_deploy) | If the lambda package in s3 has the same commit id tag as the terraform build branch, the lambda will not update automatically. Set to True if making changes to Lambda code from on the same commit for example during development | `bool` | `false` | no |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
@@ -24,6 +26,7 @@
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
 | <a name="input_pipe_event_patterns"></a> [pipe\_event\_patterns](#input\_pipe\_event\_patterns) | value | `list(string)` | `[]` | no |
+| <a name="input_pipe_log_level"></a> [pipe\_log\_level](#input\_pipe\_log\_level) | Log level for the EventBridge Pipe. | `string` | `"ERROR"` | no |
 | <a name="input_pipe_sqs_input_batch_size"></a> [pipe\_sqs\_input\_batch\_size](#input\_pipe\_sqs\_input\_batch\_size) | n/a | `number` | `1` | no |
 | <a name="input_pipe_sqs_max_batch_window"></a> [pipe\_sqs\_max\_batch\_window](#input\_pipe\_sqs\_max\_batch\_window) | n/a | `number` | `2` | no |
 | <a name="input_project"></a> [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes |
@@ -34,8 +37,9 @@
 |------|--------|---------|
 | <a name="module_client_config_bucket"></a> [client\_config\_bucket](#module\_client\_config\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.28/terraform-s3bucket.zip | n/a |
 | <a name="module_client_destination"></a> [client\_destination](#module\_client\_destination) | ../../modules/client-destination | n/a |
-| <a name="module_client_transform_filter_lambda"></a> [client\_transform\_filter\_lambda](#module\_client\_transform\_filter\_lambda) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/lambda | v2.0.29 |
+| <a name="module_client_transform_filter_lambda"></a> [client\_transform\_filter\_lambda](#module\_client\_transform\_filter\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-kms.zip | n/a |
+| <a name="module_mock_webhook_lambda"></a> [mock\_webhook\_lambda](#module\_mock\_webhook\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_sqs_inbound_event"></a> [sqs\_inbound\_event](#module\_sqs\_inbound\_event) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-sqs.zip | n/a |
 ## Outputs
 

infrastructure/terraform/components/callbacks/locals.tf

@@ -8,4 +8,23 @@ locals {
     for client in var.clients :
     client.connection_name => client
   }
+
+  # Automatic test client when mock webhook is deployed
+  mock_client = var.deploy_mock_webhook ? {
+    "mock-client" = {
+      connection_name                  = "mock-client"
+      destination_name                 = "test-destination"
+      invocation_endpoint              = aws_lambda_function_url.mock_webhook[0].function_url
+      invocation_rate_limit_per_second = 10
+      http_method                      = "POST"
+      header_name                      = "x-api-key"
+      header_value                     = random_password.mock_webhook_api_key[0].result
+      client_detail = [
+        "uk.nhs.notify.message.status.PUBLISHED.v1",
+        "uk.nhs.notify.channel.status.PUBLISHED.v1"
+      ]
+    }
+  } : {}
+
+  all_clients = merge(local.clients_by_name, local.mock_client)
 }

infrastructure/terraform/components/callbacks/module_client_destination.tf

@@ -1,6 +1,6 @@
 module "client_destination" {
   source   = "../../modules/client-destination"
-  for_each = local.clients_by_name
+  for_each = local.all_clients
 
   project         = var.project
   aws_account_id  = var.aws_account_id

infrastructure/terraform/components/callbacks/module_kms.tf

@@ -71,7 +71,8 @@ data "aws_iam_policy_document" "kms" {
       variable = "kms:EncryptionContext:aws:sqs:arn"
       values = [
         "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-inbound-event-queue",
-        "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-*-dlq" #wildcard here so that DLQs for clients can also use this key
+        "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-inbound-event-dlq",
+        "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-*-dlq-queue" #wildcard here so that DLQs for clients can also use this key
       ]
     }
   }

infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf

@@ -0,0 +1,97 @@
+module "mock_webhook_lambda" {
+  count  = var.deploy_mock_webhook ? 1 : 0
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip"
+
+  function_name = "mock-webhook"
+  description   = "Mock webhook endpoint for integration testing - logs received callbacks to CloudWatch"
+
+  aws_account_id = var.aws_account_id
+  component      = var.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  group          = var.group
+
+  log_retention_in_days = var.log_retention_in_days
+  kms_key_arn           = module.kms.key_arn
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.mock_webhook_lambda[0].json
+  }
+
+  function_s3_bucket      = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
+  function_code_base_path = local.aws_lambda_functions_dir_path
+  function_code_dir       = "mock-webhook-lambda/dist"
+  function_include_common = true
+  handler_function_name   = "handler"
+  runtime                 = "nodejs22.x"
+  memory                  = 256
+  timeout                 = 10
+  log_level               = var.log_level
+
+  force_lambda_code_deploy = var.force_lambda_code_deploy
+  enable_lambda_insights   = false
+
+  log_destination_arn       = local.log_destination_arn
+  log_subscription_role_arn = local.acct.log_subscription_role_arn
+
+  lambda_env_vars = {
+    LOG_LEVEL = var.log_level
+    API_KEY   = random_password.mock_webhook_api_key[0].result
+  }
+}
+
+resource "random_password" "mock_webhook_api_key" {
+  count   = var.deploy_mock_webhook ? 1 : 0
+  length  = 32
+  special = false
+}
+
+data "aws_iam_policy_document" "mock_webhook_lambda" {
+  count = var.deploy_mock_webhook ? 1 : 0
+
+  statement {
+    sid    = "KMSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      module.kms.key_arn,
+    ]
+  }
+}
+
+# Lambda Function URL for mock webhook (test/dev only)
+resource "aws_lambda_function_url" "mock_webhook" {
+  count              = var.deploy_mock_webhook ? 1 : 0
+  function_name      = module.mock_webhook_lambda[0].function_name
+  authorization_type = "NONE" # Public endpoint for testing
+
+  cors {
+    allow_origins = ["*"]
+    allow_methods = ["POST"]
+    allow_headers = ["*"]
+    max_age       = 86400
+  }
+}
+
+resource "aws_lambda_permission" "mock_webhook_function_url" {
+  count                  = var.deploy_mock_webhook ? 1 : 0
+  statement_id           = "FunctionURLAllowPublicAccess"
+  action                 = "lambda:InvokeFunctionUrl"
+  function_name          = module.mock_webhook_lambda[0].function_name
+  principal              = "*"
+  function_url_auth_type = "NONE"
+}
+
+resource "aws_lambda_permission" "mock_webhook_function_invoke" {
+  count         = var.deploy_mock_webhook ? 1 : 0
+  statement_id  = "FunctionURLAllowInvokeAction"
+  action        = "lambda:InvokeFunction"
+  function_name = module.mock_webhook_lambda[0].function_name
+  principal     = "*"
+}

infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf

@@ -1,5 +1,5 @@
 module "client_transform_filter_lambda" {
-  source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/lambda?ref=v2.0.29"
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip"
 
   function_name = "client-transform-filter"
   description   = "Lambda function that transforms and filters events coming to through the eventpipe"
@@ -35,6 +35,8 @@ module "client_transform_filter_lambda" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = {
+    ENVIRONMENT       = var.environment
+    METRICS_NAMESPACE = "nhs-notify-client-callbacks"
   }
 }
 
@@ -65,4 +67,17 @@ data "aws_iam_policy_document" "client_transform_filter_lambda" {
       "${module.client_config_bucket.arn}/*",
     ]
   }
+
+  statement {
+    sid    = "CloudWatchMetrics"
+    effect = "Allow"
+
+    actions = [
+      "cloudwatch:PutMetricData",
+    ]
+
+    resources = [
+      "*",
+    ]
+  }
 }

infrastructure/terraform/components/callbacks/pipes_pipe_main.tf

@@ -6,7 +6,7 @@ resource "aws_pipes_pipe" "main" {
   enrichment         = module.client_transform_filter_lambda.function_arn
   kms_key_identifier = module.kms.key_arn
   log_configuration {
-    level = "ERROR"
+    level = var.pipe_log_level
     cloudwatch_logs_log_destination {
       log_group_arn = aws_cloudwatch_log_group.main_pipe.arn
     }
@@ -25,8 +25,8 @@ resource "aws_pipes_pipe" "main" {
 
     input_template = <<EOF
 {
-  "dataschemaversion": <$.body.dataschemaversion>,
-  "type": <$.body.type>
+  "type": <$.type>,
+  "transformedPayload": <$.transformedPayload>
 }
 EOF
   }

infrastructure/terraform/components/callbacks/variables.tf

@@ -103,6 +103,17 @@ variable "clients" {
 
 }
 
+variable "pipe_log_level" {
+  type        = string
+  description = "Log level for the EventBridge Pipe."
+  default     = "ERROR"
+
+  validation {
+    condition     = contains(["OFF", "ERROR", "INFO", "TRACE"], var.pipe_log_level)
+    error_message = "pipe_log_level must be one of: OFF, ERROR, INFO, TRACE."
+  }
+}
+
 variable "pipe_sqs_input_batch_size" {
   type    = number
   default = 1
@@ -112,3 +123,9 @@ variable "pipe_sqs_max_batch_window" {
   type    = number
   default = 2
 }
+
+variable "deploy_mock_webhook" {
+  type        = bool
+  description = "Flag to deploy mock webhook lambda for integration testing (test/dev environments only)"
+  default     = false
+}

infrastructure/terraform/components/callbacks/versions.tf

@@ -4,7 +4,10 @@ terraform {
       source  = "hashicorp/aws"
       version = "6.13"
     }
-
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.0"
+    }
   }
 
   required_version = ">= 1.10.1"

infrastructure/terraform/modules/client-destination/cloudwatch_event_rule_main.tf

@@ -5,10 +5,7 @@ resource "aws_cloudwatch_event_rule" "main" {
 
   event_pattern = jsonencode({
     "detail" : {
-      "type" : var.client_detail,
-      "dataschemaversion" : [{
-        "prefix" : "1."
-      }]
+      "type" : var.client_detail
     }
   })
 }
@@ -19,8 +16,14 @@ resource "aws_cloudwatch_event_target" "main" {
   target_id      = "${local.csi}-${var.connection_name}"
   role_arn       = aws_iam_role.api_target_role.arn
   event_bus_name = var.client_bus_name
+  input_path     = "$.detail.transformedPayload"
 
   dead_letter_config {
     arn = module.target_dlq.sqs_queue_arn
   }
+
+  retry_policy {
+    maximum_retry_attempts       = 3
+    maximum_event_age_in_seconds = 3600
+  }
 }

infrastructure/terraform/modules/client-destination/module_target_dlq.tf

@@ -32,7 +32,7 @@ data "aws_iam_policy_document" "target_dlq" {
     ]
 
     resources = [
-      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${var.component}-${var.connection_name}-dlq"
+      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${var.component}-${var.connection_name}-dlq-queue"
     ]
   }
 }