From 03a813f7ee2338d3d40d322a59903b228f81a41e Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 8 Oct 2026 12:04:18 +0100 Subject: [PATCH 01/28] add state machine, popluate supplier config, and dummy lambdas --- eslint.config.mjs | 2 +- .../terraform/components/api/README.md | 37 +- .../api/iam_role_letter_sla_breaches.tf | 17 + .../iam_role_policy_letter_sla_breaches.tf | 25 + .../terraform/components/api/locals.tf | 3 + ...module_lambda_populate_sameday_breaches.tf | 55 ++ ...odule_lambda_populate_standard_breaches.tf | 55 ++ ...e_lambda_populate_supplier_config_cache.tf | 72 ++ .../api/resources/letter_sla_breaches.json | 77 ++ .../sfn_state_machine_letter_sla_breaches.tf | 11 + .../populate-sameday-breaches/.gitignore | 5 + .../populate-sameday-breaches/jest.config.ts | 46 + .../populate-sameday-breaches/package.json | 15 + .../populate-sameday-breaches.test.ts | 12 + .../src/handler/populate-sameday-breaches.ts | 18 + .../populate-sameday-breaches/src/index.ts | 2 + .../populate-sameday-breaches/tsconfig.json | 11 + .../populate-standard-breaches/.gitignore | 5 + .../populate-standard-breaches/jest.config.ts | 46 + .../populate-standard-breaches/package.json | 15 + .../populate-standard-breaches.test.ts | 12 + .../src/handler/populate-standard-breaches.ts | 18 + .../populate-standard-breaches/src/index.ts | 2 + .../populate-standard-breaches/tsconfig.json | 11 + .../populate-supplier-config-cache/.gitignore | 5 + .../jest.config.ts | 46 + .../package.json | 18 + .../src/__tests__/entity-cache.test.ts | 72 ++ .../populate-supplier-config-cache.test.ts | 32 + .../src/handler/deps.ts | 15 + .../src/handler/entity-cache.ts | 49 + .../handler/populate-supplier-config-cache.ts | 34 + .../src/index.ts | 7 + .../tsconfig.json | 11 + package-lock.json | 846 ++++-------------- package.json | 2 +- 36 files changed, 998 insertions(+), 711 deletions(-) create mode 100644 infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf create mode 100644 infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf create mode 100644 infrastructure/terraform/components/api/resources/letter_sla_breaches.json create mode 100644 infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/package.json create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/package.json create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json diff --git a/eslint.config.mjs b/eslint.config.mjs index 09a55d787..029ba22cd 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -115,7 +115,7 @@ export default defineConfig([ eslintImportResolverTypescript.createTypeScriptImportResolver({ project: [ 'lambdas/*/tsconfig.json', - 'tests/tsconfig.json', + 'lambdas/*/*/tsconfig.json', 'tests/tsconfig.json', 'internal/*/tsconfig.json', ], }), diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index b21ae6675..2d1508de4 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -60,9 +60,9 @@ No requirements. | Name | Source | Version | | ---- | ------ | ------- | -| [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | +| [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [amendments\_queue](#module\_amendments\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | -| [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | +| [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [ddb\_alarms\_idempotency](#module\_ddb\_alarms\_idempotency) | ../../modules/alarms-ddb | n/a | | [ddb\_alarms\_letter\_queue](#module\_ddb\_alarms\_letter\_queue) | ../../modules/alarms-ddb | n/a | | [ddb\_alarms\_letters](#module\_ddb\_alarms\_letters) | ../../modules/alarms-ddb | n/a | @@ -71,30 +71,33 @@ No requirements. | [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a | | [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/5.0.6/terraform-eventpub.zip | n/a | | [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a | -| [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [get\_letters](#module\_get\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [get\_mi](#module\_get\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | -| [get\_status](#module\_get\_status) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | +| [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [get\_letters](#module\_get\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [get\_mi](#module\_get\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [get\_status](#module\_get\_status) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-kms.zip | n/a | | [lambda\_alarms](#module\_lambda\_alarms) | ../../modules/alarms-lambda | n/a | | [letter\_status\_updates\_queue](#module\_letter\_status\_updates\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | -| [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [patch\_letter](#module\_patch\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | +| [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [patch\_letter](#module\_patch\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [populate\_sameday\_breaches](#module\_populate\_sameday\_breaches) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [populate\_standard\_breaches](#module\_populate\_standard\_breaches) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a | | [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a | | [sqs\_letter\_updates](#module\_sqs\_letter\_updates) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | | [sqs\_supplier\_allocator](#module\_sqs\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | | [sqs\_supplier\_config](#module\_sqs\_supplier\_config) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | -| [supplier\_allocator](#module\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [supplier\_config\_ingress](#module\_supplier\_config\_ingress) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | -| [supplier\_mock](#module\_supplier\_mock) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.5/terraform-lambda.zip | n/a | +| [supplier\_allocator](#module\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [supplier\_config\_ingress](#module\_supplier\_config\_ingress) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [supplier\_mock](#module\_supplier\_mock) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [supplier\_ssl](#module\_supplier\_ssl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-ssl.zip | n/a | -| [update\_letter\_queue](#module\_update\_letter\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | -| [upsert\_letter](#module\_upsert\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/4.0.2/terraform-lambda.zip | n/a | +| [update\_letter\_queue](#module\_update\_letter\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [upsert\_letter](#module\_upsert\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | ## Outputs | Name | Description | diff --git a/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf new file mode 100644 index 000000000..97a35bad2 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf @@ -0,0 +1,17 @@ +resource "aws_iam_role" "letter_sla_breaches" { + name = "${local.csi}-letter-sla-breaches-sfn" + description = "Execution role for the letter SLA breaches state machine" + assume_role_policy = data.aws_iam_policy_document.letter_sla_breaches_trust_policy.json +} + +data "aws_iam_policy_document" "letter_sla_breaches_trust_policy" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["states.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf new file mode 100644 index 000000000..d893cd8b0 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf @@ -0,0 +1,25 @@ +resource "aws_iam_role_policy" "letter_sla_breaches" { + name = "${local.csi}-letter-sla-breaches-sfn" + role = aws_iam_role.letter_sla_breaches.id + policy = data.aws_iam_policy_document.letter_sla_breaches_policy.json +} + +data "aws_iam_policy_document" "letter_sla_breaches_policy" { + statement { + sid = "AllowListOwnExecutions" + effect = "Allow" + actions = ["states:ListExecutions"] + resources = [local.letter_sla_breaches_state_machine_arn] + } + + statement { + sid = "AllowInvokeLambdas" + effect = "Allow" + actions = ["lambda:InvokeFunction"] + resources = [ + module.populate_supplier_config_cache.function_arn, + module.populate_sameday_breaches.function_arn, + module.populate_standard_breaches.function_arn, + ] + } +} diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 6b4d90305..18c52e680 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -52,4 +52,7 @@ locals { var.region, var.csoc_destination_account ) + + letter_sla_breaches_state_machine_name = "${local.csi}-letter-sla-breaches" + letter_sla_breaches_state_machine_arn = "arn:aws:states:${var.region}:${var.aws_account_id}:stateMachine:${local.letter_sla_breaches_state_machine_name}" } diff --git a/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf new file mode 100644 index 000000000..020c4dd1c --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf @@ -0,0 +1,55 @@ +module "populate_sameday_breaches" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "populate_sameday_breaches" + description = "Populates same-day breaches for the letter SLA breaches state machine" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.populate_sameday_breaches_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "letter-sla-breaches/populate-sameday-breaches/dist" + function_include_common = true + handler_function_name = "populateSamedayBreachesHandler" + runtime = "nodejs22.x" + memory = 128 + timeout = 29 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = {} +} + +data "aws_iam_policy_document" "populate_sameday_breaches_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } +} diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf new file mode 100644 index 000000000..f5665bb82 --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -0,0 +1,55 @@ +module "populate_standard_breaches" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "populate_standard_breaches" + description = "Populates standard breaches for the letter SLA breaches state machine" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.populate_standard_breaches_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "letter-sla-breaches/populate-standard-breaches/dist" + function_include_common = true + handler_function_name = "populateStandardBreachesHandler" + runtime = "nodejs22.x" + memory = 128 + timeout = 29 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = {} +} + +data "aws_iam_policy_document" "populate_standard_breaches_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } +} diff --git a/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf new file mode 100644 index 000000000..57bf707fa --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf @@ -0,0 +1,72 @@ +module "populate_supplier_config_cache" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "populate_supplier_config_cache" + description = "Populates the supplier config cache for the letter SLA breaches state machine" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.populate_supplier_config_cache_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "letter-sla-breaches/populate-supplier-config-cache/dist" + function_include_common = true + handler_function_name = "populateSupplierConfigCacheHandler" + runtime = "nodejs22.x" + memory = 128 + timeout = 29 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = { + SUPPLIER_CONFIG_TABLE_NAME = aws_dynamodb_table.supplier-configuration.name + EVENT_CACHE_BUCKET_NAME = local.eventsub_event_cache_bucket_name + } +} + +data "aws_iam_policy_document" "populate_supplier_config_cache_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } + + statement { + sid = "AllowQuerySupplierConfigTable" + effect = "Allow" + actions = ["dynamodb:Query"] + resources = [aws_dynamodb_table.supplier-configuration.arn] + } + + statement { + sid = "AllowWriteSupplierConfigCache" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/supplier_config_cache/*"] + } +} diff --git a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json new file mode 100644 index 000000000..725b8eaf3 --- /dev/null +++ b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json @@ -0,0 +1,77 @@ +{ + "Comment": "Identifies letters in breach of SLA for a given date (defaults to today)", + "QueryLanguage": "JSONata", + "StartAt": "CheckForRunningExecutions", + "States": { + "AlreadyRunning?": { + "Choices": [ + { + "Condition": "{% $count($states.input.Executions) > 1 %}", + "Next": "ExitAlreadyRunning" + } + ], + "Default": "PopulateSupplierConfigCache", + "Type": "Choice" + }, + "CheckForRunningExecutions": { + "Arguments": { + "StateMachineArn": "${STATE_MACHINE_ARN}", + "StatusFilter": "RUNNING" + }, + "Assign": { + "date": "{% $exists($states.context.Execution.Input.date) ? $states.context.Execution.Input.date : $substring($states.context.Execution.StartTime, 0, 10) %}" + }, + "Next": "AlreadyRunning?", + "Resource": "arn:aws:states:::aws-sdk:sfn:listExecutions", + "Type": "Task" + }, + "ExitAlreadyRunning": { + "Type": "Succeed" + }, + "PopulateSamedayBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "Next": "PopulateStandardBreaches", + "Output": { + "populateSamedayBreaches": "{% $states.result.Payload %}", + "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + }, + "PopulateStandardBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_STANDARD_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "End": true, + "Output": { + "populateSamedayBreaches": "{% $states.input.populateSamedayBreaches %}", + "populateStandardBreaches": "{% $states.result.Payload %}", + "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + }, + "PopulateSupplierConfigCache": { + "Arguments": { + "FunctionName": "${POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "Next": "PopulateSamedayBreaches", + "Output": { + "populateSupplierConfigCache": "{% $states.result.Payload %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + } + } +} diff --git a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf new file mode 100644 index 000000000..19d726103 --- /dev/null +++ b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf @@ -0,0 +1,11 @@ +resource "aws_sfn_state_machine" "letter_sla_breaches" { + name = local.letter_sla_breaches_state_machine_name + role_arn = aws_iam_role.letter_sla_breaches.arn + + definition = templatefile("${path.module}/resources/letter_sla_breaches.json", { + STATE_MACHINE_ARN = local.letter_sla_breaches_state_machine_arn + POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN = module.populate_supplier_config_cache.function_arn + POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN = module.populate_sameday_breaches.function_arn + POPULATE_STANDARD_BREACHES_FUNCTION_ARN = module.populate_standard_breaches.function_arn + }) +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore b/lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore new file mode 100644 index 000000000..fc5a7859a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore @@ -0,0 +1,5 @@ +coverage +node_modules +dist +.reports +.aws-sam diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts new file mode 100644 index 000000000..ec5a312bb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts @@ -0,0 +1,46 @@ +export const baseJestConfig = { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + testEnvironment: "node", +}; + +export default baseJestConfig; diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json new file mode 100644 index 000000000..127c212c3 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json @@ -0,0 +1,15 @@ +{ + "dependencies": { + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-populate-sameday-breaches", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts new file mode 100644 index 000000000..50492a04a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts @@ -0,0 +1,12 @@ +import populateSamedayBreaches from "../handler/populate-sameday-breaches"; + +describe("populateSamedayBreaches", () => { + it("returns a success result", async () => { + const result = await populateSamedayBreaches({ date: "2026-10-08" }); + + expect(result).toEqual({ + status: "SUCCESS", + message: "populate-sameday-breaches invoked successfully", + }); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts new file mode 100644 index 000000000..f63d70492 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts @@ -0,0 +1,18 @@ +export type PopulateSamedayBreachesEvent = { + date: string; +}; + +export type PopulateSamedayBreachesResult = { + status: "SUCCESS"; + message: string; +}; + +// Placeholder: cache population is not yet implemented. +export default async function populateSamedayBreaches( + _event: PopulateSamedayBreachesEvent, +): Promise { + return { + status: "SUCCESS", + message: "populate-sameday-breaches invoked successfully", + }; +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts new file mode 100644 index 000000000..3143567ed --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts @@ -0,0 +1,2 @@ +// eslint-disable-next-line import-x/prefer-default-export +export { default as populateSamedayBreachesHandler } from "./handler/populate-sameday-breaches"; diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore b/lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore new file mode 100644 index 000000000..fc5a7859a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore @@ -0,0 +1,5 @@ +coverage +node_modules +dist +.reports +.aws-sam diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts new file mode 100644 index 000000000..ec5a312bb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts @@ -0,0 +1,46 @@ +export const baseJestConfig = { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + testEnvironment: "node", +}; + +export default baseJestConfig; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json new file mode 100644 index 000000000..8875e4acc --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json @@ -0,0 +1,15 @@ +{ + "dependencies": { + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-populate-standard-breaches", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts new file mode 100644 index 000000000..f64093251 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -0,0 +1,12 @@ +import populateStandardBreaches from "../handler/populate-standard-breaches"; + +describe("populateStandardBreaches", () => { + it("returns a success result", async () => { + const result = await populateStandardBreaches({ date: "2026-10-08" }); + + expect(result).toEqual({ + status: "SUCCESS", + message: "populate-standard-breaches invoked successfully", + }); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts new file mode 100644 index 000000000..1bcc2318c --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -0,0 +1,18 @@ +export type PopulateStandardBreachesEvent = { + date: string; +}; + +export type PopulateStandardBreachesResult = { + status: "SUCCESS"; + message: string; +}; + +// Placeholder: cache population is not yet implemented. +export default async function populateStandardBreaches( + _event: PopulateStandardBreachesEvent, +): Promise { + return { + status: "SUCCESS", + message: "populate-standard-breaches invoked successfully", + }; +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts new file mode 100644 index 000000000..5516fb886 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts @@ -0,0 +1,2 @@ +// eslint-disable-next-line import-x/prefer-default-export +export { default as populateStandardBreachesHandler } from "./handler/populate-standard-breaches"; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json b/lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore b/lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore new file mode 100644 index 000000000..fc5a7859a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore @@ -0,0 +1,5 @@ +coverage +node_modules +dist +.reports +.aws-sam diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts new file mode 100644 index 000000000..ec5a312bb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts @@ -0,0 +1,46 @@ +export const baseJestConfig = { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + testEnvironment: "node", +}; + +export default baseJestConfig; diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json new file mode 100644 index 000000000..034e0301b --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -0,0 +1,18 @@ +{ + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.1138.0", + "@aws-sdk/client-s3": "^3.1138.0", + "@aws-sdk/lib-dynamodb": "^3.1138.0", + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-populate-supplier-config-cache", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts new file mode 100644 index 000000000..1dac234a9 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -0,0 +1,72 @@ +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; + +function makeDeps(ddbSend: jest.Mock, s3Send: jest.Mock): EntityCacheDeps { + return { + ddbClient: { send: ddbSend } as unknown as DynamoDBDocumentClient, + s3Client: { send: s3Send } as unknown as S3Client, + env: { + SUPPLIER_CONFIG_TABLE_NAME: "config-table", + EVENT_CACHE_BUCKET_NAME: "cache-bucket", + }, + }; +} + +describe("populateEntityCache", () => { + it("queries all pages for the entity and writes them to S3", async () => { + const ddbSend = jest + .fn() + .mockResolvedValueOnce({ + Items: [{ pk: "ENTITY#supplier", sk: "ID#1" }], + LastEvaluatedKey: { pk: "ENTITY#supplier", sk: "ID#1" }, + }) + .mockResolvedValueOnce({ + Items: [{ pk: "ENTITY#supplier", sk: "ID#2" }], + }); + const s3Send = jest.fn().mockResolvedValue({}); + + const count = await populateEntityCache( + makeDeps(ddbSend, s3Send), + "supplier", + ); + + expect(count).toBe(2); + expect(ddbSend).toHaveBeenCalledTimes(2); + const firstQuery = ddbSend.mock.calls[0][0] as QueryCommand; + expect(firstQuery.input).toEqual({ + TableName: "config-table", + KeyConditionExpression: "pk = :pk", + ExpressionAttributeValues: { ":pk": "ENTITY#supplier" }, + ExclusiveStartKey: undefined, + }); + const secondQuery = ddbSend.mock.calls[1][0] as QueryCommand; + expect(secondQuery.input.ExclusiveStartKey).toEqual({ + pk: "ENTITY#supplier", + sk: "ID#1", + }); + + const put = s3Send.mock.calls[0][0] as PutObjectCommand; + expect(put.input.Bucket).toBe("cache-bucket"); + expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); + expect(JSON.parse(put.input.Body as string)).toEqual([ + { pk: "ENTITY#supplier", sk: "ID#1" }, + { pk: "ENTITY#supplier", sk: "ID#2" }, + ]); + }); + + it("writes an empty file when no rows are returned", async () => { + const ddbSend = jest.fn().mockResolvedValue({}); + const s3Send = jest.fn().mockResolvedValue({}); + + const count = await populateEntityCache( + makeDeps(ddbSend, s3Send), + "volume-group", + ); + + expect(count).toBe(0); + const put = s3Send.mock.calls[0][0] as PutObjectCommand; + expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); + expect(put.input.Body).toBe("[]"); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts new file mode 100644 index 000000000..54374eb2f --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -0,0 +1,32 @@ +import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; +import createHandler, { + ENTITY_TYPES, +} from "../handler/populate-supplier-config-cache"; + +jest.mock("../handler/entity-cache"); + +describe("populateSupplierConfigCache", () => { + it("populates the cache once for each entity type and returns success", async () => { + const deps = {} as EntityCacheDeps; + const handler = createHandler(deps); + + const result = await handler({ date: "2026-10-08" }); + + expect(ENTITY_TYPES).toEqual([ + "volume-group", + "supplier-pack", + "letter-variant", + "pack-specification", + "supplier-allocation", + "supplier", + ]); + expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); + for (const entityType of ENTITY_TYPES) { + expect(populateEntityCache).toHaveBeenCalledWith(deps, entityType); + } + expect(result).toEqual({ + status: "SUCCESS", + message: "populate-supplier-config-cache invoked successfully", + }); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts new file mode 100644 index 000000000..6fdf2f926 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts @@ -0,0 +1,15 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { EntityCacheDeps } from "./entity-cache"; + +export default function createDependenciesContainer(): EntityCacheDeps { + return { + ddbClient: DynamoDBDocumentClient.from(new DynamoDBClient()), + s3Client: new S3Client(), + env: { + SUPPLIER_CONFIG_TABLE_NAME: process.env.SUPPLIER_CONFIG_TABLE_NAME!, + EVENT_CACHE_BUCKET_NAME: process.env.EVENT_CACHE_BUCKET_NAME!, + }, + }; +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts new file mode 100644 index 000000000..eab1beef6 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -0,0 +1,49 @@ +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { + DynamoDBDocumentClient, + QueryCommand, + QueryCommandOutput, +} from "@aws-sdk/lib-dynamodb"; + +export type EntityCacheDeps = { + ddbClient: DynamoDBDocumentClient; + s3Client: S3Client; + env: { + SUPPLIER_CONFIG_TABLE_NAME: string; + EVENT_CACHE_BUCKET_NAME: string; + }; +}; + +export const CACHE_FOLDER = "supplier_config_cache"; + +export default async function populateEntityCache( + deps: EntityCacheDeps, + entityType: string, +): Promise { + const items: Record[] = []; + let exclusiveStartKey: QueryCommandOutput["LastEvaluatedKey"]; + + do { + const page: QueryCommandOutput = await deps.ddbClient.send( + new QueryCommand({ + TableName: deps.env.SUPPLIER_CONFIG_TABLE_NAME, + KeyConditionExpression: "pk = :pk", + ExpressionAttributeValues: { ":pk": `ENTITY#${entityType}` }, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + items.push(...(page.Items ?? [])); + exclusiveStartKey = page.LastEvaluatedKey; + } while (exclusiveStartKey); + + await deps.s3Client.send( + new PutObjectCommand({ + Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, + Key: `${CACHE_FOLDER}/${entityType}.json`, + Body: JSON.stringify(items), + ContentType: "application/json", + }), + ); + + return items.length; +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts new file mode 100644 index 000000000..b3c3d57fb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -0,0 +1,34 @@ +import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; + +export type PopulateSupplierConfigCacheEvent = { + date: string; +}; + +export type PopulateSupplierConfigCacheResult = { + status: "SUCCESS"; + message: string; +}; + +export const ENTITY_TYPES = [ + "volume-group", + "supplier-pack", + "letter-variant", + "pack-specification", + "supplier-allocation", + "supplier", +]; + +export default function createHandler(deps: EntityCacheDeps) { + return async ( + _event: PopulateSupplierConfigCacheEvent, + ): Promise => { + await Promise.all( + ENTITY_TYPES.map((entityType) => populateEntityCache(deps, entityType)), + ); + + return { + status: "SUCCESS", + message: "populate-supplier-config-cache invoked successfully", + }; + }; +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts new file mode 100644 index 000000000..fd06c3b3f --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts @@ -0,0 +1,7 @@ +import createDependenciesContainer from "./handler/deps"; +import createHandler from "./handler/populate-supplier-config-cache"; + +// eslint-disable-next-line import-x/prefer-default-export +export const populateSupplierConfigCacheHandler = createHandler( + createDependenciesContainer(), +); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index cae5e3cea..a80f25d9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "docs", "internal/*", "lambdas/*", + "lambdas/letter-sla-breaches/*", "pact-contracts", "scripts/utilities/*", "tests", @@ -155,30 +156,6 @@ "node": ">=20.0.0" } }, - "internal/datastore/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "internal/datastore/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "internal/datastore/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -196,33 +173,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "internal/datastore/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "internal/datastore/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "internal/datastore/node_modules/@nhsdigital/nhs-notify-event-schemas-supplier-config": { "version": "1.1.1", "dependencies": { @@ -241,7 +191,7 @@ }, "internal/events": { "name": "@nhsdigital/nhs-notify-event-schemas-supplier-api", - "version": "1.0.21", + "version": "1.0.22", "license": "MIT", "dependencies": { "@asyncapi/bundler": "^1.0.2", @@ -365,30 +315,6 @@ "node": ">=20.0.0" } }, - "lambdas/api-handler/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/api-handler/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/api-handler/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -406,20 +332,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/api-handler/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/api-handler/node_modules/@aws-sdk/middleware-sdk-sqs": { "version": "3.972.43", "license": "Apache-2.0", @@ -433,19 +345,6 @@ "node": ">=20.0.0" } }, - "lambdas/api-handler/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/authorizer": { "name": "nhs-notify-supplier-authorizer", "version": "0.0.1", @@ -480,30 +379,6 @@ "node": ">=20.0.0" } }, - "lambdas/authorizer/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/authorizer/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/authorizer/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -521,31 +396,28 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/authorizer/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", + "lambdas/letter-sla-breaches/populate-sameday-breaches": { + "name": "nhs-notify-supplier-api-populate-sameday-breaches", + "version": "0.0.1", "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" + "esbuild": "0.28.2" } }, - "lambdas/authorizer/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", + "lambdas/letter-sla-breaches/populate-standard-breaches": { + "name": "nhs-notify-supplier-api-populate-standard-breaches", + "version": "0.0.1", "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" + "esbuild": "0.28.2" + } + }, + "lambdas/letter-sla-breaches/populate-supplier-config-cache": { + "name": "nhs-notify-supplier-api-populate-supplier-config-cache", + "version": "0.0.1", + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.1138.0", + "@aws-sdk/client-s3": "^3.1138.0", + "@aws-sdk/lib-dynamodb": "^3.1138.0", + "esbuild": "0.28.2" } }, "lambdas/letter-updates-transformer": { @@ -565,90 +437,6 @@ "zod": "^4.6.5" } }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1146.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1146.0.tgz", - "integrity": "sha512-3Mc8YkSu2KlgNNSuspaW+Fo0pTwX+fLYlC5ovH3yX3OoDg2RPtp3Py8KuQJr0FDlQj1tHw0K3+AtaH4NXlI3aQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-node": "^3.972.84", - "@aws-sdk/dynamodb-codec": "^3.973.46", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/fetch-http-handler": "^5.8.0", - "@smithy/node-http-handler": "^4.12.1", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.46.tgz", - "integrity": "sha512-ApnSubn+5C6HQGP7025q9w/YiNzKwimIMksP76xVij779BCCqcmJ9poB6hQ4syY9NUxqmEQLBqY0PktX53L8iQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", - "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.31.tgz", - "integrity": "sha512-JqOIXupzP6cpAmCCmuEaacbXo/bVCcg1NkglCJC6q4wDAvJO09qIs7POf50JNzmhOhWPtfXvzgWxb5nMQl8QiA==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", - "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/mi-updates-transformer": { "name": "nhs-notify-supplier-api-mi-updates-transformer", "version": "0.0.1", @@ -665,90 +453,6 @@ "zod": "^4.6.5" } }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1146.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1146.0.tgz", - "integrity": "sha512-3Mc8YkSu2KlgNNSuspaW+Fo0pTwX+fLYlC5ovH3yX3OoDg2RPtp3Py8KuQJr0FDlQj1tHw0K3+AtaH4NXlI3aQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-node": "^3.972.84", - "@aws-sdk/dynamodb-codec": "^3.973.46", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/fetch-http-handler": "^5.8.0", - "@smithy/node-http-handler": "^4.12.1", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.46.tgz", - "integrity": "sha512-ApnSubn+5C6HQGP7025q9w/YiNzKwimIMksP76xVij779BCCqcmJ9poB6hQ4syY9NUxqmEQLBqY0PktX53L8iQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", - "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.31.tgz", - "integrity": "sha512-JqOIXupzP6cpAmCCmuEaacbXo/bVCcg1NkglCJC6q4wDAvJO09qIs7POf50JNzmhOhWPtfXvzgWxb5nMQl8QiA==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", - "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/supplier-allocator": { "name": "nhs-notify-supplier-api-allocate-letter", "version": "0.0.1", @@ -856,30 +560,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/supplier-allocator/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -897,20 +577,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/supplier-allocator/node_modules/@aws-sdk/middleware-sdk-sqs": { "version": "3.972.43", "license": "Apache-2.0", @@ -924,19 +590,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/supplier-allocator/node_modules/@aws/lambda-invoke-store": { "version": "0.3.0", "license": "Apache-2.0", @@ -1014,30 +667,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/supplier-config-ingress/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -1055,33 +684,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/supplier-config-ingress/node_modules/@nhsdigital/nhs-notify-event-schemas-supplier-config": { "version": "1.1.1", "dependencies": { @@ -1167,30 +769,6 @@ "node": ">=20.0.0" } }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/update-letter-queue/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -1208,33 +786,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/upsert-letter": { "name": "nhs-notify-supplier-api-upsert-letter", "version": "0.0.1", @@ -1295,79 +846,24 @@ } }, "lambdas/upsert-letter/node_modules/@aws-lambda-powertools/jmespath": { - "version": "2.35.0", - "license": "MIT-0", - "dependencies": { - "@aws-lambda-powertools/commons": "2.35.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1145.0", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-node": "^3.972.84", - "@aws-sdk/dynamodb-codec": "^3.973.46", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/fetch-http-handler": "^5.8.0", - "@smithy/node-http-handler": "^4.12.1", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/lib-dynamodb": { - "version": "3.1145.0", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/util-dynamodb": "^3.996.9", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1145.0" + "version": "2.35.0", + "license": "MIT-0", + "dependencies": { + "@aws-lambda-powertools/commons": "2.35.0" } }, - "lambdas/upsert-letter/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", + "lambdas/upsert-letter/node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1145.0", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/credential-provider-node": "^3.972.84", + "@aws-sdk/dynamodb-codec": "^3.973.46", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1375,17 +871,21 @@ "node": ">=20.0.0" } }, - "lambdas/upsert-letter/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", + "lambdas/upsert-letter/node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.1145.0", "license": "Apache-2.0", "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/util-dynamodb": "^3.996.9", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" + "@aws-sdk/client-dynamodb": "^3.1145.0" } }, "lambdas/upsert-letter/node_modules/@aws/lambda-invoke-store": { @@ -1536,6 +1036,27 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1147.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1147.0.tgz", + "integrity": "sha512-lONixsTyj2jt5bb14ir4VYuBEy8kMNMZt7DSVxZ4HHE89hEfND5lO4Qj+F3Z74sENdFCaSAgK8RYQaiQWm5pfw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/credential-provider-node": "^3.972.84", + "@aws-sdk/dynamodb-codec": "^3.973.46", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-kinesis": { "version": "3.1145.0", "license": "Apache-2.0", @@ -1780,6 +1301,69 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/dynamodb-codec": { + "version": "3.973.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.46.tgz", + "integrity": "sha512-ApnSubn+5C6HQGP7025q9w/YiNzKwimIMksP76xVij779BCCqcmJ9poB6hQ4syY9NUxqmEQLBqY0PktX53L8iQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", + "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", + "license": "Apache-2.0", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.1147.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1147.0.tgz", + "integrity": "sha512-232+7ND08nb0YZrCyCviYSpySiYDusCZo8mdovBo9eOHDqBvndiP7n5/CrtN5HjegGuwJYQ/0AedWu1ORVUGDA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/util-dynamodb": "^3.996.9", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.1147.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.31.tgz", + "integrity": "sha512-JqOIXupzP6cpAmCCmuEaacbXo/bVCcg1NkglCJC6q4wDAvJO09qIs7POf50JNzmhOhWPtfXvzgWxb5nMQl8QiA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/middleware-sdk-api-gateway": { "version": "3.972.30", "license": "Apache-2.0", @@ -1879,6 +1463,21 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/util-dynamodb": { + "version": "3.996.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", + "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.1111.0" + } + }, "node_modules/@aws-sdk/xml-builder": { "version": "3.972.41", "license": "Apache-2.0", @@ -12379,6 +11978,8 @@ }, "node_modules/mnemonist": { "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", "license": "MIT", "dependencies": { "obliterator": "^1.6.1" @@ -12537,6 +12138,18 @@ "resolved": "lambdas/mi-updates-transformer", "link": true }, + "node_modules/nhs-notify-supplier-api-populate-sameday-breaches": { + "resolved": "lambdas/letter-sla-breaches/populate-sameday-breaches", + "link": true + }, + "node_modules/nhs-notify-supplier-api-populate-standard-breaches": { + "resolved": "lambdas/letter-sla-breaches/populate-standard-breaches", + "link": true + }, + "node_modules/nhs-notify-supplier-api-populate-supplier-config-cache": { + "resolved": "lambdas/letter-sla-breaches/populate-supplier-config-cache", + "link": true + }, "node_modules/nhs-notify-supplier-api-supplier-config-ingress": { "resolved": "lambdas/supplier-config-ingress", "link": true @@ -12818,6 +12431,8 @@ }, "node_modules/obliterator": { "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", "license": "MIT" }, "node_modules/obug": { @@ -16760,30 +16375,6 @@ "node": ">=20.0.0" } }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "scripts/utilities/letter-test-data/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -16801,33 +16392,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "scripts/utilities/letter-test-data/node_modules/ansi-regex": { "version": "6.4.0", "license": "MIT", @@ -16989,30 +16553,6 @@ "node": ">=20.0.0" } }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "scripts/utilities/supplier-data/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -17030,33 +16570,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "scripts/utilities/supplier-data/node_modules/ansi-regex": { "version": "6.4.0", "license": "MIT", @@ -17274,30 +16787,6 @@ "node": ">=20.0.0" } }, - "tests/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "tests/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "tests/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -17315,20 +16804,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "tests/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "tests/node_modules/@aws-sdk/middleware-sdk-sqs": { "version": "3.972.43", "license": "Apache-2.0", @@ -17342,19 +16817,6 @@ "node": ">=20.0.0" } }, - "tests/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "tests/node_modules/@babel/code-frame": { "version": "8.0.6", "dev": true, diff --git a/package.json b/package.json index 4e7f0e70d..d6e0a3e9b 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,7 @@ "docs", "internal/*", "lambdas/*", - "pact-contracts", + "lambdas/letter-sla-breaches/*", "pact-contracts", "scripts/utilities/*", "tests", "tests/contracts/*" From 729b9e2c0650b91f18ba0760198b78c6d7d9a70b Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 11:14:07 +0100 Subject: [PATCH 02/28] add glue definitions --- ...ue_catalog_tables_supplier_config_cache.tf | 108 ++++++++++++++++++ .../src/__tests__/entity-cache.test.ts | 6 +- .../src/handler/entity-cache.ts | 2 +- 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf diff --git a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf new file mode 100644 index 000000000..7f27de168 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf @@ -0,0 +1,108 @@ +locals { + supplier_config_cache_table_columns = { + supplier = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "channelType", type = "string" }, + { name = "dailyCapacity", type = "bigint" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "status", type = "string" }, + ] + volume-group = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "description", type = "string" }, + { name = "endDate", type = "string" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "startDate", type = "string" }, + { name = "status", type = "string" }, + ] + supplier-pack = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "approval", type = "string" }, + { name = "id", type = "string" }, + { name = "packSpecificationId", type = "string" }, + { name = "status", type = "string" }, + { name = "supplierId", type = "string" }, + ] + letter-variant = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "campaignIds", type = "array" }, + { name = "clientId", type = "string" }, + { name = "constraints", type = "struct,colourCoveragePercentage:struct,deliveryDays:struct,sheets:struct,sides:struct>" }, + { name = "description", type = "string" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "packSpecificationIds", type = "array" }, + { name = "priority", type = "bigint" }, + { name = "status", type = "string" }, + { name = "supplierId", type = "string" }, + { name = "type", type = "string" }, + { name = "volumeGroupId", type = "string" }, + ] + pack-specification = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "assembly", type = "struct,insertIds:array,paper:struct,printColour:string>" }, + { name = "billingId", type = "string" }, + { name = "constraints", type = "struct,colourCoveragePercentage:struct,deliveryDays:struct,sheets:struct,sides:struct>" }, + { name = "createdAt", type = "string" }, + { name = "description", type = "string" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "postage", type = "struct" }, + { name = "status", type = "string" }, + { name = "updatedAt", type = "string" }, + { name = "version", type = "bigint" }, + ] + supplier-allocation = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "allocationPercentage", type = "double" }, + { name = "id", type = "string" }, + { name = "status", type = "string" }, + { name = "supplier", type = "string" }, + { name = "volumeGroup", type = "string" }, + ] + } +} + +resource "aws_glue_catalog_table" "supplier_config_cache" { + for_each = local.eventsub_event_cache_bucket_name != null ? local.supplier_config_cache_table_columns : {} + name = "${replace(local.csi, "-", "_")}_${replace(each.key, "-", "_")}" + database_name = aws_glue_catalog_database.supplier.name + table_type = "EXTERNAL_TABLE" + + parameters = { + EXTERNAL = "TRUE" + classification = "json" + } + + storage_descriptor { + location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}.json" + input_format = "org.apache.hadoop.mapred.TextInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" + + ser_de_info { + name = "OpenXJsonSerDe" + serialization_library = "org.openx.data.jsonserde.JsonSerDe" + + parameters = { + "case.insensitive" = "false" + } + } + + dynamic "columns" { + for_each = each.value + + content { + name = columns.value.name + type = columns.value.type + } + } + } +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index 1dac234a9..ecb44a5fc 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -49,7 +49,9 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Bucket).toBe("cache-bucket"); expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); - expect(JSON.parse(put.input.Body as string)).toEqual([ + expect( + (put.input.Body as string).split("\n").map((line) => JSON.parse(line)), + ).toEqual([ { pk: "ENTITY#supplier", sk: "ID#1" }, { pk: "ENTITY#supplier", sk: "ID#2" }, ]); @@ -67,6 +69,6 @@ describe("populateEntityCache", () => { expect(count).toBe(0); const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); - expect(put.input.Body).toBe("[]"); + expect(put.input.Body).toBe(""); }); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts index eab1beef6..3dcfdfa1e 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -40,7 +40,7 @@ export default async function populateEntityCache( new PutObjectCommand({ Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, Key: `${CACHE_FOLDER}/${entityType}.json`, - Body: JSON.stringify(items), + Body: items.map((item) => JSON.stringify(item)).join("\n"), ContentType: "application/json", }), ); From 2159b2924e93738c2b929b3129869033561c57f0 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 11:18:40 +0100 Subject: [PATCH 03/28] run SLA lambdas in parallel --- .../api/resources/letter_sla_breaches.json | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json index 725b8eaf3..4544a8e26 100644 --- a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json +++ b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json @@ -28,36 +28,54 @@ "ExitAlreadyRunning": { "Type": "Succeed" }, - "PopulateSamedayBreaches": { - "Arguments": { - "FunctionName": "${POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN}", - "Payload": { - "date": "{% $date %}" - } - }, - "Next": "PopulateStandardBreaches", - "Output": { - "populateSamedayBreaches": "{% $states.result.Payload %}", - "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" - }, - "Resource": "arn:aws:states:::lambda:invoke", - "Type": "Task" - }, - "PopulateStandardBreaches": { - "Arguments": { - "FunctionName": "${POPULATE_STANDARD_BREACHES_FUNCTION_ARN}", - "Payload": { - "date": "{% $date %}" + "PopulateBreachesInParallel": { + "Branches": [ + { + "StartAt": "PopulateSamedayBreaches", + "States": { + "PopulateSamedayBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "End": true, + "Output": { + "populateSamedayBreaches": "{% $states.result.Payload %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + } + } + }, + { + "StartAt": "PopulateStandardBreaches", + "States": { + "PopulateStandardBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_STANDARD_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "End": true, + "Output": { + "populateStandardBreaches": "{% $states.result.Payload %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + } + } } - }, + ], "End": true, "Output": { - "populateSamedayBreaches": "{% $states.input.populateSamedayBreaches %}", - "populateStandardBreaches": "{% $states.result.Payload %}", + "populateSamedayBreaches": "{% $states.result[0].populateSamedayBreaches %}", + "populateStandardBreaches": "{% $states.result[1].populateStandardBreaches %}", "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" }, - "Resource": "arn:aws:states:::lambda:invoke", - "Type": "Task" + "Type": "Parallel" }, "PopulateSupplierConfigCache": { "Arguments": { @@ -66,7 +84,7 @@ "date": "{% $date %}" } }, - "Next": "PopulateSamedayBreaches", + "Next": "PopulateBreachesInParallel", "Output": { "populateSupplierConfigCache": "{% $states.result.Payload %}" }, From c51e94cc59e4ba6839952bbdcd4abd0d2529cc24 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 14:29:00 +0100 Subject: [PATCH 04/28] add logging to config cache lambda --- .../src/__tests__/entity-cache.test.ts | 29 +++++++++++++++++-- .../src/handler/deps.ts | 2 ++ .../src/handler/entity-cache.ts | 23 ++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index ecb44a5fc..a87958f26 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -6,6 +6,7 @@ function makeDeps(ddbSend: jest.Mock, s3Send: jest.Mock): EntityCacheDeps { return { ddbClient: { send: ddbSend } as unknown as DynamoDBDocumentClient, s3Client: { send: s3Send } as unknown as S3Client, + logger: { info: jest.fn() } as unknown as EntityCacheDeps["logger"], env: { SUPPLIER_CONFIG_TABLE_NAME: "config-table", EVENT_CACHE_BUCKET_NAME: "cache-bucket", @@ -25,9 +26,10 @@ describe("populateEntityCache", () => { Items: [{ pk: "ENTITY#supplier", sk: "ID#2" }], }); const s3Send = jest.fn().mockResolvedValue({}); + const deps = makeDeps(ddbSend, s3Send); const count = await populateEntityCache( - makeDeps(ddbSend, s3Send), + deps, "supplier", ); @@ -49,6 +51,24 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Bucket).toBe("cache-bucket"); expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); + expect(deps.logger.info).toHaveBeenCalledWith( + { entityType: "supplier" }, + "Loading supplier config entity", + ); + expect(deps.logger.info).toHaveBeenCalledWith( + { entityType: "supplier", itemCount: 2, hasItems: true }, + "Loaded supplier config entity", + ); + expect(deps.logger.info).toHaveBeenCalledWith( + { + entityType: "supplier", + itemCount: 2, + bucket: "cache-bucket", + key: "supplier_config_cache/supplier.json", + s3Uri: "s3://cache-bucket/supplier_config_cache/supplier.json", + }, + "Created supplier config cache file", + ); expect( (put.input.Body as string).split("\n").map((line) => JSON.parse(line)), ).toEqual([ @@ -60,9 +80,10 @@ describe("populateEntityCache", () => { it("writes an empty file when no rows are returned", async () => { const ddbSend = jest.fn().mockResolvedValue({}); const s3Send = jest.fn().mockResolvedValue({}); + const deps = makeDeps(ddbSend, s3Send); const count = await populateEntityCache( - makeDeps(ddbSend, s3Send), + deps, "volume-group", ); @@ -70,5 +91,9 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); expect(put.input.Body).toBe(""); + expect(deps.logger.info).toHaveBeenCalledWith( + { entityType: "volume-group", itemCount: 0, hasItems: false }, + "Loaded supplier config entity", + ); }); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts index 6fdf2f926..ced5d4c5e 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts @@ -1,12 +1,14 @@ import { S3Client } from "@aws-sdk/client-s3"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { createLogger } from "@internal/helpers"; import { EntityCacheDeps } from "./entity-cache"; export default function createDependenciesContainer(): EntityCacheDeps { return { ddbClient: DynamoDBDocumentClient.from(new DynamoDBClient()), s3Client: new S3Client(), + logger: createLogger({ logLevel: process.env.PINO_LOG_LEVEL }), env: { SUPPLIER_CONFIG_TABLE_NAME: process.env.SUPPLIER_CONFIG_TABLE_NAME!, EVENT_CACHE_BUCKET_NAME: process.env.EVENT_CACHE_BUCKET_NAME!, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts index 3dcfdfa1e..381a33856 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -4,10 +4,12 @@ import { QueryCommand, QueryCommandOutput, } from "@aws-sdk/lib-dynamodb"; +import { Logger } from "pino"; export type EntityCacheDeps = { ddbClient: DynamoDBDocumentClient; s3Client: S3Client; + logger: Logger; env: { SUPPLIER_CONFIG_TABLE_NAME: string; EVENT_CACHE_BUCKET_NAME: string; @@ -20,6 +22,8 @@ export default async function populateEntityCache( deps: EntityCacheDeps, entityType: string, ): Promise { + deps.logger.info({ entityType }, "Loading supplier config entity"); + const items: Record[] = []; let exclusiveStartKey: QueryCommandOutput["LastEvaluatedKey"]; @@ -36,14 +40,31 @@ export default async function populateEntityCache( exclusiveStartKey = page.LastEvaluatedKey; } while (exclusiveStartKey); + deps.logger.info( + { entityType, itemCount: items.length, hasItems: items.length > 0 }, + "Loaded supplier config entity", + ); + + const key = `${CACHE_FOLDER}/${entityType}.json`; await deps.s3Client.send( new PutObjectCommand({ Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, - Key: `${CACHE_FOLDER}/${entityType}.json`, + Key: key, Body: items.map((item) => JSON.stringify(item)).join("\n"), ContentType: "application/json", }), ); + deps.logger.info( + { + entityType, + itemCount: items.length, + bucket: deps.env.EVENT_CACHE_BUCKET_NAME, + key, + s3Uri: `s3://${deps.env.EVENT_CACHE_BUCKET_NAME}/${key}`, + }, + "Created supplier config cache file", + ); + return items.length; } From a52a1eb218f48e2056684d6d5bfb315ec6450fad Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 14:41:45 +0100 Subject: [PATCH 05/28] fix lint issues --- .../populate-supplier-config-cache/package.json | 4 +++- .../src/__tests__/entity-cache.test.ts | 10 ++-------- package-lock.json | 4 +++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json index 034e0301b..b10a522d2 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -3,7 +3,9 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", - "esbuild": "0.28.2" + "@internal/helpers": "^0.1.0", + "esbuild": "0.28.2", + "pino": "^10.3.1" }, "name": "nhs-notify-supplier-api-populate-supplier-config-cache", "private": true, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index a87958f26..125ff82b0 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -28,10 +28,7 @@ describe("populateEntityCache", () => { const s3Send = jest.fn().mockResolvedValue({}); const deps = makeDeps(ddbSend, s3Send); - const count = await populateEntityCache( - deps, - "supplier", - ); + const count = await populateEntityCache(deps, "supplier"); expect(count).toBe(2); expect(ddbSend).toHaveBeenCalledTimes(2); @@ -82,10 +79,7 @@ describe("populateEntityCache", () => { const s3Send = jest.fn().mockResolvedValue({}); const deps = makeDeps(ddbSend, s3Send); - const count = await populateEntityCache( - deps, - "volume-group", - ); + const count = await populateEntityCache(deps, "volume-group"); expect(count).toBe(0); const put = s3Send.mock.calls[0][0] as PutObjectCommand; diff --git a/package-lock.json b/package-lock.json index a80f25d9a..2d80007dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -417,7 +417,9 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", - "esbuild": "0.28.2" + "@internal/helpers": "^0.1.0", + "esbuild": "0.28.2", + "pino": "^10.3.1" } }, "lambdas/letter-updates-transformer": { From 12dae1618f8423b720fe81aadbac9fceffee4eb4 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 15:21:32 +0100 Subject: [PATCH 06/28] amend glue config --- .../api/glue_catalog_tables_supplier_config_cache.tf | 2 +- .../src/__tests__/entity-cache.test.ts | 10 ++++++---- .../src/handler/entity-cache.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf index 7f27de168..c44077cc9 100644 --- a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf +++ b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf @@ -83,7 +83,7 @@ resource "aws_glue_catalog_table" "supplier_config_cache" { } storage_descriptor { - location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}.json" + location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}/" input_format = "org.apache.hadoop.mapred.TextInputFormat" output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index 125ff82b0..2d68709f7 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -47,7 +47,7 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Bucket).toBe("cache-bucket"); - expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); + expect(put.input.Key).toBe("supplier_config_cache/supplier/records.json"); expect(deps.logger.info).toHaveBeenCalledWith( { entityType: "supplier" }, "Loading supplier config entity", @@ -61,8 +61,8 @@ describe("populateEntityCache", () => { entityType: "supplier", itemCount: 2, bucket: "cache-bucket", - key: "supplier_config_cache/supplier.json", - s3Uri: "s3://cache-bucket/supplier_config_cache/supplier.json", + key: "supplier_config_cache/supplier/records.json", + s3Uri: "s3://cache-bucket/supplier_config_cache/supplier/records.json", }, "Created supplier config cache file", ); @@ -83,7 +83,9 @@ describe("populateEntityCache", () => { expect(count).toBe(0); const put = s3Send.mock.calls[0][0] as PutObjectCommand; - expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); + expect(put.input.Key).toBe( + "supplier_config_cache/volume-group/records.json", + ); expect(put.input.Body).toBe(""); expect(deps.logger.info).toHaveBeenCalledWith( { entityType: "volume-group", itemCount: 0, hasItems: false }, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts index 381a33856..eb4333e9a 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -45,7 +45,7 @@ export default async function populateEntityCache( "Loaded supplier config entity", ); - const key = `${CACHE_FOLDER}/${entityType}.json`; + const key = `${CACHE_FOLDER}/${entityType}/records.json`; await deps.s3Client.send( new PutObjectCommand({ Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, From acb9ce29319ead054c2f4932686285dbbc2851e5 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 16:04:23 +0100 Subject: [PATCH 07/28] amend return information from populate-supplier-config-cache --- .../api/resources/letter_sla_breaches.json | 5 +- .../populate-supplier-config-cache.test.ts | 43 ++++++++++++++-- .../handler/populate-supplier-config-cache.ts | 51 ++++++++++++++----- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json index 4544a8e26..456a54ff1 100644 --- a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json +++ b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json @@ -79,10 +79,7 @@ }, "PopulateSupplierConfigCache": { "Arguments": { - "FunctionName": "${POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN}", - "Payload": { - "date": "{% $date %}" - } + "FunctionName": "${POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN}" }, "Next": "PopulateBreachesInParallel", "Output": { diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index 54374eb2f..b2d7dab0b 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -7,10 +7,14 @@ jest.mock("../handler/entity-cache"); describe("populateSupplierConfigCache", () => { it("populates the cache once for each entity type and returns success", async () => { - const deps = {} as EntityCacheDeps; + const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; + jest.mocked(populateEntityCache).mockReset(); + for (const [index] of ENTITY_TYPES.entries()) { + jest.mocked(populateEntityCache).mockResolvedValueOnce(index + 1); + } const handler = createHandler(deps); - const result = await handler({ date: "2026-10-08" }); + const result = await handler(); expect(ENTITY_TYPES).toEqual([ "volume-group", @@ -26,7 +30,40 @@ describe("populateSupplierConfigCache", () => { } expect(result).toEqual({ status: "SUCCESS", - message: "populate-supplier-config-cache invoked successfully", + entities: ENTITY_TYPES.map((entity, index) => ({ + entity, + recordsRetrieved: index + 1, + })), + }); + }); + + it("returns failure and counts for successful entities when an entity fails", async () => { + const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; + jest + .mocked(populateEntityCache) + .mockReset() + .mockResolvedValueOnce(1) + .mockRejectedValueOnce(new Error("query failed")) + .mockResolvedValueOnce(3) + .mockResolvedValueOnce(4) + .mockResolvedValueOnce(5) + .mockResolvedValueOnce(6); + const handler = createHandler(deps); + + const result = await handler(); + + expect(result).toEqual({ + status: "FAILURE", + failureReason: + "Failed to populate supplier config cache for: supplier-pack", + entities: ENTITY_TYPES.map((entity, index) => ({ + entity, + recordsRetrieved: index === 1 ? 0 : index + 1, + })), }); + expect(deps.logger.error).toHaveBeenCalledWith( + { entityType: "supplier-pack", error: expect.any(Error) }, + "Failed to populate supplier config entity cache", + ); }); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index b3c3d57fb..cd83290a0 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -1,12 +1,12 @@ import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; -export type PopulateSupplierConfigCacheEvent = { - date: string; -}; - export type PopulateSupplierConfigCacheResult = { - status: "SUCCESS"; - message: string; + status: "SUCCESS" | "FAILURE"; + failureReason?: string; + entities: { + entity: string; + recordsRetrieved: number; + }[]; }; export const ENTITY_TYPES = [ @@ -19,16 +19,43 @@ export const ENTITY_TYPES = [ ]; export default function createHandler(deps: EntityCacheDeps) { - return async ( - _event: PopulateSupplierConfigCacheEvent, - ): Promise => { - await Promise.all( - ENTITY_TYPES.map((entityType) => populateEntityCache(deps, entityType)), + return async (): Promise => { + const results = await Promise.all( + ENTITY_TYPES.map(async (entityType) => { + try { + return { + entity: entityType, + recordsRetrieved: await populateEntityCache(deps, entityType), + failed: false, + }; + } catch (error) { + deps.logger.error( + { entityType, error }, + "Failed to populate supplier config entity cache", + ); + return { entity: entityType, recordsRetrieved: 0, failed: true }; + } + }), ); + const entities = results.map(({ entity, recordsRetrieved }) => ({ + entity, + recordsRetrieved, + })); + const failedEntities = results + .filter((result) => result.failed) + .map((result) => result.entity); + + if (failedEntities.length > 0) { + return { + status: "FAILURE", + failureReason: `Failed to populate supplier config cache for: ${failedEntities.join(", ")}`, + entities, + }; + } return { status: "SUCCESS", - message: "populate-supplier-config-cache invoked successfully", + entities, }; }; } From 41bfa405ed6bf7825469b80b1c9af437a4f5bbe1 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 16:09:42 +0100 Subject: [PATCH 08/28] alphabetise entity types --- .../src/handler/populate-supplier-config-cache.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index cd83290a0..7aa7b0953 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -10,12 +10,12 @@ export type PopulateSupplierConfigCacheResult = { }; export const ENTITY_TYPES = [ - "volume-group", - "supplier-pack", "letter-variant", "pack-specification", - "supplier-allocation", "supplier", + "supplier-allocation", + "supplier-pack", + "volume-group", ]; export default function createHandler(deps: EntityCacheDeps) { From a70b984bf42bc50ddbdec229c437f0c0c2c3832e Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 16:41:08 +0100 Subject: [PATCH 09/28] fix failing test --- .../__tests__/populate-supplier-config-cache.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index b2d7dab0b..784c75f7d 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -17,12 +17,12 @@ describe("populateSupplierConfigCache", () => { const result = await handler(); expect(ENTITY_TYPES).toEqual([ - "volume-group", - "supplier-pack", "letter-variant", "pack-specification", - "supplier-allocation", "supplier", + "supplier-allocation", + "supplier-pack", + "volume-group", ]); expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); for (const entityType of ENTITY_TYPES) { @@ -55,14 +55,14 @@ describe("populateSupplierConfigCache", () => { expect(result).toEqual({ status: "FAILURE", failureReason: - "Failed to populate supplier config cache for: supplier-pack", + "Failed to populate supplier config cache for: pack-specification", entities: ENTITY_TYPES.map((entity, index) => ({ entity, recordsRetrieved: index === 1 ? 0 : index + 1, })), }); expect(deps.logger.error).toHaveBeenCalledWith( - { entityType: "supplier-pack", error: expect.any(Error) }, + { entityType: "pack-specification", error: expect.any(Error) }, "Failed to populate supplier config entity cache", ); }); From 5fb19f550636b38b383f1080b1d29da2d1aa1858 Mon Sep 17 00:00:00 2001 From: Mark Slowey <113013138+masl2@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:47:30 +0100 Subject: [PATCH 10/28] CCM-24327: parquet catalog (#734) --- docs/generate-includes.sh | 1 + .../terraform/components/api/README.md | 4 + .../api/athena_workgroup_reporting.tf | 19 ++ .../glue_catalog_table_eventpub_parquet.tf | 91 +++++++ ...rehose_delivery_stream_eventpub_parquet.tf | 241 ++++++++++++++++++ ...ule_lambda_eventpub_parquet_transformer.tf | 35 +++ .../api/module_s3bucket_reporting.tf | 79 ++++++ .../terraform/components/api/variables.tf | 12 + .../eventpub-parquet-transformer/README.md | 17 ++ .../jest.config.ts | 63 +++++ .../eventpub-parquet-transformer/package.json | 16 ++ .../src/__tests__/transform-record.test.ts | 91 +++++++ .../eventpub-parquet-transformer/src/index.ts | 7 + .../src/transform-record.ts | 82 ++++++ .../tsconfig.json | 10 + package-lock.json | 12 + 16 files changed, 780 insertions(+) create mode 100644 infrastructure/terraform/components/api/athena_workgroup_reporting.tf create mode 100644 infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf create mode 100644 infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf create mode 100644 infrastructure/terraform/components/api/module_s3bucket_reporting.tf create mode 100644 lambdas/eventpub-parquet-transformer/README.md create mode 100644 lambdas/eventpub-parquet-transformer/jest.config.ts create mode 100644 lambdas/eventpub-parquet-transformer/package.json create mode 100644 lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts create mode 100644 lambdas/eventpub-parquet-transformer/src/index.ts create mode 100644 lambdas/eventpub-parquet-transformer/src/transform-record.ts create mode 100644 lambdas/eventpub-parquet-transformer/tsconfig.json diff --git a/docs/generate-includes.sh b/docs/generate-includes.sh index 75b217a97..2df5e62c4 100755 --- a/docs/generate-includes.sh +++ b/docs/generate-includes.sh @@ -24,6 +24,7 @@ cp ./lambdas/upsert-letter/README.md ./docs/_includes/components/generated/readm cp ./lambdas/update-letter-queue/README.md ./docs/_includes/components/generated/readmes/lambda-update-letter-queue.md cp ./lambdas/letter-updates-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-letter-updates-transformer.md cp ./lambdas/mi-updates-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-mi-updates-transformer.md +cp ./lambdas/eventpub-parquet-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-eventpub-parquet-transformer.md cp ./lambdas/supplier-config-ingress/README.md ./docs/_includes/components/generated/readmes/lambda-supplier-config-ingress.md # Function documentation (internal packages) diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 2d1508de4..48f35a193 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -53,6 +53,8 @@ No requirements. | [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no | | [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes | | [region](#input\_region) | The AWS Region | `string` | n/a | yes | +| [reports\_data\_retention\_non\_current\_days](#input\_reports\_data\_retention\_non\_current\_days) | Number of days to retain noncurrent reporting bucket object versions | `number` | `14` | no | +| [reports\_data\_retention\_policy\_days](#input\_reports\_data\_retention\_policy\_days) | Number of days to retain reporting bucket data | `number` | `90` | no | | [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Account ID of the shared infrastructure account | `string` | `"000000000000"` | no | | [sns\_success\_logging\_sample\_percent](#input\_sns\_success\_logging\_sample\_percent) | Enable SNS Delivery Successful Sample Percentage | `number` | `0` | no | | [supplier\_mock\_schedule\_expression](#input\_supplier\_mock\_schedule\_expression) | Schedule expression for supplier mock scheduler | `string` | `"rate(1 minute)"` | no | @@ -70,6 +72,7 @@ No requirements. | [ddb\_alarms\_suppliers](#module\_ddb\_alarms\_suppliers) | ../../modules/alarms-ddb | n/a | | [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a | | [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/5.0.6/terraform-eventpub.zip | n/a | +| [eventpub\_parquet\_transformer](#module\_eventpub\_parquet\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a | | [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | @@ -87,6 +90,7 @@ No requirements. | [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [s3bucket\_reporting](#module\_s3bucket\_reporting) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip | n/a | | [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a | | [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a | | [sqs\_letter\_updates](#module\_sqs\_letter\_updates) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | diff --git a/infrastructure/terraform/components/api/athena_workgroup_reporting.tf b/infrastructure/terraform/components/api/athena_workgroup_reporting.tf new file mode 100644 index 000000000..08d7fa3ea --- /dev/null +++ b/infrastructure/terraform/components/api/athena_workgroup_reporting.tf @@ -0,0 +1,19 @@ +resource "aws_athena_workgroup" "reporting" { + name = local.csi + description = "Supplier API Athena Workgroup for ${var.environment}" + force_destroy = true + + configuration { + enforce_workgroup_configuration = true + + result_configuration { + expected_bucket_owner = var.aws_account_id + output_location = "s3://${module.s3bucket_reporting.bucket}/athena-output/" + + encryption_configuration { + encryption_option = "SSE_KMS" + kms_key_arn = module.kms.key_arn + } + } + } +} diff --git a/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf new file mode 100644 index 000000000..630d3233b --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf @@ -0,0 +1,91 @@ +locals { + eventpub_parquet_enabled = local.event_cache_bucket_name != null + + # Column names must be lowercase; OpenXJsonSerDe matches JSON keys case-insensitively. + eventpub_parquet_table_columns = [ + { name = "specversion", type = "string" }, + { name = "id", type = "string" }, + { name = "type", type = "string" }, + { name = "plane", type = "string" }, + { name = "dataschema", type = "string" }, + { name = "dataschemaversion", type = "string" }, + { name = "source", type = "string" }, + { name = "subject", type = "string" }, + { name = "time", type = "string" }, + { name = "recordedtime", type = "string" }, + { name = "datacontenttype", type = "string" }, + { name = "traceparent", type = "string" }, + { name = "tracestate", type = "string" }, + { name = "partitionkey", type = "string" }, + { name = "sampledrate", type = "bigint" }, + { name = "sequence", type = "string" }, + { name = "severitytext", type = "string" }, + { name = "data_json", type = "string" }, + { name = "domainid", type = "string" }, + { name = "supplierid", type = "string" }, + { name = "groupid", type = "string" }, + { name = "campaignid", type = "string" }, + { name = "clientid", type = "string" }, + ] +} + +resource "aws_glue_catalog_table" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + name = "${replace(local.csi, "-", "_")}_eventpub" + database_name = aws_glue_catalog_database.supplier.name + table_type = "EXTERNAL_TABLE" + + parameters = { + EXTERNAL = "TRUE" + classification = "parquet" + "parquet.compress" = "SNAPPY" + + # Athena derives partitions from these properties, so none are registered in Glue + "projection.enabled" = "true" + "projection.__year.type" = "integer" + "projection.__year.range" = "2025,2050" + "projection.__year.digits" = "4" + "projection.__month.type" = "integer" + "projection.__month.range" = "1,12" + "projection.__month.digits" = "2" + "projection.__day.type" = "integer" + "projection.__day.range" = "1,31" + "projection.__day.digits" = "2" + "storage.location.template" = "s3://${module.s3bucket_reporting.bucket}/parquet/eventpub/__year=$${__year}/__month=$${__month}/__day=$${__day}/" + } + + storage_descriptor { + location = "s3://${module.s3bucket_reporting.bucket}/parquet/eventpub/" + input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" + + ser_de_info { + name = "ParquetHiveSerDe" + serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + } + + dynamic "columns" { + for_each = local.eventpub_parquet_table_columns + + content { + name = columns.value.name + type = columns.value.type + } + } + } + + partition_keys { + name = "__year" + type = "string" + } + + partition_keys { + name = "__month" + type = "string" + } + + partition_keys { + name = "__day" + type = "string" + } +} diff --git a/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf new file mode 100644 index 000000000..e5227e5cc --- /dev/null +++ b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf @@ -0,0 +1,241 @@ +resource "aws_kinesis_firehose_delivery_stream" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet" + destination = "extended_s3" + + server_side_encryption { + enabled = true + key_type = "CUSTOMER_MANAGED_CMK" + key_arn = module.kms.key_arn + } + + extended_s3_configuration { + role_arn = aws_iam_role.eventpub_parquet_firehose[0].arn + bucket_arn = module.s3bucket_reporting.arn + + prefix = "parquet/eventpub/__year=!{timestamp:yyyy}/__month=!{timestamp:MM}/__day=!{timestamp:dd}/" + error_output_prefix = "parquet/errors/eventpub/!{firehose:error-output-type}/!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}/" + + buffering_size = 128 + buffering_interval = 300 + + processing_configuration { + enabled = true + + processors { + type = "Lambda" + + parameters { + parameter_name = "LambdaArn" + parameter_value = "${module.eventpub_parquet_transformer.function_arn}:$LATEST" + } + parameters { + parameter_name = "RoleArn" + parameter_value = aws_iam_role.eventpub_parquet_firehose[0].arn + } + } + } + + data_format_conversion_configuration { + input_format_configuration { + deserializer { + open_x_json_ser_de {} + } + } + + output_format_configuration { + serializer { + parquet_ser_de {} + } + } + + schema_configuration { + database_name = aws_glue_catalog_database.supplier.name + role_arn = aws_iam_role.eventpub_parquet_firehose[0].arn + table_name = aws_glue_catalog_table.eventpub_parquet[0].name + } + } + + cloudwatch_logging_options { + enabled = true + log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name + log_stream_name = aws_cloudwatch_log_stream.eventpub_parquet_firehose[0].name + } + } +} + +resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "/aws/firehose/${local.csi}-eventpub-parquet" + kms_key_id = module.kms.key_arn + retention_in_days = var.log_retention_in_days +} + +resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "extended_s3" + log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name +} + +# Raw delivery so Firehose receives the bare event rather than the SNS envelope +resource "aws_sns_topic_subscription" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + topic_arn = module.eventpub.sns_topic.arn + protocol = "firehose" + subscription_role_arn = aws_iam_role.eventpub_parquet_sns[0].arn + endpoint = aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn + raw_message_delivery = true +} + +resource "aws_iam_role" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + role = aws_iam_role.eventpub_parquet_firehose[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_firehose[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowDestinationS3Access" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/parquet/*", + ] + } + + statement { + sid = "AllowKms" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + ] + + resources = [module.kms.key_arn] + } + + statement { + sid = "AllowInvokeTransformer" + effect = "Allow" + + actions = [ + "lambda:InvokeFunction", + "lambda:GetFunctionConfiguration", + ] + + resources = ["${module.eventpub_parquet_transformer.function_arn}:$LATEST"] + } + + statement { + sid = "AllowGlueSchemaRead" + effect = "Allow" + + actions = [ + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + ] + + resources = [ + aws_glue_catalog_table.eventpub_parquet[0].arn, + aws_glue_catalog_database.supplier.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + ] + } + + statement { + sid = "AllowLogging" + effect = "Allow" + actions = ["logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.eventpub_parquet_firehose[0].arn}:*"] + } +} + +resource "aws_iam_role" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + role = aws_iam_role.eventpub_parquet_sns[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_sns[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowFirehoseDelivery" + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn] + } + + statement { + sid = "AllowKms" + effect = "Allow" + actions = ["kms:GenerateDataKey", "kms:Decrypt"] + resources = [module.kms.key_arn] + } +} diff --git a/infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf b/infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf new file mode 100644 index 000000000..4f0aa4be2 --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf @@ -0,0 +1,35 @@ +module "eventpub_parquet_transformer" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "eventpub-parquet-transformer" + description = "Flattens EventPub events for parquet delivery by Firehose" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "eventpub-parquet-transformer/dist" + function_include_common = true + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = 256 + timeout = 60 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = local.common_lambda_env_vars +} diff --git a/infrastructure/terraform/components/api/module_s3bucket_reporting.tf b/infrastructure/terraform/components/api/module_s3bucket_reporting.tf new file mode 100644 index 000000000..f3e3d0a35 --- /dev/null +++ b/infrastructure/terraform/components/api/module_s3bucket_reporting.tf @@ -0,0 +1,79 @@ +module "s3bucket_reporting" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip" + + name = "supapi-reporting" + + aws_account_id = var.aws_account_id + region = var.region + project = var.project + environment = var.environment + component = var.component + + kms_key_arn = module.kms.key_arn + + policy_documents = [data.aws_iam_policy_document.s3bucket_reporting.json] + + force_destroy = var.force_destroy + + lifecycle_rules = [ + { + prefix = "athena-output" + enabled = true + + expiration = { + days = var.reports_data_retention_policy_days + } + + noncurrent_version_expiration = { + noncurrent_days = var.reports_data_retention_non_current_days + } + }, + ] + + default_tags = { + NHSE-Enable-S3-Backup-Acct = "True", + } +} + +data "aws_iam_policy_document" "s3bucket_reporting" { + statement { + sid = "AllowManagedAccountsToList" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + module.s3bucket_reporting.arn, + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } + + statement { + sid = "AllowManagedAccountsToGet" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + + resources = [ + "${module.s3bucket_reporting.arn}/*", + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } +} diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index 8bc9eedde..8a22e264d 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -291,3 +291,15 @@ variable "csoc_destination_account" { description = "value of the CSOC destination account, if applicable. If null, CSOC destination account will not be added as a resource in the logging policy" default = "000000000000" } + +variable "reports_data_retention_policy_days" { + type = number + description = "Number of days to retain reporting bucket data" + default = 90 +} + +variable "reports_data_retention_non_current_days" { + type = number + description = "Number of days to retain noncurrent reporting bucket object versions" + default = 14 +} diff --git a/lambdas/eventpub-parquet-transformer/README.md b/lambdas/eventpub-parquet-transformer/README.md new file mode 100644 index 000000000..eb1a5c458 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/README.md @@ -0,0 +1,17 @@ + + +# EventPub Parquet Transformer Lambda + +## Purpose + +Firehose data-transformation processor for the EventPub parquet delivery stream. Flattens each CloudEvent so Firehose can convert it to Parquet for Athena. + +## General Flow + +1. Firehose invokes the Lambda with a batch of base64 encoded CloudEvents. +2. Each event is parsed and returned as a flat JSON record: the envelope attributes, the raw `data` payload as `data_json`, and the promoted `domainid`, `supplierid`, `groupid`, `campaignid` and `clientid` properties. +3. Records that are not valid JSON objects are returned as `ProcessingFailed` and are written to the stream's error prefix. + +## Key Integration Points + +- **Kinesis Data Firehose**: the `eventpub-parquet` delivery stream. diff --git a/lambdas/eventpub-parquet-transformer/jest.config.ts b/lambdas/eventpub-parquet-transformer/jest.config.ts new file mode 100644 index 000000000..93ec73a53 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/jest.config.ts @@ -0,0 +1,63 @@ +import type { Config } from "jest"; + +export const baseJestConfig: Config = { + preset: "ts-jest", + + // Automatically clear mock calls, instances, contexts and results before every test + clearMocks: true, + + // Indicates whether the coverage information should be collected while executing the test + collectCoverage: true, + + // The directory where Jest should output its coverage files + coverageDirectory: "./.reports/unit/coverage", + + // Indicates which provider should be used to instrument code for coverage + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + transform: { "^.+\\.ts$": "ts-jest" }, + transformIgnorePatterns: [ + "node_modules/(?!(@nhsdigital/nhs-notify-event-schemas-supplier-config)/)", + ], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + // Use this configuration option to add custom reporters to Jest + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + // The test environment that will be used for testing + testEnvironment: "jsdom", +}; + +const utilsJestConfig = { + ...baseJestConfig, + + testEnvironment: "node", + + coveragePathIgnorePatterns: [ + ...(baseJestConfig.coveragePathIgnorePatterns ?? []), + "zod-validators.ts", + ], +}; + +export default utilsJestConfig; diff --git a/lambdas/eventpub-parquet-transformer/package.json b/lambdas/eventpub-parquet-transformer/package.json new file mode 100644 index 000000000..7512fb336 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/package.json @@ -0,0 +1,16 @@ +{ + "dependencies": { + "aws-lambda": "^1.0.7", + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-eventpub-parquet-transformer", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts new file mode 100644 index 000000000..48db8e4b5 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts @@ -0,0 +1,91 @@ +import { FirehoseTransformationEvent } from "aws-lambda"; +import { handler } from ".."; +import transformRecord from "../transform-record"; + +const encode = (value: unknown) => + Buffer.from( + typeof value === "string" ? value : JSON.stringify(value), + "utf8", + ).toString("base64"); + +const decode = (data: string) => + JSON.parse(Buffer.from(data, "base64").toString("utf8")); + +describe("transformRecord", () => { + it("flattens the envelope, keeps data as JSON and promotes properties", () => { + const data = { + domainId: "d1", + supplierId: "s1", + groupId: "g1", + campaignId: "c1", + clientId: "cl1", + status: "PRINTED", + }; + + const out = transformRecord({ + recordId: "r1", + data: encode({ id: "e1", type: "t", time: "now", data }), + }); + + expect(out.result).toBe("Ok"); + expect(out.recordId).toBe("r1"); + expect(Buffer.from(out.data!, "base64").toString().endsWith("\n")).toBe( + true, + ); + expect(decode(out.data!)).toEqual({ + id: "e1", + type: "t", + time: "now", + data_json: JSON.stringify(data), + domainid: "d1", + supplierid: "s1", + groupid: "g1", + campaignid: "c1", + clientid: "cl1", + }); + }); + + it("leaves promoted properties empty when data is missing or not an object", () => { + const missing = decode( + transformRecord({ recordId: "r", data: encode({ id: "e" }) }).data!, + ); + const scalar = decode( + transformRecord({ recordId: "r", data: encode({ id: "e", data: 1 }) }) + .data!, + ); + + expect(missing).toEqual({ id: "e", data_json: "null" }); + expect(scalar).toEqual({ id: "e", data_json: "1" }); + }); + + it.each([["not json"], ["[1]"], ["null"]])( + "returns ProcessingFailed for %s", + (payload) => { + const record = { recordId: "r", data: encode(payload) }; + + expect(transformRecord(record)).toEqual({ + recordId: "r", + result: "ProcessingFailed", + data: record.data, + }); + }, + ); +}); + +describe("handler", () => { + it("transforms every record in the batch", async () => { + const event = { + records: [ + { recordId: "1", data: encode({ id: "a" }) }, + { recordId: "2", data: encode("bad") }, + ], + } as unknown as FirehoseTransformationEvent; + + const res = await handler(event, {} as never, () => {}); + + expect(res!.records.map((r) => r.result)).toEqual([ + "Ok", + "ProcessingFailed", + ]); + }); +}); diff --git a/lambdas/eventpub-parquet-transformer/src/index.ts b/lambdas/eventpub-parquet-transformer/src/index.ts new file mode 100644 index 000000000..b310292f4 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/src/index.ts @@ -0,0 +1,7 @@ +import { FirehoseTransformationHandler } from "aws-lambda"; +import transformRecord from "./transform-record"; + +// eslint-disable-next-line import-x/prefer-default-export +export const handler: FirehoseTransformationHandler = async (event) => ({ + records: event.records.map((record) => transformRecord(record)), +}); diff --git a/lambdas/eventpub-parquet-transformer/src/transform-record.ts b/lambdas/eventpub-parquet-transformer/src/transform-record.ts new file mode 100644 index 000000000..97a10a1b5 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/src/transform-record.ts @@ -0,0 +1,82 @@ +import type { + FirehoseRecordTransformationStatus, + FirehoseTransformationResultRecord, +} from "aws-lambda"; + +const ENVELOPE_FIELDS = [ + "specversion", + "id", + "type", + "plane", + "dataschema", + "dataschemaversion", + "source", + "subject", + "time", + "recordedtime", + "datacontenttype", + "traceparent", + "tracestate", + "partitionkey", + "sampledrate", + "sequence", + "severitytext", +] as const; + +const PROMOTED_DATA_FIELDS = [ + "domainId", + "supplierId", + "groupId", + "campaignId", + "clientId", +] as const; + +type Json = Record; + +function isObject(value: unknown): value is Json { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function flatten(event: Json): Json { + const data = isObject(event.data) ? event.data : {}; + + const flat: Json = {}; + for (const field of ENVELOPE_FIELDS) { + flat[field] = event[field]; + } + flat.data_json = JSON.stringify(event.data ?? null); + for (const field of PROMOTED_DATA_FIELDS) { + flat[field.toLowerCase()] = data[field]; + } + return flat; +} + +function result( + recordId: string, + status: FirehoseRecordTransformationStatus, + data: string, +): FirehoseTransformationResultRecord { + return { recordId, result: status, data }; +} + +export default function transformRecord(record: { + recordId: string; + data: string; +}): FirehoseTransformationResultRecord { + try { + const event: unknown = JSON.parse( + Buffer.from(record.data, "base64").toString("utf8"), + ); + if (!isObject(event)) { + return result(record.recordId, "ProcessingFailed", record.data); + } + const output = `${JSON.stringify(flatten(event))}\n`; + return result( + record.recordId, + "Ok", + Buffer.from(output, "utf8").toString("base64"), + ); + } catch { + return result(record.recordId, "ProcessingFailed", record.data); + } +} diff --git a/lambdas/eventpub-parquet-transformer/tsconfig.json b/lambdas/eventpub-parquet-transformer/tsconfig.json new file mode 100644 index 000000000..bb8177b74 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "esModuleInterop": true + }, + "extends": "../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index 2d80007dc..c13521172 100644 --- a/package-lock.json +++ b/package-lock.json @@ -396,6 +396,14 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, + "lambdas/eventpub-parquet-transformer": { + "name": "nhs-notify-supplier-api-eventpub-parquet-transformer", + "version": "0.0.1", + "dependencies": { + "aws-lambda": "^1.0.7", + "esbuild": "0.28.2" + } + }, "lambdas/letter-sla-breaches/populate-sameday-breaches": { "name": "nhs-notify-supplier-api-populate-sameday-breaches", "version": "0.0.1", @@ -12124,6 +12132,10 @@ "resolved": "lambdas/supplier-allocator", "link": true }, + "node_modules/nhs-notify-supplier-api-eventpub-parquet-transformer": { + "resolved": "lambdas/eventpub-parquet-transformer", + "link": true + }, "node_modules/nhs-notify-supplier-api-handler": { "resolved": "lambdas/api-handler", "link": true From 1010ed96753c2487d891b430c26d9de338dad51b Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:06:26 +0100 Subject: [PATCH 11/28] follow zod schema pattern for env variables in breach config lambda --- .../package.json | 3 +- .../src/__tests__/env.test.ts | 48 +++++++++++++++++++ .../src/handler/deps.ts | 7 +-- .../src/handler/env.ts | 11 +++++ package-lock.json | 3 +- 5 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json index b10a522d2..cd62c3eca 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -5,7 +5,8 @@ "@aws-sdk/lib-dynamodb": "^3.1138.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", - "pino": "^10.3.1" + "pino": "^10.3.1", + "zod": "^4.6.5" }, "name": "nhs-notify-supplier-api-populate-supplier-config-cache", "private": true, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts new file mode 100644 index 000000000..ee6aafe5c --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/no-require-imports */ +/* Allow require imports to enable re-import of modules */ + +import { ZodError } from "zod"; + +describe("lambdaEnv", () => { + const OLD_ENV = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { ...OLD_ENV }; + }); + + afterAll(() => { + process.env = OLD_ENV; + }); + + it("loads required and optional environment variables", () => { + process.env.SUPPLIER_CONFIG_TABLE_NAME = "supplier-config-table"; + process.env.EVENT_CACHE_BUCKET_NAME = "event-cache-bucket"; + process.env.PINO_LOG_LEVEL = "debug"; + + const { envVars } = require("../handler/env"); + + expect(envVars).toEqual({ + SUPPLIER_CONFIG_TABLE_NAME: "supplier-config-table", + EVENT_CACHE_BUCKET_NAME: "event-cache-bucket", + PINO_LOG_LEVEL: "debug", + }); + }); + + it("throws when a required environment variable is missing", () => { + process.env.SUPPLIER_CONFIG_TABLE_NAME = undefined; + process.env.EVENT_CACHE_BUCKET_NAME = "event-cache-bucket"; + + expect(() => require("../handler/env")).toThrow(ZodError); + }); + + it("allows the optional log level to be unset", () => { + process.env.SUPPLIER_CONFIG_TABLE_NAME = "supplier-config-table"; + process.env.EVENT_CACHE_BUCKET_NAME = "event-cache-bucket"; + process.env.PINO_LOG_LEVEL = undefined; + + const { envVars } = require("../handler/env"); + + expect(envVars.PINO_LOG_LEVEL).toBeUndefined(); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts index ced5d4c5e..9be803265 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts @@ -3,15 +3,16 @@ import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; import { createLogger } from "@internal/helpers"; import { EntityCacheDeps } from "./entity-cache"; +import { envVars } from "./env"; export default function createDependenciesContainer(): EntityCacheDeps { return { ddbClient: DynamoDBDocumentClient.from(new DynamoDBClient()), s3Client: new S3Client(), - logger: createLogger({ logLevel: process.env.PINO_LOG_LEVEL }), + logger: createLogger({ logLevel: envVars.PINO_LOG_LEVEL }), env: { - SUPPLIER_CONFIG_TABLE_NAME: process.env.SUPPLIER_CONFIG_TABLE_NAME!, - EVENT_CACHE_BUCKET_NAME: process.env.EVENT_CACHE_BUCKET_NAME!, + SUPPLIER_CONFIG_TABLE_NAME: envVars.SUPPLIER_CONFIG_TABLE_NAME, + EVENT_CACHE_BUCKET_NAME: envVars.EVENT_CACHE_BUCKET_NAME, }, }; } diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts new file mode 100644 index 000000000..03be4f276 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts @@ -0,0 +1,11 @@ +import { z } from "zod"; + +const EnvVarsSchema = z.object({ + SUPPLIER_CONFIG_TABLE_NAME: z.string(), + EVENT_CACHE_BUCKET_NAME: z.string(), + PINO_LOG_LEVEL: z.string().optional(), +}); + +export type EnvVars = z.infer; + +export const envVars = EnvVarsSchema.parse(process.env); diff --git a/package-lock.json b/package-lock.json index c13521172..969f956f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -427,7 +427,8 @@ "@aws-sdk/lib-dynamodb": "^3.1138.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", - "pino": "^10.3.1" + "pino": "^10.3.1", + "zod": "^4.6.5" } }, "lambdas/letter-updates-transformer": { From 23ced3286877f285c27fa98b22f923f83b735cbd Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:20:05 +0100 Subject: [PATCH 12/28] retrieve entity type from internal datastore --- .../package.json | 1 + .../populate-supplier-config-cache.test.ts | 27 +++++++++++-------- .../handler/populate-supplier-config-cache.ts | 10 ++----- package-lock.json | 1 + 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json index cd62c3eca..5667e2485 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -3,6 +3,7 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", + "@internal/datastore": "^0.1.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", "pino": "^10.3.1", diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index 784c75f7d..a867152b6 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -1,8 +1,21 @@ +import { $SupplierConfigEntity } from "@internal/datastore"; import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; import createHandler, { ENTITY_TYPES, } from "../handler/populate-supplier-config-cache"; +jest.mock("@internal/datastore", () => ({ + $SupplierConfigEntity: { + options: [ + "letter-variant", + "volume-group", + "supplier-allocation", + "supplier", + "pack-specification", + "supplier-pack", + ], + }, +})); jest.mock("../handler/entity-cache"); describe("populateSupplierConfigCache", () => { @@ -16,14 +29,7 @@ describe("populateSupplierConfigCache", () => { const result = await handler(); - expect(ENTITY_TYPES).toEqual([ - "letter-variant", - "pack-specification", - "supplier", - "supplier-allocation", - "supplier-pack", - "volume-group", - ]); + expect(ENTITY_TYPES).toEqual($SupplierConfigEntity.options); expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); for (const entityType of ENTITY_TYPES) { expect(populateEntityCache).toHaveBeenCalledWith(deps, entityType); @@ -54,15 +60,14 @@ describe("populateSupplierConfigCache", () => { expect(result).toEqual({ status: "FAILURE", - failureReason: - "Failed to populate supplier config cache for: pack-specification", + failureReason: `Failed to populate supplier config cache for: ${ENTITY_TYPES[1]}`, entities: ENTITY_TYPES.map((entity, index) => ({ entity, recordsRetrieved: index === 1 ? 0 : index + 1, })), }); expect(deps.logger.error).toHaveBeenCalledWith( - { entityType: "pack-specification", error: expect.any(Error) }, + { entityType: ENTITY_TYPES[1], error: expect.any(Error) }, "Failed to populate supplier config entity cache", ); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 7aa7b0953..5eff5a1a4 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -1,3 +1,4 @@ +import { $SupplierConfigEntity } from "@internal/datastore"; import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; export type PopulateSupplierConfigCacheResult = { @@ -9,14 +10,7 @@ export type PopulateSupplierConfigCacheResult = { }[]; }; -export const ENTITY_TYPES = [ - "letter-variant", - "pack-specification", - "supplier", - "supplier-allocation", - "supplier-pack", - "volume-group", -]; +export const ENTITY_TYPES = $SupplierConfigEntity.options; export default function createHandler(deps: EntityCacheDeps) { return async (): Promise => { diff --git a/package-lock.json b/package-lock.json index 969f956f7..0912227cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -425,6 +425,7 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", + "@internal/datastore": "^0.1.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", "pino": "^10.3.1", From e43b8544723a804ed4db386110af1f585d93234a Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:40:33 +0100 Subject: [PATCH 13/28] remove entities from return of config cache lambda --- .../populate-sameday-breaches/tsconfig.json | 3 +-- .../__tests__/populate-supplier-config-cache.test.ts | 10 +--------- .../src/handler/populate-supplier-config-cache.ts | 6 ------ 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json index 4a11b255a..f4059d5e6 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json @@ -1,7 +1,6 @@ { "compilerOptions": { - "esModuleInterop": true, - "ignoreDeprecations": "5.0" + "esModuleInterop": true }, "extends": "../../../tsconfig.base.json", "include": [ diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index a867152b6..911f78f7f 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -36,14 +36,10 @@ describe("populateSupplierConfigCache", () => { } expect(result).toEqual({ status: "SUCCESS", - entities: ENTITY_TYPES.map((entity, index) => ({ - entity, - recordsRetrieved: index + 1, - })), }); }); - it("returns failure and counts for successful entities when an entity fails", async () => { + it("returns failure when an entity fails", async () => { const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; jest .mocked(populateEntityCache) @@ -61,10 +57,6 @@ describe("populateSupplierConfigCache", () => { expect(result).toEqual({ status: "FAILURE", failureReason: `Failed to populate supplier config cache for: ${ENTITY_TYPES[1]}`, - entities: ENTITY_TYPES.map((entity, index) => ({ - entity, - recordsRetrieved: index === 1 ? 0 : index + 1, - })), }); expect(deps.logger.error).toHaveBeenCalledWith( { entityType: ENTITY_TYPES[1], error: expect.any(Error) }, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 5eff5a1a4..17b87022a 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -4,10 +4,6 @@ import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; export type PopulateSupplierConfigCacheResult = { status: "SUCCESS" | "FAILURE"; failureReason?: string; - entities: { - entity: string; - recordsRetrieved: number; - }[]; }; export const ENTITY_TYPES = $SupplierConfigEntity.options; @@ -43,13 +39,11 @@ export default function createHandler(deps: EntityCacheDeps) { return { status: "FAILURE", failureReason: `Failed to populate supplier config cache for: ${failedEntities.join(", ")}`, - entities, }; } return { status: "SUCCESS", - entities, }; }; } From 027236f5b16816a1f11e1020f616a8a83c5443de Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:56:21 +0100 Subject: [PATCH 14/28] missing newlines and terraform restructuring --- eslint.config.mjs | 3 +- ...tch_log_group_eventpub_parquet_firehose.tf | 7 + ...ch_log_stream_eventpub_parquet_firehose.tf | 6 + .../glue_catalog_table_eventpub_parquet.tf | 31 ---- .../api/iam_role_eventpub_parquet_firehose.tf | 18 ++ .../api/iam_role_eventpub_parquet_sns.tf | 18 ++ ...m_role_policy_eventpub_parquet_firehose.tf | 79 ++++++++ .../iam_role_policy_eventpub_parquet_sns.tf | 30 +++ ...rehose_delivery_stream_eventpub_parquet.tf | 175 ------------------ .../terraform/components/api/locals.tf | 29 +++ ...sns_topic_subscription_eventpub_parquet.tf | 10 + package.json | 3 +- 12 files changed, 201 insertions(+), 208 deletions(-) create mode 100644 infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf create mode 100644 infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf create mode 100644 infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf diff --git a/eslint.config.mjs b/eslint.config.mjs index 029ba22cd..65dc77b54 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -115,7 +115,8 @@ export default defineConfig([ eslintImportResolverTypescript.createTypeScriptImportResolver({ project: [ 'lambdas/*/tsconfig.json', - 'lambdas/*/*/tsconfig.json', 'tests/tsconfig.json', + 'lambdas/*/*/tsconfig.json', + 'tests/tsconfig.json', 'internal/*/tsconfig.json', ], }), diff --git a/infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..95a9fc627 --- /dev/null +++ b/infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf @@ -0,0 +1,7 @@ +resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "/aws/firehose/${local.csi}-eventpub-parquet" + kms_key_id = module.kms.key_arn + retention_in_days = var.log_retention_in_days +} diff --git a/infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..e634886b4 --- /dev/null +++ b/infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf @@ -0,0 +1,6 @@ +resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "extended_s3" + log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name +} diff --git a/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf index 630d3233b..233e27a0c 100644 --- a/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf +++ b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf @@ -1,34 +1,3 @@ -locals { - eventpub_parquet_enabled = local.event_cache_bucket_name != null - - # Column names must be lowercase; OpenXJsonSerDe matches JSON keys case-insensitively. - eventpub_parquet_table_columns = [ - { name = "specversion", type = "string" }, - { name = "id", type = "string" }, - { name = "type", type = "string" }, - { name = "plane", type = "string" }, - { name = "dataschema", type = "string" }, - { name = "dataschemaversion", type = "string" }, - { name = "source", type = "string" }, - { name = "subject", type = "string" }, - { name = "time", type = "string" }, - { name = "recordedtime", type = "string" }, - { name = "datacontenttype", type = "string" }, - { name = "traceparent", type = "string" }, - { name = "tracestate", type = "string" }, - { name = "partitionkey", type = "string" }, - { name = "sampledrate", type = "bigint" }, - { name = "sequence", type = "string" }, - { name = "severitytext", type = "string" }, - { name = "data_json", type = "string" }, - { name = "domainid", type = "string" }, - { name = "supplierid", type = "string" }, - { name = "groupid", type = "string" }, - { name = "campaignid", type = "string" }, - { name = "clientid", type = "string" }, - ] -} - resource "aws_glue_catalog_table" "eventpub_parquet" { count = local.eventpub_parquet_enabled ? 1 : 0 name = "${replace(local.csi, "-", "_")}_eventpub" diff --git a/infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..c39074016 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf @@ -0,0 +1,18 @@ +resource "aws_iam_role" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf new file mode 100644 index 000000000..50363d68b --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf @@ -0,0 +1,18 @@ +resource "aws_iam_role" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..ae8273301 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf @@ -0,0 +1,79 @@ +resource "aws_iam_role_policy" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + role = aws_iam_role.eventpub_parquet_firehose[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_firehose[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowDestinationS3Access" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/parquet/*", + ] + } + + statement { + sid = "AllowKms" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + ] + + resources = [module.kms.key_arn] + } + + statement { + sid = "AllowInvokeTransformer" + effect = "Allow" + + actions = [ + "lambda:InvokeFunction", + "lambda:GetFunctionConfiguration", + ] + + resources = ["${module.eventpub_parquet_transformer.function_arn}:$LATEST"] + } + + statement { + sid = "AllowGlueSchemaRead" + effect = "Allow" + + actions = [ + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + ] + + resources = [ + aws_glue_catalog_table.eventpub_parquet[0].arn, + aws_glue_catalog_database.supplier.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + ] + } + + statement { + sid = "AllowLogging" + effect = "Allow" + actions = ["logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.eventpub_parquet_firehose[0].arn}:*"] + } +} diff --git a/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf new file mode 100644 index 000000000..b7e41a764 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf @@ -0,0 +1,30 @@ +resource "aws_iam_role_policy" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + role = aws_iam_role.eventpub_parquet_sns[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_sns[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowFirehoseDelivery" + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn] + } + + statement { + sid = "AllowKms" + effect = "Allow" + actions = ["kms:GenerateDataKey", "kms:Decrypt"] + resources = [module.kms.key_arn] + } +} diff --git a/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf index e5227e5cc..4dcdec6e2 100644 --- a/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf +++ b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf @@ -64,178 +64,3 @@ resource "aws_kinesis_firehose_delivery_stream" "eventpub_parquet" { } } } - -resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "/aws/firehose/${local.csi}-eventpub-parquet" - kms_key_id = module.kms.key_arn - retention_in_days = var.log_retention_in_days -} - -resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "extended_s3" - log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name -} - -# Raw delivery so Firehose receives the bare event rather than the SNS envelope -resource "aws_sns_topic_subscription" "eventpub_parquet" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - topic_arn = module.eventpub.sns_topic.arn - protocol = "firehose" - subscription_role_arn = aws_iam_role.eventpub_parquet_sns[0].arn - endpoint = aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn - raw_message_delivery = true -} - -resource "aws_iam_role" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-firehose" - assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json -} - -data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["firehose.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-firehose" - role = aws_iam_role.eventpub_parquet_firehose[0].id - policy = data.aws_iam_policy_document.eventpub_parquet_firehose[0].json -} - -data "aws_iam_policy_document" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - statement { - sid = "AllowDestinationS3Access" - effect = "Allow" - - actions = [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject", - ] - - resources = [ - module.s3bucket_reporting.arn, - "${module.s3bucket_reporting.arn}/parquet/*", - ] - } - - statement { - sid = "AllowKms" - effect = "Allow" - - actions = [ - "kms:Decrypt", - "kms:DescribeKey", - "kms:GenerateDataKey*", - ] - - resources = [module.kms.key_arn] - } - - statement { - sid = "AllowInvokeTransformer" - effect = "Allow" - - actions = [ - "lambda:InvokeFunction", - "lambda:GetFunctionConfiguration", - ] - - resources = ["${module.eventpub_parquet_transformer.function_arn}:$LATEST"] - } - - statement { - sid = "AllowGlueSchemaRead" - effect = "Allow" - - actions = [ - "glue:GetTable", - "glue:GetTableVersion", - "glue:GetTableVersions", - ] - - resources = [ - aws_glue_catalog_table.eventpub_parquet[0].arn, - aws_glue_catalog_database.supplier.arn, - "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", - ] - } - - statement { - sid = "AllowLogging" - effect = "Allow" - actions = ["logs:PutLogEvents"] - resources = ["${aws_cloudwatch_log_group.eventpub_parquet_firehose[0].arn}:*"] - } -} - -resource "aws_iam_role" "eventpub_parquet_sns" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-sns" - assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json -} - -data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["sns.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy" "eventpub_parquet_sns" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-sns" - role = aws_iam_role.eventpub_parquet_sns[0].id - policy = data.aws_iam_policy_document.eventpub_parquet_sns[0].json -} - -data "aws_iam_policy_document" "eventpub_parquet_sns" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - statement { - sid = "AllowFirehoseDelivery" - effect = "Allow" - - actions = [ - "firehose:PutRecord", - "firehose:PutRecordBatch", - ] - - resources = [aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn] - } - - statement { - sid = "AllowKms" - effect = "Allow" - actions = ["kms:GenerateDataKey", "kms:Decrypt"] - resources = [module.kms.key_arn] - } -} diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 18c52e680..7c2dbf46b 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -1,4 +1,33 @@ locals { + eventpub_parquet_enabled = local.event_cache_bucket_name != null + + # Column names must be lowercase; OpenXJsonSerDe matches JSON keys case-insensitively. + eventpub_parquet_table_columns = [ + { name = "specversion", type = "string" }, + { name = "id", type = "string" }, + { name = "type", type = "string" }, + { name = "plane", type = "string" }, + { name = "dataschema", type = "string" }, + { name = "dataschemaversion", type = "string" }, + { name = "source", type = "string" }, + { name = "subject", type = "string" }, + { name = "time", type = "string" }, + { name = "recordedtime", type = "string" }, + { name = "datacontenttype", type = "string" }, + { name = "traceparent", type = "string" }, + { name = "tracestate", type = "string" }, + { name = "partitionkey", type = "string" }, + { name = "sampledrate", type = "bigint" }, + { name = "sequence", type = "string" }, + { name = "severitytext", type = "string" }, + { name = "data_json", type = "string" }, + { name = "domainid", type = "string" }, + { name = "supplierid", type = "string" }, + { name = "groupid", type = "string" }, + { name = "campaignid", type = "string" }, + { name = "clientid", type = "string" }, + ] + aws_lambda_functions_dir_path = "../../../../lambdas" root_domain_name = "${var.environment}.${local.acct.route53_zone_names["supplier-api"]}" # e.g. [main|dev|abxy0].supplier-api.[dev|nonprod|prod].nhsnotify.national.nhs.uk root_domain_id = local.acct.route53_zone_ids["supplier-api"] diff --git a/infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf b/infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf new file mode 100644 index 000000000..da3237198 --- /dev/null +++ b/infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf @@ -0,0 +1,10 @@ +# Raw delivery so Firehose receives the bare event rather than the SNS envelope +resource "aws_sns_topic_subscription" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + topic_arn = module.eventpub.sns_topic.arn + protocol = "firehose" + subscription_role_arn = aws_iam_role.eventpub_parquet_sns[0].arn + endpoint = aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn + raw_message_delivery = true +} diff --git a/package.json b/package.json index d6e0a3e9b..68a2c9b28 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,8 @@ "docs", "internal/*", "lambdas/*", - "lambdas/letter-sla-breaches/*", "pact-contracts", + "lambdas/letter-sla-breaches/*", + "pact-contracts", "scripts/utilities/*", "tests", "tests/contracts/*" From 94f18f17e3f730ab74278f8f67f41dfa3ba204c0 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 11:12:53 +0100 Subject: [PATCH 15/28] fix lint issue and change S3 bucket used for config cache --- .../api/glue_catalog_tables_supplier_config_cache.tf | 4 ++-- .../api/module_lambda_populate_supplier_config_cache.tf | 4 ++-- .../src/handler/populate-supplier-config-cache.ts | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf index c44077cc9..d17c10f33 100644 --- a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf +++ b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf @@ -72,7 +72,7 @@ locals { } resource "aws_glue_catalog_table" "supplier_config_cache" { - for_each = local.eventsub_event_cache_bucket_name != null ? local.supplier_config_cache_table_columns : {} + for_each = local.supplier_config_cache_table_columns name = "${replace(local.csi, "-", "_")}_${replace(each.key, "-", "_")}" database_name = aws_glue_catalog_database.supplier.name table_type = "EXTERNAL_TABLE" @@ -83,7 +83,7 @@ resource "aws_glue_catalog_table" "supplier_config_cache" { } storage_descriptor { - location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}/" + location = "s3://${module.s3bucket_reporting.bucket}/supplier_config_cache/${each.key}/" input_format = "org.apache.hadoop.mapred.TextInputFormat" output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" diff --git a/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf index 57bf707fa..86339dda2 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf @@ -37,7 +37,7 @@ module "populate_supplier_config_cache" { lambda_env_vars = { SUPPLIER_CONFIG_TABLE_NAME = aws_dynamodb_table.supplier-configuration.name - EVENT_CACHE_BUCKET_NAME = local.eventsub_event_cache_bucket_name + EVENT_CACHE_BUCKET_NAME = module.s3bucket_reporting.bucket } } @@ -67,6 +67,6 @@ data "aws_iam_policy_document" "populate_supplier_config_cache_lambda" { sid = "AllowWriteSupplierConfigCache" effect = "Allow" actions = ["s3:PutObject"] - resources = ["arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/supplier_config_cache/*"] + resources = ["${module.s3bucket_reporting.arn}/supplier_config_cache/*"] } } diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 17b87022a..1f8b50250 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -27,10 +27,6 @@ export default function createHandler(deps: EntityCacheDeps) { } }), ); - const entities = results.map(({ entity, recordsRetrieved }) => ({ - entity, - recordsRetrieved, - })); const failedEntities = results .filter((result) => result.failed) .map((result) => result.entity); From f0146df08d4b6234f5437395a5c6efed797ffdab Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 12:13:13 +0100 Subject: [PATCH 16/28] promote status --- infrastructure/terraform/components/api/locals.tf | 1 + .../src/__tests__/transform-record.test.ts | 1 + lambdas/eventpub-parquet-transformer/src/transform-record.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 7c2dbf46b..1b7db3ec6 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -26,6 +26,7 @@ locals { { name = "groupid", type = "string" }, { name = "campaignid", type = "string" }, { name = "clientid", type = "string" }, + { name = "status", type = "string" }, ] aws_lambda_functions_dir_path = "../../../../lambdas" diff --git a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts index 48db8e4b5..677cf5752 100644 --- a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts +++ b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts @@ -42,6 +42,7 @@ describe("transformRecord", () => { groupid: "g1", campaignid: "c1", clientid: "cl1", + status: "PRINTED", }); }); diff --git a/lambdas/eventpub-parquet-transformer/src/transform-record.ts b/lambdas/eventpub-parquet-transformer/src/transform-record.ts index 97a10a1b5..b743de50f 100644 --- a/lambdas/eventpub-parquet-transformer/src/transform-record.ts +++ b/lambdas/eventpub-parquet-transformer/src/transform-record.ts @@ -29,6 +29,7 @@ const PROMOTED_DATA_FIELDS = [ "groupId", "campaignId", "clientId", + "status", ] as const; type Json = Record; From fcb95e27522f8d749cf39540ca3cf70a2394113e Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 14:07:45 +0100 Subject: [PATCH 17/28] remove unnecessary export --- .../populate-supplier-config-cache.test.ts | 36 +++++++++---------- .../handler/populate-supplier-config-cache.ts | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index 911f78f7f..047694df6 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -1,8 +1,6 @@ import { $SupplierConfigEntity } from "@internal/datastore"; import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; -import createHandler, { - ENTITY_TYPES, -} from "../handler/populate-supplier-config-cache"; +import createHandler from "../handler/populate-supplier-config-cache"; jest.mock("@internal/datastore", () => ({ $SupplierConfigEntity: { @@ -21,17 +19,17 @@ jest.mock("../handler/entity-cache"); describe("populateSupplierConfigCache", () => { it("populates the cache once for each entity type and returns success", async () => { const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; + const entityTypes = $SupplierConfigEntity.options; jest.mocked(populateEntityCache).mockReset(); - for (const [index] of ENTITY_TYPES.entries()) { + for (const [index] of entityTypes.entries()) { jest.mocked(populateEntityCache).mockResolvedValueOnce(index + 1); } const handler = createHandler(deps); const result = await handler(); - expect(ENTITY_TYPES).toEqual($SupplierConfigEntity.options); - expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); - for (const entityType of ENTITY_TYPES) { + expect(populateEntityCache).toHaveBeenCalledTimes(entityTypes.length); + for (const entityType of entityTypes) { expect(populateEntityCache).toHaveBeenCalledWith(deps, entityType); } expect(result).toEqual({ @@ -41,25 +39,27 @@ describe("populateSupplierConfigCache", () => { it("returns failure when an entity fails", async () => { const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; - jest - .mocked(populateEntityCache) - .mockReset() - .mockResolvedValueOnce(1) - .mockRejectedValueOnce(new Error("query failed")) - .mockResolvedValueOnce(3) - .mockResolvedValueOnce(4) - .mockResolvedValueOnce(5) - .mockResolvedValueOnce(6); + const entityTypes = $SupplierConfigEntity.options; + jest.mocked(populateEntityCache).mockReset(); + for (const [index] of entityTypes.entries()) { + if (index === 1) { + jest + .mocked(populateEntityCache) + .mockRejectedValueOnce(new Error("query failed")); + } else { + jest.mocked(populateEntityCache).mockResolvedValueOnce(index + 1); + } + } const handler = createHandler(deps); const result = await handler(); expect(result).toEqual({ status: "FAILURE", - failureReason: `Failed to populate supplier config cache for: ${ENTITY_TYPES[1]}`, + failureReason: `Failed to populate supplier config cache for: ${entityTypes[1]}`, }); expect(deps.logger.error).toHaveBeenCalledWith( - { entityType: ENTITY_TYPES[1], error: expect.any(Error) }, + { entityType: entityTypes[1], error: expect.any(Error) }, "Failed to populate supplier config entity cache", ); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 1f8b50250..8ce432a6c 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -6,7 +6,7 @@ export type PopulateSupplierConfigCacheResult = { failureReason?: string; }; -export const ENTITY_TYPES = $SupplierConfigEntity.options; +const ENTITY_TYPES = $SupplierConfigEntity.options; export default function createHandler(deps: EntityCacheDeps) { return async (): Promise => { From b8628885e8b751386db3a0e9cdaf8a2038d64e64 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Wed, 14 Oct 2026 15:57:40 +0100 Subject: [PATCH 18/28] Create common package --- .../letter-sla-breaches/common/jest.config.ts | 28 ++++ .../letter-sla-breaches/common/package.json | 19 +++ .../__tests__/athena-date-predicate.test.ts | 74 +++++++++ .../common/src/__tests__/athena.test.ts | 154 ++++++++++++++++++ .../common/src/athena-date-predicate.ts | 45 +++++ .../letter-sla-breaches/common/src/athena.ts | 132 +++++++++++++++ .../letter-sla-breaches/common/src/deps.ts | 25 +++ .../letter-sla-breaches/common/src/index.ts | 4 + .../letter-sla-breaches/common/tsconfig.json | 11 ++ .../populate-standard-breaches/package.json | 1 + .../populate-standard-breaches.test.ts | 16 ++ .../src/handler/populate-standard-breaches.ts | 6 + .../src/handler/standard-breaches-query.ts | 39 +++++ .../populate-standard-breaches/src/index.ts | 3 + package-lock.json | 64 ++++++++ 15 files changed, 621 insertions(+) create mode 100644 lambdas/letter-sla-breaches/common/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/common/package.json create mode 100644 lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts create mode 100644 lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts create mode 100644 lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts create mode 100644 lambdas/letter-sla-breaches/common/src/athena.ts create mode 100644 lambdas/letter-sla-breaches/common/src/deps.ts create mode 100644 lambdas/letter-sla-breaches/common/src/index.ts create mode 100644 lambdas/letter-sla-breaches/common/tsconfig.json create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts diff --git a/lambdas/letter-sla-breaches/common/jest.config.ts b/lambdas/letter-sla-breaches/common/jest.config.ts new file mode 100644 index 000000000..d38a979df --- /dev/null +++ b/lambdas/letter-sla-breaches/common/jest.config.ts @@ -0,0 +1,28 @@ +export default { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + testEnvironment: "node", +}; diff --git a/lambdas/letter-sla-breaches/common/package.json b/lambdas/letter-sla-breaches/common/package.json new file mode 100644 index 000000000..3e5758ac4 --- /dev/null +++ b/lambdas/letter-sla-breaches/common/package.json @@ -0,0 +1,19 @@ +{ + "dependencies": { + "@aws-sdk/client-athena": "^3.1149.0", + "@internal/helpers": "^0.1.0", + "pino": "^10.4.0" + }, + "description": "Common utilities for letter SLA breach lambdas", + "license": "MIT", + "main": "src/index.ts", + "name": "@internal/letter-sla-breaches-common", + "private": true, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts new file mode 100644 index 000000000..a0a4f545c --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts @@ -0,0 +1,74 @@ +import buildAthenaDatePredicate from "../athena-date-predicate"; + +describe("buildAthenaDatePredicate", () => { + it("includes only the requested UTC date when no days are subtracted", () => { + expect( + buildAthenaDatePredicate(new Date("2026-10-14T23:59:59.999Z"), 0), + ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('14'))"); + }); + + it("groups consecutive dates in the same month into one partition predicate", () => { + expect( + buildAthenaDatePredicate(new Date("2026-10-14T12:00:00.000Z"), 2), + ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13', '14'))"); + }); + + it("creates separate predicates when the date range crosses a month", () => { + expect( + buildAthenaDatePredicate(new Date("2026-11-01T00:00:00.000Z"), 2), + ).toBe( + "(_year = '2026' AND _month = '10' AND _day IN ('30', '31'))\n" + + " OR (_year = '2026' AND _month = '11' AND _day IN ('01'))", + ); + }); + + it("creates separate predicates when the date range crosses a year", () => { + expect( + buildAthenaDatePredicate(new Date("2026-01-01T00:00:00.000Z"), 1), + ).toBe( + "(_year = '2025' AND _month = '12' AND _day IN ('31'))\n" + + " OR (_year = '2026' AND _month = '01' AND _day IN ('01'))", + ); + }); + + it("includes leap days in the date range", () => { + expect( + buildAthenaDatePredicate(new Date("2024-03-01T00:00:00.000Z"), 2), + ).toBe( + "(_year = '2024' AND _month = '02' AND _day IN ('28', '29'))\n" + + " OR (_year = '2024' AND _month = '03' AND _day IN ('01'))", + ); + }); + + it("uses UTC calendar dates regardless of the time component", () => { + expect( + buildAthenaDatePredicate(new Date("2026-10-14T00:30:00+02:00"), 1), + ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13'))"); + }); + + it("throws when given an invalid date", () => { + expect(() => buildAthenaDatePredicate(new Date(Number.NaN), 0)).toThrow( + new RangeError("date must be valid"), + ); + }); + + it.each([ + -1, + 1.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ])( + "throws when daysToSubtract is not a non-negative safe integer (%s)", + (daysToSubtract) => { + expect(() => + buildAthenaDatePredicate( + new Date("2026-10-14T00:00:00.000Z"), + daysToSubtract, + ), + ).toThrow( + new RangeError("daysToSubtract must be a non-negative integer"), + ); + }, + ); +}); diff --git a/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts new file mode 100644 index 000000000..b11ff60ae --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts @@ -0,0 +1,154 @@ +import { + AthenaClient, + GetQueryExecutionCommand, + GetQueryResultsCommand, + StartQueryExecutionCommand, +} from "@aws-sdk/client-athena"; +import runAthenaQuery from "../athena"; +import { type Deps } from "../deps"; + +const query = { queryString: "SELECT ?", executionParameters: ["'a'"] }; + +function makeDeps(responses: unknown[]) { + const send = jest.fn(); + for (const response of responses) { + send.mockResolvedValueOnce(response); + } + const deps = { + athenaClient: { send } as unknown as AthenaClient, + logger: { info: jest.fn(), debug: jest.fn() }, + env: { + ATHENA_WORKGROUP: "workgroup", + ATHENA_DATABASE: "database", + EVENT_CACHE_TABLE_NAME: "event_cache", + }, + } as unknown as Deps; + return { deps, send }; +} + +const columnInfo = { ColumnInfo: [{ Name: "id" }, { Name: "status" }] }; + +describe("runAthenaQuery", () => { + it("starts the query, polls until complete and returns paginated rows", async () => { + const { deps, send } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "QUEUED" } } }, + { QueryExecution: { Status: { State: "RUNNING" } } }, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + { + ResultSet: { + ResultSetMetadata: columnInfo, + Rows: [ + { Data: [{ VarCharValue: "id" }, { VarCharValue: "status" }] }, + { Data: [{ VarCharValue: "1" }, { VarCharValue: "PENDING" }] }, + ], + }, + NextToken: "next", + }, + { + ResultSet: { + ResultSetMetadata: columnInfo, + Rows: [{ Data: [{ VarCharValue: "2" }, {}] }], + }, + }, + ]); + + const rows = await runAthenaQuery(deps, query, { pollIntervalMs: 0 }); + + expect(rows).toEqual([ + { id: "1", status: "PENDING" }, + { id: "2", status: undefined }, + ]); + const commands = send.mock.calls.map(([command]) => command); + expect(commands[0]).toBeInstanceOf(StartQueryExecutionCommand); + expect(commands[0].input).toEqual({ + QueryString: "SELECT ?", + ExecutionParameters: ["'a'"], + WorkGroup: "workgroup", + QueryExecutionContext: { Database: "database" }, + }); + expect(commands[1]).toBeInstanceOf(GetQueryExecutionCommand); + expect(commands[4]).toBeInstanceOf(GetQueryResultsCommand); + expect(commands[5].input).toEqual({ + QueryExecutionId: "qid", + NextToken: "next", + }); + }); + + it("handles empty result sets and missing metadata", async () => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + { + ResultSet: { ResultSetMetadata: { ColumnInfo: [{}] }, Rows: [{}, {}] }, + }, + ]); + + await expect(runAthenaQuery(deps, query)).resolves.toEqual([ + { "": undefined }, + ]); + }); + + it("handles a response with no result set", async () => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + {}, + ]); + + await expect(runAthenaQuery(deps, query)).resolves.toEqual([]); + }); + + it("uses the default poll interval when none is given", async () => { + jest.useFakeTimers(); + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + {}, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + {}, + ]); + + const promise = runAthenaQuery(deps, query); + await jest.advanceTimersByTimeAsync(1000); + + await expect(promise).resolves.toEqual([]); + jest.useRealTimers(); + }); + + it("throws when no query execution id is returned", async () => { + const { deps } = makeDeps([{}]); + + await expect(runAthenaQuery(deps, query)).rejects.toThrow( + "Athena did not return a query execution id", + ); + }); + + it.each(["FAILED", "CANCELLED"])( + "throws when the query is %s", + async (state) => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { + QueryExecution: { + Status: { State: state, StateChangeReason: "bad query" }, + }, + }, + ]); + + await expect(runAthenaQuery(deps, query)).rejects.toThrow( + `Athena query qid ${state}: bad query`, + ); + }, + ); + + it("throws with a default reason when none is given", async () => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "FAILED" } } }, + ]); + + await expect(runAthenaQuery(deps, query)).rejects.toThrow( + "Athena query qid FAILED: no reason given", + ); + }); +}); diff --git a/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts new file mode 100644 index 000000000..0ca204bac --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts @@ -0,0 +1,45 @@ +export default function buildAthenaDatePredicate( + date: Date, + daysToSubtract: number, +): string { + if (Number.isNaN(date.getTime())) { + throw new RangeError("date must be valid"); + } + + if (!Number.isSafeInteger(daysToSubtract) || daysToSubtract < 0) { + throw new RangeError("daysToSubtract must be a non-negative integer"); + } + + const endDate = new Date(date); + endDate.setUTCHours(0, 0, 0, 0); + + const currentDate = new Date(endDate); + currentDate.setUTCDate(currentDate.getUTCDate() - daysToSubtract); + + const groups: { year: string; month: string; days: string[] }[] = []; + + while (currentDate <= endDate) { + const year = String(currentDate.getUTCFullYear()).padStart(4, "0"); + const month = String(currentDate.getUTCMonth() + 1).padStart(2, "0"); + const day = String(currentDate.getUTCDate()).padStart(2, "0"); + const lastGroup = groups.at(-1); + + if (lastGroup?.year === year && lastGroup.month === month) { + lastGroup.days.push(day); + } else { + groups.push({ year, month, days: [day] }); + } + + currentDate.setUTCDate(currentDate.getUTCDate() + 1); + } + + return groups + .map( + // eslint-disable-next-line sort-destructure-keys/sort-destructure-keys + ({ year, month, days }) => + `(_year = '${year}' AND _month = '${month}' AND _day IN (${days + .map((day) => `'${day}'`) + .join(", ")}))`, + ) + .join("\n OR "); +} diff --git a/lambdas/letter-sla-breaches/common/src/athena.ts b/lambdas/letter-sla-breaches/common/src/athena.ts new file mode 100644 index 000000000..59cf6c338 --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/athena.ts @@ -0,0 +1,132 @@ +import { + GetQueryExecutionCommand, + GetQueryResultsCommand, + GetQueryResultsCommandOutput, + QueryExecutionState, + StartQueryExecutionCommand, +} from "@aws-sdk/client-athena"; +import { Deps } from "./deps"; + +export type AthenaQuery = { + queryString: string; + executionParameters?: string[]; +}; + +export type AthenaQueryOptions = { + pollIntervalMs?: number; +}; + +export type AthenaRow = Record; + +const DEFAULT_POLL_INTERVAL_MS = 1000; + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +async function startQuery(deps: Deps, query: AthenaQuery): Promise { + const { QueryExecutionId } = await deps.athenaClient.send( + new StartQueryExecutionCommand({ + QueryString: query.queryString, + ExecutionParameters: query.executionParameters, + WorkGroup: deps.env.ATHENA_WORKGROUP, + QueryExecutionContext: { Database: deps.env.ATHENA_DATABASE }, + }), + ); + + if (!QueryExecutionId) { + throw new Error("Athena did not return a query execution id"); + } + + return QueryExecutionId; +} + +async function waitForQuery( + deps: Deps, + queryExecutionId: string, + pollIntervalMs: number, +): Promise { + for (;;) { + const { QueryExecution } = await deps.athenaClient.send( + new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId }), + ); + const state = QueryExecution?.Status?.State; + + switch (state) { + case QueryExecutionState.SUCCEEDED: { + return; + } + case QueryExecutionState.FAILED: + case QueryExecutionState.CANCELLED: { + throw new Error( + `Athena query ${queryExecutionId} ${state}: ${QueryExecution?.Status?.StateChangeReason ?? "no reason given"}`, + ); + } + default: { + deps.logger.debug({ queryExecutionId, state }, "Waiting for query"); + await sleep(pollIntervalMs); + } + } + } +} + +function toRows( + page: GetQueryResultsCommandOutput, + isFirstPage: boolean, +): AthenaRow[] { + const columns = + page.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((c) => c.Name ?? "") ?? + []; + // For SELECT queries the first row of the first page contains the column headers. + const rows = (page.ResultSet?.Rows ?? []).slice(isFirstPage ? 1 : 0); + + return rows.map((row) => + Object.fromEntries( + columns.map((column, index) => [ + column, + row.Data?.at(index)?.VarCharValue, + ]), + ), + ); +} + +async function getResults( + deps: Deps, + queryExecutionId: string, +): Promise { + const results: AthenaRow[] = []; + let nextToken: string | undefined; + let isFirstPage = true; + + do { + const page = await deps.athenaClient.send( + new GetQueryResultsCommand({ + QueryExecutionId: queryExecutionId, + NextToken: nextToken, + }), + ); + results.push(...toRows(page, isFirstPage)); + nextToken = page.NextToken; + isFirstPage = false; + } while (nextToken); + + return results; +} + +export default async function runAthenaQuery( + deps: Deps, + query: AthenaQuery, + options: AthenaQueryOptions = {}, +): Promise { + const queryExecutionId = await startQuery(deps, query); + deps.logger.info({ queryExecutionId }, "Started Athena query"); + + await waitForQuery( + deps, + queryExecutionId, + options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, + ); + + return getResults(deps, queryExecutionId); +} diff --git a/lambdas/letter-sla-breaches/common/src/deps.ts b/lambdas/letter-sla-breaches/common/src/deps.ts new file mode 100644 index 000000000..fb801c886 --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/deps.ts @@ -0,0 +1,25 @@ +import { AthenaClient } from "@aws-sdk/client-athena"; +import { createLogger } from "@internal/helpers"; +import { Logger } from "pino"; + +export type Deps = { + athenaClient: AthenaClient; + logger: Logger; + env: { + ATHENA_WORKGROUP: string; + ATHENA_DATABASE: string; + EVENT_CACHE_TABLE_NAME: string; + }; +}; + +export default function createDependenciesContainer(): Deps { + return { + athenaClient: new AthenaClient(), + logger: createLogger({ logLevel: process.env.PINO_LOG_LEVEL }), + env: { + ATHENA_WORKGROUP: process.env.ATHENA_WORKGROUP!, + ATHENA_DATABASE: process.env.ATHENA_DATABASE!, + EVENT_CACHE_TABLE_NAME: process.env.EVENT_CACHE_TABLE_NAME!, + }, + }; +} diff --git a/lambdas/letter-sla-breaches/common/src/index.ts b/lambdas/letter-sla-breaches/common/src/index.ts new file mode 100644 index 000000000..3abc594ec --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/index.ts @@ -0,0 +1,4 @@ +export { default as runAthenaQuery } from "./athena"; +export { default as createDependenciesContainer } from "./deps"; +export type { AthenaQuery, AthenaQueryOptions, AthenaRow } from "./athena"; +export type { Deps } from "./deps"; diff --git a/lambdas/letter-sla-breaches/common/tsconfig.json b/lambdas/letter-sla-breaches/common/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/common/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json index 8875e4acc..a8a5775a2 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@internal/letter-sla-breaches-common": "*", "esbuild": "0.28.2" }, "name": "nhs-notify-supplier-api-populate-standard-breaches", diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index f64093251..dfa03f528 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -1,4 +1,20 @@ +<<<<<<< HEAD import populateStandardBreaches from "../handler/populate-standard-breaches"; +======= +import { + type Deps, + runAthenaQuery, +} from "@internal/letter-sla-breaches-common"; +import createHandler from "../handler/populate-standard-breaches"; + +jest.mock("@internal/letter-sla-breaches-common"); + +const makeDeps = () => + ({ + logger: { info: jest.fn(), error: jest.fn() }, + env: { EVENT_CACHE_TABLE_NAME: "event_cache" }, + }) as unknown as Deps; +>>>>>>> 68020529 (Create common package) describe("populateStandardBreaches", () => { it("returns a success result", async () => { diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index 1bcc2318c..14decf6a5 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -1,3 +1,9 @@ +import { + type Deps, + runAthenaQuery, +} from "@internal/letter-sla-breaches-common"; +import buildStandardBreachesQuery from "./standard-breaches-query"; + export type PopulateStandardBreachesEvent = { date: string; }; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts new file mode 100644 index 000000000..1130385bc --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -0,0 +1,39 @@ +import { type AthenaQuery } from "@internal/letter-sla-breaches-common"; + +export default function buildStandardBreachesQuery( + tableName: string, + now: Date, +): AthenaQuery { + return { + queryString: `WITH now AS ( + SELECT from_iso8601_timestamp('?') AS iso8601 +), +recent_letter_events AS ( + SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time + FROM nhs_pr731_supapi_eventpub + WHERE __year = '2026' AND __month = '10' AND __day IN ('12', '13', '14') +), +allocated_letters AS ( + SELECT domainid, supplierid + FROM recent_letter_events + WHERE status = 'PENDING' + GROUP BY domainid, supplierid + HAVING min(event_time) < (SELECT iso8601 FROM now) - + CASE WHEN day_of_week((SELECT iso8601 AT TIME ZONE 'Europe/London' FROM now)) <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END +), +completed_letters AS ( + SELECT domainid, supplierid + FROM recent_letter_events + WHERE status in ('REJECTED', 'FAILED', 'CANCELLED', 'FORWARDED', 'DISPATCHED') + GROUP BY domainid, supplierid +) +SELECT supplierid, count(*) +FROM ( + SELECT al.domainid, al.supplierid + FROM allocated_letters al + LEFT JOIN completed_letters cl ON al.domainid = cl.domainid WHERE cl.domainid IS NULL +) +GROUP BY supplierid`, + executionParameters: [now.toISOString()], + }; +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts index 5516fb886..c50b040ad 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts @@ -1,2 +1,5 @@ +import { createDependenciesContainer } from "@internal/letter-sla-breaches-common"; +import createHandler from "./handler/populate-standard-breaches"; + // eslint-disable-next-line import-x/prefer-default-export export { default as populateStandardBreachesHandler } from "./handler/populate-standard-breaches"; diff --git a/package-lock.json b/package-lock.json index 0912227cc..81fb75152 100644 --- a/package-lock.json +++ b/package-lock.json @@ -404,6 +404,65 @@ "esbuild": "0.28.2" } }, + "lambdas/letter-sla-breaches/common": { + "name": "@internal/letter-sla-breaches-common", + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "@aws-sdk/client-athena": "^3.1149.0", + "@internal/helpers": "^0.1.0", + "pino": "^10.4.0" + } + }, + "lambdas/letter-sla-breaches/common/node_modules/pino": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.4.0.tgz", + "integrity": "sha512-bk1ZMTwG/Vymx+zPoa28MhNzucFvXJWp3csrSuxuHPSMeUettaCoQ5Jvg07mtC4JGNpcfTwz41oVCnNgBK9+vA==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^1.0.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "lambdas/letter-sla-breaches/common/node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "lambdas/letter-sla-breaches/common/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, + "lambdas/letter-sla-breaches/common/node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" + } + }, "lambdas/letter-sla-breaches/populate-sameday-breaches": { "name": "nhs-notify-supplier-api-populate-sameday-breaches", "version": "0.0.1", @@ -415,6 +474,7 @@ "name": "nhs-notify-supplier-api-populate-standard-breaches", "version": "0.0.1", "dependencies": { + "@internal/letter-sla-breaches-common": "*", "esbuild": "0.28.2" } }, @@ -2436,6 +2496,10 @@ "resolved": "internal/helpers", "link": true }, + "node_modules/@internal/letter-sla-breaches-common": { + "resolved": "lambdas/letter-sla-breaches/common", + "link": true + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "license": "ISC", From 89c4dda2897b2ce35cb0ee3aa97ae81f887f84ec Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 15 Oct 2026 10:59:23 +0100 Subject: [PATCH 19/28] add query and flesh out lambda to produce count --- .../letter-sla-breaches/common/src/index.ts | 1 + .../populate-sameday-breaches/package.json | 1 + .../populate-sameday-breaches.test.ts | 51 +++++++++++++++-- .../__tests__/sameday-breaches-query.test.ts | 53 ++++++++++++++++++ .../src/handler/populate-sameday-breaches.ts | 55 +++++++++++++++---- .../src/handler/sameday-breaches-query.ts | 47 ++++++++++++++++ .../populate-sameday-breaches/src/index.ts | 7 ++- .../populate-standard-breaches.test.ts | 41 +++++++++++--- .../__tests__/standard-breaches-query.test.ts | 16 ++++++ .../src/handler/populate-standard-breaches.ts | 45 ++++++++++++--- .../src/handler/standard-breaches-query.ts | 14 +++-- .../populate-standard-breaches/src/index.ts | 4 +- package-lock.json | 1 + 13 files changed, 296 insertions(+), 40 deletions(-) create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts diff --git a/lambdas/letter-sla-breaches/common/src/index.ts b/lambdas/letter-sla-breaches/common/src/index.ts index 3abc594ec..87d4356e7 100644 --- a/lambdas/letter-sla-breaches/common/src/index.ts +++ b/lambdas/letter-sla-breaches/common/src/index.ts @@ -1,4 +1,5 @@ export { default as runAthenaQuery } from "./athena"; +export { default as buildAthenaDatePredicate } from "./athena-date-predicate"; export { default as createDependenciesContainer } from "./deps"; export type { AthenaQuery, AthenaQueryOptions, AthenaRow } from "./athena"; export type { Deps } from "./deps"; diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json index 127c212c3..43d7dc0da 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@internal/letter-sla-breaches-common": "*", "esbuild": "0.28.2" }, "name": "nhs-notify-supplier-api-populate-sameday-breaches", diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts index 50492a04a..716ec5b3e 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts @@ -1,12 +1,53 @@ -import populateSamedayBreaches from "../handler/populate-sameday-breaches"; +import { + type Deps, + runAthenaQuery, +} from "@internal/letter-sla-breaches-common"; +import createHandler from "../handler/populate-sameday-breaches"; + +jest.mock("@internal/letter-sla-breaches-common"); + +const makeDeps = () => + ({ + logger: { info: jest.fn(), error: jest.fn() }, + env: { EVENT_CACHE_TABLE_NAME: "event_cache" }, + }) as unknown as Deps; describe("populateSamedayBreaches", () => { - it("returns a success result", async () => { - const result = await populateSamedayBreaches({ date: "2026-10-08" }); + it("runs the sameday breaches query and returns success", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([{ breach_count: "1" }]); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(runAthenaQuery).toHaveBeenCalled(); + expect(result).toEqual({ status: "SUCCESS", breachCount: "1" }); + }); + + it("runs the sameday breaches query and returns failure", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockRejectedValueOnce(new Error("boom")); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(runAthenaQuery).toHaveBeenCalled(); + expect(result).toEqual({ + failureReason: "boom", + status: "FAILURE", + breachCount: "0", + }); + }); + + it("runs the sameday breaches query and returns failure when not an error", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockRejectedValueOnce("boom"); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + expect(runAthenaQuery).toHaveBeenCalled(); expect(result).toEqual({ - status: "SUCCESS", - message: "populate-sameday-breaches invoked successfully", + failureReason: "boom", + status: "FAILURE", + breachCount: "0", }); }); }); diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts new file mode 100644 index 000000000..a83957668 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts @@ -0,0 +1,53 @@ +import buildSamedayBreachesQuery from "../handler/sameday-breaches-query"; + +describe("buildSamedayBreachesQuery", () => { + it("selects from the event cache table filtered by the date partitions", () => { + expect( + buildSamedayBreachesQuery("event_cache", new Date("2026-10-08"), 0), + ).toEqual( + expect.objectContaining({ + queryString: expect.stringContaining( + "(_year = '2026' AND _month = '10' AND _day IN ('08'))", + ), + executionParameters: ["2026-10-08T00:00:00.000Z"], + }), + ); + }); + + it("applies a 24 hour SLA to letters allocated at or after 16:00 UK time", () => { + const { queryString } = buildSamedayBreachesQuery( + "event_cache", + new Date("2026-10-08"), + 0, + ); + + expect(queryString).toContain( + "WHEN hour(min(event_time) AT TIME ZONE 'Europe/London') >= 16", + ); + expect(queryString).toContain("THEN min(event_time) + INTERVAL '24' HOUR"); + }); + + it("applies an end of day SLA to letters allocated before 16:00 UK time", () => { + const { queryString } = buildSamedayBreachesQuery( + "event_cache", + new Date("2026-10-08"), + 0, + ); + + expect(queryString).toContain( + "ELSE date_trunc('day', min(event_time) AT TIME ZONE 'Europe/London') + INTERVAL '1' DAY", + ); + }); + + it("returns the breach count per supplier as breach_count", () => { + const { queryString } = buildSamedayBreachesQuery( + "event_cache", + new Date("2026-10-08"), + 0, + ); + + expect(queryString).toContain( + "SELECT supplierid, count(*) AS breach_count", + ); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts index f63d70492..43130ce2e 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts @@ -1,18 +1,51 @@ -export type PopulateSamedayBreachesEvent = { +import { + type Deps, + runAthenaQuery, +} from "@internal/letter-sla-breaches-common"; +import buildSamedayBreachesQuery from "./sameday-breaches-query"; + +export type PopulatesamedayBreachesEvent = { date: string; }; -export type PopulateSamedayBreachesResult = { - status: "SUCCESS"; - message: string; +export type PopulatesamedayBreachesResult = { + status: "SUCCESS" | "FAILURE"; + failureReason?: string; + breachCount: string; }; -// Placeholder: cache population is not yet implemented. -export default async function populateSamedayBreaches( - _event: PopulateSamedayBreachesEvent, -): Promise { - return { - status: "SUCCESS", - message: "populate-sameday-breaches invoked successfully", +export default function createHandler(deps: Deps) { + return async ( + event: PopulatesamedayBreachesEvent, + ): Promise => { + try { + const rows = await runAthenaQuery( + deps, + buildSamedayBreachesQuery( + deps.env.EVENT_CACHE_TABLE_NAME, + new Date(), + 14, + ), + ); + + // TODO: CCM-24041 process the query results to populate sameday breaches. + deps.logger.info( + { date: event.date, breachCount: rows[0].breach_count }, + "sameday breaches query completed", + ); + + return { status: "SUCCESS", breachCount: rows[0].breach_count! }; + } catch (error) { + deps.logger.error( + { date: event.date, error }, + "Failed to populate sameday breaches", + ); + + return { + status: "FAILURE", + failureReason: error instanceof Error ? error.message : String(error), + breachCount: "0", + }; + } }; } diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts new file mode 100644 index 000000000..6a11a40b8 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts @@ -0,0 +1,47 @@ +import { + type AthenaQuery, + buildAthenaDatePredicate, +} from "@internal/letter-sla-breaches-common"; + +export default function buildSamedayBreachesQuery( + tableName: string, + now: Date, + daysHistory: number, +): AthenaQuery { + return { + queryString: `WITH now AS ( + SELECT from_iso8601_timestamp('?') AS iso8601 +), +recent_letter_events AS ( + SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time + FROM ${tableName} + WHERE ${buildAthenaDatePredicate(now, daysHistory)} +), +allocated_letters AS ( + SELECT domainid, supplierid + FROM recent_letter_events + WHERE status = 'PENDING' + GROUP BY domainid, supplierid + HAVING (SELECT iso8601 FROM now) > + CASE + WHEN hour(min(event_time) AT TIME ZONE 'Europe/London') >= 16 + THEN min(event_time) + INTERVAL '24' HOUR + ELSE date_trunc('day', min(event_time) AT TIME ZONE 'Europe/London') + INTERVAL '1' DAY + END +), +completed_letters AS ( + SELECT domainid, supplierid + FROM recent_letter_events + WHERE status in ('REJECTED', 'FAILED', 'CANCELLED', 'FORWARDED', 'DISPATCHED') + GROUP BY domainid, supplierid +) +SELECT supplierid, count(*) AS breach_count +FROM ( + SELECT al.domainid, al.supplierid + FROM allocated_letters al + LEFT JOIN completed_letters cl ON al.domainid = cl.domainid WHERE cl.domainid IS NULL +) +GROUP BY supplierid`, + executionParameters: [now.toISOString()], + }; +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts index 3143567ed..65429ac8b 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts @@ -1,2 +1,7 @@ +import { createDependenciesContainer } from "@internal/letter-sla-breaches-common"; +import createHandler from "./handler/populate-sameday-breaches"; + // eslint-disable-next-line import-x/prefer-default-export -export { default as populateSamedayBreachesHandler } from "./handler/populate-sameday-breaches"; +export const populateSamedayBreachesHandler = createHandler( + createDependenciesContainer(), +); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index dfa03f528..5516860f4 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -1,6 +1,3 @@ -<<<<<<< HEAD -import populateStandardBreaches from "../handler/populate-standard-breaches"; -======= import { type Deps, runAthenaQuery, @@ -14,15 +11,43 @@ const makeDeps = () => logger: { info: jest.fn(), error: jest.fn() }, env: { EVENT_CACHE_TABLE_NAME: "event_cache" }, }) as unknown as Deps; ->>>>>>> 68020529 (Create common package) describe("populateStandardBreaches", () => { - it("returns a success result", async () => { - const result = await populateStandardBreaches({ date: "2026-10-08" }); + it("runs the standard breaches query and returns success", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([{ breach_count: "1" }]); + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(runAthenaQuery).toHaveBeenCalled(); + expect(result).toEqual({ status: "SUCCESS", breachCount: "1" }); + }); + + it("runs the standard breaches query and returns failure", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockRejectedValueOnce(new Error("boom")); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(runAthenaQuery).toHaveBeenCalled(); + expect(result).toEqual({ + failureReason: "boom", + status: "FAILURE", + breachCount: "0", + }); + }); + + it("runs the standard breaches query and returns failure when not an error", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockRejectedValueOnce("boom"); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(runAthenaQuery).toHaveBeenCalled(); expect(result).toEqual({ - status: "SUCCESS", - message: "populate-standard-breaches invoked successfully", + failureReason: "boom", + status: "FAILURE", + breachCount: "0", }); }); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts new file mode 100644 index 000000000..1e4b3ff03 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -0,0 +1,16 @@ +import buildStandardBreachesQuery from "../handler/standard-breaches-query"; + +describe("buildStandardBreachesQuery", () => { + it("selects from the event cache table filtered by the date partitions", () => { + expect( + buildStandardBreachesQuery("event_cache", new Date("2026-10-08"), 0), + ).toEqual( + expect.objectContaining({ + queryString: expect.stringContaining( + "(_year = '2026' AND _month = '10' AND _day IN ('08'))", + ), + executionParameters: ["2026-10-08T00:00:00.000Z"], + }), + ); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index 14decf6a5..72950dde6 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -9,16 +9,43 @@ export type PopulateStandardBreachesEvent = { }; export type PopulateStandardBreachesResult = { - status: "SUCCESS"; - message: string; + status: "SUCCESS" | "FAILURE"; + failureReason?: string; + breachCount: string; }; -// Placeholder: cache population is not yet implemented. -export default async function populateStandardBreaches( - _event: PopulateStandardBreachesEvent, -): Promise { - return { - status: "SUCCESS", - message: "populate-standard-breaches invoked successfully", +export default function createHandler(deps: Deps) { + return async ( + event: PopulateStandardBreachesEvent, + ): Promise => { + try { + const rows = await runAthenaQuery( + deps, + buildStandardBreachesQuery( + deps.env.EVENT_CACHE_TABLE_NAME, + new Date(), + 14, + ), + ); + + // TODO: CCM-24041 process the query results to populate standard breaches. + deps.logger.info( + { date: event.date, breachCount: rows[0].breach_count }, + "Standard breaches query completed", + ); + + return { status: "SUCCESS", breachCount: rows[0].breach_count! }; + } catch (error) { + deps.logger.error( + { date: event.date, error }, + "Failed to populate standard breaches", + ); + + return { + status: "FAILURE", + failureReason: error instanceof Error ? error.message : String(error), + breachCount: "0", + }; + } }; } diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 1130385bc..871db1363 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -1,17 +1,21 @@ -import { type AthenaQuery } from "@internal/letter-sla-breaches-common"; +import { + type AthenaQuery, + buildAthenaDatePredicate, +} from "@internal/letter-sla-breaches-common"; export default function buildStandardBreachesQuery( tableName: string, now: Date, + daysHistory: number, ): AthenaQuery { return { queryString: `WITH now AS ( - SELECT from_iso8601_timestamp('?') AS iso8601 + SELECT from_iso8601_timestamp(?) AS iso8601 ), recent_letter_events AS ( SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time - FROM nhs_pr731_supapi_eventpub - WHERE __year = '2026' AND __month = '10' AND __day IN ('12', '13', '14') + FROM ${tableName} + WHERE ${buildAthenaDatePredicate(now, daysHistory)} ), allocated_letters AS ( SELECT domainid, supplierid @@ -27,7 +31,7 @@ completed_letters AS ( WHERE status in ('REJECTED', 'FAILED', 'CANCELLED', 'FORWARDED', 'DISPATCHED') GROUP BY domainid, supplierid ) -SELECT supplierid, count(*) +SELECT supplierid, count(*) AS breach_count FROM ( SELECT al.domainid, al.supplierid FROM allocated_letters al diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts index c50b040ad..f52283ddb 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts @@ -2,4 +2,6 @@ import { createDependenciesContainer } from "@internal/letter-sla-breaches-commo import createHandler from "./handler/populate-standard-breaches"; // eslint-disable-next-line import-x/prefer-default-export -export { default as populateStandardBreachesHandler } from "./handler/populate-standard-breaches"; +export const populateStandardBreachesHandler = createHandler( + createDependenciesContainer(), +); diff --git a/package-lock.json b/package-lock.json index 81fb75152..0bd0f2511 100644 --- a/package-lock.json +++ b/package-lock.json @@ -467,6 +467,7 @@ "name": "nhs-notify-supplier-api-populate-sameday-breaches", "version": "0.0.1", "dependencies": { + "@internal/letter-sla-breaches-common": "*", "esbuild": "0.28.2" } }, From d5dd5973b4f3cd73e1a81dc7eabc66aa27126520 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 15 Oct 2026 11:15:47 +0100 Subject: [PATCH 20/28] fix package-lock.json --- package-lock.json | 155 ++++++++++++++++++++++++++++++---------------- 1 file changed, 101 insertions(+), 54 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0bd0f2511..29e96e4eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1091,6 +1091,25 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-athena": { + "version": "3.1152.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.1152.0.tgz", + "integrity": "sha512-Yk6ml5I2+N2jQBfhuQ8RXYGv9Fhn/qMJIIdkMQ+UFfT2ZM0Cg3Dzr+b2ZozluHw0SsNV5S6NohXIEZBgcY2kVg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/credential-provider-node": "^3.972.86", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.1146.0", "dev": true, @@ -1219,14 +1238,16 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.978.1", + "version": "3.978.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.3.tgz", + "integrity": "sha512-nslpzUCTI9b2IQMWSd6mSAXldBexPc8rr0d3dXUutXYFLFaTdsHa2GD7sJdsn3PhyjK0O5K8ypd6zi+RXXPVlg==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.6", "@aws-sdk/xml-builder": "^3.972.41", "@aws/lambda-invoke-store": "^0.3.0", - "@smithy/core": "^3.35.0", - "@smithy/signature-v4": "^5.7.3", + "@smithy/core": "^3.35.2", + "@smithy/signature-v4": "^5.7.4", "@smithy/types": "^4.19.0", "bowser": "^2.11.0", "tslib": "^2.6.2" @@ -1243,12 +1264,14 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.72", + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.74.tgz", + "integrity": "sha512-rsRJcW2iFUhCgclyD6OSbMEaPl8Eu0qSeRZb2uuCK1ViZ66hl5wOe/p6Vr6MO3b+ydyKgFeqza2p6tZThvILlw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", + "@aws-sdk/core": "^3.978.3", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1257,12 +1280,14 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.74", + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.76.tgz", + "integrity": "sha512-pnKdh0Br7OJPj+7qKjzgTfDTFW0RhEMKXNDR2jv9CAM4oHuG/4z3hRh8KtutLfpmItuRtDbKBmwWkDqHwwAWXw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", + "@aws-sdk/core": "^3.978.3", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", @@ -1273,19 +1298,21 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.973.17", + "version": "3.973.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.19.tgz", + "integrity": "sha512-kcjvAxqQEV8yuwkX58RelNn3ne65o7JgRWf3Z/gVvbi7Xuw1qSDLJlatrLD76+40cmRxmNkLxBoyp/vAQChQ5g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-env": "^3.972.72", - "@aws-sdk/credential-provider-http": "^3.972.74", - "@aws-sdk/credential-provider-login": "^3.972.79", - "@aws-sdk/credential-provider-process": "^3.972.72", - "@aws-sdk/credential-provider-sso": "^3.973.16", - "@aws-sdk/credential-provider-web-identity": "^3.972.78", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/credential-provider-env": "^3.972.74", + "@aws-sdk/credential-provider-http": "^3.972.76", + "@aws-sdk/credential-provider-login": "^3.972.81", + "@aws-sdk/credential-provider-process": "^3.972.74", + "@aws-sdk/credential-provider-sso": "^3.973.18", + "@aws-sdk/credential-provider-web-identity": "^3.972.80", + "@aws-sdk/nested-clients": "^3.997.48", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/credential-provider-imds": "^4.5.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" @@ -1295,13 +1322,15 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.79", + "version": "3.972.81", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.81.tgz", + "integrity": "sha512-CeQHSbE2CnVdaHIJ/NayRaCecIER+Pm9faUPkmRATO54rCoMTEPSbVieGHzEwvm1dZM2HIs6qQZCznhayNTWJQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1310,17 +1339,19 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.84", + "version": "3.972.86", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.86.tgz", + "integrity": "sha512-WNttSyr6e8innPtBSty//CdnqJcgZN6eAbtw3v/CR2IObz6MYZEo8eusdtuCHJ2ujwFqC2bSPHVyhXGHx8HruQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.72", - "@aws-sdk/credential-provider-http": "^3.972.74", - "@aws-sdk/credential-provider-ini": "^3.973.17", - "@aws-sdk/credential-provider-process": "^3.972.72", - "@aws-sdk/credential-provider-sso": "^3.973.16", - "@aws-sdk/credential-provider-web-identity": "^3.972.78", + "@aws-sdk/credential-provider-env": "^3.972.74", + "@aws-sdk/credential-provider-http": "^3.972.76", + "@aws-sdk/credential-provider-ini": "^3.973.19", + "@aws-sdk/credential-provider-process": "^3.972.74", + "@aws-sdk/credential-provider-sso": "^3.973.18", + "@aws-sdk/credential-provider-web-identity": "^3.972.80", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/credential-provider-imds": "^4.5.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" @@ -1330,12 +1361,14 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.72", + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.74.tgz", + "integrity": "sha512-Yj6kZFJQdbR+PygTkqdVLPHHpdDpOS9GCYVcDCuweCRRig9LF24ELQBMfyUw1vspvrZJUfnOGVyMjhaL094ahw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", + "@aws-sdk/core": "^3.978.3", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1344,14 +1377,16 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.973.16", + "version": "3.973.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.18.tgz", + "integrity": "sha512-FnpPhdY4qUGtX/8X6eKxh+uD+9Y19i9FYibgk4SBe0M935s9vocHkHzaxaMNk+YkHmvzgdu8cHA5SCoeU+kkew==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", - "@aws-sdk/token-providers": "3.1138.0", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", + "@aws-sdk/token-providers": "3.1150.0", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1360,13 +1395,15 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.78", + "version": "3.972.80", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.80.tgz", + "integrity": "sha512-TJF25Ndwy2HxcVj6oxMgp8qVlJAawlExHYBXqhuC6Zt2m+A7WpdkjBD6QglNbiArlteYljZHEY3R7KKmno6ISA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1466,13 +1503,15 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.997.46", + "version": "3.997.48", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.48.tgz", + "integrity": "sha512-B01jdXJtAW/XOiUzGdI2mz67WyEHbGEc9X6Euzw3bjLAa+txTqdZDER4X2FsIEcXzV2ABqudVZ5thali8tCCzQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/signature-v4-multi-region": "^3.996.47", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.48", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", @@ -1498,11 +1537,13 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.47", + "version": "3.996.48", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.48.tgz", + "integrity": "sha512-cFDXfEbXaGwbnpvR/DyqfxfjxpjXVRVthQLgkwvcfDS5u3+8G9WSj1X1Ag/pTRg/QOeOno1Tq3ycEdrKsCUcOA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.6", - "@smithy/signature-v4": "^5.7.3", + "@smithy/signature-v4": "^5.7.4", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1511,13 +1552,15 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1138.0", + "version": "3.1150.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1150.0.tgz", + "integrity": "sha512-wDU36ofhabZYu+d2KSriyDbwzxNoaOp+4dG0p3qkabk7HeALR5nIWyQbjoyJZLn3iqWkv/xNdyUfbfevDVDzzg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.3", + "@aws-sdk/nested-clients": "^3.997.48", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -3682,7 +3725,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.35.1", + "version": "3.36.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.36.0.tgz", + "integrity": "sha512-3hJiwG1F5t1SoXqp/yOCmYXziyx7ZVqF63S5W6YU201e/qwVaHPex+E/3vvyY9a3ORq/b+BydeNZpP2saQ9X2g==", "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.19.0", @@ -3694,6 +3739,8 @@ }, "node_modules/@smithy/credential-provider-imds": { "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", "license": "Apache-2.0", "dependencies": { "@smithy/core": "^3.33.2", From 2846a0a2b5934b1e403e013599230433dc73f096 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 15 Oct 2026 11:53:34 +0100 Subject: [PATCH 21/28] setup environment variables --- ...module_lambda_populate_sameday_breaches.tf | 67 ++++++++++++++++++- ...odule_lambda_populate_standard_breaches.tf | 67 ++++++++++++++++++- 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf index 020c4dd1c..f423e8ae7 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf @@ -35,7 +35,11 @@ module "populate_sameday_breaches" { odin_log_destination_arn = local.odin_destination_arn log_subscription_role_arn = local.acct.log_subscription_role_arn - lambda_env_vars = {} + lambda_env_vars = { + ATHENA_WORKGROUP = aws_athena_workgroup.reporting.name + ATHENA_DATABASE = aws_glue_catalog_database.supplier.name + EVENT_CACHE_TABLE_NAME = "${replace(local.csi, "-", "_")}_eventpub" + } } data "aws_iam_policy_document" "populate_sameday_breaches_lambda" { @@ -52,4 +56,65 @@ data "aws_iam_policy_document" "populate_sameday_breaches_lambda" { module.kms.key_arn, ] } + statement { + sid = "AllowAthenaQueryExecution" + effect = "Allow" + + actions = [ + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:StartQueryExecution", + ] + + resources = [aws_athena_workgroup.reporting.arn] + } + + statement { + sid = "AllowGlueCatalogRead" + effect = "Allow" + + actions = [ + "glue:GetDatabase", + "glue:GetPartitions", + "glue:GetTable", + ] + + resources = [ + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + "arn:aws:glue:${var.region}:${var.aws_account_id}:database/${aws_glue_catalog_database.supplier.name}", + "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.supplier.name}/*", + ] + } + + statement { + sid = "AllowReadReportingData" + effect = "Allow" + + actions = [ + "s3:GetBucketLocation", + "s3:ListBucket", + ] + + resources = [module.s3bucket_reporting.arn] + } + + statement { + sid = "AllowReadEventData" + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["${module.s3bucket_reporting.arn}/parquet/*"] + } + + statement { + sid = "AllowAthenaOutput" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetObject", + "s3:PutObject", + ] + + resources = ["${module.s3bucket_reporting.arn}/athena-output/*"] + } } diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index f5665bb82..4b6475c91 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -35,7 +35,11 @@ module "populate_standard_breaches" { odin_log_destination_arn = local.odin_destination_arn log_subscription_role_arn = local.acct.log_subscription_role_arn - lambda_env_vars = {} + lambda_env_vars = { + ATHENA_WORKGROUP = aws_athena_workgroup.reporting.name + ATHENA_DATABASE = aws_glue_catalog_database.supplier.name + EVENT_CACHE_TABLE_NAME = "${replace(local.csi, "-", "_")}_eventpub" + } } data "aws_iam_policy_document" "populate_standard_breaches_lambda" { @@ -52,4 +56,65 @@ data "aws_iam_policy_document" "populate_standard_breaches_lambda" { module.kms.key_arn, ] } + statement { + sid = "AllowAthenaQueryExecution" + effect = "Allow" + + actions = [ + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:StartQueryExecution", + ] + + resources = [aws_athena_workgroup.reporting.arn] + } + + statement { + sid = "AllowGlueCatalogRead" + effect = "Allow" + + actions = [ + "glue:GetDatabase", + "glue:GetPartitions", + "glue:GetTable", + ] + + resources = [ + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + "arn:aws:glue:${var.region}:${var.aws_account_id}:database/${aws_glue_catalog_database.supplier.name}", + "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.supplier.name}/*", + ] + } + + statement { + sid = "AllowReadReportingData" + effect = "Allow" + + actions = [ + "s3:GetBucketLocation", + "s3:ListBucket", + ] + + resources = [module.s3bucket_reporting.arn] + } + + statement { + sid = "AllowReadEventData" + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["${module.s3bucket_reporting.arn}/parquet/*"] + } + + statement { + sid = "AllowAthenaOutput" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetObject", + "s3:PutObject", + ] + + resources = ["${module.s3bucket_reporting.arn}/athena-output/*"] + } } From e74a28324cc9947fbe56c105ed52166236bcbf82 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 15 Oct 2026 12:46:53 +0100 Subject: [PATCH 22/28] fix predicate and sameday parameter --- .../__tests__/athena-date-predicate.test.ts | 20 ++++++++++--------- .../common/src/athena-date-predicate.ts | 2 +- .../__tests__/sameday-breaches-query.test.ts | 2 +- .../src/handler/sameday-breaches-query.ts | 2 +- .../__tests__/standard-breaches-query.test.ts | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts index a0a4f545c..932387f95 100644 --- a/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts @@ -4,21 +4,23 @@ describe("buildAthenaDatePredicate", () => { it("includes only the requested UTC date when no days are subtracted", () => { expect( buildAthenaDatePredicate(new Date("2026-10-14T23:59:59.999Z"), 0), - ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('14'))"); + ).toBe("(__year = '2026' AND __month = '10' AND __day IN ('14'))"); }); it("groups consecutive dates in the same month into one partition predicate", () => { expect( buildAthenaDatePredicate(new Date("2026-10-14T12:00:00.000Z"), 2), - ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13', '14'))"); + ).toBe( + "(__year = '2026' AND __month = '10' AND __day IN ('12', '13', '14'))", + ); }); it("creates separate predicates when the date range crosses a month", () => { expect( buildAthenaDatePredicate(new Date("2026-11-01T00:00:00.000Z"), 2), ).toBe( - "(_year = '2026' AND _month = '10' AND _day IN ('30', '31'))\n" + - " OR (_year = '2026' AND _month = '11' AND _day IN ('01'))", + "(__year = '2026' AND __month = '10' AND __day IN ('30', '31'))\n" + + " OR (__year = '2026' AND __month = '11' AND __day IN ('01'))", ); }); @@ -26,8 +28,8 @@ describe("buildAthenaDatePredicate", () => { expect( buildAthenaDatePredicate(new Date("2026-01-01T00:00:00.000Z"), 1), ).toBe( - "(_year = '2025' AND _month = '12' AND _day IN ('31'))\n" + - " OR (_year = '2026' AND _month = '01' AND _day IN ('01'))", + "(__year = '2025' AND __month = '12' AND __day IN ('31'))\n" + + " OR (__year = '2026' AND __month = '01' AND __day IN ('01'))", ); }); @@ -35,15 +37,15 @@ describe("buildAthenaDatePredicate", () => { expect( buildAthenaDatePredicate(new Date("2024-03-01T00:00:00.000Z"), 2), ).toBe( - "(_year = '2024' AND _month = '02' AND _day IN ('28', '29'))\n" + - " OR (_year = '2024' AND _month = '03' AND _day IN ('01'))", + "(__year = '2024' AND __month = '02' AND __day IN ('28', '29'))\n" + + " OR (__year = '2024' AND __month = '03' AND __day IN ('01'))", ); }); it("uses UTC calendar dates regardless of the time component", () => { expect( buildAthenaDatePredicate(new Date("2026-10-14T00:30:00+02:00"), 1), - ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13'))"); + ).toBe("(__year = '2026' AND __month = '10' AND __day IN ('12', '13'))"); }); it("throws when given an invalid date", () => { diff --git a/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts index 0ca204bac..17b74ed1d 100644 --- a/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts +++ b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts @@ -37,7 +37,7 @@ export default function buildAthenaDatePredicate( .map( // eslint-disable-next-line sort-destructure-keys/sort-destructure-keys ({ year, month, days }) => - `(_year = '${year}' AND _month = '${month}' AND _day IN (${days + `(__year = '${year}' AND __month = '${month}' AND __day IN (${days .map((day) => `'${day}'`) .join(", ")}))`, ) diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts index a83957668..011bae6bb 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/sameday-breaches-query.test.ts @@ -7,7 +7,7 @@ describe("buildSamedayBreachesQuery", () => { ).toEqual( expect.objectContaining({ queryString: expect.stringContaining( - "(_year = '2026' AND _month = '10' AND _day IN ('08'))", + "(__year = '2026' AND __month = '10' AND __day IN ('08'))", ), executionParameters: ["2026-10-08T00:00:00.000Z"], }), diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts index 6a11a40b8..411f6d33d 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/sameday-breaches-query.ts @@ -10,7 +10,7 @@ export default function buildSamedayBreachesQuery( ): AthenaQuery { return { queryString: `WITH now AS ( - SELECT from_iso8601_timestamp('?') AS iso8601 + SELECT from_iso8601_timestamp(?) AS iso8601 ), recent_letter_events AS ( SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts index 1e4b3ff03..ff1f807e1 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -7,7 +7,7 @@ describe("buildStandardBreachesQuery", () => { ).toEqual( expect.objectContaining({ queryString: expect.stringContaining( - "(_year = '2026' AND _month = '10' AND _day IN ('08'))", + "(__year = '2026' AND __month = '10' AND __day IN ('08'))", ), executionParameters: ["2026-10-08T00:00:00.000Z"], }), From 88ef8274262e2223feb12096df77542e40b5205d Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 15 Oct 2026 13:42:42 +0100 Subject: [PATCH 23/28] fix breach_count error in both breach lambdas --- .../populate-sameday-breaches.test.ts | 24 +++++++++++++++++++ .../src/handler/populate-sameday-breaches.ts | 8 +++++-- .../populate-standard-breaches.test.ts | 24 +++++++++++++++++++ .../src/handler/populate-standard-breaches.ts | 8 +++++-- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts index 716ec5b3e..949e1e26f 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts @@ -23,6 +23,30 @@ describe("populateSamedayBreaches", () => { expect(result).toEqual({ status: "SUCCESS", breachCount: "1" }); }); + it("sums the breach count across suppliers", async () => { + const deps = makeDeps(); + jest + .mocked(runAthenaQuery) + .mockResolvedValueOnce([ + { supplierid: "a", breach_count: "2" }, + { supplierid: "b", breach_count: "3" }, + { supplierid: "c" }, + ]); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(result).toEqual({ status: "SUCCESS", breachCount: "5" }); + }); + + it("returns a zero breach count when the query returns no rows", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([]); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(result).toEqual({ status: "SUCCESS", breachCount: "0" }); + }); + it("runs the sameday breaches query and returns failure", async () => { const deps = makeDeps(); jest.mocked(runAthenaQuery).mockRejectedValueOnce(new Error("boom")); diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts index 43130ce2e..33bf56b53 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts @@ -28,13 +28,17 @@ export default function createHandler(deps: Deps) { ), ); + const breachCount = String( + rows.reduce((total, row) => total + Number(row.breach_count ?? 0), 0), + ); + // TODO: CCM-24041 process the query results to populate sameday breaches. deps.logger.info( - { date: event.date, breachCount: rows[0].breach_count }, + { date: event.date, breachCount }, "sameday breaches query completed", ); - return { status: "SUCCESS", breachCount: rows[0].breach_count! }; + return { status: "SUCCESS", breachCount }; } catch (error) { deps.logger.error( { date: event.date, error }, diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index 5516860f4..5843c2073 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -23,6 +23,30 @@ describe("populateStandardBreaches", () => { expect(result).toEqual({ status: "SUCCESS", breachCount: "1" }); }); + it("sums the breach count across suppliers", async () => { + const deps = makeDeps(); + jest + .mocked(runAthenaQuery) + .mockResolvedValueOnce([ + { supplierid: "a", breach_count: "2" }, + { supplierid: "b", breach_count: "3" }, + { supplierid: "c" }, + ]); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(result).toEqual({ status: "SUCCESS", breachCount: "5" }); + }); + + it("returns a zero breach count when the query returns no rows", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([]); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(result).toEqual({ status: "SUCCESS", breachCount: "0" }); + }); + it("runs the standard breaches query and returns failure", async () => { const deps = makeDeps(); jest.mocked(runAthenaQuery).mockRejectedValueOnce(new Error("boom")); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index 72950dde6..b93666ac9 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -28,13 +28,17 @@ export default function createHandler(deps: Deps) { ), ); + const breachCount = String( + rows.reduce((total, row) => total + Number(row.breach_count ?? 0), 0), + ); + // TODO: CCM-24041 process the query results to populate standard breaches. deps.logger.info( - { date: event.date, breachCount: rows[0].breach_count }, + { date: event.date, breachCount }, "Standard breaches query completed", ); - return { status: "SUCCESS", breachCount: rows[0].breach_count! }; + return { status: "SUCCESS", breachCount }; } catch (error) { deps.logger.error( { date: event.date, error }, From 77d773f7168700b8ed2c2dcd5e70c7abaab4aec1 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 15 Oct 2026 14:15:27 +0100 Subject: [PATCH 24/28] use passed in date in breach lambdas --- .../populate-sameday-breaches.test.ts | 25 +++++++++++++++++++ .../src/handler/populate-sameday-breaches.ts | 2 +- .../populate-standard-breaches.test.ts | 25 +++++++++++++++++++ .../src/handler/populate-standard-breaches.ts | 2 +- 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts index 949e1e26f..fece836ce 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts @@ -38,6 +38,31 @@ describe("populateSamedayBreaches", () => { expect(result).toEqual({ status: "SUCCESS", breachCount: "5" }); }); + it("builds the query from the date component of the event date", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([]); + + await createHandler(deps)({ date: "2026-10-08T15:30:00.000Z" }); + + expect(runAthenaQuery).toHaveBeenCalledWith( + deps, + expect.objectContaining({ + executionParameters: ["2026-10-08T00:00:00.000Z"], + }), + ); + }); + + it("returns failure when the event date is invalid", async () => { + const deps = makeDeps(); + + const result = await createHandler(deps)({ date: "not-a-date" }); + + expect(runAthenaQuery).not.toHaveBeenCalled(); + expect(result).toEqual( + expect.objectContaining({ status: "FAILURE", breachCount: "0" }), + ); + }); + it("returns a zero breach count when the query returns no rows", async () => { const deps = makeDeps(); jest.mocked(runAthenaQuery).mockResolvedValueOnce([]); diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts index 33bf56b53..49e6f9d8e 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts @@ -23,7 +23,7 @@ export default function createHandler(deps: Deps) { deps, buildSamedayBreachesQuery( deps.env.EVENT_CACHE_TABLE_NAME, - new Date(), + new Date(event.date.slice(0, 10)), 14, ), ); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index 5843c2073..5b6941f68 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -38,6 +38,31 @@ describe("populateStandardBreaches", () => { expect(result).toEqual({ status: "SUCCESS", breachCount: "5" }); }); + it("builds the query from the date component of the event date", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([]); + + await createHandler(deps)({ date: "2026-10-08T15:30:00.000Z" }); + + expect(runAthenaQuery).toHaveBeenCalledWith( + deps, + expect.objectContaining({ + executionParameters: ["2026-10-08T00:00:00.000Z"], + }), + ); + }); + + it("returns failure when the event date is invalid", async () => { + const deps = makeDeps(); + + const result = await createHandler(deps)({ date: "not-a-date" }); + + expect(runAthenaQuery).not.toHaveBeenCalled(); + expect(result).toEqual( + expect.objectContaining({ status: "FAILURE", breachCount: "0" }), + ); + }); + it("returns a zero breach count when the query returns no rows", async () => { const deps = makeDeps(); jest.mocked(runAthenaQuery).mockResolvedValueOnce([]); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index b93666ac9..873490b34 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -23,7 +23,7 @@ export default function createHandler(deps: Deps) { deps, buildStandardBreachesQuery( deps.env.EVENT_CACHE_TABLE_NAME, - new Date(), + new Date(event.date.slice(0, 10)), 14, ), ); From 39dbb992db66ba6bc0663702b952c3996787082f Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Fri, 16 Oct 2026 10:02:12 +0100 Subject: [PATCH 25/28] change transformer to extract clientid and campaignid from groupid --- .../src/__tests__/transform-record.test.ts | 46 +++++++++++++++++-- .../src/transform-record.ts | 20 +++++++- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts index 677cf5752..f08104dd1 100644 --- a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts +++ b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts @@ -16,9 +16,7 @@ describe("transformRecord", () => { const data = { domainId: "d1", supplierId: "s1", - groupId: "g1", - campaignId: "c1", - clientId: "cl1", + groupId: "cl1-c1-t1", status: "PRINTED", }; @@ -39,13 +37,53 @@ describe("transformRecord", () => { data_json: JSON.stringify(data), domainid: "d1", supplierid: "s1", - groupid: "g1", + groupid: "cl1-c1-t1", campaignid: "c1", clientid: "cl1", status: "PRINTED", }); }); + it.each([ + ["cl1-c1-t1", "cl1", "c1"], + ["cl1-c1", "cl1", "c1"], + ["cl1", "cl1", undefined], + ["cl1--t1", "cl1", undefined], + ["-c1-t1", undefined, "c1"], + [" - ", undefined, undefined], + ["", undefined, undefined], + [undefined, undefined, undefined], + [123, undefined, undefined], + ])( + "derives clientid and campaignid from groupId %p", + (groupId, clientid, campaignid) => { + const out = decode( + transformRecord({ + recordId: "r", + data: encode({ id: "e", data: { groupId } }), + }).data!, + ); + + expect(out.clientid).toBe(clientid); + expect(out.campaignid).toBe(campaignid); + }, + ); + + it("ignores clientId and campaignId properties on the data", () => { + const out = decode( + transformRecord({ + recordId: "r", + data: encode({ + id: "e", + data: { groupId: "cl1-c1-t1", clientId: "x", campaignId: "y" }, + }), + }).data!, + ); + + expect(out.clientid).toBe("cl1"); + expect(out.campaignid).toBe("c1"); + }); + it("leaves promoted properties empty when data is missing or not an object", () => { const missing = decode( transformRecord({ recordId: "r", data: encode({ id: "e" }) }).data!, diff --git a/lambdas/eventpub-parquet-transformer/src/transform-record.ts b/lambdas/eventpub-parquet-transformer/src/transform-record.ts index b743de50f..a9800e750 100644 --- a/lambdas/eventpub-parquet-transformer/src/transform-record.ts +++ b/lambdas/eventpub-parquet-transformer/src/transform-record.ts @@ -27,8 +27,6 @@ const PROMOTED_DATA_FIELDS = [ "domainId", "supplierId", "groupId", - "campaignId", - "clientId", "status", ] as const; @@ -38,6 +36,21 @@ function isObject(value: unknown): value is Json { return typeof value === "object" && value !== null && !Array.isArray(value); } +// groupId is "--"; missing or blank parts stay undefined +function parseGroupId(groupId: unknown): { + clientId?: string; + campaignId?: string; +} { + if (typeof groupId !== "string") { + return {}; + } + const [clientId, campaignId] = groupId.split("-"); + return { + clientId: clientId?.trim() || undefined, + campaignId: campaignId?.trim() || undefined, + }; +} + function flatten(event: Json): Json { const data = isObject(event.data) ? event.data : {}; @@ -49,6 +62,9 @@ function flatten(event: Json): Json { for (const field of PROMOTED_DATA_FIELDS) { flat[field.toLowerCase()] = data[field]; } + const { campaignId, clientId } = parseGroupId(data.groupId); + flat.clientid = clientId; + flat.campaignid = campaignId; return flat; } From 775394bac1036220cc8df30a95cf69f94a0948b1 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Fri, 16 Oct 2026 10:32:31 +0100 Subject: [PATCH 26/28] fix broken terraform --- infrastructure/terraform/components/api/variables.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index 0ff8df276..2d0b63bfa 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -302,6 +302,8 @@ variable "reports_data_retention_non_current_days" { type = number description = "Number of days to retain noncurrent reporting bucket object versions" default = 14 +} + variable "event_source_filter" { type = list(string) description = "SNS subscription source filter values" From 937e8b41081e02f5d33be53c11a00ba55ec7503d Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Fri, 16 Oct 2026 13:54:38 +0100 Subject: [PATCH 27/28] use correct eventpub name and introduce count in terraform --- .../components/api/iam_role_letter_sla_breaches.tf | 1 + .../api/iam_role_policy_letter_sla_breaches.tf | 11 +++++++---- .../api/module_lambda_populate_sameday_breaches.tf | 8 +++++++- .../api/module_lambda_populate_standard_breaches.tf | 8 +++++++- .../api/sfn_state_machine_letter_sla_breaches.tf | 7 ++++--- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf index 97a35bad2..b56042514 100644 --- a/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf @@ -1,4 +1,5 @@ resource "aws_iam_role" "letter_sla_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 name = "${local.csi}-letter-sla-breaches-sfn" description = "Execution role for the letter SLA breaches state machine" assume_role_policy = data.aws_iam_policy_document.letter_sla_breaches_trust_policy.json diff --git a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf index d893cd8b0..25d483c6c 100644 --- a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf @@ -1,10 +1,13 @@ resource "aws_iam_role_policy" "letter_sla_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 name = "${local.csi}-letter-sla-breaches-sfn" - role = aws_iam_role.letter_sla_breaches.id - policy = data.aws_iam_policy_document.letter_sla_breaches_policy.json + role = aws_iam_role.letter_sla_breaches[0].id + policy = data.aws_iam_policy_document.letter_sla_breaches_policy[0].json } data "aws_iam_policy_document" "letter_sla_breaches_policy" { + count = local.eventpub_parquet_enabled ? 1 : 0 + statement { sid = "AllowListOwnExecutions" effect = "Allow" @@ -18,8 +21,8 @@ data "aws_iam_policy_document" "letter_sla_breaches_policy" { actions = ["lambda:InvokeFunction"] resources = [ module.populate_supplier_config_cache.function_arn, - module.populate_sameday_breaches.function_arn, - module.populate_standard_breaches.function_arn, + module.populate_sameday_breaches[0].function_arn, + module.populate_standard_breaches[0].function_arn, ] } } diff --git a/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf index f423e8ae7..da6140233 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf @@ -1,4 +1,10 @@ +moved { + from = module.populate_sameday_breaches + to = module.populate_sameday_breaches[0] +} + module "populate_sameday_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" function_name = "populate_sameday_breaches" @@ -38,7 +44,7 @@ module "populate_sameday_breaches" { lambda_env_vars = { ATHENA_WORKGROUP = aws_athena_workgroup.reporting.name ATHENA_DATABASE = aws_glue_catalog_database.supplier.name - EVENT_CACHE_TABLE_NAME = "${replace(local.csi, "-", "_")}_eventpub" + EVENT_CACHE_TABLE_NAME = aws_glue_catalog_table.eventpub_parquet[0].name } } diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index 4b6475c91..d07120334 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -1,4 +1,10 @@ +moved { + from = module.populate_standard_breaches + to = module.populate_standard_breaches[0] +} + module "populate_standard_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" function_name = "populate_standard_breaches" @@ -38,7 +44,7 @@ module "populate_standard_breaches" { lambda_env_vars = { ATHENA_WORKGROUP = aws_athena_workgroup.reporting.name ATHENA_DATABASE = aws_glue_catalog_database.supplier.name - EVENT_CACHE_TABLE_NAME = "${replace(local.csi, "-", "_")}_eventpub" + EVENT_CACHE_TABLE_NAME = aws_glue_catalog_table.eventpub_parquet[0].name } } diff --git a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf index 19d726103..ccb5a8ef9 100644 --- a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf @@ -1,11 +1,12 @@ resource "aws_sfn_state_machine" "letter_sla_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 name = local.letter_sla_breaches_state_machine_name - role_arn = aws_iam_role.letter_sla_breaches.arn + role_arn = aws_iam_role.letter_sla_breaches[0].arn definition = templatefile("${path.module}/resources/letter_sla_breaches.json", { STATE_MACHINE_ARN = local.letter_sla_breaches_state_machine_arn POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN = module.populate_supplier_config_cache.function_arn - POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN = module.populate_sameday_breaches.function_arn - POPULATE_STANDARD_BREACHES_FUNCTION_ARN = module.populate_standard_breaches.function_arn + POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN = module.populate_sameday_breaches[0].function_arn + POPULATE_STANDARD_BREACHES_FUNCTION_ARN = module.populate_standard_breaches[0].function_arn }) } From d49677faa5b6b85d2565e668bdd3352b26644c9d Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Fri, 16 Oct 2026 14:35:48 +0100 Subject: [PATCH 28/28] remove moved section --- .../api/module_lambda_populate_standard_breaches.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index d07120334..083f5ddfe 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -1,8 +1,3 @@ -moved { - from = module.populate_standard_breaches - to = module.populate_standard_breaches[0] -} - module "populate_standard_breaches" { count = local.eventpub_parquet_enabled ? 1 : 0 source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip"