From be02e8daca5392964d7d5a48ff3c773ebe3f591e Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 8 Oct 2026 12:04:18 +0100 Subject: [PATCH 01/37] add state machine, popluate supplier config, and dummy lambdas --- eslint.config.mjs | 2 +- .../terraform/components/api/README.md | 4 + .../api/iam_role_letter_sla_breaches.tf | 17 + .../iam_role_policy_letter_sla_breaches.tf | 25 + .../terraform/components/api/locals.tf | 3 + ...module_lambda_populate_sameday_breaches.tf | 55 ++ ...odule_lambda_populate_standard_breaches.tf | 55 ++ ...e_lambda_populate_supplier_config_cache.tf | 72 ++ .../api/resources/letter_sla_breaches.json | 77 ++ .../sfn_state_machine_letter_sla_breaches.tf | 11 + .../populate-sameday-breaches/.gitignore | 5 + .../populate-sameday-breaches/jest.config.ts | 46 + .../populate-sameday-breaches/package.json | 15 + .../populate-sameday-breaches.test.ts | 12 + .../src/handler/populate-sameday-breaches.ts | 18 + .../populate-sameday-breaches/src/index.ts | 2 + .../populate-sameday-breaches/tsconfig.json | 11 + .../populate-standard-breaches/.gitignore | 5 + .../populate-standard-breaches/jest.config.ts | 46 + .../populate-standard-breaches/package.json | 15 + .../populate-standard-breaches.test.ts | 12 + .../src/handler/populate-standard-breaches.ts | 18 + .../populate-standard-breaches/src/index.ts | 2 + .../populate-standard-breaches/tsconfig.json | 11 + .../populate-supplier-config-cache/.gitignore | 5 + .../jest.config.ts | 46 + .../package.json | 18 + .../src/__tests__/entity-cache.test.ts | 72 ++ .../populate-supplier-config-cache.test.ts | 32 + .../src/handler/deps.ts | 15 + .../src/handler/entity-cache.ts | 49 + .../handler/populate-supplier-config-cache.ts | 34 + .../src/index.ts | 7 + .../tsconfig.json | 11 + package-lock.json | 846 ++++-------------- package.json | 2 +- 36 files changed, 982 insertions(+), 694 deletions(-) create mode 100644 infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf create mode 100644 infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf create mode 100644 infrastructure/terraform/components/api/resources/letter_sla_breaches.json create mode 100644 infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/package.json create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts create mode 100644 lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/package.json create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json diff --git a/eslint.config.mjs b/eslint.config.mjs index 09a55d787..029ba22cd 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -115,7 +115,7 @@ export default defineConfig([ eslintImportResolverTypescript.createTypeScriptImportResolver({ project: [ 'lambdas/*/tsconfig.json', - 'tests/tsconfig.json', + 'lambdas/*/*/tsconfig.json', 'tests/tsconfig.json', 'internal/*/tsconfig.json', ], }), diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index b4d8adfb4..5fe8966cf 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -62,6 +62,7 @@ No requirements. | Name | Source | Version | | ---- | ------ | ------- | | [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [amendments\_queue](#module\_amendments\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | | [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [ddb\_alarms\_idempotency](#module\_ddb\_alarms\_idempotency) | ../../modules/alarms-ddb | n/a | @@ -83,6 +84,9 @@ No requirements. | [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [patch\_letter](#module\_patch\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [populate\_sameday\_breaches](#module\_populate\_sameday\_breaches) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [populate\_standard\_breaches](#module\_populate\_standard\_breaches) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a | diff --git a/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf new file mode 100644 index 000000000..97a35bad2 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf @@ -0,0 +1,17 @@ +resource "aws_iam_role" "letter_sla_breaches" { + name = "${local.csi}-letter-sla-breaches-sfn" + description = "Execution role for the letter SLA breaches state machine" + assume_role_policy = data.aws_iam_policy_document.letter_sla_breaches_trust_policy.json +} + +data "aws_iam_policy_document" "letter_sla_breaches_trust_policy" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["states.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf new file mode 100644 index 000000000..d893cd8b0 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf @@ -0,0 +1,25 @@ +resource "aws_iam_role_policy" "letter_sla_breaches" { + name = "${local.csi}-letter-sla-breaches-sfn" + role = aws_iam_role.letter_sla_breaches.id + policy = data.aws_iam_policy_document.letter_sla_breaches_policy.json +} + +data "aws_iam_policy_document" "letter_sla_breaches_policy" { + statement { + sid = "AllowListOwnExecutions" + effect = "Allow" + actions = ["states:ListExecutions"] + resources = [local.letter_sla_breaches_state_machine_arn] + } + + statement { + sid = "AllowInvokeLambdas" + effect = "Allow" + actions = ["lambda:InvokeFunction"] + resources = [ + module.populate_supplier_config_cache.function_arn, + module.populate_sameday_breaches.function_arn, + module.populate_standard_breaches.function_arn, + ] + } +} diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 6b4d90305..18c52e680 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -52,4 +52,7 @@ locals { var.region, var.csoc_destination_account ) + + letter_sla_breaches_state_machine_name = "${local.csi}-letter-sla-breaches" + letter_sla_breaches_state_machine_arn = "arn:aws:states:${var.region}:${var.aws_account_id}:stateMachine:${local.letter_sla_breaches_state_machine_name}" } diff --git a/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf new file mode 100644 index 000000000..020c4dd1c --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_populate_sameday_breaches.tf @@ -0,0 +1,55 @@ +module "populate_sameday_breaches" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "populate_sameday_breaches" + description = "Populates same-day breaches for the letter SLA breaches state machine" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.populate_sameday_breaches_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "letter-sla-breaches/populate-sameday-breaches/dist" + function_include_common = true + handler_function_name = "populateSamedayBreachesHandler" + runtime = "nodejs22.x" + memory = 128 + timeout = 29 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = {} +} + +data "aws_iam_policy_document" "populate_sameday_breaches_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } +} diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf new file mode 100644 index 000000000..f5665bb82 --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -0,0 +1,55 @@ +module "populate_standard_breaches" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "populate_standard_breaches" + description = "Populates standard breaches for the letter SLA breaches state machine" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.populate_standard_breaches_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "letter-sla-breaches/populate-standard-breaches/dist" + function_include_common = true + handler_function_name = "populateStandardBreachesHandler" + runtime = "nodejs22.x" + memory = 128 + timeout = 29 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = {} +} + +data "aws_iam_policy_document" "populate_standard_breaches_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } +} diff --git a/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf new file mode 100644 index 000000000..57bf707fa --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf @@ -0,0 +1,72 @@ +module "populate_supplier_config_cache" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "populate_supplier_config_cache" + description = "Populates the supplier config cache for the letter SLA breaches state machine" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.populate_supplier_config_cache_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "letter-sla-breaches/populate-supplier-config-cache/dist" + function_include_common = true + handler_function_name = "populateSupplierConfigCacheHandler" + runtime = "nodejs22.x" + memory = 128 + timeout = 29 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = { + SUPPLIER_CONFIG_TABLE_NAME = aws_dynamodb_table.supplier-configuration.name + EVENT_CACHE_BUCKET_NAME = local.eventsub_event_cache_bucket_name + } +} + +data "aws_iam_policy_document" "populate_supplier_config_cache_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } + + statement { + sid = "AllowQuerySupplierConfigTable" + effect = "Allow" + actions = ["dynamodb:Query"] + resources = [aws_dynamodb_table.supplier-configuration.arn] + } + + statement { + sid = "AllowWriteSupplierConfigCache" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/supplier_config_cache/*"] + } +} diff --git a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json new file mode 100644 index 000000000..725b8eaf3 --- /dev/null +++ b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json @@ -0,0 +1,77 @@ +{ + "Comment": "Identifies letters in breach of SLA for a given date (defaults to today)", + "QueryLanguage": "JSONata", + "StartAt": "CheckForRunningExecutions", + "States": { + "AlreadyRunning?": { + "Choices": [ + { + "Condition": "{% $count($states.input.Executions) > 1 %}", + "Next": "ExitAlreadyRunning" + } + ], + "Default": "PopulateSupplierConfigCache", + "Type": "Choice" + }, + "CheckForRunningExecutions": { + "Arguments": { + "StateMachineArn": "${STATE_MACHINE_ARN}", + "StatusFilter": "RUNNING" + }, + "Assign": { + "date": "{% $exists($states.context.Execution.Input.date) ? $states.context.Execution.Input.date : $substring($states.context.Execution.StartTime, 0, 10) %}" + }, + "Next": "AlreadyRunning?", + "Resource": "arn:aws:states:::aws-sdk:sfn:listExecutions", + "Type": "Task" + }, + "ExitAlreadyRunning": { + "Type": "Succeed" + }, + "PopulateSamedayBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "Next": "PopulateStandardBreaches", + "Output": { + "populateSamedayBreaches": "{% $states.result.Payload %}", + "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + }, + "PopulateStandardBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_STANDARD_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "End": true, + "Output": { + "populateSamedayBreaches": "{% $states.input.populateSamedayBreaches %}", + "populateStandardBreaches": "{% $states.result.Payload %}", + "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + }, + "PopulateSupplierConfigCache": { + "Arguments": { + "FunctionName": "${POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "Next": "PopulateSamedayBreaches", + "Output": { + "populateSupplierConfigCache": "{% $states.result.Payload %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + } + } +} diff --git a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf new file mode 100644 index 000000000..19d726103 --- /dev/null +++ b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf @@ -0,0 +1,11 @@ +resource "aws_sfn_state_machine" "letter_sla_breaches" { + name = local.letter_sla_breaches_state_machine_name + role_arn = aws_iam_role.letter_sla_breaches.arn + + definition = templatefile("${path.module}/resources/letter_sla_breaches.json", { + STATE_MACHINE_ARN = local.letter_sla_breaches_state_machine_arn + POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN = module.populate_supplier_config_cache.function_arn + POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN = module.populate_sameday_breaches.function_arn + POPULATE_STANDARD_BREACHES_FUNCTION_ARN = module.populate_standard_breaches.function_arn + }) +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore b/lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore new file mode 100644 index 000000000..fc5a7859a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/.gitignore @@ -0,0 +1,5 @@ +coverage +node_modules +dist +.reports +.aws-sam diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts new file mode 100644 index 000000000..ec5a312bb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/jest.config.ts @@ -0,0 +1,46 @@ +export const baseJestConfig = { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + testEnvironment: "node", +}; + +export default baseJestConfig; diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json new file mode 100644 index 000000000..127c212c3 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/package.json @@ -0,0 +1,15 @@ +{ + "dependencies": { + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-populate-sameday-breaches", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts new file mode 100644 index 000000000..50492a04a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/__tests__/populate-sameday-breaches.test.ts @@ -0,0 +1,12 @@ +import populateSamedayBreaches from "../handler/populate-sameday-breaches"; + +describe("populateSamedayBreaches", () => { + it("returns a success result", async () => { + const result = await populateSamedayBreaches({ date: "2026-10-08" }); + + expect(result).toEqual({ + status: "SUCCESS", + message: "populate-sameday-breaches invoked successfully", + }); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts new file mode 100644 index 000000000..f63d70492 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/handler/populate-sameday-breaches.ts @@ -0,0 +1,18 @@ +export type PopulateSamedayBreachesEvent = { + date: string; +}; + +export type PopulateSamedayBreachesResult = { + status: "SUCCESS"; + message: string; +}; + +// Placeholder: cache population is not yet implemented. +export default async function populateSamedayBreaches( + _event: PopulateSamedayBreachesEvent, +): Promise { + return { + status: "SUCCESS", + message: "populate-sameday-breaches invoked successfully", + }; +} diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts new file mode 100644 index 000000000..3143567ed --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/src/index.ts @@ -0,0 +1,2 @@ +// eslint-disable-next-line import-x/prefer-default-export +export { default as populateSamedayBreachesHandler } from "./handler/populate-sameday-breaches"; diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore b/lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore new file mode 100644 index 000000000..fc5a7859a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/.gitignore @@ -0,0 +1,5 @@ +coverage +node_modules +dist +.reports +.aws-sam diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts new file mode 100644 index 000000000..ec5a312bb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/jest.config.ts @@ -0,0 +1,46 @@ +export const baseJestConfig = { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + testEnvironment: "node", +}; + +export default baseJestConfig; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json new file mode 100644 index 000000000..8875e4acc --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json @@ -0,0 +1,15 @@ +{ + "dependencies": { + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-populate-standard-breaches", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts new file mode 100644 index 000000000..f64093251 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -0,0 +1,12 @@ +import populateStandardBreaches from "../handler/populate-standard-breaches"; + +describe("populateStandardBreaches", () => { + it("returns a success result", async () => { + const result = await populateStandardBreaches({ date: "2026-10-08" }); + + expect(result).toEqual({ + status: "SUCCESS", + message: "populate-standard-breaches invoked successfully", + }); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts new file mode 100644 index 000000000..1bcc2318c --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -0,0 +1,18 @@ +export type PopulateStandardBreachesEvent = { + date: string; +}; + +export type PopulateStandardBreachesResult = { + status: "SUCCESS"; + message: string; +}; + +// Placeholder: cache population is not yet implemented. +export default async function populateStandardBreaches( + _event: PopulateStandardBreachesEvent, +): Promise { + return { + status: "SUCCESS", + message: "populate-standard-breaches invoked successfully", + }; +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts new file mode 100644 index 000000000..5516fb886 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts @@ -0,0 +1,2 @@ +// eslint-disable-next-line import-x/prefer-default-export +export { default as populateStandardBreachesHandler } from "./handler/populate-standard-breaches"; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json b/lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore b/lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore new file mode 100644 index 000000000..fc5a7859a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/.gitignore @@ -0,0 +1,5 @@ +coverage +node_modules +dist +.reports +.aws-sam diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts new file mode 100644 index 000000000..ec5a312bb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/jest.config.ts @@ -0,0 +1,46 @@ +export const baseJestConfig = { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + testEnvironment: "node", +}; + +export default baseJestConfig; diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json new file mode 100644 index 000000000..034e0301b --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -0,0 +1,18 @@ +{ + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.1138.0", + "@aws-sdk/client-s3": "^3.1138.0", + "@aws-sdk/lib-dynamodb": "^3.1138.0", + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-populate-supplier-config-cache", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts new file mode 100644 index 000000000..1dac234a9 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -0,0 +1,72 @@ +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; + +function makeDeps(ddbSend: jest.Mock, s3Send: jest.Mock): EntityCacheDeps { + return { + ddbClient: { send: ddbSend } as unknown as DynamoDBDocumentClient, + s3Client: { send: s3Send } as unknown as S3Client, + env: { + SUPPLIER_CONFIG_TABLE_NAME: "config-table", + EVENT_CACHE_BUCKET_NAME: "cache-bucket", + }, + }; +} + +describe("populateEntityCache", () => { + it("queries all pages for the entity and writes them to S3", async () => { + const ddbSend = jest + .fn() + .mockResolvedValueOnce({ + Items: [{ pk: "ENTITY#supplier", sk: "ID#1" }], + LastEvaluatedKey: { pk: "ENTITY#supplier", sk: "ID#1" }, + }) + .mockResolvedValueOnce({ + Items: [{ pk: "ENTITY#supplier", sk: "ID#2" }], + }); + const s3Send = jest.fn().mockResolvedValue({}); + + const count = await populateEntityCache( + makeDeps(ddbSend, s3Send), + "supplier", + ); + + expect(count).toBe(2); + expect(ddbSend).toHaveBeenCalledTimes(2); + const firstQuery = ddbSend.mock.calls[0][0] as QueryCommand; + expect(firstQuery.input).toEqual({ + TableName: "config-table", + KeyConditionExpression: "pk = :pk", + ExpressionAttributeValues: { ":pk": "ENTITY#supplier" }, + ExclusiveStartKey: undefined, + }); + const secondQuery = ddbSend.mock.calls[1][0] as QueryCommand; + expect(secondQuery.input.ExclusiveStartKey).toEqual({ + pk: "ENTITY#supplier", + sk: "ID#1", + }); + + const put = s3Send.mock.calls[0][0] as PutObjectCommand; + expect(put.input.Bucket).toBe("cache-bucket"); + expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); + expect(JSON.parse(put.input.Body as string)).toEqual([ + { pk: "ENTITY#supplier", sk: "ID#1" }, + { pk: "ENTITY#supplier", sk: "ID#2" }, + ]); + }); + + it("writes an empty file when no rows are returned", async () => { + const ddbSend = jest.fn().mockResolvedValue({}); + const s3Send = jest.fn().mockResolvedValue({}); + + const count = await populateEntityCache( + makeDeps(ddbSend, s3Send), + "volume-group", + ); + + expect(count).toBe(0); + const put = s3Send.mock.calls[0][0] as PutObjectCommand; + expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); + expect(put.input.Body).toBe("[]"); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts new file mode 100644 index 000000000..54374eb2f --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -0,0 +1,32 @@ +import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; +import createHandler, { + ENTITY_TYPES, +} from "../handler/populate-supplier-config-cache"; + +jest.mock("../handler/entity-cache"); + +describe("populateSupplierConfigCache", () => { + it("populates the cache once for each entity type and returns success", async () => { + const deps = {} as EntityCacheDeps; + const handler = createHandler(deps); + + const result = await handler({ date: "2026-10-08" }); + + expect(ENTITY_TYPES).toEqual([ + "volume-group", + "supplier-pack", + "letter-variant", + "pack-specification", + "supplier-allocation", + "supplier", + ]); + expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); + for (const entityType of ENTITY_TYPES) { + expect(populateEntityCache).toHaveBeenCalledWith(deps, entityType); + } + expect(result).toEqual({ + status: "SUCCESS", + message: "populate-supplier-config-cache invoked successfully", + }); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts new file mode 100644 index 000000000..6fdf2f926 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts @@ -0,0 +1,15 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { EntityCacheDeps } from "./entity-cache"; + +export default function createDependenciesContainer(): EntityCacheDeps { + return { + ddbClient: DynamoDBDocumentClient.from(new DynamoDBClient()), + s3Client: new S3Client(), + env: { + SUPPLIER_CONFIG_TABLE_NAME: process.env.SUPPLIER_CONFIG_TABLE_NAME!, + EVENT_CACHE_BUCKET_NAME: process.env.EVENT_CACHE_BUCKET_NAME!, + }, + }; +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts new file mode 100644 index 000000000..eab1beef6 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -0,0 +1,49 @@ +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { + DynamoDBDocumentClient, + QueryCommand, + QueryCommandOutput, +} from "@aws-sdk/lib-dynamodb"; + +export type EntityCacheDeps = { + ddbClient: DynamoDBDocumentClient; + s3Client: S3Client; + env: { + SUPPLIER_CONFIG_TABLE_NAME: string; + EVENT_CACHE_BUCKET_NAME: string; + }; +}; + +export const CACHE_FOLDER = "supplier_config_cache"; + +export default async function populateEntityCache( + deps: EntityCacheDeps, + entityType: string, +): Promise { + const items: Record[] = []; + let exclusiveStartKey: QueryCommandOutput["LastEvaluatedKey"]; + + do { + const page: QueryCommandOutput = await deps.ddbClient.send( + new QueryCommand({ + TableName: deps.env.SUPPLIER_CONFIG_TABLE_NAME, + KeyConditionExpression: "pk = :pk", + ExpressionAttributeValues: { ":pk": `ENTITY#${entityType}` }, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + items.push(...(page.Items ?? [])); + exclusiveStartKey = page.LastEvaluatedKey; + } while (exclusiveStartKey); + + await deps.s3Client.send( + new PutObjectCommand({ + Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, + Key: `${CACHE_FOLDER}/${entityType}.json`, + Body: JSON.stringify(items), + ContentType: "application/json", + }), + ); + + return items.length; +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts new file mode 100644 index 000000000..b3c3d57fb --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -0,0 +1,34 @@ +import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; + +export type PopulateSupplierConfigCacheEvent = { + date: string; +}; + +export type PopulateSupplierConfigCacheResult = { + status: "SUCCESS"; + message: string; +}; + +export const ENTITY_TYPES = [ + "volume-group", + "supplier-pack", + "letter-variant", + "pack-specification", + "supplier-allocation", + "supplier", +]; + +export default function createHandler(deps: EntityCacheDeps) { + return async ( + _event: PopulateSupplierConfigCacheEvent, + ): Promise => { + await Promise.all( + ENTITY_TYPES.map((entityType) => populateEntityCache(deps, entityType)), + ); + + return { + status: "SUCCESS", + message: "populate-supplier-config-cache invoked successfully", + }; + }; +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts new file mode 100644 index 000000000..fd06c3b3f --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/index.ts @@ -0,0 +1,7 @@ +import createDependenciesContainer from "./handler/deps"; +import createHandler from "./handler/populate-supplier-config-cache"; + +// eslint-disable-next-line import-x/prefer-default-export +export const populateSupplierConfigCacheHandler = createHandler( + createDependenciesContainer(), +); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index cae5e3cea..a80f25d9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "docs", "internal/*", "lambdas/*", + "lambdas/letter-sla-breaches/*", "pact-contracts", "scripts/utilities/*", "tests", @@ -155,30 +156,6 @@ "node": ">=20.0.0" } }, - "internal/datastore/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "internal/datastore/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "internal/datastore/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -196,33 +173,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "internal/datastore/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "internal/datastore/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "internal/datastore/node_modules/@nhsdigital/nhs-notify-event-schemas-supplier-config": { "version": "1.1.1", "dependencies": { @@ -241,7 +191,7 @@ }, "internal/events": { "name": "@nhsdigital/nhs-notify-event-schemas-supplier-api", - "version": "1.0.21", + "version": "1.0.22", "license": "MIT", "dependencies": { "@asyncapi/bundler": "^1.0.2", @@ -365,30 +315,6 @@ "node": ">=20.0.0" } }, - "lambdas/api-handler/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/api-handler/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/api-handler/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -406,20 +332,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/api-handler/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/api-handler/node_modules/@aws-sdk/middleware-sdk-sqs": { "version": "3.972.43", "license": "Apache-2.0", @@ -433,19 +345,6 @@ "node": ">=20.0.0" } }, - "lambdas/api-handler/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/authorizer": { "name": "nhs-notify-supplier-authorizer", "version": "0.0.1", @@ -480,30 +379,6 @@ "node": ">=20.0.0" } }, - "lambdas/authorizer/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/authorizer/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/authorizer/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -521,31 +396,28 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/authorizer/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", + "lambdas/letter-sla-breaches/populate-sameday-breaches": { + "name": "nhs-notify-supplier-api-populate-sameday-breaches", + "version": "0.0.1", "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" + "esbuild": "0.28.2" } }, - "lambdas/authorizer/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", + "lambdas/letter-sla-breaches/populate-standard-breaches": { + "name": "nhs-notify-supplier-api-populate-standard-breaches", + "version": "0.0.1", "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" + "esbuild": "0.28.2" + } + }, + "lambdas/letter-sla-breaches/populate-supplier-config-cache": { + "name": "nhs-notify-supplier-api-populate-supplier-config-cache", + "version": "0.0.1", + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.1138.0", + "@aws-sdk/client-s3": "^3.1138.0", + "@aws-sdk/lib-dynamodb": "^3.1138.0", + "esbuild": "0.28.2" } }, "lambdas/letter-updates-transformer": { @@ -565,90 +437,6 @@ "zod": "^4.6.5" } }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1146.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1146.0.tgz", - "integrity": "sha512-3Mc8YkSu2KlgNNSuspaW+Fo0pTwX+fLYlC5ovH3yX3OoDg2RPtp3Py8KuQJr0FDlQj1tHw0K3+AtaH4NXlI3aQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-node": "^3.972.84", - "@aws-sdk/dynamodb-codec": "^3.973.46", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/fetch-http-handler": "^5.8.0", - "@smithy/node-http-handler": "^4.12.1", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.46.tgz", - "integrity": "sha512-ApnSubn+5C6HQGP7025q9w/YiNzKwimIMksP76xVij779BCCqcmJ9poB6hQ4syY9NUxqmEQLBqY0PktX53L8iQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", - "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.31.tgz", - "integrity": "sha512-JqOIXupzP6cpAmCCmuEaacbXo/bVCcg1NkglCJC6q4wDAvJO09qIs7POf50JNzmhOhWPtfXvzgWxb5nMQl8QiA==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/letter-updates-transformer/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", - "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/mi-updates-transformer": { "name": "nhs-notify-supplier-api-mi-updates-transformer", "version": "0.0.1", @@ -665,90 +453,6 @@ "zod": "^4.6.5" } }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1146.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1146.0.tgz", - "integrity": "sha512-3Mc8YkSu2KlgNNSuspaW+Fo0pTwX+fLYlC5ovH3yX3OoDg2RPtp3Py8KuQJr0FDlQj1tHw0K3+AtaH4NXlI3aQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-node": "^3.972.84", - "@aws-sdk/dynamodb-codec": "^3.973.46", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/fetch-http-handler": "^5.8.0", - "@smithy/node-http-handler": "^4.12.1", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.46.tgz", - "integrity": "sha512-ApnSubn+5C6HQGP7025q9w/YiNzKwimIMksP76xVij779BCCqcmJ9poB6hQ4syY9NUxqmEQLBqY0PktX53L8iQ==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", - "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.31.tgz", - "integrity": "sha512-JqOIXupzP6cpAmCCmuEaacbXo/bVCcg1NkglCJC6q4wDAvJO09qIs7POf50JNzmhOhWPtfXvzgWxb5nMQl8QiA==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/mi-updates-transformer/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", - "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/supplier-allocator": { "name": "nhs-notify-supplier-api-allocate-letter", "version": "0.0.1", @@ -856,30 +560,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/supplier-allocator/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -897,20 +577,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/supplier-allocator/node_modules/@aws-sdk/middleware-sdk-sqs": { "version": "3.972.43", "license": "Apache-2.0", @@ -924,19 +590,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-allocator/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/supplier-allocator/node_modules/@aws/lambda-invoke-store": { "version": "0.3.0", "license": "Apache-2.0", @@ -1014,30 +667,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/supplier-config-ingress/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -1055,33 +684,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/supplier-config-ingress/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/supplier-config-ingress/node_modules/@nhsdigital/nhs-notify-event-schemas-supplier-config": { "version": "1.1.1", "dependencies": { @@ -1167,30 +769,6 @@ "node": ">=20.0.0" } }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "lambdas/update-letter-queue/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -1208,33 +786,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/update-letter-queue/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "lambdas/upsert-letter": { "name": "nhs-notify-supplier-api-upsert-letter", "version": "0.0.1", @@ -1295,79 +846,24 @@ } }, "lambdas/upsert-letter/node_modules/@aws-lambda-powertools/jmespath": { - "version": "2.35.0", - "license": "MIT-0", - "dependencies": { - "@aws-lambda-powertools/commons": "2.35.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1145.0", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-node": "^3.972.84", - "@aws-sdk/dynamodb-codec": "^3.973.46", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/fetch-http-handler": "^5.8.0", - "@smithy/node-http-handler": "^4.12.1", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "lambdas/upsert-letter/node_modules/@aws-sdk/lib-dynamodb": { - "version": "3.1145.0", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/util-dynamodb": "^3.996.9", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1145.0" + "version": "2.35.0", + "license": "MIT-0", + "dependencies": { + "@aws-lambda-powertools/commons": "2.35.0" } }, - "lambdas/upsert-letter/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", + "lambdas/upsert-letter/node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1145.0", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/credential-provider-node": "^3.972.84", + "@aws-sdk/dynamodb-codec": "^3.973.46", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1375,17 +871,21 @@ "node": ">=20.0.0" } }, - "lambdas/upsert-letter/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", + "lambdas/upsert-letter/node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.1145.0", "license": "Apache-2.0", "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/util-dynamodb": "^3.996.9", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" + "@aws-sdk/client-dynamodb": "^3.1145.0" } }, "lambdas/upsert-letter/node_modules/@aws/lambda-invoke-store": { @@ -1536,6 +1036,27 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1147.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1147.0.tgz", + "integrity": "sha512-lONixsTyj2jt5bb14ir4VYuBEy8kMNMZt7DSVxZ4HHE89hEfND5lO4Qj+F3Z74sENdFCaSAgK8RYQaiQWm5pfw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/credential-provider-node": "^3.972.84", + "@aws-sdk/dynamodb-codec": "^3.973.46", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.31", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-kinesis": { "version": "3.1145.0", "license": "Apache-2.0", @@ -1780,6 +1301,69 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/dynamodb-codec": { + "version": "3.973.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.46.tgz", + "integrity": "sha512-ApnSubn+5C6HQGP7025q9w/YiNzKwimIMksP76xVij779BCCqcmJ9poB6hQ4syY9NUxqmEQLBqY0PktX53L8iQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", + "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", + "license": "Apache-2.0", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.1147.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1147.0.tgz", + "integrity": "sha512-232+7ND08nb0YZrCyCviYSpySiYDusCZo8mdovBo9eOHDqBvndiP7n5/CrtN5HjegGuwJYQ/0AedWu1ORVUGDA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/util-dynamodb": "^3.996.9", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.1147.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.31.tgz", + "integrity": "sha512-JqOIXupzP6cpAmCCmuEaacbXo/bVCcg1NkglCJC6q4wDAvJO09qIs7POf50JNzmhOhWPtfXvzgWxb5nMQl8QiA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/middleware-sdk-api-gateway": { "version": "3.972.30", "license": "Apache-2.0", @@ -1879,6 +1463,21 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/util-dynamodb": { + "version": "3.996.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", + "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.1111.0" + } + }, "node_modules/@aws-sdk/xml-builder": { "version": "3.972.41", "license": "Apache-2.0", @@ -12379,6 +11978,8 @@ }, "node_modules/mnemonist": { "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", "license": "MIT", "dependencies": { "obliterator": "^1.6.1" @@ -12537,6 +12138,18 @@ "resolved": "lambdas/mi-updates-transformer", "link": true }, + "node_modules/nhs-notify-supplier-api-populate-sameday-breaches": { + "resolved": "lambdas/letter-sla-breaches/populate-sameday-breaches", + "link": true + }, + "node_modules/nhs-notify-supplier-api-populate-standard-breaches": { + "resolved": "lambdas/letter-sla-breaches/populate-standard-breaches", + "link": true + }, + "node_modules/nhs-notify-supplier-api-populate-supplier-config-cache": { + "resolved": "lambdas/letter-sla-breaches/populate-supplier-config-cache", + "link": true + }, "node_modules/nhs-notify-supplier-api-supplier-config-ingress": { "resolved": "lambdas/supplier-config-ingress", "link": true @@ -12818,6 +12431,8 @@ }, "node_modules/obliterator": { "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", "license": "MIT" }, "node_modules/obug": { @@ -16760,30 +16375,6 @@ "node": ">=20.0.0" } }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "scripts/utilities/letter-test-data/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -16801,33 +16392,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/letter-test-data/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "scripts/utilities/letter-test-data/node_modules/ansi-regex": { "version": "6.4.0", "license": "MIT", @@ -16989,30 +16553,6 @@ "node": ">=20.0.0" } }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "scripts/utilities/supplier-data/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -17030,33 +16570,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "scripts/utilities/supplier-data/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "scripts/utilities/supplier-data/node_modules/ansi-regex": { "version": "6.4.0", "license": "MIT", @@ -17274,30 +16787,6 @@ "node": ">=20.0.0" } }, - "tests/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.46", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "tests/node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "tests/node_modules/@aws-sdk/lib-dynamodb": { "version": "3.1145.0", "license": "Apache-2.0", @@ -17315,20 +16804,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "tests/node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.31", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", - "@smithy/types": "^4.19.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "tests/node_modules/@aws-sdk/middleware-sdk-sqs": { "version": "3.972.43", "license": "Apache-2.0", @@ -17342,19 +16817,6 @@ "node": ">=20.0.0" } }, - "tests/node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, "tests/node_modules/@babel/code-frame": { "version": "8.0.6", "dev": true, diff --git a/package.json b/package.json index 4e7f0e70d..d6e0a3e9b 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,7 @@ "docs", "internal/*", "lambdas/*", - "pact-contracts", + "lambdas/letter-sla-breaches/*", "pact-contracts", "scripts/utilities/*", "tests", "tests/contracts/*" From 92fe1c6bbf85438fa06c09770ce176b6ffaba185 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 11:14:07 +0100 Subject: [PATCH 02/37] add glue definitions --- ...ue_catalog_tables_supplier_config_cache.tf | 108 ++++++++++++++++++ .../src/__tests__/entity-cache.test.ts | 6 +- .../src/handler/entity-cache.ts | 2 +- 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf diff --git a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf new file mode 100644 index 000000000..7f27de168 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf @@ -0,0 +1,108 @@ +locals { + supplier_config_cache_table_columns = { + supplier = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "channelType", type = "string" }, + { name = "dailyCapacity", type = "bigint" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "status", type = "string" }, + ] + volume-group = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "description", type = "string" }, + { name = "endDate", type = "string" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "startDate", type = "string" }, + { name = "status", type = "string" }, + ] + supplier-pack = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "approval", type = "string" }, + { name = "id", type = "string" }, + { name = "packSpecificationId", type = "string" }, + { name = "status", type = "string" }, + { name = "supplierId", type = "string" }, + ] + letter-variant = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "campaignIds", type = "array" }, + { name = "clientId", type = "string" }, + { name = "constraints", type = "struct,colourCoveragePercentage:struct,deliveryDays:struct,sheets:struct,sides:struct>" }, + { name = "description", type = "string" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "packSpecificationIds", type = "array" }, + { name = "priority", type = "bigint" }, + { name = "status", type = "string" }, + { name = "supplierId", type = "string" }, + { name = "type", type = "string" }, + { name = "volumeGroupId", type = "string" }, + ] + pack-specification = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "assembly", type = "struct,insertIds:array,paper:struct,printColour:string>" }, + { name = "billingId", type = "string" }, + { name = "constraints", type = "struct,colourCoveragePercentage:struct,deliveryDays:struct,sheets:struct,sides:struct>" }, + { name = "createdAt", type = "string" }, + { name = "description", type = "string" }, + { name = "id", type = "string" }, + { name = "name", type = "string" }, + { name = "postage", type = "struct" }, + { name = "status", type = "string" }, + { name = "updatedAt", type = "string" }, + { name = "version", type = "bigint" }, + ] + supplier-allocation = [ + { name = "pk", type = "string" }, + { name = "sk", type = "string" }, + { name = "allocationPercentage", type = "double" }, + { name = "id", type = "string" }, + { name = "status", type = "string" }, + { name = "supplier", type = "string" }, + { name = "volumeGroup", type = "string" }, + ] + } +} + +resource "aws_glue_catalog_table" "supplier_config_cache" { + for_each = local.eventsub_event_cache_bucket_name != null ? local.supplier_config_cache_table_columns : {} + name = "${replace(local.csi, "-", "_")}_${replace(each.key, "-", "_")}" + database_name = aws_glue_catalog_database.supplier.name + table_type = "EXTERNAL_TABLE" + + parameters = { + EXTERNAL = "TRUE" + classification = "json" + } + + storage_descriptor { + location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}.json" + input_format = "org.apache.hadoop.mapred.TextInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" + + ser_de_info { + name = "OpenXJsonSerDe" + serialization_library = "org.openx.data.jsonserde.JsonSerDe" + + parameters = { + "case.insensitive" = "false" + } + } + + dynamic "columns" { + for_each = each.value + + content { + name = columns.value.name + type = columns.value.type + } + } + } +} diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index 1dac234a9..ecb44a5fc 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -49,7 +49,9 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Bucket).toBe("cache-bucket"); expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); - expect(JSON.parse(put.input.Body as string)).toEqual([ + expect( + (put.input.Body as string).split("\n").map((line) => JSON.parse(line)), + ).toEqual([ { pk: "ENTITY#supplier", sk: "ID#1" }, { pk: "ENTITY#supplier", sk: "ID#2" }, ]); @@ -67,6 +69,6 @@ describe("populateEntityCache", () => { expect(count).toBe(0); const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); - expect(put.input.Body).toBe("[]"); + expect(put.input.Body).toBe(""); }); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts index eab1beef6..3dcfdfa1e 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -40,7 +40,7 @@ export default async function populateEntityCache( new PutObjectCommand({ Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, Key: `${CACHE_FOLDER}/${entityType}.json`, - Body: JSON.stringify(items), + Body: items.map((item) => JSON.stringify(item)).join("\n"), ContentType: "application/json", }), ); From 5cbb81c7f8eb6e88956d43aaee511e26d9d1f75d Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 11:18:40 +0100 Subject: [PATCH 03/37] run SLA lambdas in parallel --- .../api/resources/letter_sla_breaches.json | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json index 725b8eaf3..4544a8e26 100644 --- a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json +++ b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json @@ -28,36 +28,54 @@ "ExitAlreadyRunning": { "Type": "Succeed" }, - "PopulateSamedayBreaches": { - "Arguments": { - "FunctionName": "${POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN}", - "Payload": { - "date": "{% $date %}" - } - }, - "Next": "PopulateStandardBreaches", - "Output": { - "populateSamedayBreaches": "{% $states.result.Payload %}", - "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" - }, - "Resource": "arn:aws:states:::lambda:invoke", - "Type": "Task" - }, - "PopulateStandardBreaches": { - "Arguments": { - "FunctionName": "${POPULATE_STANDARD_BREACHES_FUNCTION_ARN}", - "Payload": { - "date": "{% $date %}" + "PopulateBreachesInParallel": { + "Branches": [ + { + "StartAt": "PopulateSamedayBreaches", + "States": { + "PopulateSamedayBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "End": true, + "Output": { + "populateSamedayBreaches": "{% $states.result.Payload %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + } + } + }, + { + "StartAt": "PopulateStandardBreaches", + "States": { + "PopulateStandardBreaches": { + "Arguments": { + "FunctionName": "${POPULATE_STANDARD_BREACHES_FUNCTION_ARN}", + "Payload": { + "date": "{% $date %}" + } + }, + "End": true, + "Output": { + "populateStandardBreaches": "{% $states.result.Payload %}" + }, + "Resource": "arn:aws:states:::lambda:invoke", + "Type": "Task" + } + } } - }, + ], "End": true, "Output": { - "populateSamedayBreaches": "{% $states.input.populateSamedayBreaches %}", - "populateStandardBreaches": "{% $states.result.Payload %}", + "populateSamedayBreaches": "{% $states.result[0].populateSamedayBreaches %}", + "populateStandardBreaches": "{% $states.result[1].populateStandardBreaches %}", "populateSupplierConfigCache": "{% $states.input.populateSupplierConfigCache %}" }, - "Resource": "arn:aws:states:::lambda:invoke", - "Type": "Task" + "Type": "Parallel" }, "PopulateSupplierConfigCache": { "Arguments": { @@ -66,7 +84,7 @@ "date": "{% $date %}" } }, - "Next": "PopulateSamedayBreaches", + "Next": "PopulateBreachesInParallel", "Output": { "populateSupplierConfigCache": "{% $states.result.Payload %}" }, From 85317d5561d4105c30e058217d36860302ae2c26 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 14:29:00 +0100 Subject: [PATCH 04/37] add logging to config cache lambda --- .../src/__tests__/entity-cache.test.ts | 29 +++++++++++++++++-- .../src/handler/deps.ts | 2 ++ .../src/handler/entity-cache.ts | 23 ++++++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index ecb44a5fc..a87958f26 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -6,6 +6,7 @@ function makeDeps(ddbSend: jest.Mock, s3Send: jest.Mock): EntityCacheDeps { return { ddbClient: { send: ddbSend } as unknown as DynamoDBDocumentClient, s3Client: { send: s3Send } as unknown as S3Client, + logger: { info: jest.fn() } as unknown as EntityCacheDeps["logger"], env: { SUPPLIER_CONFIG_TABLE_NAME: "config-table", EVENT_CACHE_BUCKET_NAME: "cache-bucket", @@ -25,9 +26,10 @@ describe("populateEntityCache", () => { Items: [{ pk: "ENTITY#supplier", sk: "ID#2" }], }); const s3Send = jest.fn().mockResolvedValue({}); + const deps = makeDeps(ddbSend, s3Send); const count = await populateEntityCache( - makeDeps(ddbSend, s3Send), + deps, "supplier", ); @@ -49,6 +51,24 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Bucket).toBe("cache-bucket"); expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); + expect(deps.logger.info).toHaveBeenCalledWith( + { entityType: "supplier" }, + "Loading supplier config entity", + ); + expect(deps.logger.info).toHaveBeenCalledWith( + { entityType: "supplier", itemCount: 2, hasItems: true }, + "Loaded supplier config entity", + ); + expect(deps.logger.info).toHaveBeenCalledWith( + { + entityType: "supplier", + itemCount: 2, + bucket: "cache-bucket", + key: "supplier_config_cache/supplier.json", + s3Uri: "s3://cache-bucket/supplier_config_cache/supplier.json", + }, + "Created supplier config cache file", + ); expect( (put.input.Body as string).split("\n").map((line) => JSON.parse(line)), ).toEqual([ @@ -60,9 +80,10 @@ describe("populateEntityCache", () => { it("writes an empty file when no rows are returned", async () => { const ddbSend = jest.fn().mockResolvedValue({}); const s3Send = jest.fn().mockResolvedValue({}); + const deps = makeDeps(ddbSend, s3Send); const count = await populateEntityCache( - makeDeps(ddbSend, s3Send), + deps, "volume-group", ); @@ -70,5 +91,9 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); expect(put.input.Body).toBe(""); + expect(deps.logger.info).toHaveBeenCalledWith( + { entityType: "volume-group", itemCount: 0, hasItems: false }, + "Loaded supplier config entity", + ); }); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts index 6fdf2f926..ced5d4c5e 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts @@ -1,12 +1,14 @@ import { S3Client } from "@aws-sdk/client-s3"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { createLogger } from "@internal/helpers"; import { EntityCacheDeps } from "./entity-cache"; export default function createDependenciesContainer(): EntityCacheDeps { return { ddbClient: DynamoDBDocumentClient.from(new DynamoDBClient()), s3Client: new S3Client(), + logger: createLogger({ logLevel: process.env.PINO_LOG_LEVEL }), env: { SUPPLIER_CONFIG_TABLE_NAME: process.env.SUPPLIER_CONFIG_TABLE_NAME!, EVENT_CACHE_BUCKET_NAME: process.env.EVENT_CACHE_BUCKET_NAME!, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts index 3dcfdfa1e..381a33856 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -4,10 +4,12 @@ import { QueryCommand, QueryCommandOutput, } from "@aws-sdk/lib-dynamodb"; +import { Logger } from "pino"; export type EntityCacheDeps = { ddbClient: DynamoDBDocumentClient; s3Client: S3Client; + logger: Logger; env: { SUPPLIER_CONFIG_TABLE_NAME: string; EVENT_CACHE_BUCKET_NAME: string; @@ -20,6 +22,8 @@ export default async function populateEntityCache( deps: EntityCacheDeps, entityType: string, ): Promise { + deps.logger.info({ entityType }, "Loading supplier config entity"); + const items: Record[] = []; let exclusiveStartKey: QueryCommandOutput["LastEvaluatedKey"]; @@ -36,14 +40,31 @@ export default async function populateEntityCache( exclusiveStartKey = page.LastEvaluatedKey; } while (exclusiveStartKey); + deps.logger.info( + { entityType, itemCount: items.length, hasItems: items.length > 0 }, + "Loaded supplier config entity", + ); + + const key = `${CACHE_FOLDER}/${entityType}.json`; await deps.s3Client.send( new PutObjectCommand({ Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, - Key: `${CACHE_FOLDER}/${entityType}.json`, + Key: key, Body: items.map((item) => JSON.stringify(item)).join("\n"), ContentType: "application/json", }), ); + deps.logger.info( + { + entityType, + itemCount: items.length, + bucket: deps.env.EVENT_CACHE_BUCKET_NAME, + key, + s3Uri: `s3://${deps.env.EVENT_CACHE_BUCKET_NAME}/${key}`, + }, + "Created supplier config cache file", + ); + return items.length; } From 8a55d36e7aa1aaf6430b162affe95dccb2f367ad Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 14:41:45 +0100 Subject: [PATCH 05/37] fix lint issues --- .../populate-supplier-config-cache/package.json | 4 +++- .../src/__tests__/entity-cache.test.ts | 10 ++-------- package-lock.json | 4 +++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json index 034e0301b..b10a522d2 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -3,7 +3,9 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", - "esbuild": "0.28.2" + "@internal/helpers": "^0.1.0", + "esbuild": "0.28.2", + "pino": "^10.3.1" }, "name": "nhs-notify-supplier-api-populate-supplier-config-cache", "private": true, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index a87958f26..125ff82b0 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -28,10 +28,7 @@ describe("populateEntityCache", () => { const s3Send = jest.fn().mockResolvedValue({}); const deps = makeDeps(ddbSend, s3Send); - const count = await populateEntityCache( - deps, - "supplier", - ); + const count = await populateEntityCache(deps, "supplier"); expect(count).toBe(2); expect(ddbSend).toHaveBeenCalledTimes(2); @@ -82,10 +79,7 @@ describe("populateEntityCache", () => { const s3Send = jest.fn().mockResolvedValue({}); const deps = makeDeps(ddbSend, s3Send); - const count = await populateEntityCache( - deps, - "volume-group", - ); + const count = await populateEntityCache(deps, "volume-group"); expect(count).toBe(0); const put = s3Send.mock.calls[0][0] as PutObjectCommand; diff --git a/package-lock.json b/package-lock.json index a80f25d9a..2d80007dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -417,7 +417,9 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", - "esbuild": "0.28.2" + "@internal/helpers": "^0.1.0", + "esbuild": "0.28.2", + "pino": "^10.3.1" } }, "lambdas/letter-updates-transformer": { From c9e86390117a4c17148d9a1263e225be7987c822 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 15:21:32 +0100 Subject: [PATCH 06/37] amend glue config --- .../api/glue_catalog_tables_supplier_config_cache.tf | 2 +- .../src/__tests__/entity-cache.test.ts | 10 ++++++---- .../src/handler/entity-cache.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf index 7f27de168..c44077cc9 100644 --- a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf +++ b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf @@ -83,7 +83,7 @@ resource "aws_glue_catalog_table" "supplier_config_cache" { } storage_descriptor { - location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}.json" + location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}/" input_format = "org.apache.hadoop.mapred.TextInputFormat" output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts index 125ff82b0..2d68709f7 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/entity-cache.test.ts @@ -47,7 +47,7 @@ describe("populateEntityCache", () => { const put = s3Send.mock.calls[0][0] as PutObjectCommand; expect(put.input.Bucket).toBe("cache-bucket"); - expect(put.input.Key).toBe("supplier_config_cache/supplier.json"); + expect(put.input.Key).toBe("supplier_config_cache/supplier/records.json"); expect(deps.logger.info).toHaveBeenCalledWith( { entityType: "supplier" }, "Loading supplier config entity", @@ -61,8 +61,8 @@ describe("populateEntityCache", () => { entityType: "supplier", itemCount: 2, bucket: "cache-bucket", - key: "supplier_config_cache/supplier.json", - s3Uri: "s3://cache-bucket/supplier_config_cache/supplier.json", + key: "supplier_config_cache/supplier/records.json", + s3Uri: "s3://cache-bucket/supplier_config_cache/supplier/records.json", }, "Created supplier config cache file", ); @@ -83,7 +83,9 @@ describe("populateEntityCache", () => { expect(count).toBe(0); const put = s3Send.mock.calls[0][0] as PutObjectCommand; - expect(put.input.Key).toBe("supplier_config_cache/volume-group.json"); + expect(put.input.Key).toBe( + "supplier_config_cache/volume-group/records.json", + ); expect(put.input.Body).toBe(""); expect(deps.logger.info).toHaveBeenCalledWith( { entityType: "volume-group", itemCount: 0, hasItems: false }, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts index 381a33856..eb4333e9a 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/entity-cache.ts @@ -45,7 +45,7 @@ export default async function populateEntityCache( "Loaded supplier config entity", ); - const key = `${CACHE_FOLDER}/${entityType}.json`; + const key = `${CACHE_FOLDER}/${entityType}/records.json`; await deps.s3Client.send( new PutObjectCommand({ Bucket: deps.env.EVENT_CACHE_BUCKET_NAME, From 80b49ebbb3b6f6c5bf8e95663611af88c81a6f7b Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 16:04:23 +0100 Subject: [PATCH 07/37] amend return information from populate-supplier-config-cache --- .../api/resources/letter_sla_breaches.json | 5 +- .../populate-supplier-config-cache.test.ts | 43 ++++++++++++++-- .../handler/populate-supplier-config-cache.ts | 51 ++++++++++++++----- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json index 4544a8e26..456a54ff1 100644 --- a/infrastructure/terraform/components/api/resources/letter_sla_breaches.json +++ b/infrastructure/terraform/components/api/resources/letter_sla_breaches.json @@ -79,10 +79,7 @@ }, "PopulateSupplierConfigCache": { "Arguments": { - "FunctionName": "${POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN}", - "Payload": { - "date": "{% $date %}" - } + "FunctionName": "${POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN}" }, "Next": "PopulateBreachesInParallel", "Output": { diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index 54374eb2f..b2d7dab0b 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -7,10 +7,14 @@ jest.mock("../handler/entity-cache"); describe("populateSupplierConfigCache", () => { it("populates the cache once for each entity type and returns success", async () => { - const deps = {} as EntityCacheDeps; + const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; + jest.mocked(populateEntityCache).mockReset(); + for (const [index] of ENTITY_TYPES.entries()) { + jest.mocked(populateEntityCache).mockResolvedValueOnce(index + 1); + } const handler = createHandler(deps); - const result = await handler({ date: "2026-10-08" }); + const result = await handler(); expect(ENTITY_TYPES).toEqual([ "volume-group", @@ -26,7 +30,40 @@ describe("populateSupplierConfigCache", () => { } expect(result).toEqual({ status: "SUCCESS", - message: "populate-supplier-config-cache invoked successfully", + entities: ENTITY_TYPES.map((entity, index) => ({ + entity, + recordsRetrieved: index + 1, + })), + }); + }); + + it("returns failure and counts for successful entities when an entity fails", async () => { + const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; + jest + .mocked(populateEntityCache) + .mockReset() + .mockResolvedValueOnce(1) + .mockRejectedValueOnce(new Error("query failed")) + .mockResolvedValueOnce(3) + .mockResolvedValueOnce(4) + .mockResolvedValueOnce(5) + .mockResolvedValueOnce(6); + const handler = createHandler(deps); + + const result = await handler(); + + expect(result).toEqual({ + status: "FAILURE", + failureReason: + "Failed to populate supplier config cache for: supplier-pack", + entities: ENTITY_TYPES.map((entity, index) => ({ + entity, + recordsRetrieved: index === 1 ? 0 : index + 1, + })), }); + expect(deps.logger.error).toHaveBeenCalledWith( + { entityType: "supplier-pack", error: expect.any(Error) }, + "Failed to populate supplier config entity cache", + ); }); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index b3c3d57fb..cd83290a0 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -1,12 +1,12 @@ import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; -export type PopulateSupplierConfigCacheEvent = { - date: string; -}; - export type PopulateSupplierConfigCacheResult = { - status: "SUCCESS"; - message: string; + status: "SUCCESS" | "FAILURE"; + failureReason?: string; + entities: { + entity: string; + recordsRetrieved: number; + }[]; }; export const ENTITY_TYPES = [ @@ -19,16 +19,43 @@ export const ENTITY_TYPES = [ ]; export default function createHandler(deps: EntityCacheDeps) { - return async ( - _event: PopulateSupplierConfigCacheEvent, - ): Promise => { - await Promise.all( - ENTITY_TYPES.map((entityType) => populateEntityCache(deps, entityType)), + return async (): Promise => { + const results = await Promise.all( + ENTITY_TYPES.map(async (entityType) => { + try { + return { + entity: entityType, + recordsRetrieved: await populateEntityCache(deps, entityType), + failed: false, + }; + } catch (error) { + deps.logger.error( + { entityType, error }, + "Failed to populate supplier config entity cache", + ); + return { entity: entityType, recordsRetrieved: 0, failed: true }; + } + }), ); + const entities = results.map(({ entity, recordsRetrieved }) => ({ + entity, + recordsRetrieved, + })); + const failedEntities = results + .filter((result) => result.failed) + .map((result) => result.entity); + + if (failedEntities.length > 0) { + return { + status: "FAILURE", + failureReason: `Failed to populate supplier config cache for: ${failedEntities.join(", ")}`, + entities, + }; + } return { status: "SUCCESS", - message: "populate-supplier-config-cache invoked successfully", + entities, }; }; } From b137e2679ef16b2805896a556b167387decef7c0 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 16:09:42 +0100 Subject: [PATCH 08/37] alphabetise entity types --- .../src/handler/populate-supplier-config-cache.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index cd83290a0..7aa7b0953 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -10,12 +10,12 @@ export type PopulateSupplierConfigCacheResult = { }; export const ENTITY_TYPES = [ - "volume-group", - "supplier-pack", "letter-variant", "pack-specification", - "supplier-allocation", "supplier", + "supplier-allocation", + "supplier-pack", + "volume-group", ]; export default function createHandler(deps: EntityCacheDeps) { From 3ef13ef1ceed0db37271282f1908ba83b55c86d5 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Mon, 12 Oct 2026 16:41:08 +0100 Subject: [PATCH 09/37] fix failing test --- .../__tests__/populate-supplier-config-cache.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index b2d7dab0b..784c75f7d 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -17,12 +17,12 @@ describe("populateSupplierConfigCache", () => { const result = await handler(); expect(ENTITY_TYPES).toEqual([ - "volume-group", - "supplier-pack", "letter-variant", "pack-specification", - "supplier-allocation", "supplier", + "supplier-allocation", + "supplier-pack", + "volume-group", ]); expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); for (const entityType of ENTITY_TYPES) { @@ -55,14 +55,14 @@ describe("populateSupplierConfigCache", () => { expect(result).toEqual({ status: "FAILURE", failureReason: - "Failed to populate supplier config cache for: supplier-pack", + "Failed to populate supplier config cache for: pack-specification", entities: ENTITY_TYPES.map((entity, index) => ({ entity, recordsRetrieved: index === 1 ? 0 : index + 1, })), }); expect(deps.logger.error).toHaveBeenCalledWith( - { entityType: "supplier-pack", error: expect.any(Error) }, + { entityType: "pack-specification", error: expect.any(Error) }, "Failed to populate supplier config entity cache", ); }); From bba8e21bd1f02611cb3d6808c8f6b7f835349f72 Mon Sep 17 00:00:00 2001 From: Mark Slowey <113013138+masl2@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:47:30 +0100 Subject: [PATCH 10/37] CCM-24327: parquet catalog (#734) --- docs/generate-includes.sh | 1 + .../terraform/components/api/README.md | 4 + .../api/athena_workgroup_reporting.tf | 19 ++ .../glue_catalog_table_eventpub_parquet.tf | 91 +++++++ ...rehose_delivery_stream_eventpub_parquet.tf | 241 ++++++++++++++++++ ...ule_lambda_eventpub_parquet_transformer.tf | 35 +++ .../api/module_s3bucket_reporting.tf | 79 ++++++ .../terraform/components/api/variables.tf | 12 + .../eventpub-parquet-transformer/README.md | 17 ++ .../jest.config.ts | 63 +++++ .../eventpub-parquet-transformer/package.json | 16 ++ .../src/__tests__/transform-record.test.ts | 91 +++++++ .../eventpub-parquet-transformer/src/index.ts | 7 + .../src/transform-record.ts | 82 ++++++ .../tsconfig.json | 10 + package-lock.json | 12 + 16 files changed, 780 insertions(+) create mode 100644 infrastructure/terraform/components/api/athena_workgroup_reporting.tf create mode 100644 infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf create mode 100644 infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf create mode 100644 infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf create mode 100644 infrastructure/terraform/components/api/module_s3bucket_reporting.tf create mode 100644 lambdas/eventpub-parquet-transformer/README.md create mode 100644 lambdas/eventpub-parquet-transformer/jest.config.ts create mode 100644 lambdas/eventpub-parquet-transformer/package.json create mode 100644 lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts create mode 100644 lambdas/eventpub-parquet-transformer/src/index.ts create mode 100644 lambdas/eventpub-parquet-transformer/src/transform-record.ts create mode 100644 lambdas/eventpub-parquet-transformer/tsconfig.json diff --git a/docs/generate-includes.sh b/docs/generate-includes.sh index 75b217a97..2df5e62c4 100755 --- a/docs/generate-includes.sh +++ b/docs/generate-includes.sh @@ -24,6 +24,7 @@ cp ./lambdas/upsert-letter/README.md ./docs/_includes/components/generated/readm cp ./lambdas/update-letter-queue/README.md ./docs/_includes/components/generated/readmes/lambda-update-letter-queue.md cp ./lambdas/letter-updates-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-letter-updates-transformer.md cp ./lambdas/mi-updates-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-mi-updates-transformer.md +cp ./lambdas/eventpub-parquet-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-eventpub-parquet-transformer.md cp ./lambdas/supplier-config-ingress/README.md ./docs/_includes/components/generated/readmes/lambda-supplier-config-ingress.md # Function documentation (internal packages) diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 5fe8966cf..2c50eb284 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -54,6 +54,8 @@ No requirements. | [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no | | [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes | | [region](#input\_region) | The AWS Region | `string` | n/a | yes | +| [reports\_data\_retention\_non\_current\_days](#input\_reports\_data\_retention\_non\_current\_days) | Number of days to retain noncurrent reporting bucket object versions | `number` | `14` | no | +| [reports\_data\_retention\_policy\_days](#input\_reports\_data\_retention\_policy\_days) | Number of days to retain reporting bucket data | `number` | `90` | no | | [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Account ID of the shared infrastructure account | `string` | `"000000000000"` | no | | [sns\_success\_logging\_sample\_percent](#input\_sns\_success\_logging\_sample\_percent) | Enable SNS Delivery Successful Sample Percentage | `number` | `0` | no | | [supplier\_mock\_schedule\_expression](#input\_supplier\_mock\_schedule\_expression) | Schedule expression for supplier mock scheduler | `string` | `"rate(1 minute)"` | no | @@ -72,6 +74,7 @@ No requirements. | [ddb\_alarms\_suppliers](#module\_ddb\_alarms\_suppliers) | ../../modules/alarms-ddb | n/a | | [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a | | [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/5.0.6/terraform-eventpub.zip | n/a | +| [eventpub\_parquet\_transformer](#module\_eventpub\_parquet\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a | | [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | @@ -89,6 +92,7 @@ No requirements. | [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [s3bucket\_reporting](#module\_s3bucket\_reporting) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip | n/a | | [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a | | [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a | | [sqs\_letter\_updates](#module\_sqs\_letter\_updates) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | diff --git a/infrastructure/terraform/components/api/athena_workgroup_reporting.tf b/infrastructure/terraform/components/api/athena_workgroup_reporting.tf new file mode 100644 index 000000000..08d7fa3ea --- /dev/null +++ b/infrastructure/terraform/components/api/athena_workgroup_reporting.tf @@ -0,0 +1,19 @@ +resource "aws_athena_workgroup" "reporting" { + name = local.csi + description = "Supplier API Athena Workgroup for ${var.environment}" + force_destroy = true + + configuration { + enforce_workgroup_configuration = true + + result_configuration { + expected_bucket_owner = var.aws_account_id + output_location = "s3://${module.s3bucket_reporting.bucket}/athena-output/" + + encryption_configuration { + encryption_option = "SSE_KMS" + kms_key_arn = module.kms.key_arn + } + } + } +} diff --git a/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf new file mode 100644 index 000000000..630d3233b --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf @@ -0,0 +1,91 @@ +locals { + eventpub_parquet_enabled = local.event_cache_bucket_name != null + + # Column names must be lowercase; OpenXJsonSerDe matches JSON keys case-insensitively. + eventpub_parquet_table_columns = [ + { name = "specversion", type = "string" }, + { name = "id", type = "string" }, + { name = "type", type = "string" }, + { name = "plane", type = "string" }, + { name = "dataschema", type = "string" }, + { name = "dataschemaversion", type = "string" }, + { name = "source", type = "string" }, + { name = "subject", type = "string" }, + { name = "time", type = "string" }, + { name = "recordedtime", type = "string" }, + { name = "datacontenttype", type = "string" }, + { name = "traceparent", type = "string" }, + { name = "tracestate", type = "string" }, + { name = "partitionkey", type = "string" }, + { name = "sampledrate", type = "bigint" }, + { name = "sequence", type = "string" }, + { name = "severitytext", type = "string" }, + { name = "data_json", type = "string" }, + { name = "domainid", type = "string" }, + { name = "supplierid", type = "string" }, + { name = "groupid", type = "string" }, + { name = "campaignid", type = "string" }, + { name = "clientid", type = "string" }, + ] +} + +resource "aws_glue_catalog_table" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + name = "${replace(local.csi, "-", "_")}_eventpub" + database_name = aws_glue_catalog_database.supplier.name + table_type = "EXTERNAL_TABLE" + + parameters = { + EXTERNAL = "TRUE" + classification = "parquet" + "parquet.compress" = "SNAPPY" + + # Athena derives partitions from these properties, so none are registered in Glue + "projection.enabled" = "true" + "projection.__year.type" = "integer" + "projection.__year.range" = "2025,2050" + "projection.__year.digits" = "4" + "projection.__month.type" = "integer" + "projection.__month.range" = "1,12" + "projection.__month.digits" = "2" + "projection.__day.type" = "integer" + "projection.__day.range" = "1,31" + "projection.__day.digits" = "2" + "storage.location.template" = "s3://${module.s3bucket_reporting.bucket}/parquet/eventpub/__year=$${__year}/__month=$${__month}/__day=$${__day}/" + } + + storage_descriptor { + location = "s3://${module.s3bucket_reporting.bucket}/parquet/eventpub/" + input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" + + ser_de_info { + name = "ParquetHiveSerDe" + serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + } + + dynamic "columns" { + for_each = local.eventpub_parquet_table_columns + + content { + name = columns.value.name + type = columns.value.type + } + } + } + + partition_keys { + name = "__year" + type = "string" + } + + partition_keys { + name = "__month" + type = "string" + } + + partition_keys { + name = "__day" + type = "string" + } +} diff --git a/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf new file mode 100644 index 000000000..e5227e5cc --- /dev/null +++ b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf @@ -0,0 +1,241 @@ +resource "aws_kinesis_firehose_delivery_stream" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet" + destination = "extended_s3" + + server_side_encryption { + enabled = true + key_type = "CUSTOMER_MANAGED_CMK" + key_arn = module.kms.key_arn + } + + extended_s3_configuration { + role_arn = aws_iam_role.eventpub_parquet_firehose[0].arn + bucket_arn = module.s3bucket_reporting.arn + + prefix = "parquet/eventpub/__year=!{timestamp:yyyy}/__month=!{timestamp:MM}/__day=!{timestamp:dd}/" + error_output_prefix = "parquet/errors/eventpub/!{firehose:error-output-type}/!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}/" + + buffering_size = 128 + buffering_interval = 300 + + processing_configuration { + enabled = true + + processors { + type = "Lambda" + + parameters { + parameter_name = "LambdaArn" + parameter_value = "${module.eventpub_parquet_transformer.function_arn}:$LATEST" + } + parameters { + parameter_name = "RoleArn" + parameter_value = aws_iam_role.eventpub_parquet_firehose[0].arn + } + } + } + + data_format_conversion_configuration { + input_format_configuration { + deserializer { + open_x_json_ser_de {} + } + } + + output_format_configuration { + serializer { + parquet_ser_de {} + } + } + + schema_configuration { + database_name = aws_glue_catalog_database.supplier.name + role_arn = aws_iam_role.eventpub_parquet_firehose[0].arn + table_name = aws_glue_catalog_table.eventpub_parquet[0].name + } + } + + cloudwatch_logging_options { + enabled = true + log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name + log_stream_name = aws_cloudwatch_log_stream.eventpub_parquet_firehose[0].name + } + } +} + +resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "/aws/firehose/${local.csi}-eventpub-parquet" + kms_key_id = module.kms.key_arn + retention_in_days = var.log_retention_in_days +} + +resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "extended_s3" + log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name +} + +# Raw delivery so Firehose receives the bare event rather than the SNS envelope +resource "aws_sns_topic_subscription" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + topic_arn = module.eventpub.sns_topic.arn + protocol = "firehose" + subscription_role_arn = aws_iam_role.eventpub_parquet_sns[0].arn + endpoint = aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn + raw_message_delivery = true +} + +resource "aws_iam_role" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + role = aws_iam_role.eventpub_parquet_firehose[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_firehose[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowDestinationS3Access" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/parquet/*", + ] + } + + statement { + sid = "AllowKms" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + ] + + resources = [module.kms.key_arn] + } + + statement { + sid = "AllowInvokeTransformer" + effect = "Allow" + + actions = [ + "lambda:InvokeFunction", + "lambda:GetFunctionConfiguration", + ] + + resources = ["${module.eventpub_parquet_transformer.function_arn}:$LATEST"] + } + + statement { + sid = "AllowGlueSchemaRead" + effect = "Allow" + + actions = [ + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + ] + + resources = [ + aws_glue_catalog_table.eventpub_parquet[0].arn, + aws_glue_catalog_database.supplier.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + ] + } + + statement { + sid = "AllowLogging" + effect = "Allow" + actions = ["logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.eventpub_parquet_firehose[0].arn}:*"] + } +} + +resource "aws_iam_role" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + role = aws_iam_role.eventpub_parquet_sns[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_sns[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowFirehoseDelivery" + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn] + } + + statement { + sid = "AllowKms" + effect = "Allow" + actions = ["kms:GenerateDataKey", "kms:Decrypt"] + resources = [module.kms.key_arn] + } +} diff --git a/infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf b/infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf new file mode 100644 index 000000000..4f0aa4be2 --- /dev/null +++ b/infrastructure/terraform/components/api/module_lambda_eventpub_parquet_transformer.tf @@ -0,0 +1,35 @@ +module "eventpub_parquet_transformer" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" + + function_name = "eventpub-parquet-transformer" + description = "Flattens EventPub events for parquet delivery by Firehose" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "eventpub-parquet-transformer/dist" + function_include_common = true + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = 256 + timeout = 60 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.destination_arn + odin_log_destination_arn = local.odin_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = local.common_lambda_env_vars +} diff --git a/infrastructure/terraform/components/api/module_s3bucket_reporting.tf b/infrastructure/terraform/components/api/module_s3bucket_reporting.tf new file mode 100644 index 000000000..f3e3d0a35 --- /dev/null +++ b/infrastructure/terraform/components/api/module_s3bucket_reporting.tf @@ -0,0 +1,79 @@ +module "s3bucket_reporting" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip" + + name = "supapi-reporting" + + aws_account_id = var.aws_account_id + region = var.region + project = var.project + environment = var.environment + component = var.component + + kms_key_arn = module.kms.key_arn + + policy_documents = [data.aws_iam_policy_document.s3bucket_reporting.json] + + force_destroy = var.force_destroy + + lifecycle_rules = [ + { + prefix = "athena-output" + enabled = true + + expiration = { + days = var.reports_data_retention_policy_days + } + + noncurrent_version_expiration = { + noncurrent_days = var.reports_data_retention_non_current_days + } + }, + ] + + default_tags = { + NHSE-Enable-S3-Backup-Acct = "True", + } +} + +data "aws_iam_policy_document" "s3bucket_reporting" { + statement { + sid = "AllowManagedAccountsToList" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + module.s3bucket_reporting.arn, + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } + + statement { + sid = "AllowManagedAccountsToGet" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + + resources = [ + "${module.s3bucket_reporting.arn}/*", + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } +} diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index f1499c3fc..0486ee22a 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -296,3 +296,15 @@ variable "event_source_filter" { type = list(string) description = "SNS subscription source filter values" } + +variable "reports_data_retention_policy_days" { + type = number + description = "Number of days to retain reporting bucket data" + default = 90 +} + +variable "reports_data_retention_non_current_days" { + type = number + description = "Number of days to retain noncurrent reporting bucket object versions" + default = 14 +} diff --git a/lambdas/eventpub-parquet-transformer/README.md b/lambdas/eventpub-parquet-transformer/README.md new file mode 100644 index 000000000..eb1a5c458 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/README.md @@ -0,0 +1,17 @@ + + +# EventPub Parquet Transformer Lambda + +## Purpose + +Firehose data-transformation processor for the EventPub parquet delivery stream. Flattens each CloudEvent so Firehose can convert it to Parquet for Athena. + +## General Flow + +1. Firehose invokes the Lambda with a batch of base64 encoded CloudEvents. +2. Each event is parsed and returned as a flat JSON record: the envelope attributes, the raw `data` payload as `data_json`, and the promoted `domainid`, `supplierid`, `groupid`, `campaignid` and `clientid` properties. +3. Records that are not valid JSON objects are returned as `ProcessingFailed` and are written to the stream's error prefix. + +## Key Integration Points + +- **Kinesis Data Firehose**: the `eventpub-parquet` delivery stream. diff --git a/lambdas/eventpub-parquet-transformer/jest.config.ts b/lambdas/eventpub-parquet-transformer/jest.config.ts new file mode 100644 index 000000000..93ec73a53 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/jest.config.ts @@ -0,0 +1,63 @@ +import type { Config } from "jest"; + +export const baseJestConfig: Config = { + preset: "ts-jest", + + // Automatically clear mock calls, instances, contexts and results before every test + clearMocks: true, + + // Indicates whether the coverage information should be collected while executing the test + collectCoverage: true, + + // The directory where Jest should output its coverage files + coverageDirectory: "./.reports/unit/coverage", + + // Indicates which provider should be used to instrument code for coverage + coverageProvider: "babel", + + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + + coveragePathIgnorePatterns: ["/__tests__/"], + transform: { "^.+\\.ts$": "ts-jest" }, + transformIgnorePatterns: [ + "node_modules/(?!(@nhsdigital/nhs-notify-event-schemas-supplier-config)/)", + ], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + + // Use this configuration option to add custom reporters to Jest + reporters: [ + "default", + [ + "jest-html-reporter", + { + pageTitle: "Test Report", + outputPath: "./.reports/unit/test-report.html", + includeFailureMsg: true, + }, + ], + ], + + // The test environment that will be used for testing + testEnvironment: "jsdom", +}; + +const utilsJestConfig = { + ...baseJestConfig, + + testEnvironment: "node", + + coveragePathIgnorePatterns: [ + ...(baseJestConfig.coveragePathIgnorePatterns ?? []), + "zod-validators.ts", + ], +}; + +export default utilsJestConfig; diff --git a/lambdas/eventpub-parquet-transformer/package.json b/lambdas/eventpub-parquet-transformer/package.json new file mode 100644 index 000000000..7512fb336 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/package.json @@ -0,0 +1,16 @@ +{ + "dependencies": { + "aws-lambda": "^1.0.7", + "esbuild": "0.28.2" + }, + "name": "nhs-notify-supplier-api-eventpub-parquet-transformer", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts new file mode 100644 index 000000000..48db8e4b5 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts @@ -0,0 +1,91 @@ +import { FirehoseTransformationEvent } from "aws-lambda"; +import { handler } from ".."; +import transformRecord from "../transform-record"; + +const encode = (value: unknown) => + Buffer.from( + typeof value === "string" ? value : JSON.stringify(value), + "utf8", + ).toString("base64"); + +const decode = (data: string) => + JSON.parse(Buffer.from(data, "base64").toString("utf8")); + +describe("transformRecord", () => { + it("flattens the envelope, keeps data as JSON and promotes properties", () => { + const data = { + domainId: "d1", + supplierId: "s1", + groupId: "g1", + campaignId: "c1", + clientId: "cl1", + status: "PRINTED", + }; + + const out = transformRecord({ + recordId: "r1", + data: encode({ id: "e1", type: "t", time: "now", data }), + }); + + expect(out.result).toBe("Ok"); + expect(out.recordId).toBe("r1"); + expect(Buffer.from(out.data!, "base64").toString().endsWith("\n")).toBe( + true, + ); + expect(decode(out.data!)).toEqual({ + id: "e1", + type: "t", + time: "now", + data_json: JSON.stringify(data), + domainid: "d1", + supplierid: "s1", + groupid: "g1", + campaignid: "c1", + clientid: "cl1", + }); + }); + + it("leaves promoted properties empty when data is missing or not an object", () => { + const missing = decode( + transformRecord({ recordId: "r", data: encode({ id: "e" }) }).data!, + ); + const scalar = decode( + transformRecord({ recordId: "r", data: encode({ id: "e", data: 1 }) }) + .data!, + ); + + expect(missing).toEqual({ id: "e", data_json: "null" }); + expect(scalar).toEqual({ id: "e", data_json: "1" }); + }); + + it.each([["not json"], ["[1]"], ["null"]])( + "returns ProcessingFailed for %s", + (payload) => { + const record = { recordId: "r", data: encode(payload) }; + + expect(transformRecord(record)).toEqual({ + recordId: "r", + result: "ProcessingFailed", + data: record.data, + }); + }, + ); +}); + +describe("handler", () => { + it("transforms every record in the batch", async () => { + const event = { + records: [ + { recordId: "1", data: encode({ id: "a" }) }, + { recordId: "2", data: encode("bad") }, + ], + } as unknown as FirehoseTransformationEvent; + + const res = await handler(event, {} as never, () => {}); + + expect(res!.records.map((r) => r.result)).toEqual([ + "Ok", + "ProcessingFailed", + ]); + }); +}); diff --git a/lambdas/eventpub-parquet-transformer/src/index.ts b/lambdas/eventpub-parquet-transformer/src/index.ts new file mode 100644 index 000000000..b310292f4 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/src/index.ts @@ -0,0 +1,7 @@ +import { FirehoseTransformationHandler } from "aws-lambda"; +import transformRecord from "./transform-record"; + +// eslint-disable-next-line import-x/prefer-default-export +export const handler: FirehoseTransformationHandler = async (event) => ({ + records: event.records.map((record) => transformRecord(record)), +}); diff --git a/lambdas/eventpub-parquet-transformer/src/transform-record.ts b/lambdas/eventpub-parquet-transformer/src/transform-record.ts new file mode 100644 index 000000000..97a10a1b5 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/src/transform-record.ts @@ -0,0 +1,82 @@ +import type { + FirehoseRecordTransformationStatus, + FirehoseTransformationResultRecord, +} from "aws-lambda"; + +const ENVELOPE_FIELDS = [ + "specversion", + "id", + "type", + "plane", + "dataschema", + "dataschemaversion", + "source", + "subject", + "time", + "recordedtime", + "datacontenttype", + "traceparent", + "tracestate", + "partitionkey", + "sampledrate", + "sequence", + "severitytext", +] as const; + +const PROMOTED_DATA_FIELDS = [ + "domainId", + "supplierId", + "groupId", + "campaignId", + "clientId", +] as const; + +type Json = Record; + +function isObject(value: unknown): value is Json { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function flatten(event: Json): Json { + const data = isObject(event.data) ? event.data : {}; + + const flat: Json = {}; + for (const field of ENVELOPE_FIELDS) { + flat[field] = event[field]; + } + flat.data_json = JSON.stringify(event.data ?? null); + for (const field of PROMOTED_DATA_FIELDS) { + flat[field.toLowerCase()] = data[field]; + } + return flat; +} + +function result( + recordId: string, + status: FirehoseRecordTransformationStatus, + data: string, +): FirehoseTransformationResultRecord { + return { recordId, result: status, data }; +} + +export default function transformRecord(record: { + recordId: string; + data: string; +}): FirehoseTransformationResultRecord { + try { + const event: unknown = JSON.parse( + Buffer.from(record.data, "base64").toString("utf8"), + ); + if (!isObject(event)) { + return result(record.recordId, "ProcessingFailed", record.data); + } + const output = `${JSON.stringify(flatten(event))}\n`; + return result( + record.recordId, + "Ok", + Buffer.from(output, "utf8").toString("base64"), + ); + } catch { + return result(record.recordId, "ProcessingFailed", record.data); + } +} diff --git a/lambdas/eventpub-parquet-transformer/tsconfig.json b/lambdas/eventpub-parquet-transformer/tsconfig.json new file mode 100644 index 000000000..bb8177b74 --- /dev/null +++ b/lambdas/eventpub-parquet-transformer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "esModuleInterop": true + }, + "extends": "../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index 2d80007dc..c13521172 100644 --- a/package-lock.json +++ b/package-lock.json @@ -396,6 +396,14 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, + "lambdas/eventpub-parquet-transformer": { + "name": "nhs-notify-supplier-api-eventpub-parquet-transformer", + "version": "0.0.1", + "dependencies": { + "aws-lambda": "^1.0.7", + "esbuild": "0.28.2" + } + }, "lambdas/letter-sla-breaches/populate-sameday-breaches": { "name": "nhs-notify-supplier-api-populate-sameday-breaches", "version": "0.0.1", @@ -12124,6 +12132,10 @@ "resolved": "lambdas/supplier-allocator", "link": true }, + "node_modules/nhs-notify-supplier-api-eventpub-parquet-transformer": { + "resolved": "lambdas/eventpub-parquet-transformer", + "link": true + }, "node_modules/nhs-notify-supplier-api-handler": { "resolved": "lambdas/api-handler", "link": true From da4b451e45b47cad18cd577cc3e70a26394da973 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:06:26 +0100 Subject: [PATCH 11/37] follow zod schema pattern for env variables in breach config lambda --- .../package.json | 3 +- .../src/__tests__/env.test.ts | 48 +++++++++++++++++++ .../src/handler/deps.ts | 7 +-- .../src/handler/env.ts | 11 +++++ package-lock.json | 3 +- 5 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json index b10a522d2..cd62c3eca 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -5,7 +5,8 @@ "@aws-sdk/lib-dynamodb": "^3.1138.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", - "pino": "^10.3.1" + "pino": "^10.3.1", + "zod": "^4.6.5" }, "name": "nhs-notify-supplier-api-populate-supplier-config-cache", "private": true, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts new file mode 100644 index 000000000..ee6aafe5c --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/env.test.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/no-require-imports */ +/* Allow require imports to enable re-import of modules */ + +import { ZodError } from "zod"; + +describe("lambdaEnv", () => { + const OLD_ENV = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { ...OLD_ENV }; + }); + + afterAll(() => { + process.env = OLD_ENV; + }); + + it("loads required and optional environment variables", () => { + process.env.SUPPLIER_CONFIG_TABLE_NAME = "supplier-config-table"; + process.env.EVENT_CACHE_BUCKET_NAME = "event-cache-bucket"; + process.env.PINO_LOG_LEVEL = "debug"; + + const { envVars } = require("../handler/env"); + + expect(envVars).toEqual({ + SUPPLIER_CONFIG_TABLE_NAME: "supplier-config-table", + EVENT_CACHE_BUCKET_NAME: "event-cache-bucket", + PINO_LOG_LEVEL: "debug", + }); + }); + + it("throws when a required environment variable is missing", () => { + process.env.SUPPLIER_CONFIG_TABLE_NAME = undefined; + process.env.EVENT_CACHE_BUCKET_NAME = "event-cache-bucket"; + + expect(() => require("../handler/env")).toThrow(ZodError); + }); + + it("allows the optional log level to be unset", () => { + process.env.SUPPLIER_CONFIG_TABLE_NAME = "supplier-config-table"; + process.env.EVENT_CACHE_BUCKET_NAME = "event-cache-bucket"; + process.env.PINO_LOG_LEVEL = undefined; + + const { envVars } = require("../handler/env"); + + expect(envVars.PINO_LOG_LEVEL).toBeUndefined(); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts index ced5d4c5e..9be803265 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/deps.ts @@ -3,15 +3,16 @@ import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; import { createLogger } from "@internal/helpers"; import { EntityCacheDeps } from "./entity-cache"; +import { envVars } from "./env"; export default function createDependenciesContainer(): EntityCacheDeps { return { ddbClient: DynamoDBDocumentClient.from(new DynamoDBClient()), s3Client: new S3Client(), - logger: createLogger({ logLevel: process.env.PINO_LOG_LEVEL }), + logger: createLogger({ logLevel: envVars.PINO_LOG_LEVEL }), env: { - SUPPLIER_CONFIG_TABLE_NAME: process.env.SUPPLIER_CONFIG_TABLE_NAME!, - EVENT_CACHE_BUCKET_NAME: process.env.EVENT_CACHE_BUCKET_NAME!, + SUPPLIER_CONFIG_TABLE_NAME: envVars.SUPPLIER_CONFIG_TABLE_NAME, + EVENT_CACHE_BUCKET_NAME: envVars.EVENT_CACHE_BUCKET_NAME, }, }; } diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts new file mode 100644 index 000000000..03be4f276 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/env.ts @@ -0,0 +1,11 @@ +import { z } from "zod"; + +const EnvVarsSchema = z.object({ + SUPPLIER_CONFIG_TABLE_NAME: z.string(), + EVENT_CACHE_BUCKET_NAME: z.string(), + PINO_LOG_LEVEL: z.string().optional(), +}); + +export type EnvVars = z.infer; + +export const envVars = EnvVarsSchema.parse(process.env); diff --git a/package-lock.json b/package-lock.json index c13521172..969f956f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -427,7 +427,8 @@ "@aws-sdk/lib-dynamodb": "^3.1138.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", - "pino": "^10.3.1" + "pino": "^10.3.1", + "zod": "^4.6.5" } }, "lambdas/letter-updates-transformer": { From ca2cc302169eeb6d1590558030e64fe5ae0ffe1f Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:20:05 +0100 Subject: [PATCH 12/37] retrieve entity type from internal datastore --- .../package.json | 1 + .../populate-supplier-config-cache.test.ts | 27 +++++++++++-------- .../handler/populate-supplier-config-cache.ts | 10 ++----- package-lock.json | 1 + 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json index cd62c3eca..5667e2485 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/package.json @@ -3,6 +3,7 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", + "@internal/datastore": "^0.1.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", "pino": "^10.3.1", diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index 784c75f7d..a867152b6 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -1,8 +1,21 @@ +import { $SupplierConfigEntity } from "@internal/datastore"; import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; import createHandler, { ENTITY_TYPES, } from "../handler/populate-supplier-config-cache"; +jest.mock("@internal/datastore", () => ({ + $SupplierConfigEntity: { + options: [ + "letter-variant", + "volume-group", + "supplier-allocation", + "supplier", + "pack-specification", + "supplier-pack", + ], + }, +})); jest.mock("../handler/entity-cache"); describe("populateSupplierConfigCache", () => { @@ -16,14 +29,7 @@ describe("populateSupplierConfigCache", () => { const result = await handler(); - expect(ENTITY_TYPES).toEqual([ - "letter-variant", - "pack-specification", - "supplier", - "supplier-allocation", - "supplier-pack", - "volume-group", - ]); + expect(ENTITY_TYPES).toEqual($SupplierConfigEntity.options); expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); for (const entityType of ENTITY_TYPES) { expect(populateEntityCache).toHaveBeenCalledWith(deps, entityType); @@ -54,15 +60,14 @@ describe("populateSupplierConfigCache", () => { expect(result).toEqual({ status: "FAILURE", - failureReason: - "Failed to populate supplier config cache for: pack-specification", + failureReason: `Failed to populate supplier config cache for: ${ENTITY_TYPES[1]}`, entities: ENTITY_TYPES.map((entity, index) => ({ entity, recordsRetrieved: index === 1 ? 0 : index + 1, })), }); expect(deps.logger.error).toHaveBeenCalledWith( - { entityType: "pack-specification", error: expect.any(Error) }, + { entityType: ENTITY_TYPES[1], error: expect.any(Error) }, "Failed to populate supplier config entity cache", ); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 7aa7b0953..5eff5a1a4 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -1,3 +1,4 @@ +import { $SupplierConfigEntity } from "@internal/datastore"; import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; export type PopulateSupplierConfigCacheResult = { @@ -9,14 +10,7 @@ export type PopulateSupplierConfigCacheResult = { }[]; }; -export const ENTITY_TYPES = [ - "letter-variant", - "pack-specification", - "supplier", - "supplier-allocation", - "supplier-pack", - "volume-group", -]; +export const ENTITY_TYPES = $SupplierConfigEntity.options; export default function createHandler(deps: EntityCacheDeps) { return async (): Promise => { diff --git a/package-lock.json b/package-lock.json index 969f956f7..0912227cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -425,6 +425,7 @@ "@aws-sdk/client-dynamodb": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/lib-dynamodb": "^3.1138.0", + "@internal/datastore": "^0.1.0", "@internal/helpers": "^0.1.0", "esbuild": "0.28.2", "pino": "^10.3.1", From 2ad34143152459f19459a22b0775a2b436381c62 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:40:33 +0100 Subject: [PATCH 13/37] remove entities from return of config cache lambda --- .../populate-sameday-breaches/tsconfig.json | 3 +-- .../__tests__/populate-supplier-config-cache.test.ts | 10 +--------- .../src/handler/populate-supplier-config-cache.ts | 6 ------ 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json index 4a11b255a..f4059d5e6 100644 --- a/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json +++ b/lambdas/letter-sla-breaches/populate-sameday-breaches/tsconfig.json @@ -1,7 +1,6 @@ { "compilerOptions": { - "esModuleInterop": true, - "ignoreDeprecations": "5.0" + "esModuleInterop": true }, "extends": "../../../tsconfig.base.json", "include": [ diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index a867152b6..911f78f7f 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -36,14 +36,10 @@ describe("populateSupplierConfigCache", () => { } expect(result).toEqual({ status: "SUCCESS", - entities: ENTITY_TYPES.map((entity, index) => ({ - entity, - recordsRetrieved: index + 1, - })), }); }); - it("returns failure and counts for successful entities when an entity fails", async () => { + it("returns failure when an entity fails", async () => { const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; jest .mocked(populateEntityCache) @@ -61,10 +57,6 @@ describe("populateSupplierConfigCache", () => { expect(result).toEqual({ status: "FAILURE", failureReason: `Failed to populate supplier config cache for: ${ENTITY_TYPES[1]}`, - entities: ENTITY_TYPES.map((entity, index) => ({ - entity, - recordsRetrieved: index === 1 ? 0 : index + 1, - })), }); expect(deps.logger.error).toHaveBeenCalledWith( { entityType: ENTITY_TYPES[1], error: expect.any(Error) }, diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 5eff5a1a4..17b87022a 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -4,10 +4,6 @@ import populateEntityCache, { EntityCacheDeps } from "./entity-cache"; export type PopulateSupplierConfigCacheResult = { status: "SUCCESS" | "FAILURE"; failureReason?: string; - entities: { - entity: string; - recordsRetrieved: number; - }[]; }; export const ENTITY_TYPES = $SupplierConfigEntity.options; @@ -43,13 +39,11 @@ export default function createHandler(deps: EntityCacheDeps) { return { status: "FAILURE", failureReason: `Failed to populate supplier config cache for: ${failedEntities.join(", ")}`, - entities, }; } return { status: "SUCCESS", - entities, }; }; } From dacb5235e9989063a3bb5d903dfe38e04f81ee81 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 10:56:21 +0100 Subject: [PATCH 14/37] missing newlines and terraform restructuring --- eslint.config.mjs | 3 +- ...tch_log_group_eventpub_parquet_firehose.tf | 7 + ...ch_log_stream_eventpub_parquet_firehose.tf | 6 + .../glue_catalog_table_eventpub_parquet.tf | 31 ---- .../api/iam_role_eventpub_parquet_firehose.tf | 18 ++ .../api/iam_role_eventpub_parquet_sns.tf | 18 ++ ...m_role_policy_eventpub_parquet_firehose.tf | 79 ++++++++ .../iam_role_policy_eventpub_parquet_sns.tf | 30 +++ ...rehose_delivery_stream_eventpub_parquet.tf | 175 ------------------ .../terraform/components/api/locals.tf | 29 +++ ...sns_topic_subscription_eventpub_parquet.tf | 10 + package.json | 3 +- 12 files changed, 201 insertions(+), 208 deletions(-) create mode 100644 infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf create mode 100644 infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf create mode 100644 infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf create mode 100644 infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf diff --git a/eslint.config.mjs b/eslint.config.mjs index 029ba22cd..65dc77b54 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -115,7 +115,8 @@ export default defineConfig([ eslintImportResolverTypescript.createTypeScriptImportResolver({ project: [ 'lambdas/*/tsconfig.json', - 'lambdas/*/*/tsconfig.json', 'tests/tsconfig.json', + 'lambdas/*/*/tsconfig.json', + 'tests/tsconfig.json', 'internal/*/tsconfig.json', ], }), diff --git a/infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..95a9fc627 --- /dev/null +++ b/infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf @@ -0,0 +1,7 @@ +resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "/aws/firehose/${local.csi}-eventpub-parquet" + kms_key_id = module.kms.key_arn + retention_in_days = var.log_retention_in_days +} diff --git a/infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..e634886b4 --- /dev/null +++ b/infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf @@ -0,0 +1,6 @@ +resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "extended_s3" + log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name +} diff --git a/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf index 630d3233b..233e27a0c 100644 --- a/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf +++ b/infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf @@ -1,34 +1,3 @@ -locals { - eventpub_parquet_enabled = local.event_cache_bucket_name != null - - # Column names must be lowercase; OpenXJsonSerDe matches JSON keys case-insensitively. - eventpub_parquet_table_columns = [ - { name = "specversion", type = "string" }, - { name = "id", type = "string" }, - { name = "type", type = "string" }, - { name = "plane", type = "string" }, - { name = "dataschema", type = "string" }, - { name = "dataschemaversion", type = "string" }, - { name = "source", type = "string" }, - { name = "subject", type = "string" }, - { name = "time", type = "string" }, - { name = "recordedtime", type = "string" }, - { name = "datacontenttype", type = "string" }, - { name = "traceparent", type = "string" }, - { name = "tracestate", type = "string" }, - { name = "partitionkey", type = "string" }, - { name = "sampledrate", type = "bigint" }, - { name = "sequence", type = "string" }, - { name = "severitytext", type = "string" }, - { name = "data_json", type = "string" }, - { name = "domainid", type = "string" }, - { name = "supplierid", type = "string" }, - { name = "groupid", type = "string" }, - { name = "campaignid", type = "string" }, - { name = "clientid", type = "string" }, - ] -} - resource "aws_glue_catalog_table" "eventpub_parquet" { count = local.eventpub_parquet_enabled ? 1 : 0 name = "${replace(local.csi, "-", "_")}_eventpub" diff --git a/infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..c39074016 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf @@ -0,0 +1,18 @@ +resource "aws_iam_role" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["firehose.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf new file mode 100644 index 000000000..50363d68b --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf @@ -0,0 +1,18 @@ +resource "aws_iam_role" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf new file mode 100644 index 000000000..ae8273301 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_firehose.tf @@ -0,0 +1,79 @@ +resource "aws_iam_role_policy" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-firehose" + role = aws_iam_role.eventpub_parquet_firehose[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_firehose[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_firehose" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowDestinationS3Access" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/parquet/*", + ] + } + + statement { + sid = "AllowKms" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + ] + + resources = [module.kms.key_arn] + } + + statement { + sid = "AllowInvokeTransformer" + effect = "Allow" + + actions = [ + "lambda:InvokeFunction", + "lambda:GetFunctionConfiguration", + ] + + resources = ["${module.eventpub_parquet_transformer.function_arn}:$LATEST"] + } + + statement { + sid = "AllowGlueSchemaRead" + effect = "Allow" + + actions = [ + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + ] + + resources = [ + aws_glue_catalog_table.eventpub_parquet[0].arn, + aws_glue_catalog_database.supplier.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + ] + } + + statement { + sid = "AllowLogging" + effect = "Allow" + actions = ["logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.eventpub_parquet_firehose[0].arn}:*"] + } +} diff --git a/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf new file mode 100644 index 000000000..b7e41a764 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_policy_eventpub_parquet_sns.tf @@ -0,0 +1,30 @@ +resource "aws_iam_role_policy" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + name = "${local.csi}-eventpub-parquet-sns" + role = aws_iam_role.eventpub_parquet_sns[0].id + policy = data.aws_iam_policy_document.eventpub_parquet_sns[0].json +} + +data "aws_iam_policy_document" "eventpub_parquet_sns" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + statement { + sid = "AllowFirehoseDelivery" + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn] + } + + statement { + sid = "AllowKms" + effect = "Allow" + actions = ["kms:GenerateDataKey", "kms:Decrypt"] + resources = [module.kms.key_arn] + } +} diff --git a/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf index e5227e5cc..4dcdec6e2 100644 --- a/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf +++ b/infrastructure/terraform/components/api/kinesis_firehose_delivery_stream_eventpub_parquet.tf @@ -64,178 +64,3 @@ resource "aws_kinesis_firehose_delivery_stream" "eventpub_parquet" { } } } - -resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "/aws/firehose/${local.csi}-eventpub-parquet" - kms_key_id = module.kms.key_arn - retention_in_days = var.log_retention_in_days -} - -resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "extended_s3" - log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name -} - -# Raw delivery so Firehose receives the bare event rather than the SNS envelope -resource "aws_sns_topic_subscription" "eventpub_parquet" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - topic_arn = module.eventpub.sns_topic.arn - protocol = "firehose" - subscription_role_arn = aws_iam_role.eventpub_parquet_sns[0].arn - endpoint = aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn - raw_message_delivery = true -} - -resource "aws_iam_role" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-firehose" - assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json -} - -data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["firehose.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-firehose" - role = aws_iam_role.eventpub_parquet_firehose[0].id - policy = data.aws_iam_policy_document.eventpub_parquet_firehose[0].json -} - -data "aws_iam_policy_document" "eventpub_parquet_firehose" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - statement { - sid = "AllowDestinationS3Access" - effect = "Allow" - - actions = [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject", - ] - - resources = [ - module.s3bucket_reporting.arn, - "${module.s3bucket_reporting.arn}/parquet/*", - ] - } - - statement { - sid = "AllowKms" - effect = "Allow" - - actions = [ - "kms:Decrypt", - "kms:DescribeKey", - "kms:GenerateDataKey*", - ] - - resources = [module.kms.key_arn] - } - - statement { - sid = "AllowInvokeTransformer" - effect = "Allow" - - actions = [ - "lambda:InvokeFunction", - "lambda:GetFunctionConfiguration", - ] - - resources = ["${module.eventpub_parquet_transformer.function_arn}:$LATEST"] - } - - statement { - sid = "AllowGlueSchemaRead" - effect = "Allow" - - actions = [ - "glue:GetTable", - "glue:GetTableVersion", - "glue:GetTableVersions", - ] - - resources = [ - aws_glue_catalog_table.eventpub_parquet[0].arn, - aws_glue_catalog_database.supplier.arn, - "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", - ] - } - - statement { - sid = "AllowLogging" - effect = "Allow" - actions = ["logs:PutLogEvents"] - resources = ["${aws_cloudwatch_log_group.eventpub_parquet_firehose[0].arn}:*"] - } -} - -resource "aws_iam_role" "eventpub_parquet_sns" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-sns" - assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json -} - -data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["sns.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy" "eventpub_parquet_sns" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - name = "${local.csi}-eventpub-parquet-sns" - role = aws_iam_role.eventpub_parquet_sns[0].id - policy = data.aws_iam_policy_document.eventpub_parquet_sns[0].json -} - -data "aws_iam_policy_document" "eventpub_parquet_sns" { - count = local.eventpub_parquet_enabled ? 1 : 0 - - statement { - sid = "AllowFirehoseDelivery" - effect = "Allow" - - actions = [ - "firehose:PutRecord", - "firehose:PutRecordBatch", - ] - - resources = [aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn] - } - - statement { - sid = "AllowKms" - effect = "Allow" - actions = ["kms:GenerateDataKey", "kms:Decrypt"] - resources = [module.kms.key_arn] - } -} diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 18c52e680..7c2dbf46b 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -1,4 +1,33 @@ locals { + eventpub_parquet_enabled = local.event_cache_bucket_name != null + + # Column names must be lowercase; OpenXJsonSerDe matches JSON keys case-insensitively. + eventpub_parquet_table_columns = [ + { name = "specversion", type = "string" }, + { name = "id", type = "string" }, + { name = "type", type = "string" }, + { name = "plane", type = "string" }, + { name = "dataschema", type = "string" }, + { name = "dataschemaversion", type = "string" }, + { name = "source", type = "string" }, + { name = "subject", type = "string" }, + { name = "time", type = "string" }, + { name = "recordedtime", type = "string" }, + { name = "datacontenttype", type = "string" }, + { name = "traceparent", type = "string" }, + { name = "tracestate", type = "string" }, + { name = "partitionkey", type = "string" }, + { name = "sampledrate", type = "bigint" }, + { name = "sequence", type = "string" }, + { name = "severitytext", type = "string" }, + { name = "data_json", type = "string" }, + { name = "domainid", type = "string" }, + { name = "supplierid", type = "string" }, + { name = "groupid", type = "string" }, + { name = "campaignid", type = "string" }, + { name = "clientid", type = "string" }, + ] + aws_lambda_functions_dir_path = "../../../../lambdas" root_domain_name = "${var.environment}.${local.acct.route53_zone_names["supplier-api"]}" # e.g. [main|dev|abxy0].supplier-api.[dev|nonprod|prod].nhsnotify.national.nhs.uk root_domain_id = local.acct.route53_zone_ids["supplier-api"] diff --git a/infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf b/infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf new file mode 100644 index 000000000..da3237198 --- /dev/null +++ b/infrastructure/terraform/components/api/sns_topic_subscription_eventpub_parquet.tf @@ -0,0 +1,10 @@ +# Raw delivery so Firehose receives the bare event rather than the SNS envelope +resource "aws_sns_topic_subscription" "eventpub_parquet" { + count = local.eventpub_parquet_enabled ? 1 : 0 + + topic_arn = module.eventpub.sns_topic.arn + protocol = "firehose" + subscription_role_arn = aws_iam_role.eventpub_parquet_sns[0].arn + endpoint = aws_kinesis_firehose_delivery_stream.eventpub_parquet[0].arn + raw_message_delivery = true +} diff --git a/package.json b/package.json index d6e0a3e9b..68a2c9b28 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,8 @@ "docs", "internal/*", "lambdas/*", - "lambdas/letter-sla-breaches/*", "pact-contracts", + "lambdas/letter-sla-breaches/*", + "pact-contracts", "scripts/utilities/*", "tests", "tests/contracts/*" From 20683c7d912c4faf2e3f45d35da60e0af3147d02 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 11:12:53 +0100 Subject: [PATCH 15/37] fix lint issue and change S3 bucket used for config cache --- .../api/glue_catalog_tables_supplier_config_cache.tf | 4 ++-- .../api/module_lambda_populate_supplier_config_cache.tf | 4 ++-- .../src/handler/populate-supplier-config-cache.ts | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf index c44077cc9..d17c10f33 100644 --- a/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf +++ b/infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf @@ -72,7 +72,7 @@ locals { } resource "aws_glue_catalog_table" "supplier_config_cache" { - for_each = local.eventsub_event_cache_bucket_name != null ? local.supplier_config_cache_table_columns : {} + for_each = local.supplier_config_cache_table_columns name = "${replace(local.csi, "-", "_")}_${replace(each.key, "-", "_")}" database_name = aws_glue_catalog_database.supplier.name table_type = "EXTERNAL_TABLE" @@ -83,7 +83,7 @@ resource "aws_glue_catalog_table" "supplier_config_cache" { } storage_descriptor { - location = "s3://${local.eventsub_event_cache_bucket_name}/supplier_config_cache/${each.key}/" + location = "s3://${module.s3bucket_reporting.bucket}/supplier_config_cache/${each.key}/" input_format = "org.apache.hadoop.mapred.TextInputFormat" output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" diff --git a/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf index 57bf707fa..86339dda2 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_supplier_config_cache.tf @@ -37,7 +37,7 @@ module "populate_supplier_config_cache" { lambda_env_vars = { SUPPLIER_CONFIG_TABLE_NAME = aws_dynamodb_table.supplier-configuration.name - EVENT_CACHE_BUCKET_NAME = local.eventsub_event_cache_bucket_name + EVENT_CACHE_BUCKET_NAME = module.s3bucket_reporting.bucket } } @@ -67,6 +67,6 @@ data "aws_iam_policy_document" "populate_supplier_config_cache_lambda" { sid = "AllowWriteSupplierConfigCache" effect = "Allow" actions = ["s3:PutObject"] - resources = ["arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/supplier_config_cache/*"] + resources = ["${module.s3bucket_reporting.arn}/supplier_config_cache/*"] } } diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 17b87022a..1f8b50250 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -27,10 +27,6 @@ export default function createHandler(deps: EntityCacheDeps) { } }), ); - const entities = results.map(({ entity, recordsRetrieved }) => ({ - entity, - recordsRetrieved, - })); const failedEntities = results .filter((result) => result.failed) .map((result) => result.entity); From 1c283bc206e71c0b8db48b59703f73dfcfa37dea Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 12:13:13 +0100 Subject: [PATCH 16/37] promote status --- infrastructure/terraform/components/api/locals.tf | 1 + .../src/__tests__/transform-record.test.ts | 1 + lambdas/eventpub-parquet-transformer/src/transform-record.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 7c2dbf46b..1b7db3ec6 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -26,6 +26,7 @@ locals { { name = "groupid", type = "string" }, { name = "campaignid", type = "string" }, { name = "clientid", type = "string" }, + { name = "status", type = "string" }, ] aws_lambda_functions_dir_path = "../../../../lambdas" diff --git a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts index 48db8e4b5..677cf5752 100644 --- a/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts +++ b/lambdas/eventpub-parquet-transformer/src/__tests__/transform-record.test.ts @@ -42,6 +42,7 @@ describe("transformRecord", () => { groupid: "g1", campaignid: "c1", clientid: "cl1", + status: "PRINTED", }); }); diff --git a/lambdas/eventpub-parquet-transformer/src/transform-record.ts b/lambdas/eventpub-parquet-transformer/src/transform-record.ts index 97a10a1b5..b743de50f 100644 --- a/lambdas/eventpub-parquet-transformer/src/transform-record.ts +++ b/lambdas/eventpub-parquet-transformer/src/transform-record.ts @@ -29,6 +29,7 @@ const PROMOTED_DATA_FIELDS = [ "groupId", "campaignId", "clientId", + "status", ] as const; type Json = Record; From cec54a093db17d9dbbac083c63af750150f16da0 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 14 Oct 2026 14:07:45 +0100 Subject: [PATCH 17/37] remove unnecessary export --- .../populate-supplier-config-cache.test.ts | 36 +++++++++---------- .../handler/populate-supplier-config-cache.ts | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts index 911f78f7f..047694df6 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/__tests__/populate-supplier-config-cache.test.ts @@ -1,8 +1,6 @@ import { $SupplierConfigEntity } from "@internal/datastore"; import populateEntityCache, { EntityCacheDeps } from "../handler/entity-cache"; -import createHandler, { - ENTITY_TYPES, -} from "../handler/populate-supplier-config-cache"; +import createHandler from "../handler/populate-supplier-config-cache"; jest.mock("@internal/datastore", () => ({ $SupplierConfigEntity: { @@ -21,17 +19,17 @@ jest.mock("../handler/entity-cache"); describe("populateSupplierConfigCache", () => { it("populates the cache once for each entity type and returns success", async () => { const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; + const entityTypes = $SupplierConfigEntity.options; jest.mocked(populateEntityCache).mockReset(); - for (const [index] of ENTITY_TYPES.entries()) { + for (const [index] of entityTypes.entries()) { jest.mocked(populateEntityCache).mockResolvedValueOnce(index + 1); } const handler = createHandler(deps); const result = await handler(); - expect(ENTITY_TYPES).toEqual($SupplierConfigEntity.options); - expect(populateEntityCache).toHaveBeenCalledTimes(ENTITY_TYPES.length); - for (const entityType of ENTITY_TYPES) { + expect(populateEntityCache).toHaveBeenCalledTimes(entityTypes.length); + for (const entityType of entityTypes) { expect(populateEntityCache).toHaveBeenCalledWith(deps, entityType); } expect(result).toEqual({ @@ -41,25 +39,27 @@ describe("populateSupplierConfigCache", () => { it("returns failure when an entity fails", async () => { const deps = { logger: { error: jest.fn() } } as unknown as EntityCacheDeps; - jest - .mocked(populateEntityCache) - .mockReset() - .mockResolvedValueOnce(1) - .mockRejectedValueOnce(new Error("query failed")) - .mockResolvedValueOnce(3) - .mockResolvedValueOnce(4) - .mockResolvedValueOnce(5) - .mockResolvedValueOnce(6); + const entityTypes = $SupplierConfigEntity.options; + jest.mocked(populateEntityCache).mockReset(); + for (const [index] of entityTypes.entries()) { + if (index === 1) { + jest + .mocked(populateEntityCache) + .mockRejectedValueOnce(new Error("query failed")); + } else { + jest.mocked(populateEntityCache).mockResolvedValueOnce(index + 1); + } + } const handler = createHandler(deps); const result = await handler(); expect(result).toEqual({ status: "FAILURE", - failureReason: `Failed to populate supplier config cache for: ${ENTITY_TYPES[1]}`, + failureReason: `Failed to populate supplier config cache for: ${entityTypes[1]}`, }); expect(deps.logger.error).toHaveBeenCalledWith( - { entityType: ENTITY_TYPES[1], error: expect.any(Error) }, + { entityType: entityTypes[1], error: expect.any(Error) }, "Failed to populate supplier config entity cache", ); }); diff --git a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts index 1f8b50250..8ce432a6c 100644 --- a/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts +++ b/lambdas/letter-sla-breaches/populate-supplier-config-cache/src/handler/populate-supplier-config-cache.ts @@ -6,7 +6,7 @@ export type PopulateSupplierConfigCacheResult = { failureReason?: string; }; -export const ENTITY_TYPES = $SupplierConfigEntity.options; +const ENTITY_TYPES = $SupplierConfigEntity.options; export default function createHandler(deps: EntityCacheDeps) { return async (): Promise => { From dbfd5cf43cf945ad67d06bd094ffef806c809fed Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Wed, 14 Oct 2026 09:07:11 +0100 Subject: [PATCH 18/37] First draft --- .../terraform/components/api/README.md | 1 + .../athena_workgroup_letter_sla_breaches.tf | 19 ++ .../terraform/components/api/locals.tf | 3 + ...odule_lambda_populate_standard_breaches.tf | 78 +++++- .../components/api/s3_athena_query_results.tf | 73 ++++++ .../populate-standard-breaches/package.json | 5 +- .../src/__tests__/athena.test.ts | 154 ++++++++++++ .../populate-standard-breaches.test.ts | 67 ++++- .../__tests__/standard-breaches-query.test.ts | 26 ++ .../src/handler/athena.ts | 132 ++++++++++ .../src/handler/deps.ts | 25 ++ .../src/handler/populate-standard-breaches.ts | 45 +++- .../src/handler/standard-breaches-query.ts | 42 ++++ .../populate-standard-breaches/src/index.ts | 7 +- package-lock.json | 237 ++++++++++++------ 15 files changed, 820 insertions(+), 94 deletions(-) create mode 100644 infrastructure/terraform/components/api/athena_workgroup_letter_sla_breaches.tf create mode 100644 infrastructure/terraform/components/api/s3_athena_query_results.tf create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/athena.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/deps.ts create mode 100644 lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 2c50eb284..03404fc65 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -92,6 +92,7 @@ No requirements. | [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | +| [s3bucket\_athena\_query\_results](#module\_s3bucket\_athena\_query\_results) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a | | [s3bucket\_reporting](#module\_s3bucket\_reporting) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip | n/a | | [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a | | [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a | diff --git a/infrastructure/terraform/components/api/athena_workgroup_letter_sla_breaches.tf b/infrastructure/terraform/components/api/athena_workgroup_letter_sla_breaches.tf new file mode 100644 index 000000000..e3939d786 --- /dev/null +++ b/infrastructure/terraform/components/api/athena_workgroup_letter_sla_breaches.tf @@ -0,0 +1,19 @@ +resource "aws_athena_workgroup" "letter_sla_breaches" { + name = "${local.csi}-letter-sla-breaches" + description = "Athena workgroup for the letter SLA breaches state machine" + force_destroy = var.force_destroy + + configuration { + enforce_workgroup_configuration = true + publish_cloudwatch_metrics_enabled = true + + result_configuration { + output_location = "s3://${module.s3bucket_athena_query_results.bucket}/letter-sla-breaches/" + + encryption_configuration { + encryption_option = "SSE_KMS" + kms_key_arn = module.kms.key_arn + } + } + } +} diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 1b7db3ec6..c1fbb3063 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -78,6 +78,9 @@ locals { event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null) eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null) + # The Glue crawler names tables after the crawled bucket, replacing characters that are invalid in table names + event_cache_glue_table_name = local.event_cache_bucket_name != null ? lower(replace(local.event_cache_bucket_name, "/[^a-zA-Z0-9_]/", "_")) : null + csoc_api_gw_log_destination_arn = format("arn:aws:logs:%s:%s:destination:api_gateway_log_destination", var.region, var.csoc_destination_account diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index f5665bb82..4ef41c41f 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -25,7 +25,7 @@ module "populate_standard_breaches" { handler_function_name = "populateStandardBreachesHandler" runtime = "nodejs22.x" memory = 128 - timeout = 29 + timeout = 300 log_level = var.log_level force_lambda_code_deploy = var.force_lambda_code_deploy @@ -35,7 +35,11 @@ module "populate_standard_breaches" { odin_log_destination_arn = local.odin_destination_arn log_subscription_role_arn = local.acct.log_subscription_role_arn - lambda_env_vars = {} + lambda_env_vars = { + ATHENA_WORKGROUP = aws_athena_workgroup.letter_sla_breaches.name + ATHENA_DATABASE = aws_glue_catalog_database.supplier.name + EVENT_CACHE_TABLE_NAME = local.event_cache_glue_table_name + } } data "aws_iam_policy_document" "populate_standard_breaches_lambda" { @@ -52,4 +56,74 @@ data "aws_iam_policy_document" "populate_standard_breaches_lambda" { module.kms.key_arn, ] } + + statement { + sid = "AllowAthenaQueries" + effect = "Allow" + + actions = [ + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:StartQueryExecution", + "athena:StopQueryExecution", + ] + + resources = [ + aws_athena_workgroup.letter_sla_breaches.arn, + ] + } + + statement { + sid = "AllowGlueCatalogRead" + effect = "Allow" + + actions = [ + "glue:GetDatabase", + "glue:GetPartition", + "glue:GetPartitions", + "glue:GetTable", + "glue:GetTables", + ] + + resources = [ + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + aws_glue_catalog_database.supplier.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.supplier.name}/*", + ] + } + + statement { + sid = "AllowReadEventCacheBucket" + effect = "Allow" + + actions = [ + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + ] + + resources = [ + "arn:aws:s3:::${local.event_cache_bucket_name}", + "arn:aws:s3:::${local.event_cache_bucket_name}/*", + ] + } + + statement { + sid = "AllowAthenaQueryResultsBucket" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListMultipartUploadParts", + "s3:PutObject", + ] + + resources = [ + module.s3bucket_athena_query_results.arn, + "${module.s3bucket_athena_query_results.arn}/letter-sla-breaches/*", + ] + } } diff --git a/infrastructure/terraform/components/api/s3_athena_query_results.tf b/infrastructure/terraform/components/api/s3_athena_query_results.tf new file mode 100644 index 000000000..68d514b78 --- /dev/null +++ b/infrastructure/terraform/components/api/s3_athena_query_results.tf @@ -0,0 +1,73 @@ +module "s3bucket_athena_query_results" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip" + + name = "athena-query-results" + + aws_account_id = var.aws_account_id + region = var.region + project = var.project + environment = var.environment + component = var.component + + acl = "private" + force_destroy = var.force_destroy + versioning = false + kms_key_arn = module.kms.key_arn + + lifecycle_rules = [ + { + enabled = true + + expiration = { + days = "7" + } + + abort_incomplete_multipart_upload = { + days = "1" + } + } + ] + + policy_documents = [ + data.aws_iam_policy_document.s3bucket_athena_query_results.json + ] + + bucket_logging_target = { + bucket = local.acct.s3_buckets["access_logs"]["id"] + } + + public_access = { + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true + } + + default_tags = { + Name = "Athena Query Results" + } +} + +data "aws_iam_policy_document" "s3bucket_athena_query_results" { + statement { + sid = "DontAllowNonSecureConnection" + effect = "Deny" + actions = ["s3:*"] + + resources = [ + module.s3bucket_athena_query_results.arn, + "${module.s3bucket_athena_query_results.arn}/*", + ] + + principals { + type = "AWS" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json index 8875e4acc..87d2f1d15 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json @@ -1,6 +1,9 @@ { "dependencies": { - "esbuild": "0.28.2" + "@aws-sdk/client-athena": "^3.1149.0", + "@internal/helpers": "^0.1.0", + "esbuild": "0.28.2", + "pino": "^10.4.0" }, "name": "nhs-notify-supplier-api-populate-standard-breaches", "private": true, diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts new file mode 100644 index 000000000..d56b075f8 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts @@ -0,0 +1,154 @@ +import { + AthenaClient, + GetQueryExecutionCommand, + GetQueryResultsCommand, + StartQueryExecutionCommand, +} from "@aws-sdk/client-athena"; +import runAthenaQuery from "../handler/athena"; +import { Deps } from "../handler/deps"; + +const query = { queryString: "SELECT ?", executionParameters: ["'a'"] }; + +function makeDeps(responses: unknown[]) { + const send = jest.fn(); + for (const response of responses) { + send.mockResolvedValueOnce(response); + } + const deps = { + athenaClient: { send } as unknown as AthenaClient, + logger: { info: jest.fn(), debug: jest.fn() }, + env: { + ATHENA_WORKGROUP: "workgroup", + ATHENA_DATABASE: "database", + EVENT_CACHE_TABLE_NAME: "event_cache", + }, + } as unknown as Deps; + return { deps, send }; +} + +const columnInfo = { ColumnInfo: [{ Name: "id" }, { Name: "status" }] }; + +describe("runAthenaQuery", () => { + it("starts the query, polls until complete and returns paginated rows", async () => { + const { deps, send } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "QUEUED" } } }, + { QueryExecution: { Status: { State: "RUNNING" } } }, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + { + ResultSet: { + ResultSetMetadata: columnInfo, + Rows: [ + { Data: [{ VarCharValue: "id" }, { VarCharValue: "status" }] }, + { Data: [{ VarCharValue: "1" }, { VarCharValue: "PENDING" }] }, + ], + }, + NextToken: "next", + }, + { + ResultSet: { + ResultSetMetadata: columnInfo, + Rows: [{ Data: [{ VarCharValue: "2" }, {}] }], + }, + }, + ]); + + const rows = await runAthenaQuery(deps, query, { pollIntervalMs: 0 }); + + expect(rows).toEqual([ + { id: "1", status: "PENDING" }, + { id: "2", status: undefined }, + ]); + const commands = send.mock.calls.map(([command]) => command); + expect(commands[0]).toBeInstanceOf(StartQueryExecutionCommand); + expect(commands[0].input).toEqual({ + QueryString: "SELECT ?", + ExecutionParameters: ["'a'"], + WorkGroup: "workgroup", + QueryExecutionContext: { Database: "database" }, + }); + expect(commands[1]).toBeInstanceOf(GetQueryExecutionCommand); + expect(commands[4]).toBeInstanceOf(GetQueryResultsCommand); + expect(commands[5].input).toEqual({ + QueryExecutionId: "qid", + NextToken: "next", + }); + }); + + it("handles empty result sets and missing metadata", async () => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + { + ResultSet: { ResultSetMetadata: { ColumnInfo: [{}] }, Rows: [{}, {}] }, + }, + ]); + + await expect(runAthenaQuery(deps, query)).resolves.toEqual([ + { "": undefined }, + ]); + }); + + it("handles a response with no result set", async () => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + {}, + ]); + + await expect(runAthenaQuery(deps, query)).resolves.toEqual([]); + }); + + it("uses the default poll interval when none is given", async () => { + jest.useFakeTimers(); + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + {}, + { QueryExecution: { Status: { State: "SUCCEEDED" } } }, + {}, + ]); + + const promise = runAthenaQuery(deps, query); + await jest.advanceTimersByTimeAsync(1000); + + await expect(promise).resolves.toEqual([]); + jest.useRealTimers(); + }); + + it("throws when no query execution id is returned", async () => { + const { deps } = makeDeps([{}]); + + await expect(runAthenaQuery(deps, query)).rejects.toThrow( + "Athena did not return a query execution id", + ); + }); + + it.each(["FAILED", "CANCELLED"])( + "throws when the query is %s", + async (state) => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { + QueryExecution: { + Status: { State: state, StateChangeReason: "bad query" }, + }, + }, + ]); + + await expect(runAthenaQuery(deps, query)).rejects.toThrow( + `Athena query qid ${state}: bad query`, + ); + }, + ); + + it("throws with a default reason when none is given", async () => { + const { deps } = makeDeps([ + { QueryExecutionId: "qid" }, + { QueryExecution: { Status: { State: "FAILED" } } }, + ]); + + await expect(runAthenaQuery(deps, query)).rejects.toThrow( + "Athena query qid FAILED: no reason given", + ); + }); +}); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index f64093251..5403d3148 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -1,12 +1,69 @@ -import populateStandardBreaches from "../handler/populate-standard-breaches"; +import runAthenaQuery from "../handler/athena"; +import { Deps } from "../handler/deps"; +import createHandler from "../handler/populate-standard-breaches"; +import buildStandardBreachesQuery from "../handler/standard-breaches-query"; + +jest.mock("../handler/athena"); + +const makeDeps = () => + ({ + logger: { info: jest.fn(), error: jest.fn() }, + env: { EVENT_CACHE_TABLE_NAME: "event_cache" }, + }) as unknown as Deps; describe("populateStandardBreaches", () => { - it("returns a success result", async () => { - const result = await populateStandardBreaches({ date: "2026-10-08" }); + it("runs the standard breaches query and returns success", async () => { + const deps = makeDeps(); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([{ date: "2026-10-08" }]); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(runAthenaQuery).toHaveBeenCalledWith( + deps, + buildStandardBreachesQuery("event_cache", "2026-10-08"), + ); + expect(result).toEqual({ status: "SUCCESS", rowsReturned: 1 }); + }); + + it("returns failure when the query fails", async () => { + const deps = makeDeps(); + jest + .mocked(runAthenaQuery) + .mockRejectedValueOnce(new Error("query failed")); + + const result = await createHandler(deps)({ date: "2026-10-08" }); + + expect(result).toEqual({ + status: "FAILURE", + failureReason: "query failed", + rowsReturned: 0, + }); + expect(deps.logger.error).toHaveBeenCalledWith( + { date: "2026-10-08", error: expect.any(Error) }, + "Failed to populate standard breaches", + ); + }); + + it("returns failure without querying when the date is invalid", async () => { + const result = await createHandler(makeDeps())({ date: "not-a-date" }); + + expect(runAthenaQuery).not.toHaveBeenCalled(); + expect(result).toEqual({ + status: "FAILURE", + failureReason: 'Invalid date "not-a-date", expected YYYY-MM-DD', + rowsReturned: 0, + }); + }); + + it("returns failure when a non-Error is thrown", async () => { + jest.mocked(runAthenaQuery).mockRejectedValueOnce("boom"); + + const result = await createHandler(makeDeps())({ date: "2026-10-08" }); expect(result).toEqual({ - status: "SUCCESS", - message: "populate-standard-breaches invoked successfully", + status: "FAILURE", + failureReason: "boom", + rowsReturned: 0, }); }); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts new file mode 100644 index 000000000..118b0afe4 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -0,0 +1,26 @@ +import buildStandardBreachesQuery from "../handler/standard-breaches-query"; + +describe("buildStandardBreachesQuery", () => { + it("selects from the event cache table filtered by the date partitions", () => { + expect(buildStandardBreachesQuery("event_cache", "2026-10-08")).toEqual({ + queryString: + 'SELECT * FROM "event_cache" WHERE partition_0 = ? AND partition_1 = ? AND partition_2 = ? LIMIT 100', + executionParameters: ["'2026'", "'10'", "'08'"], + }); + }); + + it("escapes double quotes in the table name", () => { + expect( + buildStandardBreachesQuery('bad"table', "2026-10-08").queryString, + ).toContain('FROM "bad""table"'); + }); + + it.each(["", "2026-10-8", "08/10/2026", "2026-10-08' OR '1'='1"])( + "rejects invalid date %p", + (date) => { + expect(() => buildStandardBreachesQuery("event_cache", date)).toThrow( + `Invalid date "${date}", expected YYYY-MM-DD`, + ); + }, + ); +}); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/athena.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/athena.ts new file mode 100644 index 000000000..59cf6c338 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/athena.ts @@ -0,0 +1,132 @@ +import { + GetQueryExecutionCommand, + GetQueryResultsCommand, + GetQueryResultsCommandOutput, + QueryExecutionState, + StartQueryExecutionCommand, +} from "@aws-sdk/client-athena"; +import { Deps } from "./deps"; + +export type AthenaQuery = { + queryString: string; + executionParameters?: string[]; +}; + +export type AthenaQueryOptions = { + pollIntervalMs?: number; +}; + +export type AthenaRow = Record; + +const DEFAULT_POLL_INTERVAL_MS = 1000; + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +async function startQuery(deps: Deps, query: AthenaQuery): Promise { + const { QueryExecutionId } = await deps.athenaClient.send( + new StartQueryExecutionCommand({ + QueryString: query.queryString, + ExecutionParameters: query.executionParameters, + WorkGroup: deps.env.ATHENA_WORKGROUP, + QueryExecutionContext: { Database: deps.env.ATHENA_DATABASE }, + }), + ); + + if (!QueryExecutionId) { + throw new Error("Athena did not return a query execution id"); + } + + return QueryExecutionId; +} + +async function waitForQuery( + deps: Deps, + queryExecutionId: string, + pollIntervalMs: number, +): Promise { + for (;;) { + const { QueryExecution } = await deps.athenaClient.send( + new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId }), + ); + const state = QueryExecution?.Status?.State; + + switch (state) { + case QueryExecutionState.SUCCEEDED: { + return; + } + case QueryExecutionState.FAILED: + case QueryExecutionState.CANCELLED: { + throw new Error( + `Athena query ${queryExecutionId} ${state}: ${QueryExecution?.Status?.StateChangeReason ?? "no reason given"}`, + ); + } + default: { + deps.logger.debug({ queryExecutionId, state }, "Waiting for query"); + await sleep(pollIntervalMs); + } + } + } +} + +function toRows( + page: GetQueryResultsCommandOutput, + isFirstPage: boolean, +): AthenaRow[] { + const columns = + page.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((c) => c.Name ?? "") ?? + []; + // For SELECT queries the first row of the first page contains the column headers. + const rows = (page.ResultSet?.Rows ?? []).slice(isFirstPage ? 1 : 0); + + return rows.map((row) => + Object.fromEntries( + columns.map((column, index) => [ + column, + row.Data?.at(index)?.VarCharValue, + ]), + ), + ); +} + +async function getResults( + deps: Deps, + queryExecutionId: string, +): Promise { + const results: AthenaRow[] = []; + let nextToken: string | undefined; + let isFirstPage = true; + + do { + const page = await deps.athenaClient.send( + new GetQueryResultsCommand({ + QueryExecutionId: queryExecutionId, + NextToken: nextToken, + }), + ); + results.push(...toRows(page, isFirstPage)); + nextToken = page.NextToken; + isFirstPage = false; + } while (nextToken); + + return results; +} + +export default async function runAthenaQuery( + deps: Deps, + query: AthenaQuery, + options: AthenaQueryOptions = {}, +): Promise { + const queryExecutionId = await startQuery(deps, query); + deps.logger.info({ queryExecutionId }, "Started Athena query"); + + await waitForQuery( + deps, + queryExecutionId, + options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, + ); + + return getResults(deps, queryExecutionId); +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/deps.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/deps.ts new file mode 100644 index 000000000..fb801c886 --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/deps.ts @@ -0,0 +1,25 @@ +import { AthenaClient } from "@aws-sdk/client-athena"; +import { createLogger } from "@internal/helpers"; +import { Logger } from "pino"; + +export type Deps = { + athenaClient: AthenaClient; + logger: Logger; + env: { + ATHENA_WORKGROUP: string; + ATHENA_DATABASE: string; + EVENT_CACHE_TABLE_NAME: string; + }; +}; + +export default function createDependenciesContainer(): Deps { + return { + athenaClient: new AthenaClient(), + logger: createLogger({ logLevel: process.env.PINO_LOG_LEVEL }), + env: { + ATHENA_WORKGROUP: process.env.ATHENA_WORKGROUP!, + ATHENA_DATABASE: process.env.ATHENA_DATABASE!, + EVENT_CACHE_TABLE_NAME: process.env.EVENT_CACHE_TABLE_NAME!, + }, + }; +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index 1bcc2318c..a06774a09 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -1,18 +1,45 @@ +import runAthenaQuery from "./athena"; +import { Deps } from "./deps"; +import buildStandardBreachesQuery from "./standard-breaches-query"; + export type PopulateStandardBreachesEvent = { date: string; }; export type PopulateStandardBreachesResult = { - status: "SUCCESS"; - message: string; + status: "SUCCESS" | "FAILURE"; + failureReason?: string; + rowsReturned: number; }; -// Placeholder: cache population is not yet implemented. -export default async function populateStandardBreaches( - _event: PopulateStandardBreachesEvent, -): Promise { - return { - status: "SUCCESS", - message: "populate-standard-breaches invoked successfully", +export default function createHandler(deps: Deps) { + return async ( + event: PopulateStandardBreachesEvent, + ): Promise => { + try { + const rows = await runAthenaQuery( + deps, + buildStandardBreachesQuery(deps.env.EVENT_CACHE_TABLE_NAME, event.date), + ); + + // TODO: CCM-24041 process the query results to populate standard breaches. + deps.logger.info( + { date: event.date, rowsReturned: rows.length }, + "Standard breaches query completed", + ); + + return { status: "SUCCESS", rowsReturned: rows.length }; + } catch (error) { + deps.logger.error( + { date: event.date, error }, + "Failed to populate standard breaches", + ); + + return { + status: "FAILURE", + failureReason: error instanceof Error ? error.message : String(error), + rowsReturned: 0, + }; + } }; } diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts new file mode 100644 index 000000000..9e370ebbd --- /dev/null +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -0,0 +1,42 @@ +import { AthenaQuery } from "./athena"; + +export default function buildStandardBreachesQuery( + tableName: string, +): AthenaQuery { + return { + queryString: `WITH unpacked AS ( + SELECT + json_extract_scalar(message, '$.data.domainId') AS domain_id, + json_extract_scalar(message, '$.data.status') AS status, + json_extract_scalar(message, '$.data.specificationId') AS specification_id, + json_extract_scalar(message, '$.data.billingRef') AS billing_ref, + json_extract_scalar(message, '$.data.supplierId') AS supplier_id, + json_extract_scalar(message, '$.data.groupId') AS group_id, + json_extract_scalar(message, '$.data.reasonCode') AS reason_code, + json_extract_scalar(message, '$.data.reasonText') AS reason_text, + from_iso8601_timestamp(json_extract_scalar(message, '$.time')) AS event_time + FROM ${tableName} +), +allocated_letters AS ( + SELECT domain_id, supplier_id + FROM unpacked + WHERE status = 'PENDING' + GROUP BY domain_id, supplier_id + HAVING min(event_time) < current_timestamp - + CASE WHEN day_of_week(current_timestamp AT TIME ZONE 'Europe/London') <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END +), +completed_letters AS ( + SELECT domain_id, supplier_id + FROM unpacked + WHERE status in ('REJECTED', 'FAILED', 'CANCELLED', 'FORWARDED', 'DISPATCHED') + GROUP BY domain_id, supplier_id +) +SELECT supplier_id, count(*) +FROM ( + SELECT al.domain_id, al.supplier_id + FROM allocated_letters al + LEFT JOIN completed_letters cl ON al.domain_id = cl.domain_id WHERE cl.domain_id IS NULL +) +GROUP BY supplier_id`, + }; +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts index 5516fb886..37ca055cf 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts @@ -1,2 +1,7 @@ +import createDependenciesContainer from "./handler/deps"; +import createHandler from "./handler/populate-standard-breaches"; + // eslint-disable-next-line import-x/prefer-default-export -export { default as populateStandardBreachesHandler } from "./handler/populate-standard-breaches"; +export const populateStandardBreachesHandler = createHandler( + createDependenciesContainer(), +); diff --git a/package-lock.json b/package-lock.json index 0912227cc..62e3b0d8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -415,7 +415,59 @@ "name": "nhs-notify-supplier-api-populate-standard-breaches", "version": "0.0.1", "dependencies": { - "esbuild": "0.28.2" + "@aws-sdk/client-athena": "^3.1149.0", + "@internal/helpers": "^0.1.0", + "esbuild": "0.28.2", + "pino": "^10.4.0" + } + }, + "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/pino": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.4.0.tgz", + "integrity": "sha512-bk1ZMTwG/Vymx+zPoa28MhNzucFvXJWp3csrSuxuHPSMeUettaCoQ5Jvg07mtC4JGNpcfTwz41oVCnNgBK9+vA==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^1.0.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, + "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" } }, "lambdas/letter-sla-breaches/populate-supplier-config-cache": { @@ -602,13 +654,6 @@ "node": ">=20.0.0" } }, - "lambdas/supplier-allocator/node_modules/@aws/lambda-invoke-store": { - "version": "0.3.0", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, "lambdas/supplier-allocator/node_modules/@nhsdigital/nhs-notify-event-schemas-supplier-config": { "version": "1.1.1", "dependencies": { @@ -900,13 +945,6 @@ "@aws-sdk/client-dynamodb": "^3.1145.0" } }, - "lambdas/upsert-letter/node_modules/@aws/lambda-invoke-store": { - "version": "0.3.0", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.9.3", "license": "MIT", @@ -1030,6 +1068,25 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-athena": { + "version": "3.1149.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.1149.0.tgz", + "integrity": "sha512-GPHrphuEDhybsEkcCOnmadtIkCAzJ6wckZ1gllkHhkicO5EccXwMHeZNsmTWyjkrnafD4EjSz5Oaf+HfbHLIZA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/credential-provider-node": "^3.972.85", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.2", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.1146.0", "dev": true, @@ -1158,14 +1215,16 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.978.1", + "version": "3.978.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.2.tgz", + "integrity": "sha512-ViOBLuGSSyPAmkKGJ6KJXtVN7MOEp3rnI7uH+kWxnHccvo8C9cvunEAnN5/oaZH1laQfsTBw7IOnmNkWlFZl2A==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.6", "@aws-sdk/xml-builder": "^3.972.41", "@aws/lambda-invoke-store": "^0.3.0", - "@smithy/core": "^3.35.0", - "@smithy/signature-v4": "^5.7.3", + "@smithy/core": "^3.35.2", + "@smithy/signature-v4": "^5.7.4", "@smithy/types": "^4.19.0", "bowser": "^2.11.0", "tslib": "^2.6.2" @@ -1174,20 +1233,15 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/core/node_modules/@aws/lambda-invoke-store": { - "version": "0.3.0", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.72", + "version": "3.972.73", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.73.tgz", + "integrity": "sha512-PxlFZiF5IPUsPnuVSqjM9jlmRwZhL0xx40ccWfZiLQS8TRnHQpegpK1uAidatWuEqbJtbbY97A2Sezikz2AMUw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", + "@aws-sdk/core": "^3.978.2", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1196,12 +1250,14 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.74", + "version": "3.972.75", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.75.tgz", + "integrity": "sha512-Q/uIDy8eAVN7JgSKDg16F/f+hmNJj/ZyLJN8AkW2ofVdDgJVh8fO/A26iZ7YsvM2/k76hsf/u2lLAtyXsK8Ziw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", + "@aws-sdk/core": "^3.978.2", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", @@ -1212,19 +1268,21 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.973.17", + "version": "3.973.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.18.tgz", + "integrity": "sha512-KoyjZ9vdjuxEQ8obAH06xKP8H0WKqGhwUZLMqAOiS7G0xgUAht4X8f0ASLC/vhNLCWGblZOD3VQHWyTROGJHLg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/credential-provider-env": "^3.972.72", - "@aws-sdk/credential-provider-http": "^3.972.74", - "@aws-sdk/credential-provider-login": "^3.972.79", - "@aws-sdk/credential-provider-process": "^3.972.72", - "@aws-sdk/credential-provider-sso": "^3.973.16", - "@aws-sdk/credential-provider-web-identity": "^3.972.78", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/credential-provider-env": "^3.972.73", + "@aws-sdk/credential-provider-http": "^3.972.75", + "@aws-sdk/credential-provider-login": "^3.972.80", + "@aws-sdk/credential-provider-process": "^3.972.73", + "@aws-sdk/credential-provider-sso": "^3.973.17", + "@aws-sdk/credential-provider-web-identity": "^3.972.79", + "@aws-sdk/nested-clients": "^3.997.47", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/credential-provider-imds": "^4.5.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" @@ -1234,13 +1292,15 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.79", + "version": "3.972.80", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.80.tgz", + "integrity": "sha512-xF9A0gaO7k+1dYPFoo69OQatA9Ov0QuDZfqE8H0FLyIoOph+mlEC+I1Qo9UULupGuDfSYh+kVpfcDSxSYf/2fg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/nested-clients": "^3.997.47", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1249,17 +1309,19 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.84", + "version": "3.972.85", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.85.tgz", + "integrity": "sha512-CrFcML/xImiSlquH4W3V6Npz3aSkPXHDVciAU1JOBN0vqjmkcz9H9up2sCHQqN76J/WrT+TJ580L8h2joksQrg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.72", - "@aws-sdk/credential-provider-http": "^3.972.74", - "@aws-sdk/credential-provider-ini": "^3.973.17", - "@aws-sdk/credential-provider-process": "^3.972.72", - "@aws-sdk/credential-provider-sso": "^3.973.16", - "@aws-sdk/credential-provider-web-identity": "^3.972.78", + "@aws-sdk/credential-provider-env": "^3.972.73", + "@aws-sdk/credential-provider-http": "^3.972.75", + "@aws-sdk/credential-provider-ini": "^3.973.18", + "@aws-sdk/credential-provider-process": "^3.972.73", + "@aws-sdk/credential-provider-sso": "^3.973.17", + "@aws-sdk/credential-provider-web-identity": "^3.972.79", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/credential-provider-imds": "^4.5.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" @@ -1269,12 +1331,14 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.72", + "version": "3.972.73", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.73.tgz", + "integrity": "sha512-tyNnT4aVuOEXmKKKZPExBi828vRd9heshobBsKF0bov5EORHSJc1AGsHOMRP57sDt+dIuVASUOywaixiX26t4g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", + "@aws-sdk/core": "^3.978.2", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1283,14 +1347,16 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.973.16", + "version": "3.973.17", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.17.tgz", + "integrity": "sha512-gxKTWaUk2gSbBOH4m3NPRfEKcGXeDAOgHr7nNG5JktR1xZLo0R+DpMjbcleoHbB0uJREOG7hjIRE635IuBWsRQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", - "@aws-sdk/token-providers": "3.1138.0", + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/nested-clients": "^3.997.47", + "@aws-sdk/token-providers": "3.1149.0", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1299,13 +1365,15 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.78", + "version": "3.972.79", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.79.tgz", + "integrity": "sha512-/jn0Gs/qm6fs6nTIA5yncHayybz42SATZ3rNxCEBYPJ9v69PLrDRgk07E9xTT2gpRcNmDXXbi27ZbuU06N0Z6Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/nested-clients": "^3.997.47", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1405,13 +1473,15 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.997.46", + "version": "3.997.47", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.47.tgz", + "integrity": "sha512-bovCKvDsLocNTzaj6VrrMVJ2BGROqO0JlwcGseO67a+I9Bo+08nUJhSQfZUIx4+A4a7I4101GEJd/AYm7dQ68w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/signature-v4-multi-region": "^3.996.47", + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/signature-v4-multi-region": "^3.996.48", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", @@ -1437,11 +1507,13 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.47", + "version": "3.996.48", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.48.tgz", + "integrity": "sha512-cFDXfEbXaGwbnpvR/DyqfxfjxpjXVRVthQLgkwvcfDS5u3+8G9WSj1X1Ag/pTRg/QOeOno1Tq3ycEdrKsCUcOA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.974.6", - "@smithy/signature-v4": "^5.7.3", + "@smithy/signature-v4": "^5.7.4", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1450,13 +1522,15 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1138.0", + "version": "3.1149.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1149.0.tgz", + "integrity": "sha512-CAwYyA8eVk1w6h8YgLvkjIOakWPwURaUFrP7BRVgkxTFUM+O87thyKsGKJ6rVPbP8eoEV10cBWQ5YfaYl3qSVQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.978.1", - "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/core": "^3.978.2", + "@aws-sdk/nested-clients": "^3.997.47", "@aws-sdk/types": "^3.974.6", - "@smithy/core": "^3.35.0", + "@smithy/core": "^3.35.2", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" }, @@ -1501,6 +1575,15 @@ "node": ">=20.0.0" } }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "license": "MIT", @@ -3617,7 +3700,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.35.1", + "version": "3.35.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.35.2.tgz", + "integrity": "sha512-ejTukBuP/Pg3t9N8Ux+0zKsXeHWm7p6W3HbXFOyltKasSxZwCFmvTut7kBdZSCF3m9y1bXXyVu6p2xuMFcsTyQ==", "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.19.0", From 76b895e9886e4d01ed7ada823319865f10db0320 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Wed, 14 Oct 2026 14:10:41 +0100 Subject: [PATCH 19/37] test coverage --- .../populate-standard-breaches.test.ts | 38 +++++-------------- .../__tests__/standard-breaches-query.test.ts | 25 +++--------- .../src/handler/populate-standard-breaches.ts | 2 +- .../src/handler/standard-breaches-query.ts | 9 +++-- 4 files changed, 23 insertions(+), 51 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index 5403d3148..b72933621 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -1,7 +1,6 @@ import runAthenaQuery from "../handler/athena"; import { Deps } from "../handler/deps"; import createHandler from "../handler/populate-standard-breaches"; -import buildStandardBreachesQuery from "../handler/standard-breaches-query"; jest.mock("../handler/athena"); @@ -18,51 +17,34 @@ describe("populateStandardBreaches", () => { const result = await createHandler(deps)({ date: "2026-10-08" }); - expect(runAthenaQuery).toHaveBeenCalledWith( - deps, - buildStandardBreachesQuery("event_cache", "2026-10-08"), - ); + expect(runAthenaQuery).toHaveBeenCalled(); expect(result).toEqual({ status: "SUCCESS", rowsReturned: 1 }); }); - it("returns failure when the query fails", async () => { + it("runs the standard breaches query and returns failure", async () => { const deps = makeDeps(); - jest - .mocked(runAthenaQuery) - .mockRejectedValueOnce(new Error("query failed")); + jest.mocked(runAthenaQuery).mockRejectedValueOnce(new Error("boom")); const result = await createHandler(deps)({ date: "2026-10-08" }); + expect(runAthenaQuery).toHaveBeenCalled(); expect(result).toEqual({ + failureReason: "boom", status: "FAILURE", - failureReason: "query failed", - rowsReturned: 0, - }); - expect(deps.logger.error).toHaveBeenCalledWith( - { date: "2026-10-08", error: expect.any(Error) }, - "Failed to populate standard breaches", - ); - }); - - it("returns failure without querying when the date is invalid", async () => { - const result = await createHandler(makeDeps())({ date: "not-a-date" }); - - expect(runAthenaQuery).not.toHaveBeenCalled(); - expect(result).toEqual({ - status: "FAILURE", - failureReason: 'Invalid date "not-a-date", expected YYYY-MM-DD', rowsReturned: 0, }); }); - it("returns failure when a non-Error is thrown", async () => { + it("runs the standard breaches query and returns failure when not an error", async () => { + const deps = makeDeps(); jest.mocked(runAthenaQuery).mockRejectedValueOnce("boom"); - const result = await createHandler(makeDeps())({ date: "2026-10-08" }); + const result = await createHandler(deps)({ date: "2026-10-08" }); + expect(runAthenaQuery).toHaveBeenCalled(); expect(result).toEqual({ - status: "FAILURE", failureReason: "boom", + status: "FAILURE", rowsReturned: 0, }); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts index 118b0afe4..92fa7f133 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -2,25 +2,12 @@ import buildStandardBreachesQuery from "../handler/standard-breaches-query"; describe("buildStandardBreachesQuery", () => { it("selects from the event cache table filtered by the date partitions", () => { - expect(buildStandardBreachesQuery("event_cache", "2026-10-08")).toEqual({ - queryString: - 'SELECT * FROM "event_cache" WHERE partition_0 = ? AND partition_1 = ? AND partition_2 = ? LIMIT 100', - executionParameters: ["'2026'", "'10'", "'08'"], - }); - }); - - it("escapes double quotes in the table name", () => { expect( - buildStandardBreachesQuery('bad"table', "2026-10-08").queryString, - ).toContain('FROM "bad""table"'); + buildStandardBreachesQuery("event_cache", new Date("2026-10-08")), + ).toEqual( + expect.objectContaining({ + executionParameters: ["2026-10-08T00:00:00.000Z"], + }), + ); }); - - it.each(["", "2026-10-8", "08/10/2026", "2026-10-08' OR '1'='1"])( - "rejects invalid date %p", - (date) => { - expect(() => buildStandardBreachesQuery("event_cache", date)).toThrow( - `Invalid date "${date}", expected YYYY-MM-DD`, - ); - }, - ); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index a06774a09..e32db6a2f 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -19,7 +19,7 @@ export default function createHandler(deps: Deps) { try { const rows = await runAthenaQuery( deps, - buildStandardBreachesQuery(deps.env.EVENT_CACHE_TABLE_NAME, event.date), + buildStandardBreachesQuery(deps.env.EVENT_CACHE_TABLE_NAME, new Date()), ); // TODO: CCM-24041 process the query results to populate standard breaches. diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 9e370ebbd..28ac5d7bc 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -2,6 +2,7 @@ import { AthenaQuery } from "./athena"; export default function buildStandardBreachesQuery( tableName: string, + now: Date, ): AthenaQuery { return { queryString: `WITH unpacked AS ( @@ -14,7 +15,8 @@ export default function buildStandardBreachesQuery( json_extract_scalar(message, '$.data.groupId') AS group_id, json_extract_scalar(message, '$.data.reasonCode') AS reason_code, json_extract_scalar(message, '$.data.reasonText') AS reason_text, - from_iso8601_timestamp(json_extract_scalar(message, '$.time')) AS event_time + from_iso8601_timestamp(json_extract_scalar(message, '$.time')) AS event_time, + from_iso8601_timestamp(?) AS now FROM ${tableName} ), allocated_letters AS ( @@ -22,8 +24,8 @@ allocated_letters AS ( FROM unpacked WHERE status = 'PENDING' GROUP BY domain_id, supplier_id - HAVING min(event_time) < current_timestamp - - CASE WHEN day_of_week(current_timestamp AT TIME ZONE 'Europe/London') <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END + HAVING min(event_time) < unpacked.now - + CASE WHEN day_of_week(unpacked.now AT TIME ZONE 'Europe/London') <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END ), completed_letters AS ( SELECT domain_id, supplier_id @@ -38,5 +40,6 @@ FROM ( LEFT JOIN completed_letters cl ON al.domain_id = cl.domain_id WHERE cl.domain_id IS NULL ) GROUP BY supplier_id`, + executionParameters: [now.toISOString()], }; } From 0ef43698ddc9914692e98b11214a995de6391dd8 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Wed, 14 Oct 2026 15:57:40 +0100 Subject: [PATCH 20/37] Create common package --- .../letter-sla-breaches/common/jest.config.ts | 28 +++++++ .../letter-sla-breaches/common/package.json | 19 +++++ .../__tests__/athena-date-predicate.test.ts | 74 +++++++++++++++++++ .../src/__tests__/athena.test.ts | 4 +- .../common/src/athena-date-predicate.ts | 45 +++++++++++ .../src/handler => common/src}/athena.ts | 0 .../src/handler => common/src}/deps.ts | 0 .../letter-sla-breaches/common/src/index.ts | 4 + .../letter-sla-breaches/common/tsconfig.json | 11 +++ .../populate-standard-breaches/package.json | 6 +- .../populate-standard-breaches.test.ts | 8 +- .../src/handler/populate-standard-breaches.ts | 6 +- .../src/handler/standard-breaches-query.ts | 46 +++++------- .../populate-standard-breaches/src/index.ts | 2 +- package-lock.json | 40 ++++++---- 15 files changed, 241 insertions(+), 52 deletions(-) create mode 100644 lambdas/letter-sla-breaches/common/jest.config.ts create mode 100644 lambdas/letter-sla-breaches/common/package.json create mode 100644 lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts rename lambdas/letter-sla-breaches/{populate-standard-breaches => common}/src/__tests__/athena.test.ts (98%) create mode 100644 lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts rename lambdas/letter-sla-breaches/{populate-standard-breaches/src/handler => common/src}/athena.ts (100%) rename lambdas/letter-sla-breaches/{populate-standard-breaches/src/handler => common/src}/deps.ts (100%) create mode 100644 lambdas/letter-sla-breaches/common/src/index.ts create mode 100644 lambdas/letter-sla-breaches/common/tsconfig.json diff --git a/lambdas/letter-sla-breaches/common/jest.config.ts b/lambdas/letter-sla-breaches/common/jest.config.ts new file mode 100644 index 000000000..d38a979df --- /dev/null +++ b/lambdas/letter-sla-breaches/common/jest.config.ts @@ -0,0 +1,28 @@ +export default { + preset: "ts-jest", + extensionsToTreatAsEsm: [".ts"], + transform: { + "^.+\\.ts$": [ + "ts-jest", + { + useESM: true, + }, + ], + }, + clearMocks: true, + collectCoverage: true, + coverageDirectory: "./.reports/unit/coverage", + coverageProvider: "babel", + coverageThreshold: { + global: { + branches: 100, + functions: 100, + lines: 100, + statements: -10, + }, + }, + coveragePathIgnorePatterns: ["/__tests__/"], + testPathIgnorePatterns: [".build"], + testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"], + testEnvironment: "node", +}; diff --git a/lambdas/letter-sla-breaches/common/package.json b/lambdas/letter-sla-breaches/common/package.json new file mode 100644 index 000000000..3e5758ac4 --- /dev/null +++ b/lambdas/letter-sla-breaches/common/package.json @@ -0,0 +1,19 @@ +{ + "dependencies": { + "@aws-sdk/client-athena": "^3.1149.0", + "@internal/helpers": "^0.1.0", + "pino": "^10.4.0" + }, + "description": "Common utilities for letter SLA breach lambdas", + "license": "MIT", + "main": "src/index.ts", + "name": "@internal/letter-sla-breaches-common", + "private": true, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts new file mode 100644 index 000000000..a0a4f545c --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts @@ -0,0 +1,74 @@ +import buildAthenaDatePredicate from "../athena-date-predicate"; + +describe("buildAthenaDatePredicate", () => { + it("includes only the requested UTC date when no days are subtracted", () => { + expect( + buildAthenaDatePredicate(new Date("2026-10-14T23:59:59.999Z"), 0), + ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('14'))"); + }); + + it("groups consecutive dates in the same month into one partition predicate", () => { + expect( + buildAthenaDatePredicate(new Date("2026-10-14T12:00:00.000Z"), 2), + ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13', '14'))"); + }); + + it("creates separate predicates when the date range crosses a month", () => { + expect( + buildAthenaDatePredicate(new Date("2026-11-01T00:00:00.000Z"), 2), + ).toBe( + "(_year = '2026' AND _month = '10' AND _day IN ('30', '31'))\n" + + " OR (_year = '2026' AND _month = '11' AND _day IN ('01'))", + ); + }); + + it("creates separate predicates when the date range crosses a year", () => { + expect( + buildAthenaDatePredicate(new Date("2026-01-01T00:00:00.000Z"), 1), + ).toBe( + "(_year = '2025' AND _month = '12' AND _day IN ('31'))\n" + + " OR (_year = '2026' AND _month = '01' AND _day IN ('01'))", + ); + }); + + it("includes leap days in the date range", () => { + expect( + buildAthenaDatePredicate(new Date("2024-03-01T00:00:00.000Z"), 2), + ).toBe( + "(_year = '2024' AND _month = '02' AND _day IN ('28', '29'))\n" + + " OR (_year = '2024' AND _month = '03' AND _day IN ('01'))", + ); + }); + + it("uses UTC calendar dates regardless of the time component", () => { + expect( + buildAthenaDatePredicate(new Date("2026-10-14T00:30:00+02:00"), 1), + ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13'))"); + }); + + it("throws when given an invalid date", () => { + expect(() => buildAthenaDatePredicate(new Date(Number.NaN), 0)).toThrow( + new RangeError("date must be valid"), + ); + }); + + it.each([ + -1, + 1.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ])( + "throws when daysToSubtract is not a non-negative safe integer (%s)", + (daysToSubtract) => { + expect(() => + buildAthenaDatePredicate( + new Date("2026-10-14T00:00:00.000Z"), + daysToSubtract, + ), + ).toThrow( + new RangeError("daysToSubtract must be a non-negative integer"), + ); + }, + ); +}); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts similarity index 98% rename from lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts rename to lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts index d56b075f8..b11ff60ae 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/athena.test.ts +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts @@ -4,8 +4,8 @@ import { GetQueryResultsCommand, StartQueryExecutionCommand, } from "@aws-sdk/client-athena"; -import runAthenaQuery from "../handler/athena"; -import { Deps } from "../handler/deps"; +import runAthenaQuery from "../athena"; +import { type Deps } from "../deps"; const query = { queryString: "SELECT ?", executionParameters: ["'a'"] }; diff --git a/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts new file mode 100644 index 000000000..0ca204bac --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts @@ -0,0 +1,45 @@ +export default function buildAthenaDatePredicate( + date: Date, + daysToSubtract: number, +): string { + if (Number.isNaN(date.getTime())) { + throw new RangeError("date must be valid"); + } + + if (!Number.isSafeInteger(daysToSubtract) || daysToSubtract < 0) { + throw new RangeError("daysToSubtract must be a non-negative integer"); + } + + const endDate = new Date(date); + endDate.setUTCHours(0, 0, 0, 0); + + const currentDate = new Date(endDate); + currentDate.setUTCDate(currentDate.getUTCDate() - daysToSubtract); + + const groups: { year: string; month: string; days: string[] }[] = []; + + while (currentDate <= endDate) { + const year = String(currentDate.getUTCFullYear()).padStart(4, "0"); + const month = String(currentDate.getUTCMonth() + 1).padStart(2, "0"); + const day = String(currentDate.getUTCDate()).padStart(2, "0"); + const lastGroup = groups.at(-1); + + if (lastGroup?.year === year && lastGroup.month === month) { + lastGroup.days.push(day); + } else { + groups.push({ year, month, days: [day] }); + } + + currentDate.setUTCDate(currentDate.getUTCDate() + 1); + } + + return groups + .map( + // eslint-disable-next-line sort-destructure-keys/sort-destructure-keys + ({ year, month, days }) => + `(_year = '${year}' AND _month = '${month}' AND _day IN (${days + .map((day) => `'${day}'`) + .join(", ")}))`, + ) + .join("\n OR "); +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/athena.ts b/lambdas/letter-sla-breaches/common/src/athena.ts similarity index 100% rename from lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/athena.ts rename to lambdas/letter-sla-breaches/common/src/athena.ts diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/deps.ts b/lambdas/letter-sla-breaches/common/src/deps.ts similarity index 100% rename from lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/deps.ts rename to lambdas/letter-sla-breaches/common/src/deps.ts diff --git a/lambdas/letter-sla-breaches/common/src/index.ts b/lambdas/letter-sla-breaches/common/src/index.ts new file mode 100644 index 000000000..3abc594ec --- /dev/null +++ b/lambdas/letter-sla-breaches/common/src/index.ts @@ -0,0 +1,4 @@ +export { default as runAthenaQuery } from "./athena"; +export { default as createDependenciesContainer } from "./deps"; +export type { AthenaQuery, AthenaQueryOptions, AthenaRow } from "./athena"; +export type { Deps } from "./deps"; diff --git a/lambdas/letter-sla-breaches/common/tsconfig.json b/lambdas/letter-sla-breaches/common/tsconfig.json new file mode 100644 index 000000000..4a11b255a --- /dev/null +++ b/lambdas/letter-sla-breaches/common/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "ignoreDeprecations": "5.0" + }, + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json index 87d2f1d15..a8a5775a2 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json @@ -1,9 +1,7 @@ { "dependencies": { - "@aws-sdk/client-athena": "^3.1149.0", - "@internal/helpers": "^0.1.0", - "esbuild": "0.28.2", - "pino": "^10.4.0" + "@internal/letter-sla-breaches-common": "*", + "esbuild": "0.28.2" }, "name": "nhs-notify-supplier-api-populate-standard-breaches", "private": true, diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index b72933621..b76524ed6 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -1,8 +1,10 @@ -import runAthenaQuery from "../handler/athena"; -import { Deps } from "../handler/deps"; +import { + type Deps, + runAthenaQuery, +} from "@internal/letter-sla-breaches-common"; import createHandler from "../handler/populate-standard-breaches"; -jest.mock("../handler/athena"); +jest.mock("@internal/letter-sla-breaches-common"); const makeDeps = () => ({ diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index e32db6a2f..2d6f9d113 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -1,5 +1,7 @@ -import runAthenaQuery from "./athena"; -import { Deps } from "./deps"; +import { + type Deps, + runAthenaQuery, +} from "@internal/letter-sla-breaches-common"; import buildStandardBreachesQuery from "./standard-breaches-query"; export type PopulateStandardBreachesEvent = { diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 28ac5d7bc..1130385bc 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -1,45 +1,39 @@ -import { AthenaQuery } from "./athena"; +import { type AthenaQuery } from "@internal/letter-sla-breaches-common"; export default function buildStandardBreachesQuery( tableName: string, now: Date, ): AthenaQuery { return { - queryString: `WITH unpacked AS ( - SELECT - json_extract_scalar(message, '$.data.domainId') AS domain_id, - json_extract_scalar(message, '$.data.status') AS status, - json_extract_scalar(message, '$.data.specificationId') AS specification_id, - json_extract_scalar(message, '$.data.billingRef') AS billing_ref, - json_extract_scalar(message, '$.data.supplierId') AS supplier_id, - json_extract_scalar(message, '$.data.groupId') AS group_id, - json_extract_scalar(message, '$.data.reasonCode') AS reason_code, - json_extract_scalar(message, '$.data.reasonText') AS reason_text, - from_iso8601_timestamp(json_extract_scalar(message, '$.time')) AS event_time, - from_iso8601_timestamp(?) AS now - FROM ${tableName} + queryString: `WITH now AS ( + SELECT from_iso8601_timestamp('?') AS iso8601 +), +recent_letter_events AS ( + SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time + FROM nhs_pr731_supapi_eventpub + WHERE __year = '2026' AND __month = '10' AND __day IN ('12', '13', '14') ), allocated_letters AS ( - SELECT domain_id, supplier_id - FROM unpacked + SELECT domainid, supplierid + FROM recent_letter_events WHERE status = 'PENDING' - GROUP BY domain_id, supplier_id - HAVING min(event_time) < unpacked.now - - CASE WHEN day_of_week(unpacked.now AT TIME ZONE 'Europe/London') <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END + GROUP BY domainid, supplierid + HAVING min(event_time) < (SELECT iso8601 FROM now) - + CASE WHEN day_of_week((SELECT iso8601 AT TIME ZONE 'Europe/London' FROM now)) <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END ), completed_letters AS ( - SELECT domain_id, supplier_id - FROM unpacked + SELECT domainid, supplierid + FROM recent_letter_events WHERE status in ('REJECTED', 'FAILED', 'CANCELLED', 'FORWARDED', 'DISPATCHED') - GROUP BY domain_id, supplier_id + GROUP BY domainid, supplierid ) -SELECT supplier_id, count(*) +SELECT supplierid, count(*) FROM ( - SELECT al.domain_id, al.supplier_id + SELECT al.domainid, al.supplierid FROM allocated_letters al - LEFT JOIN completed_letters cl ON al.domain_id = cl.domain_id WHERE cl.domain_id IS NULL + LEFT JOIN completed_letters cl ON al.domainid = cl.domainid WHERE cl.domainid IS NULL ) -GROUP BY supplier_id`, +GROUP BY supplierid`, executionParameters: [now.toISOString()], }; } diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts index 37ca055cf..f52283ddb 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/index.ts @@ -1,4 +1,4 @@ -import createDependenciesContainer from "./handler/deps"; +import { createDependenciesContainer } from "@internal/letter-sla-breaches-common"; import createHandler from "./handler/populate-standard-breaches"; // eslint-disable-next-line import-x/prefer-default-export diff --git a/package-lock.json b/package-lock.json index 62e3b0d8e..e8ac5de45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -404,24 +404,17 @@ "esbuild": "0.28.2" } }, - "lambdas/letter-sla-breaches/populate-sameday-breaches": { - "name": "nhs-notify-supplier-api-populate-sameday-breaches", - "version": "0.0.1", - "dependencies": { - "esbuild": "0.28.2" - } - }, - "lambdas/letter-sla-breaches/populate-standard-breaches": { - "name": "nhs-notify-supplier-api-populate-standard-breaches", + "lambdas/letter-sla-breaches/common": { + "name": "@internal/letter-sla-breaches-common", "version": "0.0.1", + "license": "MIT", "dependencies": { "@aws-sdk/client-athena": "^3.1149.0", "@internal/helpers": "^0.1.0", - "esbuild": "0.28.2", "pino": "^10.4.0" } }, - "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/pino": { + "lambdas/letter-sla-breaches/common/node_modules/pino": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/pino/-/pino-10.4.0.tgz", "integrity": "sha512-bk1ZMTwG/Vymx+zPoa28MhNzucFvXJWp3csrSuxuHPSMeUettaCoQ5Jvg07mtC4JGNpcfTwz41oVCnNgBK9+vA==", @@ -443,7 +436,7 @@ "pino": "bin.js" } }, - "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/pino-abstract-transport": { + "lambdas/letter-sla-breaches/common/node_modules/pino-abstract-transport": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", @@ -452,13 +445,13 @@ "split2": "^4.0.0" } }, - "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/real-require": { + "lambdas/letter-sla-breaches/common/node_modules/real-require": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", "license": "MIT" }, - "lambdas/letter-sla-breaches/populate-standard-breaches/node_modules/thread-stream": { + "lambdas/letter-sla-breaches/common/node_modules/thread-stream": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", @@ -470,6 +463,21 @@ "node": ">=20" } }, + "lambdas/letter-sla-breaches/populate-sameday-breaches": { + "name": "nhs-notify-supplier-api-populate-sameday-breaches", + "version": "0.0.1", + "dependencies": { + "esbuild": "0.28.2" + } + }, + "lambdas/letter-sla-breaches/populate-standard-breaches": { + "name": "nhs-notify-supplier-api-populate-standard-breaches", + "version": "0.0.1", + "dependencies": { + "@internal/letter-sla-breaches-common": "*", + "esbuild": "0.28.2" + } + }, "lambdas/letter-sla-breaches/populate-supplier-config-cache": { "name": "nhs-notify-supplier-api-populate-supplier-config-cache", "version": "0.0.1", @@ -2519,6 +2527,10 @@ "resolved": "internal/helpers", "link": true }, + "node_modules/@internal/letter-sla-breaches-common": { + "resolved": "lambdas/letter-sla-breaches/common", + "link": true + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "license": "ISC", From dff7a494b65c290de1d74b76293665e083e97dac Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Wed, 14 Oct 2026 16:42:15 +0100 Subject: [PATCH 21/37] Wire it together --- lambdas/letter-sla-breaches/common/src/index.ts | 1 + .../src/__tests__/standard-breaches-query.test.ts | 5 ++++- .../src/handler/populate-standard-breaches.ts | 6 +++++- .../src/handler/standard-breaches-query.ts | 10 +++++++--- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lambdas/letter-sla-breaches/common/src/index.ts b/lambdas/letter-sla-breaches/common/src/index.ts index 3abc594ec..27aaea112 100644 --- a/lambdas/letter-sla-breaches/common/src/index.ts +++ b/lambdas/letter-sla-breaches/common/src/index.ts @@ -1,4 +1,5 @@ export { default as runAthenaQuery } from "./athena"; export { default as createDependenciesContainer } from "./deps"; +export { default as buildAthenaDatePredicate } from "./athena-date-predicate"; export type { AthenaQuery, AthenaQueryOptions, AthenaRow } from "./athena"; export type { Deps } from "./deps"; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts index 92fa7f133..1e4b3ff03 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -3,9 +3,12 @@ import buildStandardBreachesQuery from "../handler/standard-breaches-query"; describe("buildStandardBreachesQuery", () => { it("selects from the event cache table filtered by the date partitions", () => { expect( - buildStandardBreachesQuery("event_cache", new Date("2026-10-08")), + buildStandardBreachesQuery("event_cache", new Date("2026-10-08"), 0), ).toEqual( expect.objectContaining({ + queryString: expect.stringContaining( + "(_year = '2026' AND _month = '10' AND _day IN ('08'))", + ), executionParameters: ["2026-10-08T00:00:00.000Z"], }), ); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index 2d6f9d113..a3011ddd0 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -21,7 +21,11 @@ export default function createHandler(deps: Deps) { try { const rows = await runAthenaQuery( deps, - buildStandardBreachesQuery(deps.env.EVENT_CACHE_TABLE_NAME, new Date()), + buildStandardBreachesQuery( + deps.env.EVENT_CACHE_TABLE_NAME, + new Date(), + 14, + ), ); // TODO: CCM-24041 process the query results to populate standard breaches. diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 1130385bc..24db5696e 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -1,8 +1,12 @@ -import { type AthenaQuery } from "@internal/letter-sla-breaches-common"; +import { + type AthenaQuery, + buildAthenaDatePredicate, +} from "@internal/letter-sla-breaches-common"; export default function buildStandardBreachesQuery( tableName: string, now: Date, + daysHistory: number, ): AthenaQuery { return { queryString: `WITH now AS ( @@ -10,8 +14,8 @@ export default function buildStandardBreachesQuery( ), recent_letter_events AS ( SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time - FROM nhs_pr731_supapi_eventpub - WHERE __year = '2026' AND __month = '10' AND __day IN ('12', '13', '14') + FROM ${tableName} + WHERE ${buildAthenaDatePredicate(now, daysHistory)} ), allocated_letters AS ( SELECT domainid, supplierid From 93f1c0342a611fa6fd95c3fc7237a1b58e9bcb45 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Thu, 15 Oct 2026 09:28:10 +0100 Subject: [PATCH 22/37] Parameter fix --- .../src/handler/populate-standard-breaches.ts | 8 ++++---- .../src/handler/standard-breaches-query.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index a3011ddd0..72950dde6 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -11,7 +11,7 @@ export type PopulateStandardBreachesEvent = { export type PopulateStandardBreachesResult = { status: "SUCCESS" | "FAILURE"; failureReason?: string; - rowsReturned: number; + breachCount: string; }; export default function createHandler(deps: Deps) { @@ -30,11 +30,11 @@ export default function createHandler(deps: Deps) { // TODO: CCM-24041 process the query results to populate standard breaches. deps.logger.info( - { date: event.date, rowsReturned: rows.length }, + { date: event.date, breachCount: rows[0].breach_count }, "Standard breaches query completed", ); - return { status: "SUCCESS", rowsReturned: rows.length }; + return { status: "SUCCESS", breachCount: rows[0].breach_count! }; } catch (error) { deps.logger.error( { date: event.date, error }, @@ -44,7 +44,7 @@ export default function createHandler(deps: Deps) { return { status: "FAILURE", failureReason: error instanceof Error ? error.message : String(error), - rowsReturned: 0, + breachCount: "0", }; } }; diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 24db5696e..871db1363 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -10,7 +10,7 @@ export default function buildStandardBreachesQuery( ): AthenaQuery { return { queryString: `WITH now AS ( - SELECT from_iso8601_timestamp('?') AS iso8601 + SELECT from_iso8601_timestamp(?) AS iso8601 ), recent_letter_events AS ( SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time @@ -31,7 +31,7 @@ completed_letters AS ( WHERE status in ('REJECTED', 'FAILED', 'CANCELLED', 'FORWARDED', 'DISPATCHED') GROUP BY domainid, supplierid ) -SELECT supplierid, count(*) +SELECT supplierid, count(*) AS breach_count FROM ( SELECT al.domainid, al.supplierid FROM allocated_letters al From 86a0ab413ecd8b07427c43afc37cea13e879b923 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Thu, 15 Oct 2026 15:06:42 +0100 Subject: [PATCH 23/37] Add trino-based unit test --- .github/workflows/stage-2-test.yaml | 8 + internal/datastore/src/__test__/db.ts | 1 + internal/sql-test-helpers/README.md | 46 ++++ internal/sql-test-helpers/package.json | 17 ++ internal/sql-test-helpers/src/index.ts | 8 + internal/sql-test-helpers/src/trino.ts | 119 ++++++++++ internal/sql-test-helpers/tsconfig.json | 7 + .../common/src/__tests__/athena.test.ts | 49 +++-- .../letter-sla-breaches/common/src/athena.ts | 2 +- .../populate-standard-breaches/package.json | 3 + .../populate-standard-breaches.test.ts | 8 +- .../__tests__/standard-breaches-query.test.ts | 208 +++++++++++++++++- .../src/handler/standard-breaches-query.ts | 4 +- package-lock.json | 140 ++---------- scripts/config/sonar-scanner.properties | 2 +- 15 files changed, 455 insertions(+), 167 deletions(-) create mode 100644 internal/sql-test-helpers/README.md create mode 100644 internal/sql-test-helpers/package.json create mode 100644 internal/sql-test-helpers/src/index.ts create mode 100644 internal/sql-test-helpers/src/trino.ts create mode 100644 internal/sql-test-helpers/tsconfig.json diff --git a/.github/workflows/stage-2-test.yaml b/.github/workflows/stage-2-test.yaml index f1282be1f..051e5564c 100644 --- a/.github/workflows/stage-2-test.yaml +++ b/.github/workflows/stage-2-test.yaml @@ -77,6 +77,14 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: + # Pull testcontainers images in the background so the download overlaps + # the setup steps. Keep these references in sync with the images used in + # internal/datastore/src/__test__/db.ts and + # internal/sql-test-helpers/src/trino.ts. + - name: "Pre-pull test container images" + run: | + docker pull amazon/dynamodb-local > /dev/null 2>&1 & + docker pull trinodb/trino:476 > /dev/null 2>&1 & - name: "Checkout code" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup NodeJS diff --git a/internal/datastore/src/__test__/db.ts b/internal/datastore/src/__test__/db.ts index 7e5f18edf..76c245175 100644 --- a/internal/datastore/src/__test__/db.ts +++ b/internal/datastore/src/__test__/db.ts @@ -9,6 +9,7 @@ import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; import { DatastoreConfig } from "../config"; export async function setupDynamoDBContainer() { + // Pre-pulled in CI by .github/workflows/stage-2-test.yaml; keep in sync. const container = await new GenericContainer("amazon/dynamodb-local") .withExposedPorts(8000) .start(); diff --git a/internal/sql-test-helpers/README.md b/internal/sql-test-helpers/README.md new file mode 100644 index 000000000..819b06b30 --- /dev/null +++ b/internal/sql-test-helpers/README.md @@ -0,0 +1,46 @@ + + +# @internal/sql-test-helpers + +## Purpose + +Test-only helpers for running SQL queries against databases in [testcontainers](https://node.testcontainers.org/), so that query builders can be checked against a real SQL engine using known test data. + +## General Structure highlights + +- **`src/trino.ts`**: Trino helpers, which talk to Trino over its REST API using `fetch`: + - `setupTrinoContainer()` — starts a `trinodb/trino` `GenericContainer` and returns a `TrinoContext` (`container`, `endpoint`). + - `runTrinoStatement(context, sql)` — runs a statement against the built-in `memory` catalog, follows result pagination and returns rows as `Record` (NULL → `undefined`). + - `runParameterisedQueryOnTrino(context, query)` — runs an Athena-style `{ queryString, executionParameters }` query via `EXECUTE IMMEDIATE ... USING`. Execution parameters are SQL literal expressions, as in Athena (for example `"'2026-10-14T12:00:00.000Z'"`). + +## Usage + +Add the package as a **devDependency** of the consuming workspace and import it from test code only: + +```json +"devDependencies": { + "@internal/sql-test-helpers": "*" +} +``` + +```ts +import { + runParameterisedQueryOnTrino, + runTrinoStatement, + setupTrinoContainer, +} from "@internal/sql-test-helpers"; + +const context = await setupTrinoContainer(); +await runTrinoStatement(context, "CREATE TABLE event_cache (...)"); +const rows = await runParameterisedQueryOnTrino(context, buildQuery(...)); +await context.container.stop(); +``` + +## Nuances and Peculiarities + +- Must never be imported by production code. Because consumers list it under `devDependencies`, ESLint's `import-x/no-extraneous-dependencies` rule reports any import from non-test files. +- Has no unit tests of its own and is excluded from coverage (see `sonar.coverage.exclusions` in `scripts/config/sonar-scanner.properties`). It is checked through the tests that use it. +- Tests that use it require Docker. The first run pulls the Trino image (about 1 GB), so allow a generous `beforeAll` timeout (for example `180_000` ms). In CI, the image is pulled in the background at the start of the unit test job in `.github/workflows/stage-2-test.yaml`. If you change `TRINO_IMAGE`, update that step too. +- Trino SQL is close to, but not identical to, Athena's engine version. The helpers check query logic, not every Athena-specific behaviour. + + diff --git a/internal/sql-test-helpers/package.json b/internal/sql-test-helpers/package.json new file mode 100644 index 000000000..ec64c9abe --- /dev/null +++ b/internal/sql-test-helpers/package.json @@ -0,0 +1,17 @@ +{ + "dependencies": { + "testcontainers": "^12.2.0" + }, + "description": "Test-only helpers for running SQL queries against containerised databases", + "license": "MIT", + "main": "src/index.ts", + "name": "@internal/sql-test-helpers", + "private": true, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "echo \"Test helpers package has no unit tests\"", + "typecheck": "tsc --noEmit" + }, + "version": "0.1.0" +} diff --git a/internal/sql-test-helpers/src/index.ts b/internal/sql-test-helpers/src/index.ts new file mode 100644 index 000000000..79ebbf71c --- /dev/null +++ b/internal/sql-test-helpers/src/index.ts @@ -0,0 +1,8 @@ +export { + type SqlQuery, + type SqlRow, + type TrinoContext, + runParameterisedQueryOnTrino, + runTrinoStatement, + setupTrinoContainer, +} from "./trino"; diff --git a/internal/sql-test-helpers/src/trino.ts b/internal/sql-test-helpers/src/trino.ts new file mode 100644 index 000000000..543cd332b --- /dev/null +++ b/internal/sql-test-helpers/src/trino.ts @@ -0,0 +1,119 @@ +import { GenericContainer, StartedTestContainer, Wait } from "testcontainers"; + +// Pre-pulled in CI by .github/workflows/stage-2-test.yaml; keep in sync. +const TRINO_IMAGE = "trinodb/trino:476"; +const TRINO_PORT = 8080; + +// Structurally compatible with Athena's query/row shapes, so Athena query +// builders can be exercised against Trino without depending on them here. +export type SqlQuery = { + queryString: string; + executionParameters?: string[]; +}; + +export type SqlRow = Record; + +type TrinoResponse = { + columns?: { name: string }[]; + data?: unknown[][]; + nextUri?: string; + error?: { message: string }; +}; + +export type TrinoContext = { + container: StartedTestContainer; + endpoint: string; +}; + +export async function setupTrinoContainer(): Promise { + const container = await new GenericContainer(TRINO_IMAGE) + .withExposedPorts(TRINO_PORT) + .withWaitStrategy(Wait.forLogMessage("======== SERVER STARTED ========")) + .withStartupTimeout(120_000) + .start(); + + const endpoint = `http://${container.getHost()}:${container.getMappedPort(TRINO_PORT)}`; + + return { container, endpoint }; +} + +async function readTrinoResponse(response: Response): Promise { + if (!response.ok) { + throw new Error( + `Trino request failed: ${response.status} ${await response.text()}`, + ); + } + + const body = (await response.json()) as TrinoResponse; + + if (body.error) { + throw new Error(`Trino query failed: ${body.error.message}`); + } + + return body; +} + +function toRow(columns: string[], values: unknown[]): SqlRow { + return Object.fromEntries( + columns.map((name, i) => { + const value = values.at(i); + return [name, value === null ? undefined : String(value)]; + }), + ); +} + +// Runs a statement via the Trino REST API against the in-memory catalog. +export async function runTrinoStatement( + context: TrinoContext, + sql: string, +): Promise { + let body = await readTrinoResponse( + await fetch(`${context.endpoint}/v1/statement`, { + method: "POST", + headers: { + "X-Trino-User": "test", + "X-Trino-Catalog": "memory", + "X-Trino-Schema": "default", + }, + body: sql, + }), + ); + + let columns: string[] = []; + const rows: SqlRow[] = []; + + while (true) { + if (body.columns) { + columns = body.columns.map((column) => column.name); + } + + const currentColumns = columns; + rows.push( + ...(body.data ?? []).map((values) => toRow(currentColumns, values)), + ); + + if (!body.nextUri) { + return rows; + } + + body = await readTrinoResponse( + await fetch(body.nextUri, { headers: { "X-Trino-User": "test" } }), + ); + } +} + +// Emulates Athena parameterised queries: each execution parameter is +// substituted into the query as a SQL literal expression. +export async function runParameterisedQueryOnTrino( + context: TrinoContext, + query: SqlQuery, +): Promise { + const escapedQuery = query.queryString.replaceAll("'", "''"); + const parameters = query.executionParameters ?? []; + const using = parameters.length > 0 ? ` USING ${parameters.join(", ")}` : ""; + + return runTrinoStatement( + context, + `EXECUTE IMMEDIATE '${escapedQuery}'${using}`, + ); +} diff --git a/internal/sql-test-helpers/tsconfig.json b/internal/sql-test-helpers/tsconfig.json new file mode 100644 index 000000000..70ae3804b --- /dev/null +++ b/internal/sql-test-helpers/tsconfig.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": {}, + "extends": "../../tsconfig.base.json", + "include": [ + "src/**/*" + ] +} diff --git a/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts index b11ff60ae..d33a405a0 100644 --- a/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena.test.ts @@ -9,28 +9,27 @@ import { type Deps } from "../deps"; const query = { queryString: "SELECT ?", executionParameters: ["'a'"] }; -function makeDeps(responses: unknown[]) { - const send = jest.fn(); - for (const response of responses) { - send.mockResolvedValueOnce(response); - } - const deps = { - athenaClient: { send } as unknown as AthenaClient, - logger: { info: jest.fn(), debug: jest.fn() }, - env: { - ATHENA_WORKGROUP: "workgroup", - ATHENA_DATABASE: "database", - EVENT_CACHE_TABLE_NAME: "event_cache", - }, - } as unknown as Deps; - return { deps, send }; -} +const deps = { + athenaClient: { send: jest.fn() } as unknown as AthenaClient, + logger: { info: jest.fn(), debug: jest.fn() }, + env: { + ATHENA_WORKGROUP: "workgroup", + ATHENA_DATABASE: "database", + EVENT_CACHE_TABLE_NAME: "event_cache", + }, +} as unknown as Deps; const columnInfo = { ColumnInfo: [{ Name: "id" }, { Name: "status" }] }; +function mockResolvedValues(fn: any, returnValues: any[]) { + for (const value of returnValues) { + (fn as jest.Mock).mockResolvedValueOnce(value); + } +} + describe("runAthenaQuery", () => { it("starts the query, polls until complete and returns paginated rows", async () => { - const { deps, send } = makeDeps([ + mockResolvedValues(deps.athenaClient.send, [ { QueryExecutionId: "qid" }, { QueryExecution: { Status: { State: "QUEUED" } } }, { QueryExecution: { Status: { State: "RUNNING" } } }, @@ -59,7 +58,9 @@ describe("runAthenaQuery", () => { { id: "1", status: "PENDING" }, { id: "2", status: undefined }, ]); - const commands = send.mock.calls.map(([command]) => command); + const commands = (deps.athenaClient.send as jest.Mock).mock.calls.map( + ([command]) => command, + ); expect(commands[0]).toBeInstanceOf(StartQueryExecutionCommand); expect(commands[0].input).toEqual({ QueryString: "SELECT ?", @@ -76,7 +77,7 @@ describe("runAthenaQuery", () => { }); it("handles empty result sets and missing metadata", async () => { - const { deps } = makeDeps([ + mockResolvedValues(deps.athenaClient.send, [ { QueryExecutionId: "qid" }, { QueryExecution: { Status: { State: "SUCCEEDED" } } }, { @@ -90,7 +91,7 @@ describe("runAthenaQuery", () => { }); it("handles a response with no result set", async () => { - const { deps } = makeDeps([ + mockResolvedValues(deps.athenaClient.send, [ { QueryExecutionId: "qid" }, { QueryExecution: { Status: { State: "SUCCEEDED" } } }, {}, @@ -101,7 +102,7 @@ describe("runAthenaQuery", () => { it("uses the default poll interval when none is given", async () => { jest.useFakeTimers(); - const { deps } = makeDeps([ + mockResolvedValues(deps.athenaClient.send, [ { QueryExecutionId: "qid" }, {}, { QueryExecution: { Status: { State: "SUCCEEDED" } } }, @@ -116,7 +117,7 @@ describe("runAthenaQuery", () => { }); it("throws when no query execution id is returned", async () => { - const { deps } = makeDeps([{}]); + mockResolvedValues(deps.athenaClient.send, [{}]); await expect(runAthenaQuery(deps, query)).rejects.toThrow( "Athena did not return a query execution id", @@ -126,7 +127,7 @@ describe("runAthenaQuery", () => { it.each(["FAILED", "CANCELLED"])( "throws when the query is %s", async (state) => { - const { deps } = makeDeps([ + mockResolvedValues(deps.athenaClient.send, [ { QueryExecutionId: "qid" }, { QueryExecution: { @@ -142,7 +143,7 @@ describe("runAthenaQuery", () => { ); it("throws with a default reason when none is given", async () => { - const { deps } = makeDeps([ + mockResolvedValues(deps.athenaClient.send, [ { QueryExecutionId: "qid" }, { QueryExecution: { Status: { State: "FAILED" } } }, ]); diff --git a/lambdas/letter-sla-breaches/common/src/athena.ts b/lambdas/letter-sla-breaches/common/src/athena.ts index 59cf6c338..7edc2bc4f 100644 --- a/lambdas/letter-sla-breaches/common/src/athena.ts +++ b/lambdas/letter-sla-breaches/common/src/athena.ts @@ -47,7 +47,7 @@ async function waitForQuery( queryExecutionId: string, pollIntervalMs: number, ): Promise { - for (;;) { + while (true) { const { QueryExecution } = await deps.athenaClient.send( new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId }), ); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json index a8a5775a2..e92bbdc61 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/package.json +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/package.json @@ -3,6 +3,9 @@ "@internal/letter-sla-breaches-common": "*", "esbuild": "0.28.2" }, + "devDependencies": { + "@internal/sql-test-helpers": "*" + }, "name": "nhs-notify-supplier-api-populate-standard-breaches", "private": true, "scripts": { diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index b76524ed6..5516860f4 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -15,12 +15,12 @@ const makeDeps = () => describe("populateStandardBreaches", () => { it("runs the standard breaches query and returns success", async () => { const deps = makeDeps(); - jest.mocked(runAthenaQuery).mockResolvedValueOnce([{ date: "2026-10-08" }]); + jest.mocked(runAthenaQuery).mockResolvedValueOnce([{ breach_count: "1" }]); const result = await createHandler(deps)({ date: "2026-10-08" }); expect(runAthenaQuery).toHaveBeenCalled(); - expect(result).toEqual({ status: "SUCCESS", rowsReturned: 1 }); + expect(result).toEqual({ status: "SUCCESS", breachCount: "1" }); }); it("runs the standard breaches query and returns failure", async () => { @@ -33,7 +33,7 @@ describe("populateStandardBreaches", () => { expect(result).toEqual({ failureReason: "boom", status: "FAILURE", - rowsReturned: 0, + breachCount: "0", }); }); @@ -47,7 +47,7 @@ describe("populateStandardBreaches", () => { expect(result).toEqual({ failureReason: "boom", status: "FAILURE", - rowsReturned: 0, + breachCount: "0", }); }); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts index 1e4b3ff03..99a8bf135 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -1,16 +1,204 @@ +import { + TrinoContext, + runParameterisedQueryOnTrino, + runTrinoStatement, + setupTrinoContainer, +} from "@internal/sql-test-helpers"; import buildStandardBreachesQuery from "../handler/standard-breaches-query"; +type LetterEvent = { + domainid: string; + supplierid: string; + status: string; + time: string; +}; + +const TABLE_NAME = "event_cache"; + +async function insertEvents(context: TrinoContext, events: LetterEvent[]) { + const values = events + .map(({ domainid, status, supplierid, time }) => { + const [year, month, day] = time.slice(0, 10).split("-"); + return `('${domainid}', '${supplierid}', '${status}', '${time}', '${year}', '${month}', '${day}')`; + }) + .join(",\n"); + + await runTrinoStatement( + context, + `INSERT INTO ${TABLE_NAME} VALUES ${values}`, + ); +} + +function sortBySupplier(rows: Record[]) { + return rows.toSorted((a, b) => + (a.supplierid ?? "").localeCompare(b.supplierid ?? ""), + ); +} + describe("buildStandardBreachesQuery", () => { - it("selects from the event cache table filtered by the date partitions", () => { - expect( - buildStandardBreachesQuery("event_cache", new Date("2026-10-08"), 0), - ).toEqual( - expect.objectContaining({ - queryString: expect.stringContaining( - "(_year = '2026' AND _month = '10' AND _day IN ('08'))", - ), - executionParameters: ["2026-10-08T00:00:00.000Z"], - }), + let context: TrinoContext; + + beforeAll(async () => { + context = await setupTrinoContainer(); + }, 180_000); + + afterAll(async () => { + await context?.container.stop(); + }); + + beforeEach(async () => { + await runTrinoStatement( + context, + `CREATE TABLE ${TABLE_NAME} ( + domainid VARCHAR, + supplierid VARCHAR, + status VARCHAR, + time VARCHAR, + _year VARCHAR, + _month VARCHAR, + _day VARCHAR + )`, + ); + }); + + afterEach(async () => { + await runTrinoStatement(context, `DROP TABLE IF EXISTS ${TABLE_NAME}`); + }); + + it("counts letters pending for more than 48 hours without a completed status, per supplier", async () => { + // Wednesday + const now = new Date("2026-10-14T12:00:00.000Z"); + + await insertEvents(context, [ + // Breach: pending for > 48 hours, never completed + { + domainid: "letter-1", + supplierid: "supplier-a", + status: "PENDING", + time: "2026-10-11T10:00:00.000Z", + }, + // Not a breach: completed + { + domainid: "letter-2", + supplierid: "supplier-a", + status: "PENDING", + time: "2026-10-11T10:00:00.000Z", + }, + { + domainid: "letter-2", + supplierid: "supplier-a", + status: "DISPATCHED", + time: "2026-10-13T10:00:00.000Z", + }, + // Not a breach: pending for < 48 hours + { + domainid: "letter-3", + supplierid: "supplier-a", + status: "PENDING", + time: "2026-10-13T10:00:00.000Z", + }, + // Breach: a non-terminal status does not complete the letter + { + domainid: "letter-4", + supplierid: "supplier-b", + status: "PENDING", + time: "2026-10-10T09:00:00.000Z", + }, + { + domainid: "letter-4", + supplierid: "supplier-b", + status: "ACCEPTED", + time: "2026-10-10T10:00:00.000Z", + }, + // Breach: first PENDING event is used, even if repeated later + { + domainid: "letter-5", + supplierid: "supplier-b", + status: "PENDING", + time: "2026-10-12T11:00:00.000Z", + }, + { + domainid: "letter-5", + supplierid: "supplier-b", + status: "PENDING", + time: "2026-10-14T11:00:00.000Z", + }, + // Not a breach: each terminal status completes the letter + ...["REJECTED", "FAILED", "CANCELLED", "FORWARDED"].flatMap( + (status, i) => [ + { + domainid: `letter-terminal-${i}`, + supplierid: "supplier-b", + status: "PENDING", + time: "2026-10-10T10:00:00.000Z", + }, + { + domainid: `letter-terminal-${i}`, + supplierid: "supplier-b", + status, + time: "2026-10-11T10:00:00.000Z", + }, + ], + ), + // Breaching, but ignored: before the 14-day window + { + domainid: "letter-6", + supplierid: "supplier-c", + status: "PENDING", + time: "2026-09-01T10:00:00.000Z", + }, + ]); + + const rows = await runParameterisedQueryOnTrino( + context, + buildStandardBreachesQuery(TABLE_NAME, now, 14), ); + + expect(sortBySupplier(rows)).toEqual([ + { supplierid: "supplier-a", breach_count: "1" }, + { supplierid: "supplier-b", breach_count: "2" }, + ]); + }); + + it("skips non-working days", async () => { + await insertEvents(context, [ + // Received Thursday - won't breach until Monday + { + domainid: "letter-1", + supplierid: "supplier-a", + status: "PENDING", + time: "2026-10-15T10:00:00.000Z", + }, + // Received Friday - won't breach until Tuesday + { + domainid: "letter-2", + supplierid: "supplier-a", + status: "PENDING", + time: "2026-10-16T10:00:00.000Z", + }, + ]); + + const [rowsOnSaturday, rowsOnSunday, rowsOnMonday, rowsOnTuesday] = + await Promise.all( + [ + new Date("2026-10-17T22:00:00.000Z"), + new Date("2026-10-18T22:00:00.000Z"), + new Date("2026-10-19T22:00:00.000Z"), + new Date("2026-10-20T22:00:00.000Z"), + ].map(async (day) => + runParameterisedQueryOnTrino( + context, + buildStandardBreachesQuery(TABLE_NAME, day, 14), + ), + ), + ); + expect(rowsOnSaturday).toHaveLength(0); + expect(rowsOnSunday).toHaveLength(0); + expect(rowsOnMonday).toEqual([ + { supplierid: "supplier-a", breach_count: "1" }, + ]); + expect(rowsOnTuesday).toEqual([ + { supplierid: "supplier-a", breach_count: "2" }, + ]); }); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 871db1363..971045ab3 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -23,7 +23,7 @@ allocated_letters AS ( WHERE status = 'PENDING' GROUP BY domainid, supplierid HAVING min(event_time) < (SELECT iso8601 FROM now) - - CASE WHEN day_of_week((SELECT iso8601 AT TIME ZONE 'Europe/London' FROM now)) <= 2 THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END + CASE WHEN day_of_week(min(event_time) AT TIME ZONE 'Europe/London') IN (4, 5, 6) THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END ), completed_letters AS ( SELECT domainid, supplierid @@ -38,6 +38,6 @@ FROM ( LEFT JOIN completed_letters cl ON al.domainid = cl.domainid WHERE cl.domainid IS NULL ) GROUP BY supplierid`, - executionParameters: [now.toISOString()], + executionParameters: [`'${now.toISOString()}'`], }; } diff --git a/package-lock.json b/package-lock.json index e8ac5de45..1003d9a1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -257,6 +257,14 @@ "zod": "^4.6.5" } }, + "internal/sql-test-helpers": { + "name": "@internal/sql-test-helpers", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "testcontainers": "^12.2.0" + } + }, "lambdas/api-handler": { "name": "nhs-notify-supplier-api-handler", "version": "0.0.1", @@ -476,6 +484,9 @@ "dependencies": { "@internal/letter-sla-breaches-common": "*", "esbuild": "0.28.2" + }, + "devDependencies": { + "@internal/sql-test-helpers": "*" } }, "lambdas/letter-sla-breaches/populate-supplier-config-cache": { @@ -1981,7 +1992,6 @@ }, "node_modules/@balena/dockerignore": { "version": "1.0.2", - "dev": true, "license": "Apache-2.0" }, "node_modules/@bcoe/v8-coverage": { @@ -2352,7 +2362,6 @@ }, "node_modules/@grpc/grpc-js": { "version": "1.14.4", - "dev": true, "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.8.0", @@ -2364,7 +2373,6 @@ }, "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { "version": "0.8.1", - "dev": true, "license": "Apache-2.0", "dependencies": { "lodash.camelcase": "^4.3.0", @@ -2381,7 +2389,6 @@ }, "node_modules/@grpc/proto-loader": { "version": "0.7.15", - "dev": true, "license": "Apache-2.0", "dependencies": { "lodash.camelcase": "^4.3.0", @@ -2531,6 +2538,10 @@ "resolved": "lambdas/letter-sla-breaches/common", "link": true }, + "node_modules/@internal/sql-test-helpers": { + "resolved": "internal/sql-test-helpers", + "link": true + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "license": "ISC", @@ -3015,7 +3026,6 @@ }, "node_modules/@js-sdsl/ordered-map": { "version": "4.4.2", - "dev": true, "license": "MIT", "funding": { "type": "opencollective", @@ -3061,7 +3071,6 @@ }, "node_modules/@kwsites/file-exists": { "version": "1.1.1", - "dev": true, "license": "MIT", "dependencies": { "debug": "^4.1.1" @@ -3515,27 +3524,22 @@ }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/base64": { "version": "1.1.2", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/codegen": { "version": "2.0.5", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/eventemitter": { "version": "1.1.1", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/fetch": { "version": "1.1.1", - "dev": true, "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.1" @@ -3543,22 +3547,18 @@ }, "node_modules/@protobufjs/float": { "version": "1.0.2", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/path": { "version": "1.1.2", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/pool": { "version": "1.1.0", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/utf8": { "version": "1.1.2", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/@redocly/ajv": { @@ -4351,7 +4351,6 @@ }, "node_modules/@types/docker-modem": { "version": "3.0.6", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", @@ -4360,7 +4359,6 @@ }, "node_modules/@types/dockerode": { "version": "4.0.1", - "dev": true, "license": "MIT", "dependencies": { "@types/docker-modem": "*", @@ -4468,7 +4466,6 @@ }, "node_modules/@types/ssh2": { "version": "1.15.5", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "^18.11.18" @@ -4476,7 +4473,6 @@ }, "node_modules/@types/ssh2-streams": { "version": "0.1.13", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*" @@ -4484,7 +4480,6 @@ }, "node_modules/@types/ssh2/node_modules/@types/node": { "version": "18.19.130", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -4492,7 +4487,6 @@ }, "node_modules/@types/ssh2/node_modules/undici-types": { "version": "5.26.5", - "dev": true, "license": "MIT" }, "node_modules/@types/stack-utils": { @@ -4796,7 +4790,6 @@ }, "node_modules/abort-controller": { "version": "3.0.0", - "dev": true, "license": "MIT", "dependencies": { "event-target-shim": "^5.0.0" @@ -5021,7 +5014,6 @@ }, "node_modules/archiver": { "version": "7.0.1", - "dev": true, "license": "MIT", "dependencies": { "archiver-utils": "^5.0.2", @@ -5038,7 +5030,6 @@ }, "node_modules/archiver-utils": { "version": "5.0.2", - "dev": true, "license": "MIT", "dependencies": { "glob": "^10.0.0", @@ -5055,12 +5046,10 @@ }, "node_modules/archiver-utils/node_modules/balanced-match": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/archiver-utils/node_modules/brace-expansion": { "version": "2.1.7", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -5068,7 +5057,6 @@ }, "node_modules/archiver-utils/node_modules/buffer": { "version": "6.0.3", - "dev": true, "funding": [ { "type": "github", @@ -5091,7 +5079,6 @@ }, "node_modules/archiver-utils/node_modules/events": { "version": "3.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -5099,7 +5086,6 @@ }, "node_modules/archiver-utils/node_modules/glob": { "version": "10.5.0", - "dev": true, "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", @@ -5118,7 +5104,6 @@ }, "node_modules/archiver-utils/node_modules/ieee754": { "version": "1.2.1", - "dev": true, "funding": [ { "type": "github", @@ -5137,12 +5122,10 @@ }, "node_modules/archiver-utils/node_modules/lru-cache": { "version": "10.4.3", - "dev": true, "license": "ISC" }, "node_modules/archiver-utils/node_modules/minimatch": { "version": "9.0.9", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.2" @@ -5156,7 +5139,6 @@ }, "node_modules/archiver-utils/node_modules/path-scurry": { "version": "1.11.1", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "lru-cache": "^10.2.0", @@ -5171,7 +5153,6 @@ }, "node_modules/archiver-utils/node_modules/readable-stream": { "version": "4.7.0", - "dev": true, "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", @@ -5186,7 +5167,6 @@ }, "node_modules/archiver/node_modules/buffer": { "version": "6.0.3", - "dev": true, "funding": [ { "type": "github", @@ -5209,7 +5189,6 @@ }, "node_modules/archiver/node_modules/events": { "version": "3.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -5217,7 +5196,6 @@ }, "node_modules/archiver/node_modules/ieee754": { "version": "1.2.1", - "dev": true, "funding": [ { "type": "github", @@ -5236,7 +5214,6 @@ }, "node_modules/archiver/node_modules/readable-stream": { "version": "4.7.0", - "dev": true, "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", @@ -5424,7 +5401,6 @@ }, "node_modules/asn1": { "version": "0.2.6", - "dev": true, "license": "MIT", "dependencies": { "safer-buffer": "~2.1.0" @@ -5456,7 +5432,6 @@ }, "node_modules/async": { "version": "3.2.6", - "dev": true, "license": "MIT" }, "node_modules/async-function": { @@ -5469,7 +5444,6 @@ }, "node_modules/async-lock": { "version": "1.4.1", - "dev": true, "license": "MIT" }, "node_modules/asynckit": { @@ -5610,7 +5584,6 @@ }, "node_modules/b4a": { "version": "1.8.1", - "dev": true, "license": "Apache-2.0", "peerDependencies": { "react-native-b4a": "*" @@ -5718,7 +5691,6 @@ }, "node_modules/bare-events": { "version": "2.9.1", - "dev": true, "license": "Apache-2.0", "peerDependencies": { "bare-abort-controller": "*" @@ -5731,7 +5703,6 @@ }, "node_modules/bare-fs": { "version": "4.7.2", - "dev": true, "license": "Apache-2.0", "dependencies": { "bare-events": "^2.5.4", @@ -5754,7 +5725,6 @@ }, "node_modules/bare-os": { "version": "3.9.1", - "dev": true, "license": "Apache-2.0", "engines": { "bare": ">=1.14.0" @@ -5762,7 +5732,6 @@ }, "node_modules/bare-path": { "version": "3.0.1", - "dev": true, "license": "Apache-2.0", "dependencies": { "bare-os": "^3.0.1" @@ -5770,7 +5739,6 @@ }, "node_modules/bare-stream": { "version": "2.13.1", - "dev": true, "license": "Apache-2.0", "dependencies": { "streamx": "^2.25.0", @@ -5795,7 +5763,6 @@ }, "node_modules/bare-url": { "version": "2.4.5", - "dev": true, "license": "Apache-2.0", "dependencies": { "bare-path": "^3.0.0" @@ -5839,7 +5806,6 @@ }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", - "dev": true, "license": "BSD-3-Clause", "dependencies": { "tweetnacl": "^0.14.3" @@ -5847,7 +5813,6 @@ }, "node_modules/bl": { "version": "4.1.0", - "dev": true, "license": "MIT", "dependencies": { "buffer": "^5.5.0", @@ -5857,7 +5822,6 @@ }, "node_modules/bl/node_modules/buffer": { "version": "5.7.1", - "dev": true, "funding": [ { "type": "github", @@ -6124,7 +6088,6 @@ }, "node_modules/buffer-crc32": { "version": "1.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=8.0.0" @@ -6132,7 +6095,6 @@ }, "node_modules/buildcheck": { "version": "0.0.7", - "dev": true, "optional": true, "engines": { "node": ">=10.0.0" @@ -6156,7 +6118,6 @@ }, "node_modules/byline": { "version": "5.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6338,7 +6299,6 @@ }, "node_modules/chownr": { "version": "1.1.4", - "dev": true, "license": "ISC" }, "node_modules/ci-info": { @@ -6530,7 +6490,6 @@ }, "node_modules/compress-commons": { "version": "6.0.2", - "dev": true, "license": "MIT", "dependencies": { "crc-32": "^1.2.0", @@ -6545,7 +6504,6 @@ }, "node_modules/compress-commons/node_modules/buffer": { "version": "6.0.3", - "dev": true, "funding": [ { "type": "github", @@ -6568,7 +6526,6 @@ }, "node_modules/compress-commons/node_modules/events": { "version": "3.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -6576,7 +6533,6 @@ }, "node_modules/compress-commons/node_modules/ieee754": { "version": "1.2.1", - "dev": true, "funding": [ { "type": "github", @@ -6595,7 +6551,6 @@ }, "node_modules/compress-commons/node_modules/readable-stream": { "version": "4.7.0", - "dev": true, "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", @@ -6883,12 +6838,10 @@ }, "node_modules/core-util-is": { "version": "1.0.3", - "dev": true, "license": "MIT" }, "node_modules/cpu-features": { "version": "0.0.10", - "dev": true, "hasInstallScript": true, "optional": true, "dependencies": { @@ -6901,7 +6854,6 @@ }, "node_modules/crc-32": { "version": "1.2.2", - "dev": true, "license": "Apache-2.0", "bin": { "crc32": "bin/crc32.njs" @@ -6912,7 +6864,6 @@ }, "node_modules/crc32-stream": { "version": "6.0.0", - "dev": true, "license": "MIT", "dependencies": { "crc-32": "^1.2.0", @@ -6924,7 +6875,6 @@ }, "node_modules/crc32-stream/node_modules/buffer": { "version": "6.0.3", - "dev": true, "funding": [ { "type": "github", @@ -6947,7 +6897,6 @@ }, "node_modules/crc32-stream/node_modules/events": { "version": "3.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -6955,7 +6904,6 @@ }, "node_modules/crc32-stream/node_modules/ieee754": { "version": "1.2.1", - "dev": true, "funding": [ { "type": "github", @@ -6974,7 +6922,6 @@ }, "node_modules/crc32-stream/node_modules/readable-stream": { "version": "4.7.0", - "dev": true, "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", @@ -7301,7 +7248,6 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-1.5.0.tgz", "integrity": "sha512-O+eD6c63+BJORUWddXA7uAlI6H1KLDPd/aS14k73nXYpFliM48C3xrrKB3sXsR1Fp/rHUZQe7+fJ+QA7+jepyw==", - "dev": true, "license": "MIT", "dependencies": { "yaml": "^2.2.2" @@ -7312,7 +7258,6 @@ }, "node_modules/docker-modem": { "version": "5.0.7", - "dev": true, "license": "Apache-2.0", "dependencies": { "debug": "^4.1.1", @@ -7326,7 +7271,6 @@ }, "node_modules/dockerode": { "version": "5.0.1", - "dev": true, "license": "Apache-2.0", "dependencies": { "@balena/dockerignore": "^1.0.2", @@ -7342,7 +7286,6 @@ }, "node_modules/dockerode/node_modules/tar-fs": { "version": "2.1.5", - "dev": true, "license": "MIT", "dependencies": { "chownr": "^1.1.1", @@ -7353,7 +7296,6 @@ }, "node_modules/dockerode/node_modules/tar-stream": { "version": "2.2.0", - "dev": true, "license": "MIT", "dependencies": { "bl": "^4.0.3", @@ -8970,7 +8912,6 @@ }, "node_modules/event-target-shim": { "version": "5.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -8989,7 +8930,6 @@ }, "node_modules/events-universal": { "version": "1.0.1", - "dev": true, "license": "Apache-2.0", "dependencies": { "bare-events": "^2.7.0" @@ -9109,7 +9049,6 @@ }, "node_modules/fast-fifo": { "version": "1.3.2", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -9393,7 +9332,6 @@ }, "node_modules/fs-constants": { "version": "1.0.0", - "dev": true, "license": "MIT" }, "node_modules/fs-extra": { @@ -9515,7 +9453,6 @@ }, "node_modules/get-port": { "version": "5.1.1", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -10484,7 +10421,6 @@ }, "node_modules/is-stream": { "version": "2.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -11568,7 +11504,6 @@ }, "node_modules/lazystream": { "version": "1.0.1", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "^2.0.5" @@ -11579,7 +11514,6 @@ }, "node_modules/lazystream/node_modules/readable-stream": { "version": "2.3.8", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -11593,12 +11527,10 @@ }, "node_modules/lazystream/node_modules/safe-buffer": { "version": "5.1.2", - "dev": true, "license": "MIT" }, "node_modules/lazystream/node_modules/string_decoder": { "version": "1.1.1", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" @@ -11834,7 +11766,6 @@ }, "node_modules/lodash.camelcase": { "version": "4.3.0", - "dev": true, "license": "MIT" }, "node_modules/lodash.debounce": { @@ -11864,7 +11795,6 @@ }, "node_modules/long": { "version": "5.3.2", - "dev": true, "license": "Apache-2.0" }, "node_modules/loose-envify": { @@ -12033,7 +11963,6 @@ }, "node_modules/minimatch": { "version": "5.1.9", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -12044,12 +11973,10 @@ }, "node_modules/minimatch/node_modules/balanced-match": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/minimatch/node_modules/brace-expansion": { "version": "2.1.7", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -12082,7 +12009,6 @@ }, "node_modules/mkdirp-classic": { "version": "0.5.3", - "dev": true, "license": "MIT" }, "node_modules/mnemonist": { @@ -12176,7 +12102,6 @@ }, "node_modules/nan": { "version": "2.27.0", - "dev": true, "license": "MIT", "optional": true }, @@ -13226,7 +13151,6 @@ }, "node_modules/process": { "version": "0.11.10", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6.0" @@ -13234,7 +13158,6 @@ }, "node_modules/process-nextick-args": { "version": "2.0.1", - "dev": true, "license": "MIT" }, "node_modules/process-warning": { @@ -13263,7 +13186,6 @@ }, "node_modules/proper-lockfile": { "version": "4.1.2", - "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", @@ -13273,12 +13195,10 @@ }, "node_modules/proper-lockfile/node_modules/signal-exit": { "version": "3.0.7", - "dev": true, "license": "ISC" }, "node_modules/properties-reader": { "version": "3.0.1", - "dev": true, "license": "MIT", "dependencies": { "@kwsites/file-exists": "^1.1.1", @@ -13294,7 +13214,6 @@ }, "node_modules/properties-reader/node_modules/mkdirp": { "version": "3.0.1", - "dev": true, "license": "MIT", "bin": { "mkdirp": "dist/cjs/src/bin.js" @@ -13308,7 +13227,6 @@ }, "node_modules/protobufjs": { "version": "7.6.5", - "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -13547,7 +13465,6 @@ }, "node_modules/readable-stream": { "version": "3.6.2", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -13560,7 +13477,6 @@ }, "node_modules/readdir-glob": { "version": "1.1.3", - "dev": true, "license": "Apache-2.0", "dependencies": { "minimatch": "^5.1.0" @@ -13731,7 +13647,6 @@ }, "node_modules/require-directory": { "version": "2.1.1", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -13810,7 +13725,6 @@ }, "node_modules/retry": { "version": "0.12.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -14438,7 +14352,6 @@ }, "node_modules/split-ca": { "version": "1.0.1", - "dev": true, "license": "ISC" }, "node_modules/split2": { @@ -14454,7 +14367,6 @@ }, "node_modules/ssh-remote-port-forward": { "version": "1.0.4", - "dev": true, "license": "MIT", "dependencies": { "@types/ssh2": "^0.5.48", @@ -14463,7 +14375,6 @@ }, "node_modules/ssh-remote-port-forward/node_modules/@types/ssh2": { "version": "0.5.52", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", @@ -14472,7 +14383,6 @@ }, "node_modules/ssh2": { "version": "1.17.0", - "dev": true, "hasInstallScript": true, "dependencies": { "asn1": "^0.2.6", @@ -14546,7 +14456,6 @@ }, "node_modules/streamx": { "version": "2.27.0", - "dev": true, "license": "MIT", "dependencies": { "events-universal": "^1.0.0", @@ -14556,7 +14465,6 @@ }, "node_modules/string_decoder": { "version": "1.3.0", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -14836,7 +14744,6 @@ }, "node_modules/tar-fs": { "version": "3.1.3", - "dev": true, "license": "MIT", "dependencies": { "pump": "^3.0.0", @@ -14849,7 +14756,6 @@ }, "node_modules/tar-stream": { "version": "3.2.0", - "dev": true, "license": "MIT", "dependencies": { "b4a": "^1.6.4", @@ -14860,7 +14766,6 @@ }, "node_modules/teex": { "version": "1.0.1", - "dev": true, "license": "MIT", "dependencies": { "streamx": "^2.12.5" @@ -14972,7 +14877,6 @@ "version": "12.2.0", "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-12.2.0.tgz", "integrity": "sha512-Pd0c2eU0DdIuK+Pb3c+vmBUdKjDLgxZmPecQGESFAKyn5LTqSaSw8l7htfMlLwabb/QsUILe6XkxMJw8Ly/ThA==", - "dev": true, "license": "MIT", "dependencies": { "@balena/dockerignore": "^1.0.2", @@ -14997,7 +14901,6 @@ }, "node_modules/text-decoder": { "version": "1.2.7", - "dev": true, "license": "Apache-2.0", "dependencies": { "b4a": "^1.6.4" @@ -15075,7 +14978,6 @@ }, "node_modules/tmp": { "version": "0.2.7", - "dev": true, "license": "MIT", "engines": { "node": ">=14.14" @@ -15378,7 +15280,6 @@ }, "node_modules/tweetnacl": { "version": "0.14.5", - "dev": true, "license": "Unlicense" }, "node_modules/type-check": { @@ -15774,7 +15675,6 @@ }, "node_modules/util-deprecate": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/utility-types": { @@ -16252,7 +16152,6 @@ }, "node_modules/yaml": { "version": "2.9.1", - "dev": true, "license": "ISC", "bin": { "yaml": "bin.mjs" @@ -16271,7 +16170,6 @@ }, "node_modules/yargs": { "version": "17.7.2", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -16288,7 +16186,6 @@ }, "node_modules/yargs-parser": { "version": "21.1.1", - "dev": true, "license": "ISC", "engines": { "node": ">=12" @@ -16296,7 +16193,6 @@ }, "node_modules/yargs/node_modules/cliui": { "version": "8.0.1", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -16309,7 +16205,6 @@ }, "node_modules/yargs/node_modules/wrap-ansi": { "version": "7.0.0", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -16344,7 +16239,6 @@ }, "node_modules/zip-stream": { "version": "6.0.1", - "dev": true, "license": "MIT", "dependencies": { "archiver-utils": "^5.0.0", @@ -16357,7 +16251,6 @@ }, "node_modules/zip-stream/node_modules/buffer": { "version": "6.0.3", - "dev": true, "funding": [ { "type": "github", @@ -16380,7 +16273,6 @@ }, "node_modules/zip-stream/node_modules/events": { "version": "3.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -16388,7 +16280,6 @@ }, "node_modules/zip-stream/node_modules/ieee754": { "version": "1.2.1", - "dev": true, "funding": [ { "type": "github", @@ -16407,7 +16298,6 @@ }, "node_modules/zip-stream/node_modules/readable-stream": { "version": "4.7.0", - "dev": true, "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", diff --git a/scripts/config/sonar-scanner.properties b/scripts/config/sonar-scanner.properties index cda56bc1d..6ad5ec975 100644 --- a/scripts/config/sonar-scanner.properties +++ b/scripts/config/sonar-scanner.properties @@ -9,7 +9,7 @@ sonar.exclusions=lambdas/**/src/__tests__/**/* sonar.test.inclusions=**/*.test.*, **/*.spec.* sonar.terraform.provider.aws.version=5.54.1 sonar.cpd.exclusions=**.test.* -sonar.coverage.exclusions=tests/, **/*.dev.*, lambdas/**/src/__tests__, +sonar.coverage.exclusions=tests/, **/*.dev.*, lambdas/**/src/__tests__, internal/sql-test-helpers/**, #sonar.python.coverage.reportPaths=.coverage/coverage.xml sonar.javascript.lcov.reportPaths=lcov.info From be2e0f21954576370424919484765ecdab1cac39 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Thu, 15 Oct 2026 15:30:08 +0100 Subject: [PATCH 24/37] Dependency --- infrastructure/terraform/components/api/README.md | 1 - package.json | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 03404fc65..89bcc7e84 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -64,7 +64,6 @@ No requirements. | Name | Source | Version | | ---- | ------ | ------- | | [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | -| [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [amendments\_queue](#module\_amendments\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a | | [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a | | [ddb\_alarms\_idempotency](#module\_ddb\_alarms\_idempotency) | ../../modules/alarms-ddb | n/a | diff --git a/package.json b/package.json index 68a2c9b28..ca8906970 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "@aws-sdk/client-kinesis": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/client-sns": "^3.1138.0", + "@gprc/grpc-js": "^1.14.5", "@openapitools/openapi-generator-cli": "^2.41.0", "@playwright/test": "^1.63.0", "@redocly/cli": "^2.54.2", From 9bcbc2e918c2ab6f4335be5d892f1b6cc553981c Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Thu, 15 Oct 2026 15:42:10 +0100 Subject: [PATCH 25/37] Fix package dependency --- package-lock.json | 6 ++++-- package.json | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1003d9a1c..9037c20bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -191,7 +191,7 @@ }, "internal/events": { "name": "@nhsdigital/nhs-notify-event-schemas-supplier-api", - "version": "1.0.22", + "version": "1.0.23", "license": "MIT", "dependencies": { "@asyncapi/bundler": "^1.0.2", @@ -2361,7 +2361,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.14.4", + "version": "1.14.6", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.6.tgz", + "integrity": "sha512-SgC/B3/VZ8lTyOZ4Cwk6f1skWhmkNAaVfiX90qd5u1rRXGKk1//8FTUEN9GvOUq7rfWIqAYkA3CoyumYVpOS6g==", "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.8.0", diff --git a/package.json b/package.json index ca8906970..e63f7080e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "@aws-sdk/client-kinesis": "^3.1138.0", "@aws-sdk/client-s3": "^3.1138.0", "@aws-sdk/client-sns": "^3.1138.0", - "@gprc/grpc-js": "^1.14.5", "@openapitools/openapi-generator-cli": "^2.41.0", "@playwright/test": "^1.63.0", "@redocly/cli": "^2.54.2", @@ -67,6 +66,7 @@ "axios": "^1.20.0", "fast-uri": "^3.1.7", "fast-xml-parser": "^5.5.6", + "@grpc/grpc-js": "^1.14.5", "@isaacs/brace-expansion": "^5.0.1", "minimatch@^3.0.0": { "brace-expansion": "^1.1.20" From fe6de19a5dbabfcd1bc28488c2da2ccb4654ccdc Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Thu, 15 Oct 2026 16:21:25 +0100 Subject: [PATCH 26/37] Export results from lambda --- .../__tests__/athena-date-predicate.test.ts | 20 ++++++++++--------- .../common/src/athena-date-predicate.ts | 2 +- .../__tests__/standard-breaches-query.test.ts | 6 +++--- .../src/handler/populate-standard-breaches.ts | 8 ++++---- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts index a0a4f545c..932387f95 100644 --- a/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts +++ b/lambdas/letter-sla-breaches/common/src/__tests__/athena-date-predicate.test.ts @@ -4,21 +4,23 @@ describe("buildAthenaDatePredicate", () => { it("includes only the requested UTC date when no days are subtracted", () => { expect( buildAthenaDatePredicate(new Date("2026-10-14T23:59:59.999Z"), 0), - ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('14'))"); + ).toBe("(__year = '2026' AND __month = '10' AND __day IN ('14'))"); }); it("groups consecutive dates in the same month into one partition predicate", () => { expect( buildAthenaDatePredicate(new Date("2026-10-14T12:00:00.000Z"), 2), - ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13', '14'))"); + ).toBe( + "(__year = '2026' AND __month = '10' AND __day IN ('12', '13', '14'))", + ); }); it("creates separate predicates when the date range crosses a month", () => { expect( buildAthenaDatePredicate(new Date("2026-11-01T00:00:00.000Z"), 2), ).toBe( - "(_year = '2026' AND _month = '10' AND _day IN ('30', '31'))\n" + - " OR (_year = '2026' AND _month = '11' AND _day IN ('01'))", + "(__year = '2026' AND __month = '10' AND __day IN ('30', '31'))\n" + + " OR (__year = '2026' AND __month = '11' AND __day IN ('01'))", ); }); @@ -26,8 +28,8 @@ describe("buildAthenaDatePredicate", () => { expect( buildAthenaDatePredicate(new Date("2026-01-01T00:00:00.000Z"), 1), ).toBe( - "(_year = '2025' AND _month = '12' AND _day IN ('31'))\n" + - " OR (_year = '2026' AND _month = '01' AND _day IN ('01'))", + "(__year = '2025' AND __month = '12' AND __day IN ('31'))\n" + + " OR (__year = '2026' AND __month = '01' AND __day IN ('01'))", ); }); @@ -35,15 +37,15 @@ describe("buildAthenaDatePredicate", () => { expect( buildAthenaDatePredicate(new Date("2024-03-01T00:00:00.000Z"), 2), ).toBe( - "(_year = '2024' AND _month = '02' AND _day IN ('28', '29'))\n" + - " OR (_year = '2024' AND _month = '03' AND _day IN ('01'))", + "(__year = '2024' AND __month = '02' AND __day IN ('28', '29'))\n" + + " OR (__year = '2024' AND __month = '03' AND __day IN ('01'))", ); }); it("uses UTC calendar dates regardless of the time component", () => { expect( buildAthenaDatePredicate(new Date("2026-10-14T00:30:00+02:00"), 1), - ).toBe("(_year = '2026' AND _month = '10' AND _day IN ('12', '13'))"); + ).toBe("(__year = '2026' AND __month = '10' AND __day IN ('12', '13'))"); }); it("throws when given an invalid date", () => { diff --git a/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts index 0ca204bac..17b74ed1d 100644 --- a/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts +++ b/lambdas/letter-sla-breaches/common/src/athena-date-predicate.ts @@ -37,7 +37,7 @@ export default function buildAthenaDatePredicate( .map( // eslint-disable-next-line sort-destructure-keys/sort-destructure-keys ({ year, month, days }) => - `(_year = '${year}' AND _month = '${month}' AND _day IN (${days + `(__year = '${year}' AND __month = '${month}' AND __day IN (${days .map((day) => `'${day}'`) .join(", ")}))`, ) diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts index 99a8bf135..e4af0d4f1 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/standard-breaches-query.test.ts @@ -54,9 +54,9 @@ describe("buildStandardBreachesQuery", () => { supplierid VARCHAR, status VARCHAR, time VARCHAR, - _year VARCHAR, - _month VARCHAR, - _day VARCHAR + __year VARCHAR, + __month VARCHAR, + __day VARCHAR )`, ); }); diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index 72950dde6..d43a6bcdd 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -11,7 +11,7 @@ export type PopulateStandardBreachesEvent = { export type PopulateStandardBreachesResult = { status: "SUCCESS" | "FAILURE"; failureReason?: string; - breachCount: string; + breachCounts: string; }; export default function createHandler(deps: Deps) { @@ -30,11 +30,11 @@ export default function createHandler(deps: Deps) { // TODO: CCM-24041 process the query results to populate standard breaches. deps.logger.info( - { date: event.date, breachCount: rows[0].breach_count }, + { date: event.date, breachCounts: JSON.stringify(rows) }, "Standard breaches query completed", ); - return { status: "SUCCESS", breachCount: rows[0].breach_count! }; + return { status: "SUCCESS", breachCounts: JSON.stringify(rows) }; } catch (error) { deps.logger.error( { date: event.date, error }, @@ -44,7 +44,7 @@ export default function createHandler(deps: Deps) { return { status: "FAILURE", failureReason: error instanceof Error ? error.message : String(error), - breachCount: "0", + breachCounts: "-", }; } }; From 23100299d47c97a0a3240a2452d6750548963114 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Thu, 15 Oct 2026 16:39:59 +0100 Subject: [PATCH 27/37] Wait for Trino node --- internal/sql-test-helpers/src/trino.ts | 41 ++++++++++++++++++- .../populate-standard-breaches.test.ts | 9 ++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/internal/sql-test-helpers/src/trino.ts b/internal/sql-test-helpers/src/trino.ts index 543cd332b..56322e4ec 100644 --- a/internal/sql-test-helpers/src/trino.ts +++ b/internal/sql-test-helpers/src/trino.ts @@ -25,6 +25,37 @@ export type TrinoContext = { endpoint: string; }; +const NODE_READY_TIMEOUT_MS = 60_000; +const NODE_READY_POLL_MS = 500; + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +// "SERVER STARTED" is logged before the coordinator has registered its own +// node as a worker, so early queries can fail with "No nodes available to run +// query". Wait until a trivial query succeeds before handing out the context. +async function waitForTrinoNode(context: TrinoContext): Promise { + const deadline = Date.now() + NODE_READY_TIMEOUT_MS; + + for (;;) { + try { + await runTrinoStatement(context, "SELECT 1"); + return; + } catch (error) { + const retryable = + error instanceof Error && error.message.includes("No nodes available"); + + if (!retryable || Date.now() >= deadline) { + throw error; + } + } + + await sleep(NODE_READY_POLL_MS); + } +} + export async function setupTrinoContainer(): Promise { const container = await new GenericContainer(TRINO_IMAGE) .withExposedPorts(TRINO_PORT) @@ -33,8 +64,16 @@ export async function setupTrinoContainer(): Promise { .start(); const endpoint = `http://${container.getHost()}:${container.getMappedPort(TRINO_PORT)}`; + const context = { container, endpoint }; + + try { + await waitForTrinoNode(context); + } catch (error) { + await container.stop(); + throw error; + } - return { container, endpoint }; + return context; } async function readTrinoResponse(response: Response): Promise { diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts index 5516860f4..d331ed329 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/__tests__/populate-standard-breaches.test.ts @@ -20,7 +20,10 @@ describe("populateStandardBreaches", () => { const result = await createHandler(deps)({ date: "2026-10-08" }); expect(runAthenaQuery).toHaveBeenCalled(); - expect(result).toEqual({ status: "SUCCESS", breachCount: "1" }); + expect(result).toEqual({ + status: "SUCCESS", + breachCounts: JSON.stringify([{ breach_count: "1" }]), + }); }); it("runs the standard breaches query and returns failure", async () => { @@ -33,7 +36,7 @@ describe("populateStandardBreaches", () => { expect(result).toEqual({ failureReason: "boom", status: "FAILURE", - breachCount: "0", + breachCounts: "-", }); }); @@ -47,7 +50,7 @@ describe("populateStandardBreaches", () => { expect(result).toEqual({ failureReason: "boom", status: "FAILURE", - breachCount: "0", + breachCounts: "-", }); }); }); From c5a969f20dc6a4f33d10e2a695864e57154af2a3 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 09:14:43 +0100 Subject: [PATCH 28/37] Print logs on failure --- internal/sql-test-helpers/src/trino.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/sql-test-helpers/src/trino.ts b/internal/sql-test-helpers/src/trino.ts index 56322e4ec..ff51b381c 100644 --- a/internal/sql-test-helpers/src/trino.ts +++ b/internal/sql-test-helpers/src/trino.ts @@ -69,6 +69,7 @@ export async function setupTrinoContainer(): Promise { try { await waitForTrinoNode(context); } catch (error) { + await printContainerLogs(context.container); await container.stop(); throw error; } @@ -76,6 +77,21 @@ export async function setupTrinoContainer(): Promise { return context; } +async function printContainerLogs(container: StartedTestContainer) { + try { + const stream = await container.logs(); + const chunks: string[] = []; + for await (const chunk of stream) { + chunks.push(chunk.toString()); + } + // eslint-disable-next-line no-console + console.error(`Trino container logs:\n${chunks.join("")}`); + } catch (logError) { + // eslint-disable-next-line no-console + console.error("Unable to read Trino container logs", logError); + } +} + async function readTrinoResponse(response: Response): Promise { if (!response.ok) { throw new Error( From 33247b4efbf459e678c78af9396483f29329f0f7 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 10:31:49 +0100 Subject: [PATCH 29/37] Fix table name --- .../components/api/module_lambda_populate_standard_breaches.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index 4ef41c41f..c1fdc2d4b 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -38,7 +38,7 @@ module "populate_standard_breaches" { lambda_env_vars = { ATHENA_WORKGROUP = aws_athena_workgroup.letter_sla_breaches.name ATHENA_DATABASE = aws_glue_catalog_database.supplier.name - EVENT_CACHE_TABLE_NAME = local.event_cache_glue_table_name + EVENT_CACHE_TABLE_NAME = aws_glue_catalog_table.eventpub_parquet.name } } From dd84199196af1c9cbac064ad953691b52e06ae42 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 10:40:17 +0100 Subject: [PATCH 30/37] Check for parquet enabled --- .../api/module_lambda_populate_standard_breaches.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index c1fdc2d4b..1f8849016 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -1,4 +1,6 @@ module "populate_standard_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip" function_name = "populate_standard_breaches" @@ -38,7 +40,7 @@ module "populate_standard_breaches" { lambda_env_vars = { ATHENA_WORKGROUP = aws_athena_workgroup.letter_sla_breaches.name ATHENA_DATABASE = aws_glue_catalog_database.supplier.name - EVENT_CACHE_TABLE_NAME = aws_glue_catalog_table.eventpub_parquet.name + EVENT_CACHE_TABLE_NAME = aws_glue_catalog_table.eventpub_parquet[0].name } } From cf985b6cabfad0436075cd267fbdd60799fd7b99 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 10:55:04 +0100 Subject: [PATCH 31/37] Extend the use of count --- .../components/api/iam_role_policy_letter_sla_breaches.tf | 2 ++ .../components/api/sfn_state_machine_letter_sla_breaches.tf | 2 ++ 2 files changed, 4 insertions(+) diff --git a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf index d893cd8b0..ac75f4666 100644 --- a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf @@ -1,4 +1,6 @@ resource "aws_iam_role_policy" "letter_sla_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 + name = "${local.csi}-letter-sla-breaches-sfn" role = aws_iam_role.letter_sla_breaches.id policy = data.aws_iam_policy_document.letter_sla_breaches_policy.json diff --git a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf index 19d726103..0fbd89b1e 100644 --- a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf @@ -1,4 +1,6 @@ resource "aws_sfn_state_machine" "letter_sla_breaches" { + count = local.eventpub_parquet_enabled ? 1 : 0 + name = local.letter_sla_breaches_state_machine_name role_arn = aws_iam_role.letter_sla_breaches.arn From 4db97c3e0f038cf6c595b9c31e4dcc537e052df1 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 11:11:24 +0100 Subject: [PATCH 32/37] And again --- .../components/api/iam_role_policy_letter_sla_breaches.tf | 2 +- .../components/api/sfn_state_machine_letter_sla_breaches.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf index ac75f4666..f8076c49a 100644 --- a/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf @@ -21,7 +21,7 @@ data "aws_iam_policy_document" "letter_sla_breaches_policy" { resources = [ module.populate_supplier_config_cache.function_arn, module.populate_sameday_breaches.function_arn, - module.populate_standard_breaches.function_arn, + module.populate_standard_breaches[0].function_arn, ] } } diff --git a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf index 0fbd89b1e..946b56c8f 100644 --- a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf @@ -2,12 +2,12 @@ resource "aws_sfn_state_machine" "letter_sla_breaches" { count = local.eventpub_parquet_enabled ? 1 : 0 name = local.letter_sla_breaches_state_machine_name - role_arn = aws_iam_role.letter_sla_breaches.arn + role_arn = aws_iam_role.letter_sla_breaches[0].arn definition = templatefile("${path.module}/resources/letter_sla_breaches.json", { STATE_MACHINE_ARN = local.letter_sla_breaches_state_machine_arn POPULATE_SUPPLIER_CONFIG_CACHE_FUNCTION_ARN = module.populate_supplier_config_cache.function_arn POPULATE_SAMEDAY_BREACHES_FUNCTION_ARN = module.populate_sameday_breaches.function_arn - POPULATE_STANDARD_BREACHES_FUNCTION_ARN = module.populate_standard_breaches.function_arn + POPULATE_STANDARD_BREACHES_FUNCTION_ARN = module.populate_standard_breaches[0].function_arn }) } From 5dee0784a727c63eeffc037b02a9cd7a628a6f00 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 11:14:11 +0100 Subject: [PATCH 33/37] Remove from policy --- .../components/api/sfn_state_machine_letter_sla_breaches.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf index 946b56c8f..e36a69e26 100644 --- a/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf +++ b/infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf @@ -2,7 +2,7 @@ resource "aws_sfn_state_machine" "letter_sla_breaches" { count = local.eventpub_parquet_enabled ? 1 : 0 name = local.letter_sla_breaches_state_machine_name - role_arn = aws_iam_role.letter_sla_breaches[0].arn + role_arn = aws_iam_role.letter_sla_breaches.arn definition = templatefile("${path.module}/resources/letter_sla_breaches.json", { STATE_MACHINE_ARN = local.letter_sla_breaches_state_machine_arn From c248b99c8134f34f355a6cb9c9727fa04728d677 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 11:28:13 +0100 Subject: [PATCH 34/37] Trino fixes/commits --- internal/sql-test-helpers/src/trino.ts | 91 ++++++++++++++------------ 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/internal/sql-test-helpers/src/trino.ts b/internal/sql-test-helpers/src/trino.ts index ff51b381c..2040e6f3e 100644 --- a/internal/sql-test-helpers/src/trino.ts +++ b/internal/sql-test-helpers/src/trino.ts @@ -23,6 +23,7 @@ type TrinoResponse = { export type TrinoContext = { container: StartedTestContainer; endpoint: string; + logs: string[]; }; const NODE_READY_TIMEOUT_MS = 60_000; @@ -33,43 +34,39 @@ const sleep = (ms: number) => setTimeout(resolve, ms); }); -// "SERVER STARTED" is logged before the coordinator has registered its own -// node as a worker, so early queries can fail with "No nodes available to run -// query". Wait until a trivial query succeeds before handing out the context. -async function waitForTrinoNode(context: TrinoContext): Promise { - const deadline = Date.now() + NODE_READY_TIMEOUT_MS; - - for (;;) { - try { - await runTrinoStatement(context, "SELECT 1"); - return; - } catch (error) { - const retryable = - error instanceof Error && error.message.includes("No nodes available"); +// Trino can report "No nodes available to run query" when the coordinator has +// not (or no longer) registered a worker, e.g. just after "SERVER STARTED" or +// on a loaded CI runner. It is raised before execution starts, so retrying is +// safe even for non-idempotent statements. +const isNoNodesError = (error: unknown) => + error instanceof Error && error.message.includes("No nodes available"); - if (!retryable || Date.now() >= deadline) { - throw error; - } - } +export async function setupTrinoContainer(): Promise { + // Buffer logs from the start so they are available even if startup fails. + const logs: string[] = []; + let container: StartedTestContainer; - await sleep(NODE_READY_POLL_MS); + try { + container = await new GenericContainer(TRINO_IMAGE) + .withExposedPorts(TRINO_PORT) + .withWaitStrategy(Wait.forLogMessage("======== SERVER STARTED ========")) + .withStartupTimeout(120_000) + .withLogConsumer((stream) => { + stream.on("data", (chunk) => logs.push(chunk.toString())); + }) + .start(); + } catch (error) { + printLogs(logs); + throw error; } -} - -export async function setupTrinoContainer(): Promise { - const container = await new GenericContainer(TRINO_IMAGE) - .withExposedPorts(TRINO_PORT) - .withWaitStrategy(Wait.forLogMessage("======== SERVER STARTED ========")) - .withStartupTimeout(120_000) - .start(); const endpoint = `http://${container.getHost()}:${container.getMappedPort(TRINO_PORT)}`; - const context = { container, endpoint }; + const context = { container, endpoint, logs }; try { - await waitForTrinoNode(context); + await runTrinoStatement(context, "SELECT 1"); } catch (error) { - await printContainerLogs(context.container); + printLogs(logs); await container.stop(); throw error; } @@ -77,19 +74,9 @@ export async function setupTrinoContainer(): Promise { return context; } -async function printContainerLogs(container: StartedTestContainer) { - try { - const stream = await container.logs(); - const chunks: string[] = []; - for await (const chunk of stream) { - chunks.push(chunk.toString()); - } - // eslint-disable-next-line no-console - console.error(`Trino container logs:\n${chunks.join("")}`); - } catch (logError) { - // eslint-disable-next-line no-console - console.error("Unable to read Trino container logs", logError); - } +function printLogs(logs: string[]) { + // eslint-disable-next-line no-console + console.error(`Trino container logs:\n${logs.join("")}`); } async function readTrinoResponse(response: Response): Promise { @@ -121,6 +108,26 @@ function toRow(columns: string[], values: unknown[]): SqlRow { export async function runTrinoStatement( context: TrinoContext, sql: string, +): Promise { + const deadline = Date.now() + NODE_READY_TIMEOUT_MS; + + for (;;) { + try { + return await executeTrinoStatement(context, sql); + } catch (error) { + if (!isNoNodesError(error) || Date.now() >= deadline) { + printLogs(context.logs); + throw error; + } + } + + await sleep(NODE_READY_POLL_MS); + } +} + +async function executeTrinoStatement( + context: TrinoContext, + sql: string, ): Promise { let body = await readTrinoResponse( await fetch(`${context.endpoint}/v1/statement`, { From b2d8dc3b62c36c0f6d69f5daf24985b95b24e082 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 13:11:08 +0100 Subject: [PATCH 35/37] Bucket permissions --- infrastructure/terraform/components/api/locals.tf | 1 + .../api/module_lambda_populate_standard_breaches.tf | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index c1fbb3063..24900a1f2 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -77,6 +77,7 @@ locals { event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null) eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null) + reporting_bucket_name = lookup(module.s3bucket_reporting, "bucket", null) # The Glue crawler names tables after the crawled bucket, replacing characters that are invalid in table names event_cache_glue_table_name = local.event_cache_bucket_name != null ? lower(replace(local.event_cache_bucket_name, "/[^a-zA-Z0-9_]/", "_")) : null diff --git a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf index 1f8849016..b994f3af8 100644 --- a/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf +++ b/infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf @@ -95,7 +95,7 @@ data "aws_iam_policy_document" "populate_standard_breaches_lambda" { } statement { - sid = "AllowReadEventCacheBucket" + sid = "AllowReadReportingBucket" effect = "Allow" actions = [ @@ -105,8 +105,8 @@ data "aws_iam_policy_document" "populate_standard_breaches_lambda" { ] resources = [ - "arn:aws:s3:::${local.event_cache_bucket_name}", - "arn:aws:s3:::${local.event_cache_bucket_name}/*", + "arn:aws:s3:::${local.reporting_bucket_name}", + "arn:aws:s3:::${local.reporting_bucket_name}/*", ] } From c7ca99eb867d7982ab6fc7c211f5b2ec82beeef7 Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 14:16:57 +0100 Subject: [PATCH 36/37] Better logging --- .../src/handler/populate-standard-breaches.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index d43a6bcdd..e2ca053ff 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -19,28 +19,28 @@ export default function createHandler(deps: Deps) { event: PopulateStandardBreachesEvent, ): Promise => { try { - const rows = await runAthenaQuery( - deps, - buildStandardBreachesQuery( - deps.env.EVENT_CACHE_TABLE_NAME, - new Date(), - 14, - ), + const sql = buildStandardBreachesQuery( + deps.env.EVENT_CACHE_TABLE_NAME, + new Date(), + 14, ); + deps.logger.info({ description: "Running Athena query", sql }); + const rows = await runAthenaQuery(deps, sql); // TODO: CCM-24041 process the query results to populate standard breaches. - deps.logger.info( - { date: event.date, breachCounts: JSON.stringify(rows) }, - "Standard breaches query completed", - ); + deps.logger.info({ + description: "Standard breaches query completed", + date: event.date, + breachCounts: JSON.stringify(rows), + }); return { status: "SUCCESS", breachCounts: JSON.stringify(rows) }; } catch (error) { - deps.logger.error( - { date: event.date, error }, - "Failed to populate standard breaches", - ); - + deps.logger.error({ + description: "Failed to populate standard breaches", + date: event.date, + error, + }); return { status: "FAILURE", failureReason: error instanceof Error ? error.message : String(error), From 9627ce1e712d99fa21a0db9e073b302e8cd795eb Mon Sep 17 00:00:00 2001 From: Steve Buxton Date: Fri, 16 Oct 2026 14:58:10 +0100 Subject: [PATCH 37/37] Use passed in date --- lambdas/letter-sla-breaches/common/src/athena.ts | 2 +- .../src/handler/populate-standard-breaches.ts | 3 +-- .../src/handler/standard-breaches-query.ts | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lambdas/letter-sla-breaches/common/src/athena.ts b/lambdas/letter-sla-breaches/common/src/athena.ts index 7edc2bc4f..9bd2d58fb 100644 --- a/lambdas/letter-sla-breaches/common/src/athena.ts +++ b/lambdas/letter-sla-breaches/common/src/athena.ts @@ -120,7 +120,7 @@ export default async function runAthenaQuery( options: AthenaQueryOptions = {}, ): Promise { const queryExecutionId = await startQuery(deps, query); - deps.logger.info({ queryExecutionId }, "Started Athena query"); + deps.logger.info({ description: "Started Athena query", query }); await waitForQuery( deps, diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts index e2ca053ff..5ffdf4e20 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/populate-standard-breaches.ts @@ -21,10 +21,9 @@ export default function createHandler(deps: Deps) { try { const sql = buildStandardBreachesQuery( deps.env.EVENT_CACHE_TABLE_NAME, - new Date(), + new Date(event.date), 14, ); - deps.logger.info({ description: "Running Athena query", sql }); const rows = await runAthenaQuery(deps, sql); // TODO: CCM-24041 process the query results to populate standard breaches. diff --git a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts index 971045ab3..79c6aa462 100644 --- a/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts +++ b/lambdas/letter-sla-breaches/populate-standard-breaches/src/handler/standard-breaches-query.ts @@ -9,8 +9,8 @@ export default function buildStandardBreachesQuery( daysHistory: number, ): AthenaQuery { return { - queryString: `WITH now AS ( - SELECT from_iso8601_timestamp(?) AS iso8601 + queryString: `WITH parameters AS ( + SELECT from_iso8601_timestamp(?) AS now ), recent_letter_events AS ( SELECT domainid, supplierid, status, from_iso8601_timestamp(time) AS event_time @@ -22,7 +22,7 @@ allocated_letters AS ( FROM recent_letter_events WHERE status = 'PENDING' GROUP BY domainid, supplierid - HAVING min(event_time) < (SELECT iso8601 FROM now) - + HAVING min(event_time) < (SELECT now FROM parameters) - CASE WHEN day_of_week(min(event_time) AT TIME ZONE 'Europe/London') IN (4, 5, 6) THEN INTERVAL '96' HOUR ELSE INTERVAL '48' HOUR END ), completed_letters AS (