CCM-24041: standard letter breaches

.github/workflows/stage-2-test.yaml

@@ -77,6 +77,14 @@ jobs:
     env:
       NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
+      # Pull testcontainers images in the background so the download overlaps
+      # the setup steps. Keep these references in sync with the images used in
+      # internal/datastore/src/__test__/db.ts and
+      # internal/sql-test-helpers/src/trino.ts.
+      - name: "Pre-pull test container images"
+        run: |
+          docker pull amazon/dynamodb-local > /dev/null 2>&1 &
+          docker pull trinodb/trino:476 > /dev/null 2>&1 &
       - name: "Checkout code"
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Setup NodeJS

docs/generate-includes.sh

@@ -24,6 +24,7 @@ cp ./lambdas/upsert-letter/README.md ./docs/_includes/components/generated/readm
 cp ./lambdas/update-letter-queue/README.md ./docs/_includes/components/generated/readmes/lambda-update-letter-queue.md
 cp ./lambdas/letter-updates-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-letter-updates-transformer.md
 cp ./lambdas/mi-updates-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-mi-updates-transformer.md
+cp ./lambdas/eventpub-parquet-transformer/README.md ./docs/_includes/components/generated/readmes/lambda-eventpub-parquet-transformer.md
 cp ./lambdas/supplier-config-ingress/README.md ./docs/_includes/components/generated/readmes/lambda-supplier-config-ingress.md
 
 # Function documentation (internal packages)

eslint.config.mjs

@@ -115,6 +115,7 @@ export default defineConfig([
         eslintImportResolverTypescript.createTypeScriptImportResolver({
           project: [
             'lambdas/*/tsconfig.json',
+            'lambdas/*/*/tsconfig.json',
             'tests/tsconfig.json',
             'internal/*/tsconfig.json',
           ],

infrastructure/terraform/components/api/README.md

@@ -54,6 +54,8 @@ No requirements.
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The AWS Region | `string` | n/a | yes |
+| <a name="input_reports_data_retention_non_current_days"></a> [reports\_data\_retention\_non\_current\_days](#input\_reports\_data\_retention\_non\_current\_days) | Number of days to retain noncurrent reporting bucket object versions | `number` | `14` | no |
+| <a name="input_reports_data_retention_policy_days"></a> [reports\_data\_retention\_policy\_days](#input\_reports\_data\_retention\_policy\_days) | Number of days to retain reporting bucket data | `number` | `90` | no |
 | <a name="input_shared_infra_account_id"></a> [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Account ID of the shared infrastructure account | `string` | `"000000000000"` | no |
 | <a name="input_sns_success_logging_sample_percent"></a> [sns\_success\_logging\_sample\_percent](#input\_sns\_success\_logging\_sample\_percent) | Enable SNS Delivery Successful Sample Percentage | `number` | `0` | no |
 | <a name="input_supplier_mock_schedule_expression"></a> [supplier\_mock\_schedule\_expression](#input\_supplier\_mock\_schedule\_expression) | Schedule expression for supplier mock scheduler | `string` | `"rate(1 minute)"` | no |
@@ -71,6 +73,7 @@ No requirements.
 | <a name="module_ddb_alarms_suppliers"></a> [ddb\_alarms\_suppliers](#module\_ddb\_alarms\_suppliers) | ../../modules/alarms-ddb | n/a |
 | <a name="module_domain_truststore"></a> [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a |
 | <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/5.0.6/terraform-eventpub.zip | n/a |
+| <a name="module_eventpub_parquet_transformer"></a> [eventpub\_parquet\_transformer](#module\_eventpub\_parquet\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_eventsub"></a> [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a |
 | <a name="module_get_letter"></a> [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_get_letter_data"></a> [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
@@ -83,8 +86,13 @@ No requirements.
 | <a name="module_letter_updates_transformer"></a> [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_mi_updates_transformer"></a> [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_patch_letter"></a> [patch\_letter](#module\_patch\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
+| <a name="module_populate_sameday_breaches"></a> [populate\_sameday\_breaches](#module\_populate\_sameday\_breaches) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
+| <a name="module_populate_standard_breaches"></a> [populate\_standard\_breaches](#module\_populate\_standard\_breaches) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
+| <a name="module_populate_supplier_config_cache"></a> [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_post_letters"></a> [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_post_mi"></a> [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
+| <a name="module_s3bucket_athena_query_results"></a> [s3bucket\_athena\_query\_results](#module\_s3bucket\_athena\_query\_results) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a |
+| <a name="module_s3bucket_reporting"></a> [s3bucket\_reporting](#module\_s3bucket\_reporting) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip | n/a |
 | <a name="module_s3bucket_test_letters"></a> [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
 | <a name="module_sqs_alarms"></a> [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a |
 | <a name="module_sqs_letter_updates"></a> [sqs\_letter\_updates](#module\_sqs\_letter\_updates) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.5/terraform-sqs.zip | n/a |

infrastructure/terraform/components/api/athena_workgroup_letter_sla_breaches.tf

@@ -0,0 +1,19 @@
+resource "aws_athena_workgroup" "letter_sla_breaches" {
+  name          = "${local.csi}-letter-sla-breaches"
+  description   = "Athena workgroup for the letter SLA breaches state machine"
+  force_destroy = var.force_destroy
+
+  configuration {
+    enforce_workgroup_configuration    = true
+    publish_cloudwatch_metrics_enabled = true
+
+    result_configuration {
+      output_location = "s3://${module.s3bucket_athena_query_results.bucket}/letter-sla-breaches/"
+
+      encryption_configuration {
+        encryption_option = "SSE_KMS"
+        kms_key_arn       = module.kms.key_arn
+      }
+    }
+  }
+}

infrastructure/terraform/components/api/athena_workgroup_reporting.tf

@@ -0,0 +1,19 @@
+resource "aws_athena_workgroup" "reporting" {
+  name          = local.csi
+  description   = "Supplier API Athena Workgroup for ${var.environment}"
+  force_destroy = true
+
+  configuration {
+    enforce_workgroup_configuration = true
+
+    result_configuration {
+      expected_bucket_owner = var.aws_account_id
+      output_location       = "s3://${module.s3bucket_reporting.bucket}/athena-output/"
+
+      encryption_configuration {
+        encryption_option = "SSE_KMS"
+        kms_key_arn       = module.kms.key_arn
+      }
+    }
+  }
+}

infrastructure/terraform/components/api/cloudwatch_log_group_eventpub_parquet_firehose.tf

@@ -0,0 +1,7 @@
+resource "aws_cloudwatch_log_group" "eventpub_parquet_firehose" {
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
+  name              = "/aws/firehose/${local.csi}-eventpub-parquet"
+  kms_key_id        = module.kms.key_arn
+  retention_in_days = var.log_retention_in_days
+}

infrastructure/terraform/components/api/cloudwatch_log_stream_eventpub_parquet_firehose.tf

@@ -0,0 +1,6 @@
+resource "aws_cloudwatch_log_stream" "eventpub_parquet_firehose" {
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
+  name           = "extended_s3"
+  log_group_name = aws_cloudwatch_log_group.eventpub_parquet_firehose[0].name
+}

infrastructure/terraform/components/api/glue_catalog_table_eventpub_parquet.tf

@@ -0,0 +1,60 @@
+resource "aws_glue_catalog_table" "eventpub_parquet" {
+  count         = local.eventpub_parquet_enabled ? 1 : 0
+  name          = "${replace(local.csi, "-", "_")}_eventpub"
+  database_name = aws_glue_catalog_database.supplier.name
+  table_type    = "EXTERNAL_TABLE"
+
+  parameters = {
+    EXTERNAL           = "TRUE"
+    classification     = "parquet"
+    "parquet.compress" = "SNAPPY"
+
+    # Athena derives partitions from these properties, so none are registered in Glue
+    "projection.enabled"        = "true"
+    "projection.__year.type"    = "integer"
+    "projection.__year.range"   = "2025,2050"
+    "projection.__year.digits"  = "4"
+    "projection.__month.type"   = "integer"
+    "projection.__month.range"  = "1,12"
+    "projection.__month.digits" = "2"
+    "projection.__day.type"     = "integer"
+    "projection.__day.range"    = "1,31"
+    "projection.__day.digits"   = "2"
+    "storage.location.template" = "s3://${module.s3bucket_reporting.bucket}/parquet/eventpub/__year=$${__year}/__month=$${__month}/__day=$${__day}/"
+  }
+
+  storage_descriptor {
+    location      = "s3://${module.s3bucket_reporting.bucket}/parquet/eventpub/"
+    input_format  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
+
+    ser_de_info {
+      name                  = "ParquetHiveSerDe"
+      serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
+    }
+
+    dynamic "columns" {
+      for_each = local.eventpub_parquet_table_columns
+
+      content {
+        name = columns.value.name
+        type = columns.value.type
+      }
+    }
+  }
+
+  partition_keys {
+    name = "__year"
+    type = "string"
+  }
+
+  partition_keys {
+    name = "__month"
+    type = "string"
+  }
+
+  partition_keys {
+    name = "__day"
+    type = "string"
+  }
+}

infrastructure/terraform/components/api/glue_catalog_tables_supplier_config_cache.tf

@@ -0,0 +1,108 @@
+locals {
+  supplier_config_cache_table_columns = {
+    supplier = [
+      { name = "pk", type = "string" },
+      { name = "sk", type = "string" },
+      { name = "channelType", type = "string" },
+      { name = "dailyCapacity", type = "bigint" },
+      { name = "id", type = "string" },
+      { name = "name", type = "string" },
+      { name = "status", type = "string" },
+    ]
+    volume-group = [
+      { name = "pk", type = "string" },
+      { name = "sk", type = "string" },
+      { name = "description", type = "string" },
+      { name = "endDate", type = "string" },
+      { name = "id", type = "string" },
+      { name = "name", type = "string" },
+      { name = "startDate", type = "string" },
+      { name = "status", type = "string" },
+    ]
+    supplier-pack = [
+      { name = "pk", type = "string" },
+      { name = "sk", type = "string" },
+      { name = "approval", type = "string" },
+      { name = "id", type = "string" },
+      { name = "packSpecificationId", type = "string" },
+      { name = "status", type = "string" },
+      { name = "supplierId", type = "string" },
+    ]
+    letter-variant = [
+      { name = "pk", type = "string" },
+      { name = "sk", type = "string" },
+      { name = "campaignIds", type = "array<string>" },
+      { name = "clientId", type = "string" },
+      { name = "constraints", type = "struct<blackCoveragePercentage:struct<operator:string,value:double>,colourCoveragePercentage:struct<operator:string,value:double>,deliveryDays:struct<operator:string,value:double>,sheets:struct<operator:string,value:double>,sides:struct<operator:string,value:double>>" },
+      { name = "description", type = "string" },
+      { name = "id", type = "string" },
+      { name = "name", type = "string" },
+      { name = "packSpecificationIds", type = "array<string>" },
+      { name = "priority", type = "bigint" },
+      { name = "status", type = "string" },
+      { name = "supplierId", type = "string" },
+      { name = "type", type = "string" },
+      { name = "volumeGroupId", type = "string" },
+    ]
+    pack-specification = [
+      { name = "pk", type = "string" },
+      { name = "sk", type = "string" },
+      { name = "assembly", type = "struct<duplex:boolean,envelopeId:string,features:array<string>,insertIds:array<string>,paper:struct<colour:string,id:string,name:string,recycled:boolean,size:string,weightGSM:double>,printColour:string>" },
+      { name = "billingId", type = "string" },
+      { name = "constraints", type = "struct<blackCoveragePercentage:struct<operator:string,value:double>,colourCoveragePercentage:struct<operator:string,value:double>,deliveryDays:struct<operator:string,value:double>,sheets:struct<operator:string,value:double>,sides:struct<operator:string,value:double>>" },
+      { name = "createdAt", type = "string" },
+      { name = "description", type = "string" },
+      { name = "id", type = "string" },
+      { name = "name", type = "string" },
+      { name = "postage", type = "struct<deliveryDays:double,id:string,maxThicknessMm:double,maxWeightGrams:double,size:string>" },
+      { name = "status", type = "string" },
+      { name = "updatedAt", type = "string" },
+      { name = "version", type = "bigint" },
+    ]
+    supplier-allocation = [
+      { name = "pk", type = "string" },
+      { name = "sk", type = "string" },
+      { name = "allocationPercentage", type = "double" },
+      { name = "id", type = "string" },
+      { name = "status", type = "string" },
+      { name = "supplier", type = "string" },
+      { name = "volumeGroup", type = "string" },
+    ]
+  }
+}
+
+resource "aws_glue_catalog_table" "supplier_config_cache" {
+  for_each      = local.supplier_config_cache_table_columns
+  name          = "${replace(local.csi, "-", "_")}_${replace(each.key, "-", "_")}"
+  database_name = aws_glue_catalog_database.supplier.name
+  table_type    = "EXTERNAL_TABLE"
+
+  parameters = {
+    EXTERNAL       = "TRUE"
+    classification = "json"
+  }
+
+  storage_descriptor {
+    location      = "s3://${module.s3bucket_reporting.bucket}/supplier_config_cache/${each.key}/"
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    ser_de_info {
+      name                  = "OpenXJsonSerDe"
+      serialization_library = "org.openx.data.jsonserde.JsonSerDe"
+
+      parameters = {
+        "case.insensitive" = "false"
+      }
+    }
+
+    dynamic "columns" {
+      for_each = each.value
+
+      content {
+        name = columns.value.name
+        type = columns.value.type
+      }
+    }
+  }
+}

infrastructure/terraform/components/api/iam_role_eventpub_parquet_firehose.tf

@@ -0,0 +1,18 @@
+resource "aws_iam_role" "eventpub_parquet_firehose" {
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
+  name               = "${local.csi}-eventpub-parquet-firehose"
+  assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_firehose_assume_role.json
+}
+
+data "aws_iam_policy_document" "eventpub_parquet_firehose_assume_role" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+  }
+}

infrastructure/terraform/components/api/iam_role_eventpub_parquet_sns.tf

@@ -0,0 +1,18 @@
+resource "aws_iam_role" "eventpub_parquet_sns" {
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
+  name               = "${local.csi}-eventpub-parquet-sns"
+  assume_role_policy = data.aws_iam_policy_document.eventpub_parquet_sns_assume_role.json
+}
+
+data "aws_iam_policy_document" "eventpub_parquet_sns_assume_role" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["sns.amazonaws.com"]
+    }
+  }
+}

infrastructure/terraform/components/api/iam_role_letter_sla_breaches.tf

@@ -0,0 +1,17 @@
+resource "aws_iam_role" "letter_sla_breaches" {
+  name               = "${local.csi}-letter-sla-breaches-sfn"
+  description        = "Execution role for the letter SLA breaches state machine"
+  assume_role_policy = data.aws_iam_policy_document.letter_sla_breaches_trust_policy.json
+}
+
+data "aws_iam_policy_document" "letter_sla_breaches_trust_policy" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["states.amazonaws.com"]
+    }
+  }
+}