diff --git a/infrastructure/account/.terraform.lock.hcl b/infrastructure/account/.terraform.lock.hcl index c31874d86..ee2267fe6 100644 --- a/infrastructure/account/.terraform.lock.hcl +++ b/infrastructure/account/.terraform.lock.hcl @@ -23,3 +23,23 @@ provider "registry.terraform.io/hashicorp/aws" { "zh:f35ab043d7121f3a194f42f5b0e0d59fe62d94488acea07e3783405fa5838785", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.8.1" + constraints = "~> 3.0" + hashes = [ + "h1:u8AKlWVDTH5r9YLSeswoVEjiY72Rt4/ch7U+61ZDkiQ=", + "zh:08dd03b918c7b55713026037c5400c48af5b9f468f483463321bd18e17b907b4", + "zh:0eee654a5542dc1d41920bbf2419032d6f0d5625b03bd81339e5b33394a3e0ae", + "zh:229665ddf060aa0ed315597908483eee5b818a17d09b6417a0f52fd9405c4f57", + "zh:2469d2e48f28076254a2a3fc327f184914566d9e40c5780b8d96ebf7205f8bc0", + "zh:37d7eb334d9561f335e748280f5535a384a88675af9a9eac439d4cfd663bcb66", + "zh:741101426a2f2c52dee37122f0f4a2f2d6af6d852cb1db634480a86398fa3511", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a902473f08ef8df62cfe6116bd6c157070a93f66622384300de235a533e9d4a9", + "zh:b85c511a23e57a2147355932b3b6dce2a11e856b941165793a0c3d7578d94d05", + "zh:c5172226d18eaac95b1daac80172287b69d4ce32750c82ad77fa0768be4ea4b8", + "zh:dab4434dba34aad569b0bc243c2d3f3ff86dd7740def373f2a49816bd2ff819b", + "zh:f49fd62aa8c5525a5c17abd51e27ca5e213881d58882fd42fec4a545b53c9699", + ] +} diff --git a/infrastructure/account/main.tf b/infrastructure/account/main.tf index 5eef617f1..51b8a71c0 100644 --- a/infrastructure/account/main.tf +++ b/infrastructure/account/main.tf @@ -4,6 +4,10 @@ terraform { source = "hashicorp/aws" version = "~> 6" } + random = { + source = "hashicorp/random" + version = "~> 3" + } } backend "s3" { region = "eu-west-2" diff --git a/infrastructure/account/redis_cache.tf b/infrastructure/account/redis_cache.tf index 8fe759f3c..0fa37ffcb 100644 --- a/infrastructure/account/redis_cache.tf +++ b/infrastructure/account/redis_cache.tf @@ -1,12 +1,35 @@ -resource "aws_elasticache_cluster" "redis_cluster" { - cluster_id = "immunisation-redis-cluster" +resource "random_password" "redis_auth_token" { + length = 32 + special = true + override_special = "!&#$^<>-" +} + +resource "aws_secretsmanager_secret" "redis_auth_token" { + name = "imms/redis/auth-token" + description = "Auth token for the immunisation Redis cache" +} + +resource "aws_secretsmanager_secret_version" "redis_auth_token" { + secret_id = aws_secretsmanager_secret.redis_auth_token.id + secret_string = random_password.redis_auth_token.result +} + +resource "aws_elasticache_replication_group" "redis_cluster" { + replication_group_id = "immunisation-redis-cluster" + description = "Redis cache for immunisation configuration data" engine = "redis" + engine_version = "7.0" node_type = "cache.t2.micro" - num_cache_nodes = 1 + num_cache_clusters = 1 parameter_group_name = "default.redis7" port = 6379 security_group_ids = [aws_security_group.lambda_redis_sg.id] subnet_group_name = aws_elasticache_subnet_group.redis_subnet_group.name + + at_rest_encryption_enabled = true + transit_encryption_enabled = true + auth_token = random_password.redis_auth_token.result + auth_token_update_strategy = "SET" } # Subnet Group for Redis diff --git a/infrastructure/instance/ecs_batch_processor_config.tf b/infrastructure/instance/ecs_batch_processor_config.tf index 1be339c73..deef98fa0 100644 --- a/infrastructure/instance/ecs_batch_processor_config.tf +++ b/infrastructure/instance/ecs_batch_processor_config.tf @@ -114,6 +114,11 @@ resource "aws_iam_policy" "ecs_task_exec_policy" { "firehose:PutRecordBatch" ], "Resource" : "arn:aws:firehose:*:*:deliverystream/${module.splunk.firehose_stream_name}" + }, + { + Effect = "Allow", + Action = "secretsmanager:GetSecretValue", + Resource = data.aws_secretsmanager_secret.redis_auth_token.arn } ] }) @@ -147,40 +152,35 @@ resource "aws_ecs_task_definition" "ecs_task" { name = "${local.short_prefix}-process-records-container" image = var.recordprocessor_image_uri essential = true - environment = [ - { - name = "SOURCE_BUCKET_NAME" - value = aws_s3_bucket.batch_data_source_bucket.bucket - }, - { - name = "ACK_BUCKET_NAME" - value = aws_s3_bucket.batch_data_destination_bucket.bucket - }, - { - name = "KINESIS_STREAM_ARN" - value = local.kinesis_arn - }, - { - name = "KINESIS_STREAM_NAME" - value = "${local.short_prefix}-processingdata-stream" - }, - { - name = "SPLUNK_FIREHOSE_NAME" - value = module.splunk.firehose_stream_name - }, - { - name = "AUDIT_TABLE_NAME" - value = aws_dynamodb_table.audit-table.name - }, - { - name = "REDIS_HOST" - value = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - }, - { - name = "REDIS_PORT" - value = tostring(data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port) - } - ] + environment = concat( + [ + { + name = "SOURCE_BUCKET_NAME" + value = aws_s3_bucket.batch_data_source_bucket.bucket + }, + { + name = "ACK_BUCKET_NAME" + value = aws_s3_bucket.batch_data_destination_bucket.bucket + }, + { + name = "KINESIS_STREAM_ARN" + value = local.kinesis_arn + }, + { + name = "KINESIS_STREAM_NAME" + value = "${local.short_prefix}-processingdata-stream" + }, + { + name = "SPLUNK_FIREHOSE_NAME" + value = module.splunk.firehose_stream_name + }, + { + name = "AUDIT_TABLE_NAME" + value = aws_dynamodb_table.audit-table.name + } + ], + local.redis_environment + ) logConfiguration = { logDriver = "awslogs" options = { diff --git a/infrastructure/instance/endpoints.tf b/infrastructure/instance/endpoints.tf index 9ff8d56b2..36cb2b48e 100644 --- a/infrastructure/instance/endpoints.tf +++ b/infrastructure/instance/endpoints.tf @@ -23,17 +23,18 @@ locals { "get_imms", "create_imms", "update_imms", "search_imms", "delete_imms", "not_found" ] imms_table_name = aws_dynamodb_table.events-dynamodb-table.name - imms_lambda_env_vars = { - "DYNAMODB_TABLE_NAME" = local.imms_table_name, - "IMMUNIZATION_ENV" = local.resource_scope, - "IMMUNIZATION_BASE_PATH" = strcontains(var.sub_environment, "pr-") ? "immunisation-fhir-api/FHIR/R4-${var.sub_environment}" : "immunisation-fhir-api/FHIR/R4" - # except for prod and ref, any other env uses PDS int environment - "PDS_ENV" = var.pds_environment - "SPLUNK_FIREHOSE_NAME" = module.splunk.firehose_stream_name - "SQS_QUEUE_URL" = "https://sqs.${var.aws_region}.amazonaws.com/${var.immunisation_account_id}/${local.short_prefix}-ack-metadata-queue.fifo" - "REDIS_HOST" = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - "REDIS_PORT" = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port - } + imms_lambda_env_vars = merge( + { + "DYNAMODB_TABLE_NAME" = local.imms_table_name, + "IMMUNIZATION_ENV" = local.resource_scope, + "IMMUNIZATION_BASE_PATH" = strcontains(var.sub_environment, "pr-") ? "immunisation-fhir-api/FHIR/R4-${var.sub_environment}" : "immunisation-fhir-api/FHIR/R4" + # except for prod and ref, any other env uses PDS int environment + "PDS_ENV" = var.pds_environment + "SPLUNK_FIREHOSE_NAME" = module.splunk.firehose_stream_name + "SQS_QUEUE_URL" = "https://sqs.${var.aws_region}.amazonaws.com/${var.immunisation_account_id}/${local.short_prefix}-ack-metadata-queue.fifo" + }, + local.redis_env_vars + ) } data "aws_iam_policy_document" "imms_policy_document" { source_policy_documents = [ diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf index a4e0c6183..fa9558857 100644 --- a/infrastructure/instance/file_name_processor.tf +++ b/infrastructure/instance/file_name_processor.tf @@ -88,6 +88,11 @@ resource "aws_iam_policy" "filenameprocessor_lambda_exec_policy" { ], "Resource" : "arn:aws:firehose:*:*:deliverystream/${module.splunk.firehose_stream_name}" }, + { + Effect = "Allow", + Action = "secretsmanager:GetSecretValue", + Resource = data.aws_secretsmanager_secret.redis_auth_token.arn + }, { "Effect" : "Allow", "Action" : [ @@ -240,19 +245,20 @@ resource "aws_lambda_function" "file_processor_lambda" { } environment { - variables = { - ACCOUNT_ID = var.immunisation_account_id - DPS_ACCOUNT_ID = var.dspp_core_account_id - SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket - ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket - DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name - QUEUE_URL = aws_sqs_queue.batch_file_created.url - REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port - SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name - AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name - AUDIT_TABLE_TTL_DAYS = 60 - } + variables = merge( + { + ACCOUNT_ID = var.immunisation_account_id + DPS_ACCOUNT_ID = var.dspp_core_account_id + SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket + ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket + DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name + QUEUE_URL = aws_sqs_queue.batch_file_created.url + SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name + AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name + AUDIT_TABLE_TTL_DAYS = 60 + }, + local.redis_env_vars + ) } kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn reserved_concurrent_executions = local.is_temp ? -1 : 20 diff --git a/infrastructure/instance/forwarder_lambda.tf b/infrastructure/instance/forwarder_lambda.tf index 1a8ee9ca0..44579cde3 100644 --- a/infrastructure/instance/forwarder_lambda.tf +++ b/infrastructure/instance/forwarder_lambda.tf @@ -108,6 +108,11 @@ resource "aws_iam_policy" "forwarding_lambda_exec_policy" { ] Resource = aws_sqs_queue.fifo_queue.arn }, + { + Effect = "Allow" + Action = "secretsmanager:GetSecretValue" + Resource = data.aws_secretsmanager_secret.redis_auth_token.arn + }, { Effect = "Allow", Action = [ @@ -146,14 +151,15 @@ resource "aws_lambda_function" "forwarding_lambda" { } environment { - variables = { - SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket - ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket - DYNAMODB_TABLE_NAME = aws_dynamodb_table.events-dynamodb-table.name - SQS_QUEUE_URL = aws_sqs_queue.fifo_queue.url - REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port - } + variables = merge( + { + SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket + ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket + DYNAMODB_TABLE_NAME = aws_dynamodb_table.events-dynamodb-table.name + SQS_QUEUE_URL = aws_sqs_queue.fifo_queue.url + }, + local.redis_env_vars + ) } kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn depends_on = [ diff --git a/infrastructure/instance/main.tf b/infrastructure/instance/main.tf index b42598050..d4cfdfe77 100644 --- a/infrastructure/instance/main.tf +++ b/infrastructure/instance/main.tf @@ -81,8 +81,28 @@ data "aws_kms_key" "existing_dynamo_encryption_key" { key_id = "alias/imms-event-dynamodb-encryption" } -data "aws_elasticache_cluster" "existing_redis" { - cluster_id = "immunisation-redis-cluster" +data "aws_elasticache_replication_group" "existing_redis" { + replication_group_id = "immunisation-redis-cluster" +} + +data "aws_secretsmanager_secret" "redis_auth_token" { + name = "imms/redis/auth-token" +} + +locals { + redis_env_vars = { + REDIS_HOST = data.aws_elasticache_replication_group.existing_redis.primary_endpoint_address + REDIS_PORT = tostring(data.aws_elasticache_replication_group.existing_redis.port) + REDIS_SSL = "true" + REDIS_AUTH_TOKEN_SECRET_NAME = data.aws_secretsmanager_secret.redis_auth_token.name + } + + redis_environment = [ + for name, value in local.redis_env_vars : { + name = name + value = value + } + ] } data "aws_security_group" "existing_securitygroup" { diff --git a/infrastructure/instance/policies/secret_manager.json b/infrastructure/instance/policies/secret_manager.json index e40bef5d4..8b644c4dd 100644 --- a/infrastructure/instance/policies/secret_manager.json +++ b/infrastructure/instance/policies/secret_manager.json @@ -6,7 +6,8 @@ "Action": "secretsmanager:GetSecretValue", "Resource": [ "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/outbound/*/*", - "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/pds/*/*" + "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/pds/*/*", + "arn:aws:secretsmanager:eu-west-2:${account_id}:secret:imms/redis/auth-token-*" ] } ] diff --git a/infrastructure/instance/redis_sync_lambda.tf b/infrastructure/instance/redis_sync_lambda.tf index 3913b0b8b..edb124d99 100644 --- a/infrastructure/instance/redis_sync_lambda.tf +++ b/infrastructure/instance/redis_sync_lambda.tf @@ -88,6 +88,11 @@ resource "aws_iam_policy" "redis_sync_lambda_exec_policy" { ], Resource : "arn:aws:firehose:*:*:deliverystream/${module.splunk.firehose_stream_name}" }, + { + Effect = "Allow", + Action = "secretsmanager:GetSecretValue", + Resource = data.aws_secretsmanager_secret.redis_auth_token.arn + }, { Effect = "Allow" Action = "lambda:InvokeFunction" @@ -155,12 +160,13 @@ resource "aws_lambda_function" "redis_sync_lambda" { } environment { - variables = { - CONFIG_BUCKET_NAME = local.config_bucket_name - REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port - SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name - } + variables = merge( + { + CONFIG_BUCKET_NAME = local.config_bucket_name + SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name + }, + local.redis_env_vars + ) } kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn diff --git a/lambdas/shared/src/common/redis_client.py b/lambdas/shared/src/common/redis_client.py index 07c0c9e60..48e803278 100644 --- a/lambdas/shared/src/common/redis_client.py +++ b/lambdas/shared/src/common/redis_client.py @@ -2,17 +2,38 @@ import redis -from common.clients import logger +from common.clients import get_secrets_manager_client, logger REDIS_HOST = os.getenv("REDIS_HOST", "") REDIS_PORT = os.getenv("REDIS_PORT", 6379) +REDIS_SSL = os.getenv("REDIS_SSL", "false").lower() == "true" +REDIS_AUTH_TOKEN_SECRET_NAME = os.getenv("REDIS_AUTH_TOKEN_SECRET_NAME", "") redis_client = None +redis_auth_token = None + + +def get_redis_auth_token(): + global redis_auth_token + if not REDIS_AUTH_TOKEN_SECRET_NAME: + return None + + if redis_auth_token is None: + response = get_secrets_manager_client().get_secret_value(SecretId=REDIS_AUTH_TOKEN_SECRET_NAME) + redis_auth_token = response["SecretString"] + + return redis_auth_token def get_redis_client(): global redis_client if redis_client is None: - logger.info(f"Connecting to Redis at {REDIS_HOST}:{REDIS_PORT}") - redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True) + logger.info(f"Connecting to Redis at {REDIS_HOST}:{REDIS_PORT} with ssl={REDIS_SSL}") + redis_client = redis.StrictRedis( + host=REDIS_HOST, + port=REDIS_PORT, + password=get_redis_auth_token(), + ssl=REDIS_SSL, + decode_responses=True, + ) return redis_client diff --git a/lambdas/shared/tests/test_common/test_redis_client.py b/lambdas/shared/tests/test_common/test_redis_client.py index 0ac13e3f8..3c1bfd315 100644 --- a/lambdas/shared/tests/test_common/test_redis_client.py +++ b/lambdas/shared/tests/test_common/test_redis_client.py @@ -1,6 +1,6 @@ import importlib import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch import common.redis_client as redis_client @@ -8,14 +8,19 @@ class TestRedisClient(unittest.TestCase): REDIS_HOST = "mock-redis-host" REDIS_PORT = 6379 + REDIS_SSL = "true" + REDIS_AUTH_TOKEN_SECRET_NAME = "mock-redis-auth-token-secret" def setUp(self): - self.getenv_patch = patch("os.getenv") - self.mock_getenv = self.getenv_patch.start() - self.mock_getenv.side_effect = lambda key, default=None: { + self.env = { "REDIS_HOST": self.REDIS_HOST, "REDIS_PORT": self.REDIS_PORT, - }.get(key, default) + "REDIS_SSL": self.REDIS_SSL, + } + + self.getenv_patch = patch("os.getenv") + self.mock_getenv = self.getenv_patch.start() + self.mock_getenv.side_effect = lambda key, default=None: self.env.get(key, default) self.redis_patch = patch("redis.StrictRedis") self.mock_redis = self.redis_patch.start() @@ -30,6 +35,8 @@ def test_os_environ(self): importlib.reload(redis_client) self.assertEqual(redis_client.REDIS_HOST, self.REDIS_HOST) self.assertEqual(redis_client.REDIS_PORT, self.REDIS_PORT) + self.assertTrue(redis_client.REDIS_SSL) + self.assertEqual(redis_client.REDIS_AUTH_TOKEN_SECRET_NAME, "") def test_redis_client(self): """Test redis client is not initialized on import""" @@ -41,6 +48,33 @@ def test_redis_client_initialization(self): importlib.reload(redis_client) redis_client.get_redis_client() redis_client.get_redis_client() - self.mock_redis.assert_called_once_with(host=self.REDIS_HOST, port=self.REDIS_PORT, decode_responses=True) + self.mock_redis.assert_called_once_with( + host=self.REDIS_HOST, + port=self.REDIS_PORT, + password=None, + ssl=True, + decode_responses=True, + ) self.assertTrue(hasattr(redis_client, "redis_client")) self.assertIsInstance(redis_client.redis_client, self.mock_redis.return_value.__class__) + + def test_redis_client_uses_auth_token_from_secrets_manager(self): + """Test Redis auth token is fetched once from Secrets Manager""" + self.env["REDIS_AUTH_TOKEN_SECRET_NAME"] = self.REDIS_AUTH_TOKEN_SECRET_NAME + importlib.reload(redis_client) + + mock_secrets_manager_client = Mock() + mock_secrets_manager_client.get_secret_value.return_value = {"SecretString": "mock-auth-token"} + + with patch("common.redis_client.get_secrets_manager_client", return_value=mock_secrets_manager_client): + redis_client.get_redis_client() + redis_client.get_redis_client() + + mock_secrets_manager_client.get_secret_value.assert_called_once_with(SecretId=self.REDIS_AUTH_TOKEN_SECRET_NAME) + self.mock_redis.assert_called_once_with( + host=self.REDIS_HOST, + port=self.REDIS_PORT, + password="mock-auth-token", + ssl=True, + decode_responses=True, + )