From 258d7ddae36a2ff96b2077786c796e9cfe1dbb27 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 9 Jun 2026 15:40:19 -0800 Subject: [PATCH 01/31] PR-7456 empty project --- ctorm/README.md | 4 ++++ ctorm/ctorm/__init__.py | 0 ctorm/ctorm/__main__.py | 0 ctorm/ctorm/config.py | 0 ctorm/ctorm/main.py | 0 ctorm/pyproject.toml | 0 6 files changed, 4 insertions(+) create mode 100644 ctorm/README.md create mode 100644 ctorm/ctorm/__init__.py create mode 100644 ctorm/ctorm/__main__.py create mode 100644 ctorm/ctorm/config.py create mode 100644 ctorm/ctorm/main.py create mode 100644 ctorm/pyproject.toml diff --git a/ctorm/README.md b/ctorm/README.md new file mode 100644 index 0000000..4652c21 --- /dev/null +++ b/ctorm/README.md @@ -0,0 +1,4 @@ +# CTORM +Cumulus Throughput... uh... ORM? + +Load tester tool for Cumulus. diff --git a/ctorm/ctorm/__init__.py b/ctorm/ctorm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/ctorm/__main__.py b/ctorm/ctorm/__main__.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/ctorm/main.py b/ctorm/ctorm/main.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/pyproject.toml b/ctorm/pyproject.toml new file mode 100644 index 0000000..e69de29 From 1fc742124e563f933d87815096605ddbdfc08054 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 10 Jun 2026 19:36:14 -0800 Subject: [PATCH 02/31] PR-7456 basic terraform --- ctorm/infra/README.md | 17 + ctorm/infra/terraform/.terraform.lock.hcl | 46 ++ ctorm/infra/terraform/dev.tfvars | 5 + ctorm/infra/terraform/main.tf | 188 ++++++ ctorm/infra/terraform/outputs.tf | 19 + ctorm/infra/terraform/placeholder-lambda.zip | Bin 0 -> 339 bytes ctorm/infra/terraform/prod.tfvars | 5 + ctorm/infra/terraform/providers.tf | 19 + ctorm/infra/terraform/terraform.tfstate | 651 +++++++++++++++++++ ctorm/infra/terraform/terraform.tfvars | 5 + ctorm/infra/terraform/variables.tf | 20 + 11 files changed, 975 insertions(+) create mode 100644 ctorm/infra/README.md create mode 100644 ctorm/infra/terraform/.terraform.lock.hcl create mode 100644 ctorm/infra/terraform/dev.tfvars create mode 100644 ctorm/infra/terraform/main.tf create mode 100644 ctorm/infra/terraform/outputs.tf create mode 100644 ctorm/infra/terraform/placeholder-lambda.zip create mode 100644 ctorm/infra/terraform/prod.tfvars create mode 100644 ctorm/infra/terraform/providers.tf create mode 100644 ctorm/infra/terraform/terraform.tfstate create mode 100644 ctorm/infra/terraform/terraform.tfvars create mode 100644 ctorm/infra/terraform/variables.tf diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md new file mode 100644 index 0000000..1d30ec4 --- /dev/null +++ b/ctorm/infra/README.md @@ -0,0 +1,17 @@ +# CTORM infrastructure + + +```bash +export VARFILE=dev.tfvars +terraform init + +terraform plan \ + -var-file="${VARFILE}" + +terraform apply \ + -var-file="${VARFILE}" + +terrafor destroy + + +``` diff --git a/ctorm/infra/terraform/.terraform.lock.hcl b/ctorm/infra/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..e6e8e02 --- /dev/null +++ b/ctorm/infra/terraform/.terraform.lock.hcl @@ -0,0 +1,46 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.0" + constraints = "~> 2.4" + hashes = [ + "h1:WB6H5ksIZiyq1lQlD/PWeh+tn4FLsbSjVnRW3+4xe2Y=", + "zh:0d14713fdc259fb377d0b899ad3c650a34194bd52194c863303ef22a65a580e2", + "zh:369b56040c7a8085d04e7e8ffac1e2b321a3170e502f788819bc34b868ec016f", + "zh:4d1a3b983ed6af5a52bfe12794674ae55cbadfa6021b37106ade68b433ad216a", + "zh:5c547549e26e083573c78a966ca68ce6d7df6bb8f3948f66a575f07da46b74ea", + "zh:6de093e62a975eb19a5e3017ce38e6e3cb639c17b79648d2000e0a8348f0e997", + "zh:7267936c2cdbc448efeb594d73e6b56a53d6a7ae14fe88cdd2a4133adc3302f0", + "zh:7482f023050ed426b4b45116e1761643bc33b1fd4ce4a6fab207ae2571f35940", + "zh:76bbd93b234e5a2927d98b511d86565700f549b570871a194c35f944b96cefb7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:c6afc4bc1f002bac9c173007dd4da05fde788cd14c2916089f958c33fedb0dfa", + "zh:d3ba40bd806a3a08e9237dece679193c99afb2085de6b45d7f5d1f673cfcd368", + "zh:e1ad7ded53ecd6f0e5b473a3b44eae2b2e885653a56050ab583d387332be02e4", + "zh:e93e78575ce82be6084cc153c24ba8f385dc8d6880888ee66e918460c870953d", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} diff --git a/ctorm/infra/terraform/dev.tfvars b/ctorm/infra/terraform/dev.tfvars new file mode 100644 index 0000000..2d27db1 --- /dev/null +++ b/ctorm/infra/terraform/dev.tfvars @@ -0,0 +1,5 @@ +aws_profile = "cumulus-sbx-7522" +name_prefix = "ctorm-dev" + +cumulus_ingest_queue_arn = "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" +cumulus_ingest_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest" diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf new file mode 100644 index 0000000..2ed67e9 --- /dev/null +++ b/ctorm/infra/terraform/main.tf @@ -0,0 +1,188 @@ +locals { + lambda_name = "${var.name_prefix}-cnm-sender" +} + + +resource "aws_sqs_queue" "granules_dlq" { + name = "${var.name_prefix}-granule-dlq" +} + +resource "aws_sqs_queue" "granules" { + name = "${var.name_prefix}-granules" + visibility_timeout_seconds = 180 + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.granules_dlq.arn + maxReceiveCount = 5 + }) +} + +resource "aws_dynamodb_table" "state" { + name = "${var.name_prefix}-state" + billing_mode = "PAY_PER_REQUEST" + + hash_key = "pk" + range_key = "sk" + + attribute { + name = "pk" + type = "S" + } + + attribute { + name = "sk" + type = "S" + } +} + + +resource "aws_cloudwatch_log_group" "cnm-sender" { + name = "/aws/lambda/${local.lambda_name}" + retention_in_days = 14 +} + +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "cnm-sender" { + name = "${var.name_prefix}-cnm-sender-role" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json +} + +data "aws_iam_policy_document" "cnm-sender" { + statement { + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + + resources = [ + "${aws_cloudwatch_log_group.cnm-sender.arn}:*" + ] + } + + statement { + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ] + + resources = [ + aws_sqs_queue.granules.arn + ] + } + + statement { + actions = [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ] + + resources = [ + var.cumulus_ingest_queue_arn + ] + } + + statement { + actions = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:Query" + ] + + resources = [ + aws_dynamodb_table.state.arn + ] + } +} + +resource "aws_iam_role_policy" "cnm-sender" { + name = "${var.name_prefix}-cnm-sender-policy" + role = aws_iam_role.cnm-sender.id + policy = data.aws_iam_policy_document.cnm-sender.json +} + +data "archive_file" "placeholder_lambda" { + type = "zip" + output_path = "${path.module}/placeholder-lambda.zip" + + source { + filename = "index.py" + + content = <|ORE=n@940v>9OrR{^TAD>s*)h-W)3(tb7dv43*NW z%2!B5&%S0Gk=(nvJ?i;v$vgH{C0EY>=)WDheQTlbR`dOH7N2>fy0g>Gr>6Ax+`G4L zZF{%x=FDxss}HBoTW#u+(#bM={o&Pm3%i@9yL}a9VEF$(z?+@JuH@$Li$G6CF)}a& jcr!AIFdzZ~Sq?L3V8IgL&B_J}Hbx*c1Jdk3lNcBP{&;;y literal 0 HcmV?d00001 diff --git a/ctorm/infra/terraform/prod.tfvars b/ctorm/infra/terraform/prod.tfvars new file mode 100644 index 0000000..c07c055 --- /dev/null +++ b/ctorm/infra/terraform/prod.tfvars @@ -0,0 +1,5 @@ +aws_profile = "cumulus-uat-6921" +name_prefix = "ctorm" + +cumulus_ingest_queue_arn = "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" +cumulus_ingest_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest" diff --git a/ctorm/infra/terraform/providers.tf b/ctorm/infra/terraform/providers.tf new file mode 100644 index 0000000..02451ff --- /dev/null +++ b/ctorm/infra/terraform/providers.tf @@ -0,0 +1,19 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + archive = { + source = "hashicorp/archive" + version = "~> 2.4" + } + } +} + +provider "aws" { + region = "us-west-2" + profile = var.aws_profile +} diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate new file mode 100644 index 0000000..f80b91c --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate @@ -0,0 +1,651 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 12, + "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", + "outputs": { + "state_table_name": { + "value": "ctorm-dev-state", + "type": "string" + }, + "work_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "type": "string" + }, + "work_queue_arn": { + "value": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "type": "string" + }, + "work_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-dev-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "f09bd3c52a0ccf18e91ad5fb208898ddd415b903", + "output_base64sha256": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", + "output_base64sha512": "YA5YdhMOE4f3g+N2rFuq2vpYa8KXnFSH+C7bkq9ixIvMxZqDxZBI/MR5E9vrLJ8hLk/mV0kMKS1wQ269QdyIFg==", + "output_file_mode": null, + "output_md5": "de3b86299704b85f7d5621f69bdb2914", + "output_path": "./placeholder-lambda.zip", + "output_sha": "f09bd3c52a0ccf18e91ad5fb208898ddd415b903", + "output_sha256": "bbf01937538c639828cf7e2bd3fb95d4cd105dd13563565b0ced59b2ef90e291", + "output_sha512": "600e5876130e1387f783e376ac5baadafa586bc2979c5487f82edb92af62c48bccc59a83c59048fcc47913dbeb2c9f212e4fe657490c292d70436ebd41dc8816", + "output_size": 339, + "source": [ + { + "content": "import json\nimport os\n\ndef handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2455266930", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-dev-every-minute", + "is_enabled": false, + "name": "ctorm-dev-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "appsync_target": [], + "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [], + "event_bus_name": "default", + "force_destroy": false, + "http_target": [], + "id": "ctorm-dev-every-minute-terraform-20260610200222615700000001", + "input": "{\"source\":\"eventbridge\"}", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "", + "rule": "ctorm-dev-every-minute", + "run_command_targets": [], + "sagemaker_pipeline_target": [], + "sqs_target": [], + "target_id": "terraform-20260610200222615700000001" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender", + "id": "/aws/lambda/ctorm-dev-cnm-sender", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/lambda/ctorm-dev-cnm-sender", + "name_prefix": "", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "state", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", + "attribute": [ + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [], + "hash_key": "pk", + "id": "ctorm-dev-state", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-dev-state", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": null, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cnm-sender-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-10T20:01:24Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cnm-sender-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cnm-sender-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROA4VW62R3RPOQ35FULV" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", + "name": "ctorm-dev-cnm-sender-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "role": "ctorm-dev-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", + "code_sha256": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [ + { + "variables": { + "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", + "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "TABLE_NAME": "ctorm-dev-state" + } + } + ], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "./placeholder-lambda.zip", + "function_name": "ctorm-dev-cnm-sender", + "handler": "index.handler", + "id": "ctorm-dev-cnm-sender", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", + "kms_key_arn": "", + "last_modified": "2026-06-10T20:02:16.662+0000", + "layers": null, + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/ctorm-dev-cnm-sender", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::871271927522:role/ctorm-dev-cnm-sender-role", + "runtime": "python3.12", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", + "source_code_size": 339, + "tags": null, + "tags_all": {}, + "timeout": 120, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_eventbridge", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "ctorm-dev-cnm-sender", + "function_url_auth_type": null, + "id": "AllowExecutionFromEventBridge", + "principal": "events.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "statement_id": "AllowExecutionFromEventBridge", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/terraform/terraform.tfvars b/ctorm/infra/terraform/terraform.tfvars new file mode 100644 index 0000000..82288c6 --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfvars @@ -0,0 +1,5 @@ +aws_profile = "default" +name_prefix = "ctorm-dev" + +cumulus_ingest_queue_arn = "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" +cumulus_ingest_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest" diff --git a/ctorm/infra/terraform/variables.tf b/ctorm/infra/terraform/variables.tf new file mode 100644 index 0000000..155e9c3 --- /dev/null +++ b/ctorm/infra/terraform/variables.tf @@ -0,0 +1,20 @@ + +variable "aws_profile" { + type = string + description = "The named AWS profile to use for authentication" +} + +variable "name_prefix" { + type = string + default = "ctorm" +} + +variable "cumulus_ingest_queue_arn" { + type = string + description = "ARN of the target Cumulus ingest SQS queue." +} + +variable "cumulus_ingest_queue_url" { + type = string + description = "URL of the target Cumulus ingest SQS queue." +} From 5acdfbd276d159b4aa715ddb9ee783000efe3168 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 11 Jun 2026 18:04:42 -0800 Subject: [PATCH 03/31] PR-7456 A prepare() that somewhat works. --- ctorm/.python-version | 1 + ctorm/ctorm.cfg.example | 16 ++ ctorm/ctorm/__main__.py | 3 + ctorm/ctorm/config.py | 52 ++++++ ctorm/ctorm/main.py | 83 ++++++++++ ctorm/ctorm/prepare.py | 114 +++++++++++++ ctorm/pyproject.toml | 31 ++++ ctorm/uv.lock | 349 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 649 insertions(+) create mode 100644 ctorm/.python-version create mode 100644 ctorm/ctorm.cfg.example create mode 100644 ctorm/ctorm/prepare.py create mode 100644 ctorm/uv.lock diff --git a/ctorm/.python-version b/ctorm/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/ctorm/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/ctorm/ctorm.cfg.example b/ctorm/ctorm.cfg.example new file mode 100644 index 0000000..d65282f --- /dev/null +++ b/ctorm/ctorm.cfg.example @@ -0,0 +1,16 @@ +[ctorm] + +granules_sqs_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/ctorm-dev-granules" + +[[ ctorm.source_buckets ]] +bucketname = "nisar-bucket-0" +ummg_prefix = "UMMG/FOO_COLLECTION/" +share = 50 +[[ctorm.source_buckets]] +bucketname = "nisar-bucket-2" +ummg_prefix = "UMMG/BAR_COLLECTION/" +share = 45 +[[ctorm.source_buckets]] +bucketname = "opera-bucket-0" +ummg_prefix = "UMMG/BAZ_COLLECTION/" +share = 5 diff --git a/ctorm/ctorm/__main__.py b/ctorm/ctorm/__main__.py index e69de29..bb0820d 100644 --- a/ctorm/ctorm/__main__.py +++ b/ctorm/ctorm/__main__.py @@ -0,0 +1,3 @@ +from ctorm.main import main + +main() diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index e69de29..7543188 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -0,0 +1,52 @@ +import argparse +from collections.abc import Mapping +from dataclasses import MISSING, dataclass, field, fields, replace +from typing import Optional, Union + +import tomllib + +try: + from typing import Self +except ImportError: + Self = "ConfigBase" + +DEFAULT = "default" + + +class ConfigError(Exception): + pass + + +@dataclass +class CtormBucket: + bucketname: str + share: int = "" + + next_cont_token: str = None + ummg_prefix: str = "UMMG/" + + +@dataclass +class CtormConfig: + source_buckets: list + granules_sqs_queue_url: str + + @classmethod + def from_file( + cls, + cfg_file: str, + ) -> Self: + with open(cfg_file, "rb") as f: + cfg = tomllib.load(f) + if "ctorm" in cfg: + cfg = cfg["ctorm"] + else: + raise ConfigError("No 'ctorm' section in config file") + valid_fields = {f.name for f in fields(cls)} + # Filter the input dictionary + kwargs = {k: v for k, v in cfg.items() if k in valid_fields} + # create CtormBuckets for each bucket in cfg['source_buckets'] + kwargs["source_buckets"] = [CtormBucket(**b) for b in kwargs["source_buckets"]] + obj = cls(**kwargs) + + return obj diff --git a/ctorm/ctorm/main.py b/ctorm/ctorm/main.py index e69de29..9d1ac03 100644 --- a/ctorm/ctorm/main.py +++ b/ctorm/ctorm/main.py @@ -0,0 +1,83 @@ +""" +A script for load testing cumulus. + +""" + +import argparse +import json +import logging +import os +import sys +from importlib.metadata import Distribution, PackageNotFoundError +from typing import Optional + +from ctorm.config import CtormConfig +from ctorm.prepare import prepare + +log = logging.getLogger(__name__) + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + + parser.add_argument( + "--verbose", + "-v", + help="Increase verbosity level. Can be passed multiple times.", + action="count", + default=0, + ) + + parser.add_argument("--profile", help="AWS profile name") + parser.add_argument( + "--source-bucket", + help="S3 bucket source granules", + ) + parser.add_argument( + "--cfg-file", + help="Config file", + ) + return parser + + +def main(args: Optional[list[str]] = None): + parser = get_parser() + pargs = parser.parse_args(args=args) + + root_logger = logging.getLogger() + boto3_logger = logging.getLogger("boto3") + botocore_logger = logging.getLogger("botocore") + urllib3_logger = logging.getLogger("urllib3") + s3transfer_logger = logging.getLogger("s3transfer") + + root_logger.setLevel(logging.INFO) + boto3_logger.setLevel(logging.WARNING) + botocore_logger.setLevel(logging.WARNING) + urllib3_logger.setLevel(logging.WARNING) + s3transfer_logger.setLevel(logging.WARNING) + + if pargs.verbose >= 1: + root_logger.setLevel(logging.DEBUG) + if pargs.verbose >= 2: + boto3_logger.setLevel(logging.DEBUG) + botocore_logger.setLevel(logging.INFO) + urllib3_logger.setLevel(logging.DEBUG) + s3transfer_logger.setLevel(logging.DEBUG) + if pargs.verbose >= 3: + botocore_logger.setLevel(logging.DEBUG) + + base_fmt_str = "%(levelname)s: %(message)s (%(filename)s line %(lineno)d/)" + screen_fmt = logging.Formatter("%(asctime)s.%(msecs)d " + base_fmt_str, "%Y-%m-%dT%H:%M:%S") + screenlog = logging.StreamHandler() + screenlog.setFormatter(screen_fmt) + root_logger.addHandler(screenlog) + + try: + cfg = CtormConfig.from_file( + cfg_file=pargs.cfg_file, + ) + prepare(cfg) + + except Exception: + log.exception("") + sys.exit(-1) diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py new file mode 100644 index 0000000..5cd41d3 --- /dev/null +++ b/ctorm/ctorm/prepare.py @@ -0,0 +1,114 @@ +import json +import os +from logging import getLogger + +import boto3 +from botocore.paginate import PageIterator, Paginator +from mypy_boto3_s3 import S3Client +from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef + +from ctorm.config import CtormBucket, CtormConfig + +log = getLogger(__name__) + + +def get_ummg_page(s3_client: S3Client, b_cfg: CtormBucket) -> ListObjectsV2OutputTypeDef: + kwargs = { + "Bucket": b_cfg.bucketname, + "MaxKeys": b_cfg.share, + "Prefix": b_cfg.ummg_prefix, + } + if b_cfg.next_cont_token: + kwargs["ContinuationToken"] = b_cfg.next_cont_token + ret = s3_client.list_objects_v2(**kwargs) + return ret + + +def download_ummg(s3_client, bucketname: str, key: str) -> dict: + resp = s3_client.get_object(Bucket=bucketname, Key=key) + ummgfile = resp["Body"].read() + return json.loads(ummgfile) + + +def process_ummg(ummg: dict) -> dict: + outdict = {} + for f in ummg["RelatedUrls"]: + if f["Type"] == "GET DATA VIA DIRECT ACCESS" and f["Format"] == "HDF5": + outdict["u"] = f["URL"] + break + + for f in ummg["DataGranule"]["ArchiveAndDistributionInformation"]: + if f["Format"] == "HDF5": + outdict["c"] = f["Checksum"]["Value"] + break + if "c" not in outdict and "u" not in outdict: + log.error("No checksum or URL found for %s", ummg["GranuleUR"]) + outdict = {} + return outdict + + +class CtormSqsMessage: + MAX_MESSAGE_SIZE = 262144 + + def __init__(self, cfg: CtormConfig): + self.cfg = cfg + self.files = [] + + def add_file(self, file: dict): + self.files.append(file) + + def to_dict(self): + return {"files": self.files} + + def to_json(self): + return json.dumps(self.to_dict()) + + def check_message_size(self): + return len(self.to_json()) < self.MAX_MESSAGE_SIZE + + +def prepare(cfg: CtormConfig): + boto_session = boto3.Session( + region_name="us-west-2", + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + ) + s3_client = boto_session.client("s3") + sqs_client = boto_session.client("sqs") + + goal = 50 + while goal > 0: # TODO: replace with while true and an exit condition + # This is the loop that creates a SQS message from multiple objects. + sqs_msg = CtormSqsMessage(cfg) + for b in cfg.source_buckets: + # This is the loop that goes into each bucket we're interested in. + log.debug("getting objects from %s", b.bucketname) + + # get a page from obj_iterator + # We want only one page because we want to get only b.share files for this go. + page = get_ummg_page(s3_client, b) + for obj in page.get("Contents", []): + if not obj["Key"].endswith(".cmr.json"): + log.debug("skipping %s", obj["Key"]) + continue + log.debug(" %s", obj["Key"]) + # download the json object and load it into a var: + ummg = download_ummg(s3_client, b.bucketname, obj["Key"]) + file = process_ummg(ummg) + if file: + sqs_msg.add_file(file) + goal -= 1 + b.next_cont_token = page.get("NextContinuationToken") + + log.debug("goal: %d", goal) + log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) + log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) + if sqs_msg.check_message_size(): + log.debug("sqs_msg: %s", sqs_msg.to_json()) + sqs_client.send_message( + QueueUrl=cfg.granules_sqs_queue_url, + MessageBody=sqs_msg.to_json(), + ) + else: + raise Exception("Message too big") + # TODO: deal with this smarter diff --git a/ctorm/pyproject.toml b/ctorm/pyproject.toml index e69de29..60f7e03 100644 --- a/ctorm/pyproject.toml +++ b/ctorm/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "ctorm" +version = "0.1.0" +description = "Load tester for cumulus" +readme = "README.md" +authors = [ + { name = "Ben Barton", email = "bbarton@alaska.edu" } +] +requires-python = ">=3.12" +dependencies = [ + "boto3>=1.43.26", +] + +[dependency-groups] +dev = [ + "black>=26.5.1", + "boto3-stubs>=1.43.27", + "mypy-boto3-s3>=1.43.14", + "pytest>=8.0.0", + "ruff>=0.15.13", +] + +[project.scripts] +ctorm = "ctorm.main:main" + +[build-system] +requires = ["uv_build>=0.10.3,<0.11.0"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-root = "" diff --git a/ctorm/uv.lock b/ctorm/uv.lock new file mode 100644 index 0000000..153e96f --- /dev/null +++ b/ctorm/uv.lock @@ -0,0 +1,349 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "black" +version = "26.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pytokens" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/37/5628dd55bf2b34257fc7603f0fe97c40e3aaf24265f416a9c85c95ca1436/black-26.5.1.tar.gz", hash = "sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73", size = 679439, upload-time = "2026-05-18T16:53:36.107Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/7744b906703228264ef73bdd534df88ec1ef3de45c4e78f6d31b9e32d0c9/black-26.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8", size = 2012518, upload-time = "2026-05-18T17:05:20.108Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c0/c5a3b1636dfd09c42534f2b3cf33506814f6d3e066fb0879ffa16c1ae860/black-26.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217", size = 1816016, upload-time = "2026-05-18T17:05:21.84Z" }, + { url = "https://files.pythonhosted.org/packages/1f/0e/36044316b65ca471d3bb6d3703fd06fb50c6b727c3562f6a5a3153634f88/black-26.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d", size = 1884150, upload-time = "2026-05-18T17:05:23.546Z" }, + { url = "https://files.pythonhosted.org/packages/b3/33/dafc5808c2af43672912111d7c3354af1615f7e2be3bed7a878461abbe4d/black-26.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264", size = 1486825, upload-time = "2026-05-18T17:05:25.004Z" }, + { url = "https://files.pythonhosted.org/packages/82/14/b965ee6ad2a311f28bdbf692def3ee9848d2ae289dab28b27657fcee3e78/black-26.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418", size = 1288646, upload-time = "2026-05-18T17:05:26.477Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5c/c384363980e11e25ca6b93205949bb331fbf35f4e0dbec376dfa6326cec8/black-26.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3", size = 2009020, upload-time = "2026-05-18T17:05:28.132Z" }, + { url = "https://files.pythonhosted.org/packages/0b/df/9f31c5e0babbfed77d505fc5d120beb98b21b33feaeded3924ea941fe360/black-26.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0", size = 1813335, upload-time = "2026-05-18T17:05:31.266Z" }, + { url = "https://files.pythonhosted.org/packages/fb/24/8e7b9a2fa61b0afd82209efe937557d180a1fa055bd7f6161eb9defc3719/black-26.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294", size = 1881614, upload-time = "2026-05-18T17:05:32.718Z" }, + { url = "https://files.pythonhosted.org/packages/49/ad/b4e0d9365ba8ac34f6bbab62a4b1b2dd5d618fac3fa1b8db968c844201b5/black-26.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a", size = 1488925, upload-time = "2026-05-18T17:05:34.259Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4b/652b859bf5df88a751c30451b09338f7fd26a77d1271c666992f836b7711/black-26.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52", size = 1289883, upload-time = "2026-05-18T17:05:36.019Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a8da8eb208c51c7f4ce74609a45d0dcc6d8a2141e45e81ee5289d1bb0d59/black-26.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168", size = 2004800, upload-time = "2026-05-18T17:05:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/11/8a/a479296a19e383b70a725882a6cf3d786540601ff03cabbaaf1cce864c5a/black-26.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3", size = 1815576, upload-time = "2026-05-18T17:05:40.309Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/cfaf3d39f25132c156a068f6b805576c9103a84086019507c70e1911ee7d/black-26.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18", size = 1877927, upload-time = "2026-05-18T17:05:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/66/76/302e313964bcff7e28df329d39f84f5270095730d85ff0acc260610a0d82/black-26.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50", size = 1511860, upload-time = "2026-05-18T17:05:43.943Z" }, + { url = "https://files.pythonhosted.org/packages/27/4e/a3827e35e0e567f9f9ee59e2a0ab979267dca98718f25547ca8c6733afd4/black-26.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae", size = 1316632, upload-time = "2026-05-18T17:05:45.521Z" }, + { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/6a/fede22427b4e8e56bae102de50351f1a02519be86f8c8dd5347c377cfceb/boto3-1.43.26.tar.gz", hash = "sha256:33f027b7911dffe84f4896b66024c228eb856f4cf8c1ff3465be0eae1f413731", size = 113143, upload-time = "2026-06-09T19:34:20.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/39/06d683934d94972c070179952746a57adff5c1851e2396d505a9b8cd9754/boto3-1.43.26-py3-none-any.whl", hash = "sha256:7c88bfbde6abcf062d04ac410a1cf4eaf8675f025f35f3ad9accc9b0224ce6b9", size = 140537, upload-time = "2026-06-09T19:34:17.812Z" }, +] + +[[package]] +name = "boto3-stubs" +version = "1.43.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore-stubs" }, + { name = "types-s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/2e/29abbf67dc4778203f7b77ed6ad33c6fb524a6742d74af79f3d7534ebd7c/boto3_stubs-1.43.27.tar.gz", hash = "sha256:a756552f1764dbd29e255ab5c45c291246c413fbbe9047b4e085ef0f1010688d", size = 103002, upload-time = "2026-06-10T19:56:28.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/d0/0519155e5e6eb670cb03d422601e3df1b0becbc0d3c769a734c88d247909/boto3_stubs-1.43.27-py3-none-any.whl", hash = "sha256:120305c09879d927227547d0e26fb1226403c6cc983bef527ed7d1dc6157ff67", size = 70832, upload-time = "2026-06-10T19:56:17.347Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/ba/aa37cd4e72aa8b70fdb93f47129ef3b79887edcfb343bd41df2783a72f12/botocore-1.43.26.tar.gz", hash = "sha256:fd9280ac868194afcee59c7def62e0031fb4b599f7ef85360d30c7e42357c1dc", size = 15497173, upload-time = "2026-06-09T19:34:07.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/e6/5a5ec1033613e7812e5b19ec8c2a1889834fde336d8812d53019eac6e04a/botocore-1.43.26-py3-none-any.whl", hash = "sha256:eeb92265bae289555182a46341c998a656ab49c0dbdb762c65b30fe354fcc9e8", size = 15183593, upload-time = "2026-06-09T19:34:03.012Z" }, +] + +[[package]] +name = "botocore-stubs" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-awscrt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/81/79693e833291c00dc89ee610e5e915381b6f08233912e28df50106840780/botocore_stubs-1.43.14.tar.gz", hash = "sha256:9e3bc1fdd51da7473f0df726c82747a1b0ae913449d629659765c247fecc2039", size = 42738, upload-time = "2026-05-25T06:06:37.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/ca/f017727b11895908c5dedc829cf2ec35e0c4b2a26ba875db325fef2cefdf/botocore_stubs-1.43.14-py3-none-any.whl", hash = "sha256:fb98f1475c92fd718644e786b5c543a20f1b1f610e89e0a7191c3f1f429c75aa", size = 67093, upload-time = "2026-05-25T06:06:34.532Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "ctorm" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "boto3" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "boto3-stubs" }, + { name = "mypy-boto3-s3" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [{ name = "boto3", specifier = ">=1.43.26" }] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=26.5.1" }, + { name = "boto3-stubs", specifier = ">=1.43.27" }, + { name = "mypy-boto3-s3", specifier = ">=1.43.14" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "ruff", specifier = ">=0.15.13" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "mypy-boto3-s3" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/2c/fc409f9ff5904a02cf4c2c1518c34d20cb56f22b2368b35fd0adda2926f3/mypy_boto3_s3-1.43.14.tar.gz", hash = "sha256:73d54c1d0999c73c403dc9a9a3da4a9722715aba116595af08c0d4675f8bc670", size = 77078, upload-time = "2026-05-22T20:48:28.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/61/e8e74a3f4c729719efc8b1d00179bc16c302202eac32a1b56a842a997ed2/mypy_boto3_s3-1.43.14-py3-none-any.whl", hash = "sha256:ce77096d6c5f90020c45e34c83d2268ca2bb17726149ce5033751870f7fb4e97", size = 84277, upload-time = "2026-05-22T20:48:25.032Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytokens" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, + { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, + { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, + { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, + { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, + { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, + { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, + { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, + { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/5f7ec371001337d8fa61701c186ff8b613ecac1651848c5950f4c4d5f2e9/ruff-0.15.16.tar.gz", hash = "sha256:d05e78d38c78caf020b03789e25106c93017db5a0cb6e2819885018c61343b78", size = 4714267, upload-time = "2026-06-04T16:33:09.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/42/53ef1c3953f157956db9bf7861e3bc50b9b887ce93300aa48cdba8336fe6/ruff-0.15.16-py3-none-linux_armv6l.whl", hash = "sha256:6ac3c0b3969cc6cf6b158c4e2f8f682acb58e7d700d8a44b65ecdc72d66ab0b2", size = 10709025, upload-time = "2026-06-04T16:32:51.935Z" }, + { url = "https://files.pythonhosted.org/packages/93/9a/a79159346f19134a956607754e57d8d128f7a4c00f4ad2f7514d224c172c/ruff-0.15.16-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:197c207ed75ffba54a0dec23db4aa939a27a3053073e085e0042433cbdc58e4a", size = 11063550, upload-time = "2026-06-04T16:32:42.24Z" }, + { url = "https://files.pythonhosted.org/packages/bc/72/3ce2ac000a5299ec238e01f51397b3b653c93b077d9b1bfe8715bb895f20/ruff-0.15.16-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3a39fec45ab316cc23e7558f23fea4a70403ddb5648ea9a4a3854a16973d0071", size = 10421345, upload-time = "2026-06-04T16:32:37.251Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c2/cc7fad3ec9169373f5b6a18f1917b91080feec40c3f9658334a1d28e2f03/ruff-0.15.16-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba93191d79003116b95128c9d306e045200fdbd0bccb782b110f3cd1d4abc5cf", size = 10757217, upload-time = "2026-06-04T16:32:54.722Z" }, + { url = "https://files.pythonhosted.org/packages/69/d2/3474009eaa0a65b31fa7152a2fad5e2f050c640ceb1e6b02ee6922e94c82/ruff-0.15.16-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6ee4b90520630120ef032aa5cc10db483852dff950e78b1d717e2993a61ac8d", size = 10507035, upload-time = "2026-06-04T16:33:05.343Z" }, + { url = "https://files.pythonhosted.org/packages/ca/81/b7ae6ccbd11f0c8dc3d5d67fc4be9b57ff57ca86ba56152021378e1277f2/ruff-0.15.16-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e4215bc938bc3c8215c1472c1aa437e310fee20cd427335fec9d7e609563628", size = 11255291, upload-time = "2026-06-04T16:32:49.49Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e1/46e526f1a7cc90857ce6ddf25fbb77eb6568651ac38d71b033af07076dd5/ruff-0.15.16-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c8d26be963b090f10e29abc8b3e74a2a321f6fa34e02424e30b5af89350ecbb", size = 12124922, upload-time = "2026-06-04T16:33:07.821Z" }, + { url = "https://files.pythonhosted.org/packages/1a/da/5c791b088b596b24d0deb967fa28ae02ad751a140c0b9ea81c5ab915d6c0/ruff-0.15.16-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f198cf4123602a2280ed46c307bcbafe41758d6fee5b456b6b6058ca1514b3b4", size = 11332186, upload-time = "2026-06-04T16:33:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/72/11/5da87abe20047c8962361473923ebb2f62b595250126aadfad8c20649c1e/ruff-0.15.16-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb27515fa6240fb586ae82b901a59e67d24acff86f2190b433dc542fe0435aeb", size = 11373541, upload-time = "2026-06-04T16:32:47.007Z" }, + { url = "https://files.pythonhosted.org/packages/fe/2a/8554754c23a854ae3fd6b507e36ad61ddb121e298c6d5d617dec94ed0f14/ruff-0.15.16-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a267c46ba1593fc26b8eecbea050b39d40c0b6bb7781ee11c90a02cd10032951", size = 11353014, upload-time = "2026-06-04T16:32:34.795Z" }, + { url = "https://files.pythonhosted.org/packages/62/25/62ea41529ec89f742ea3fed9cb1059c72877ec7cf9b9e99ac9cf3294d1d9/ruff-0.15.16-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:528c68f39a91498a8d50e91ff5985df3d105782bab49cc378e73ac26bff083e8", size = 10737467, upload-time = "2026-06-04T16:32:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/90/17/334d3ad9de4d40f9dd58fdd09e35ce64553bb501e2f19a839e2fb6be14fc/ruff-0.15.16-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7ed55c58950df60589a9a7a5d2f8fa5f54ebd287163be805adfe6ee95a9de123", size = 10521910, upload-time = "2026-06-04T16:32:32.54Z" }, + { url = "https://files.pythonhosted.org/packages/4d/bd/3ac7c6ae77a885c1004b3dda2446ea401768d24f851c14b4ad4b24f6639c/ruff-0.15.16-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d482feaf51512b50f9790ceb417a56a61dd1e9d9bf967662b9ed27c01b34f53a", size = 10979190, upload-time = "2026-06-04T16:32:57.492Z" }, + { url = "https://files.pythonhosted.org/packages/33/d7/609546e6a413c3f216fbf2a50c928f97c80939154f6a0503114094a86191/ruff-0.15.16-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e15bc8c94513dae2a40cc9ef07c94fdd4ecc9e29dabebeebe170f952322c9e3", size = 11477014, upload-time = "2026-06-04T16:32:44.687Z" }, + { url = "https://files.pythonhosted.org/packages/74/0d/f2cd247ad32633a5c36e97141a2c21b11c6279f7957bc2ff360b1e08fddd/ruff-0.15.16-py3-none-win32.whl", hash = "sha256:580378f7bd4aa25f72e74aa54948a9622f142b1e509521dd10902e886681cc1e", size = 10735541, upload-time = "2026-06-04T16:32:30.145Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/02e845ef151b1dee585e55c4739f8e1734ae1d9f1221dff65761c162208b/ruff-0.15.16-py3-none-win_amd64.whl", hash = "sha256:408256017284eddf98fff77b29aa4fb30f586042d535b2d9befc6512f400aaec", size = 11843403, upload-time = "2026-06-04T16:32:39.76Z" }, + { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1f/12417f7f493fc45e1f9fd5d4a9b6c125cf8d2cf3f8ddbdfab3e76406e9d6/s3transfer-0.18.0.tar.gz", hash = "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd", size = 160560, upload-time = "2026-05-28T19:39:09.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/58/a58fc997655386daa2e25784e30c288aa3e3819e401f77029ee4899fb55a/s3transfer-0.18.0-py3-none-any.whl", hash = "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6", size = 88572, upload-time = "2026-05-28T19:39:07.999Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "types-awscrt" +version = "0.34.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/59/44409a8fc06b444ab1a6f71dcb29d49a6e17e02424345eb51b051bebb345/types_awscrt-0.34.1.tar.gz", hash = "sha256:559aa04250f6a419a617dfb788f3e10903aaf74700ef23e521b64a411b83b803", size = 19062, upload-time = "2026-06-05T04:40:10.689Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/b1/214b12162b452ed6acd230065e6c587cde6b96871e3ce6d653f40888f8df/types_awscrt-0.34.1-py3-none-any.whl", hash = "sha256:20c752b6031544d8f694803c35174aee129f1be5ddf886ae46d22f7ffd9b7d75", size = 45688, upload-time = "2026-06-05T04:40:09.198Z" }, +] + +[[package]] +name = "types-s3transfer" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/64/42689150509eb3e6e82b33ee3d89045de1592488842ddf23c56957786d05/types_s3transfer-0.16.0.tar.gz", hash = "sha256:b4636472024c5e2b62278c5b759661efeb52a81851cde5f092f24100b1ecb443", size = 13557, upload-time = "2025-12-08T08:13:09.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/27/e88220fe6274eccd3bdf95d9382918716d312f6f6cef6a46332d1ee2feff/types_s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:1c0cd111ecf6e21437cb410f5cddb631bfb2263b77ad973e79b9c6d0cb24e0ef", size = 19247, upload-time = "2025-12-08T08:13:08.426Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] From f591074c0cf9928259c0137f76d9cdd16c992da4 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 12 Jun 2026 13:40:41 -0800 Subject: [PATCH 04/31] PR-7456 More progress towards ctorm prepare --- ctorm/ctorm/config.py | 8 +- ctorm/ctorm/main.py | 8 +- ctorm/ctorm/prepare.py | 211 +++++++++++++++++++++++++---------------- ctorm/infra/README.md | 3 +- 4 files changed, 134 insertions(+), 96 deletions(-) diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index 7543188..dad99cd 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -1,14 +1,10 @@ -import argparse -from collections.abc import Mapping -from dataclasses import MISSING, dataclass, field, fields, replace -from typing import Optional, Union - import tomllib +from dataclasses import dataclass, fields try: from typing import Self except ImportError: - Self = "ConfigBase" + Self = "CtormConfig" DEFAULT = "default" diff --git a/ctorm/ctorm/main.py b/ctorm/ctorm/main.py index 9d1ac03..7f99f54 100644 --- a/ctorm/ctorm/main.py +++ b/ctorm/ctorm/main.py @@ -4,15 +4,12 @@ """ import argparse -import json import logging -import os import sys -from importlib.metadata import Distribution, PackageNotFoundError from typing import Optional from ctorm.config import CtormConfig -from ctorm.prepare import prepare +from ctorm.prepare import CtormPrepare log = logging.getLogger(__name__) @@ -76,7 +73,8 @@ def main(args: Optional[list[str]] = None): cfg = CtormConfig.from_file( cfg_file=pargs.cfg_file, ) - prepare(cfg) + prepare = CtormPrepare(cfg) + prepare.prepare() except Exception: log.exception("") diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 5cd41d3..1407568 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -1,52 +1,18 @@ +import hashlib import json import os +import re from logging import getLogger import boto3 -from botocore.paginate import PageIterator, Paginator -from mypy_boto3_s3 import S3Client -from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef +from botocore.exceptions import ClientError +from mypy_boto3_s3.type_defs import HeadObjectRequestTypeDef, ListObjectsV2OutputTypeDef from ctorm.config import CtormBucket, CtormConfig log = getLogger(__name__) -def get_ummg_page(s3_client: S3Client, b_cfg: CtormBucket) -> ListObjectsV2OutputTypeDef: - kwargs = { - "Bucket": b_cfg.bucketname, - "MaxKeys": b_cfg.share, - "Prefix": b_cfg.ummg_prefix, - } - if b_cfg.next_cont_token: - kwargs["ContinuationToken"] = b_cfg.next_cont_token - ret = s3_client.list_objects_v2(**kwargs) - return ret - - -def download_ummg(s3_client, bucketname: str, key: str) -> dict: - resp = s3_client.get_object(Bucket=bucketname, Key=key) - ummgfile = resp["Body"].read() - return json.loads(ummgfile) - - -def process_ummg(ummg: dict) -> dict: - outdict = {} - for f in ummg["RelatedUrls"]: - if f["Type"] == "GET DATA VIA DIRECT ACCESS" and f["Format"] == "HDF5": - outdict["u"] = f["URL"] - break - - for f in ummg["DataGranule"]["ArchiveAndDistributionInformation"]: - if f["Format"] == "HDF5": - outdict["c"] = f["Checksum"]["Value"] - break - if "c" not in outdict and "u" not in outdict: - log.error("No checksum or URL found for %s", ummg["GranuleUR"]) - outdict = {} - return outdict - - class CtormSqsMessage: MAX_MESSAGE_SIZE = 262144 @@ -67,48 +33,127 @@ def check_message_size(self): return len(self.to_json()) < self.MAX_MESSAGE_SIZE -def prepare(cfg: CtormConfig): - boto_session = boto3.Session( - region_name="us-west-2", - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - ) - s3_client = boto_session.client("s3") - sqs_client = boto_session.client("sqs") - - goal = 50 - while goal > 0: # TODO: replace with while true and an exit condition - # This is the loop that creates a SQS message from multiple objects. - sqs_msg = CtormSqsMessage(cfg) - for b in cfg.source_buckets: - # This is the loop that goes into each bucket we're interested in. - log.debug("getting objects from %s", b.bucketname) - - # get a page from obj_iterator - # We want only one page because we want to get only b.share files for this go. - page = get_ummg_page(s3_client, b) - for obj in page.get("Contents", []): - if not obj["Key"].endswith(".cmr.json"): - log.debug("skipping %s", obj["Key"]) - continue - log.debug(" %s", obj["Key"]) - # download the json object and load it into a var: - ummg = download_ummg(s3_client, b.bucketname, obj["Key"]) - file = process_ummg(ummg) - if file: - sqs_msg.add_file(file) - goal -= 1 - b.next_cont_token = page.get("NextContinuationToken") - - log.debug("goal: %d", goal) - log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) - log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) - if sqs_msg.check_message_size(): - log.debug("sqs_msg: %s", sqs_msg.to_json()) - sqs_client.send_message( - QueueUrl=cfg.granules_sqs_queue_url, - MessageBody=sqs_msg.to_json(), - ) - else: - raise Exception("Message too big") - # TODO: deal with this smarter +class CtormPrepare: + MD5_DL_CHUNK_MB = 8 + + def __init__(self, cfg: CtormConfig): + self.cfg = cfg + self.boto_session = boto3.Session( + region_name="us-west-2", + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + ) + self.s3_client = self.boto_session.client("s3") + self.sqs_client = self.boto_session.client("sqs") + + def prepare(self): + goal = 50 + while goal > 0: # TODO: replace with while true and an exit condition + # This is the loop that creates a SQS message from multiple objects. + sqs_msg = CtormSqsMessage(self.cfg) + for b in self.cfg.source_buckets: + # This is the loop that goes into each bucket we're interested in. + log.debug("getting objects from %s", b.bucketname) + + # get a page from obj_iterator + # We want only one page because we want to get only b.share files for this go. + page = self.get_ummg_page(b) + for obj in page.get("Contents", []): + if not obj["Key"].endswith(".cmr.json"): + log.debug("skipping %s", obj["Key"]) + continue + log.debug(" %s", obj["Key"]) + # download the json object and load it into a var: + ummg = self.download_ummg(b.bucketname, obj["Key"]) + file = self.process_ummg(ummg, b) + if file: + sqs_msg.add_file(file) + goal -= 1 + b.next_cont_token = page.get("NextContinuationToken") + + log.debug("goal: %d", goal) + log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) + log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) + if sqs_msg.check_message_size(): + log.debug("sqs_msg: %s", sqs_msg.to_json()) + self.sqs_client.send_message( + QueueUrl=self.cfg.granules_sqs_queue_url, + MessageBody=sqs_msg.to_json(), + ) + else: + raise Exception("Message too big") + # TODO: deal with this smarter + + def get_ummg_page(self, ct_bukt: CtormBucket) -> ListObjectsV2OutputTypeDef: + kwargs = { + "Bucket": ct_bukt.bucketname, + "MaxKeys": ct_bukt.share, + "Prefix": ct_bukt.ummg_prefix, + } + if ct_bukt.next_cont_token: + kwargs["ContinuationToken"] = ct_bukt.next_cont_token + ret = self.s3_client.list_objects_v2(**kwargs) + return ret + + def download_ummg(self, bucketname: str, key: str) -> dict: + resp = self.s3_client.get_object(Bucket=bucketname, Key=key) + ummgfile = resp["Body"].read() + return json.loads(ummgfile) + + def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: + outdict = { + "bm": {ct_bkt.bucketname: "B1"}, # bucket map + "g": ummg["GranuleUR"], + "c": ummg["CollectionReference"]["ShortName"], + "cv": ummg["CollectionReference"]["Version"], + "f": [], # list of files + } + + for f in ummg["RelatedUrls"]: + if f["URL"].startswith("s3://"): + bucket = re.sub(r"^s3://([^/]+).*$", r"\1", f["URL"]) + if bucket != ct_bkt.bucketname: + outdict["bm"][bucket] = f"B{len(outdict['bm'])}" + objloc = f["URL"].removeprefix(f"s3://{bucket}/") + + # Replace the granulename with a token for compression. Will reconstitute in the lambda + fileval = f["URL"].replace(outdict["g"], "$G").replace(bucket, f"${outdict['bm'][bucket]}") + filedict = {"f": fileval} + for m in ummg["DataGranule"]["ArchiveAndDistributionInformation"]: + if f["URL"].endswith(m["Name"]): + # We handily have the md5 and size in the ummg + filedict["m"] = m["Checksum"]["Value"] + filedict["s"] = m["SizeInBytes"] + break + else: + # We must look to S3 for the size and md5 + log.debug('getting head for "%s"', objloc) + try: + h = self.s3_client.head_object(Bucket=bucket, Key=objloc) + log.debug("head_object: %s", h) + filedict["s"] = h["ContentLength"] + filedict["m"] = h["ETag"].replace('"', "") + if filedict["m"].endswith("-1"): + # This was a multipart upload. We'll have to do something clever to get the MD5 of it. + log.debug("multipart upload detected for %s", objloc) + filedict["m"] = self.get_real_md5(h, bucket, objloc) + except ClientError as e: + log.error("head_object failed: %s", e) + # TODO: trash entire message for this granule? + + outdict["f"].append(filedict) + + return outdict + + def get_real_md5( + self, + head_obj_ret: HeadObjectRequestTypeDef, + bucket: str, + key: str, + ) -> str: + # The file is small enough we may as well download it to memory and get the MD5 that way. + resp = self.s3_client.get_object(Bucket=bucket, Key=key) + md5_accumulator = hashlib.md5() + for chunk in iter(lambda: resp["Body"].read(self.MD5_DL_CHUNK_MB * 1024 * 1024), b""): + md5_accumulator.update(chunk) + return md5_accumulator.hexdigest() diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 1d30ec4..67aaa58 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -1,6 +1,5 @@ # CTORM infrastructure - ```bash export VARFILE=dev.tfvars terraform init @@ -11,7 +10,7 @@ terraform plan \ terraform apply \ -var-file="${VARFILE}" -terrafor destroy +terraform destroy ``` From f1be9a9bde6087e060ab89fba2c3ea338712d425 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 12 Jun 2026 15:51:50 -0800 Subject: [PATCH 05/31] PR-7456 unit tests for ctorm --- .github/workflows/test.yml | 28 +++++++++++++++---------- Makefile | 4 ++++ ctorm/pyproject.toml | 16 +++++++++++++- ctorm/tests/__init__.py | 0 ctorm/tests/test_config.py | 43 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 ctorm/tests/__init__.py create mode 100644 ctorm/tests/test_config.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 30de2f6..5809c84 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,17 +8,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: [ '3.9', '3.10', '3.11', '3.12' ] fail-fast: false steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install poetry "tox<4.0.0" tox-poetry tox-gh-actions - - run: make tests + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry "tox<4.0.0" tox-poetry tox-gh-actions + - name: Run tests + run: | + make tests + if [ "${{ matrix.python-version }}" = "3.12" ]; then + # ctorm supports py312 only. + make ctorm-test + fi diff --git a/Makefile b/Makefile index 92b819c..d562f55 100644 --- a/Makefile +++ b/Makefile @@ -13,3 +13,7 @@ TOX_INIS := $(PACKAGES:%=%/tox.ini) .PHONY: tests tests: $(TOX_INIS) ss=0; $(foreach package,$(PACKAGES),cd $(package) && tox || ss=1;cd ..;) exit $$ss + +.PHONY: ctorm-test +ctorm-test: + cd ctorm && tox -c pyproject.toml diff --git a/ctorm/pyproject.toml b/ctorm/pyproject.toml index 60f7e03..faea8cd 100644 --- a/ctorm/pyproject.toml +++ b/ctorm/pyproject.toml @@ -9,13 +9,13 @@ authors = [ requires-python = ">=3.12" dependencies = [ "boto3>=1.43.26", + "mypy-boto3-s3>=1.43.14", ] [dependency-groups] dev = [ "black>=26.5.1", "boto3-stubs>=1.43.27", - "mypy-boto3-s3>=1.43.14", "pytest>=8.0.0", "ruff>=0.15.13", ] @@ -29,3 +29,17 @@ build-backend = "uv_build" [tool.uv.build-backend] module-root = "" + + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = py312 +[testenv] +deps = + mypy-boto3-s3 + pytest + pytest-mock +commands = + pytest +""" diff --git a/ctorm/tests/__init__.py b/ctorm/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/tests/test_config.py b/ctorm/tests/test_config.py new file mode 100644 index 0000000..cf6b4e5 --- /dev/null +++ b/ctorm/tests/test_config.py @@ -0,0 +1,43 @@ +from unittest.mock import mock_open, patch + +from ctorm.config import ConfigError, CtormBucket, CtormConfig + + +def test_ctorm_bucket_initialization(): + bucket = CtormBucket(bucketname="test_bucket", share=50, ummg_prefix="test_prefix/") + + assert bucket.bucketname == "test_bucket" + assert bucket.share == 50 + assert bucket.ummg_prefix == "test_prefix/" + assert bucket.next_cont_token is None + + +def test_ctorm_config_from_file_success(): + mock_data = """ + [ctorm] + source_buckets = [{"bucketname": "test_bucket", "share": 50}] + granules_sqs_queue_url = "https://sqs.queue.url/" + """ + with patch("builtins.open", mock_open(read_data=mock_data)), patch("tomllib.load") as mock_toml: + mock_toml.return_value = { + "ctorm": { + "source_buckets": [{"bucketname": "test_bucket", "share": 50}], + "granules_sqs_queue_url": "https://sqs.queue.url/", + } + } + config = CtormConfig.from_file("test.cfg") + + assert config.granules_sqs_queue_url == "https://sqs.queue.url/" + assert len(config.source_buckets) == 1 + assert isinstance(config.source_buckets[0], CtormBucket) + assert config.source_buckets[0].bucketname == "test_bucket" + + +def test_ctorm_config_from_file_missing_section(): + mock_data = "{}" + with patch("builtins.open", mock_open(read_data=mock_data)), patch("tomllib.load") as mock_toml: + mock_toml.return_value = {} + try: + CtormConfig.from_file("test.cfg") + except ConfigError as e: + assert str(e) == "No 'ctorm' section in config file" From c835c4c16cdc2bc7ea72cd5ddc62ecb856f146d8 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Mon, 15 Jun 2026 16:07:54 -0800 Subject: [PATCH 06/31] PR-7456 replace aws client member vars with cached functions, review suggestion Co-Authored-By: Cameron Showalter --- ctorm/ctorm/prepare.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 1407568..4b849ef 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -2,6 +2,7 @@ import json import os import re +from functools import cache from logging import getLogger import boto3 @@ -13,6 +14,25 @@ log = getLogger(__name__) +@cache +def get_boto_session(): + + kwargs = {"region_name": AWS_REGION} + if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"): + kwargs["aws_access_key_id"] = os.getenv("AWS_ACCESS_KEY_ID") + kwargs["aws_secret_access_key"] = os.getenv("AWS_SECRET_ACCESS_KEY") + + return boto3.Session(**kwargs) + + +@cache +def get_s3_client(): + return get_boto_session().client("s3", region_name=AWS_REGION) + + +@cache +def get_sqs_client(): + return get_boto_session().client("sqs", region_name=AWS_REGION) class CtormSqsMessage: MAX_MESSAGE_SIZE = 262144 @@ -76,7 +96,7 @@ def prepare(self): log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) if sqs_msg.check_message_size(): log.debug("sqs_msg: %s", sqs_msg.to_json()) - self.sqs_client.send_message( + get_sqs_client().send_message( QueueUrl=self.cfg.granules_sqs_queue_url, MessageBody=sqs_msg.to_json(), ) @@ -92,11 +112,11 @@ def get_ummg_page(self, ct_bukt: CtormBucket) -> ListObjectsV2OutputTypeDef: } if ct_bukt.next_cont_token: kwargs["ContinuationToken"] = ct_bukt.next_cont_token - ret = self.s3_client.list_objects_v2(**kwargs) + ret = get_s3_client().list_objects_v2(**kwargs) return ret def download_ummg(self, bucketname: str, key: str) -> dict: - resp = self.s3_client.get_object(Bucket=bucketname, Key=key) + resp = get_s3_client().get_object(Bucket=bucketname, Key=key) ummgfile = resp["Body"].read() return json.loads(ummgfile) @@ -129,11 +149,11 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: # We must look to S3 for the size and md5 log.debug('getting head for "%s"', objloc) try: - h = self.s3_client.head_object(Bucket=bucket, Key=objloc) log.debug("head_object: %s", h) filedict["s"] = h["ContentLength"] filedict["m"] = h["ETag"].replace('"', "") if filedict["m"].endswith("-1"): + headobj = get_s3_client().head_object(Bucket=bucket, Key=objloc) # This was a multipart upload. We'll have to do something clever to get the MD5 of it. log.debug("multipart upload detected for %s", objloc) filedict["m"] = self.get_real_md5(h, bucket, objloc) @@ -152,7 +172,7 @@ def get_real_md5( key: str, ) -> str: # The file is small enough we may as well download it to memory and get the MD5 that way. - resp = self.s3_client.get_object(Bucket=bucket, Key=key) + resp = get_s3_client().get_object(Bucket=bucket, Key=key) md5_accumulator = hashlib.md5() for chunk in iter(lambda: resp["Body"].read(self.MD5_DL_CHUNK_MB * 1024 * 1024), b""): md5_accumulator.update(chunk) From 79935ecf39533633ac6d3af86f7d71b73977bfb7 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Mon, 15 Jun 2026 16:09:05 -0800 Subject: [PATCH 07/31] PR-7456 review suggestions. Also verbosified some vars --- ctorm/ctorm/config.py | 7 ++-- ctorm/ctorm/prepare.py | 74 ++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index dad99cd..a774f31 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -7,10 +7,7 @@ Self = "CtormConfig" DEFAULT = "default" - - -class ConfigError(Exception): - pass +AWS_REGION = "us-west-2" # We will never not want us-west-2 @dataclass @@ -37,7 +34,7 @@ def from_file( if "ctorm" in cfg: cfg = cfg["ctorm"] else: - raise ConfigError("No 'ctorm' section in config file") + raise KeyError("No 'ctorm' section in config file") valid_fields = {f.name for f in fields(cls)} # Filter the input dictionary kwargs = {k: v for k, v in cfg.items() if k in valid_fields} diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 4b849ef..4f1fc6d 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -2,21 +2,21 @@ import json import os import re +from enum import StrEnum from functools import cache from logging import getLogger import boto3 from botocore.exceptions import ClientError -from mypy_boto3_s3.type_defs import HeadObjectRequestTypeDef, ListObjectsV2OutputTypeDef +from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef -from ctorm.config import CtormBucket, CtormConfig +from ctorm.config import AWS_REGION, CtormBucket, CtormConfig log = getLogger(__name__) @cache def get_boto_session(): - kwargs = {"region_name": AWS_REGION} if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"): kwargs["aws_access_key_id"] = os.getenv("AWS_ACCESS_KEY_ID") @@ -33,6 +33,22 @@ def get_s3_client(): @cache def get_sqs_client(): return get_boto_session().client("sqs", region_name=AWS_REGION) + + +class K(StrEnum): + """Since we want to keep the message json as light as possible, we'll keep + the keys for the message sent to the CTORM SQS queue here. + """ + + BKT_MAP = "bm" + GRANULE = "g" + COLLECTION = "c" + COLLECTION_VERSION = "cv" + FILES = "f" + MD5 = "m" + SIZE = "s" + + class CtormSqsMessage: MAX_MESSAGE_SIZE = 262144 @@ -58,13 +74,6 @@ class CtormPrepare: def __init__(self, cfg: CtormConfig): self.cfg = cfg - self.boto_session = boto3.Session( - region_name="us-west-2", - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - ) - self.s3_client = self.boto_session.client("s3") - self.sqs_client = self.boto_session.client("sqs") def prepare(self): goal = 50 @@ -122,52 +131,53 @@ def download_ummg(self, bucketname: str, key: str) -> dict: def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: outdict = { - "bm": {ct_bkt.bucketname: "B1"}, # bucket map - "g": ummg["GranuleUR"], - "c": ummg["CollectionReference"]["ShortName"], - "cv": ummg["CollectionReference"]["Version"], - "f": [], # list of files + K.BKT_MAP: {ct_bkt.bucketname: "B1"}, # bucket map + K.GRANULE: ummg["GranuleUR"], + K.COLLECTION: ummg["CollectionReference"]["ShortName"], + K.COLLECTION_VERSION: ummg["CollectionReference"]["Version"], + K.FILES: [], # list of files } - for f in ummg["RelatedUrls"]: - if f["URL"].startswith("s3://"): - bucket = re.sub(r"^s3://([^/]+).*$", r"\1", f["URL"]) + for r_urls in ummg["RelatedUrls"]: + if r_urls["URL"].startswith("s3://"): + bucket = re.sub(r"^s3://([^/]+).*$", r"\1", r_urls["URL"]) if bucket != ct_bkt.bucketname: - outdict["bm"][bucket] = f"B{len(outdict['bm'])}" - objloc = f["URL"].removeprefix(f"s3://{bucket}/") + outdict[K.BKT_MAP][bucket] = f"B{len(outdict[K.BKT_MAP])}" + objloc = r_urls["URL"].removeprefix(f"s3://{bucket}/") # Replace the granulename with a token for compression. Will reconstitute in the lambda - fileval = f["URL"].replace(outdict["g"], "$G").replace(bucket, f"${outdict['bm'][bucket]}") + fileval = ( + r_urls["URL"].replace(outdict[K.GRANULE], "$G").replace(bucket, f"${outdict[K.BKT_MAP][bucket]}") + ) filedict = {"f": fileval} - for m in ummg["DataGranule"]["ArchiveAndDistributionInformation"]: - if f["URL"].endswith(m["Name"]): + for distr_file in ummg["DataGranule"]["ArchiveAndDistributionInformation"]: + if r_urls["URL"].endswith(distr_file["Name"]): # We handily have the md5 and size in the ummg - filedict["m"] = m["Checksum"]["Value"] - filedict["s"] = m["SizeInBytes"] + filedict[K.MD5] = distr_file["Checksum"]["Value"] + filedict[K.SIZE] = distr_file["SizeInBytes"] break else: # We must look to S3 for the size and md5 log.debug('getting head for "%s"', objloc) try: - log.debug("head_object: %s", h) - filedict["s"] = h["ContentLength"] - filedict["m"] = h["ETag"].replace('"', "") - if filedict["m"].endswith("-1"): headobj = get_s3_client().head_object(Bucket=bucket, Key=objloc) + log.debug("head_object: %s", headobj) + filedict[K.SIZE] = headobj["ContentLength"] + filedict[K.MD5] = headobj["ETag"].replace('"', "") + if filedict[K.MD5].endswith("-1"): # This was a multipart upload. We'll have to do something clever to get the MD5 of it. log.debug("multipart upload detected for %s", objloc) - filedict["m"] = self.get_real_md5(h, bucket, objloc) + filedict[K.MD5] = self.get_real_md5(bucket, objloc) except ClientError as e: log.error("head_object failed: %s", e) # TODO: trash entire message for this granule? - outdict["f"].append(filedict) + outdict[K.FILES].append(filedict) return outdict def get_real_md5( self, - head_obj_ret: HeadObjectRequestTypeDef, bucket: str, key: str, ) -> str: From f5c1bca4ca3cbc6796f5d716658294ffb869649b Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 18 Jun 2026 10:33:46 -0800 Subject: [PATCH 08/31] PR-7501 review suggestions Co-Authored-By: Cameron Showalter --- ctorm/ctorm/config.py | 9 +++------ ctorm/ctorm/prepare.py | 15 ++------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index a774f31..e6b84fd 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -1,6 +1,7 @@ -import tomllib from dataclasses import dataclass, fields +import tomllib + try: from typing import Self except ImportError: @@ -31,13 +32,9 @@ def from_file( ) -> Self: with open(cfg_file, "rb") as f: cfg = tomllib.load(f) - if "ctorm" in cfg: - cfg = cfg["ctorm"] - else: - raise KeyError("No 'ctorm' section in config file") valid_fields = {f.name for f in fields(cls)} # Filter the input dictionary - kwargs = {k: v for k, v in cfg.items() if k in valid_fields} + kwargs = {k: v for k, v in cfg["ctorm"].items() if k in valid_fields} # create CtormBuckets for each bucket in cfg['source_buckets'] kwargs["source_buckets"] = [CtormBucket(**b) for b in kwargs["source_buckets"]] obj = cls(**kwargs) diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 4f1fc6d..0508485 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -1,6 +1,5 @@ import hashlib import json -import os import re from enum import StrEnum from functools import cache @@ -15,24 +14,14 @@ log = getLogger(__name__) -@cache -def get_boto_session(): - kwargs = {"region_name": AWS_REGION} - if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"): - kwargs["aws_access_key_id"] = os.getenv("AWS_ACCESS_KEY_ID") - kwargs["aws_secret_access_key"] = os.getenv("AWS_SECRET_ACCESS_KEY") - - return boto3.Session(**kwargs) - - @cache def get_s3_client(): - return get_boto_session().client("s3", region_name=AWS_REGION) + return boto3.client("s3", region_name=AWS_REGION) @cache def get_sqs_client(): - return get_boto_session().client("sqs", region_name=AWS_REGION) + return boto3.client("sqs", region_name=AWS_REGION) class K(StrEnum): From e46bd3d49e63d0dea83e8767482b66df3d4976ab Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 18 Jun 2026 12:19:58 -0800 Subject: [PATCH 09/31] PR-7502 stubs for command to send cnm --- ctorm/ctorm/cnm_sender.py | 6 +++++ ctorm/ctorm/commands/__init__.py | 27 ++++++++++++++++++++++ ctorm/ctorm/config.py | 3 +-- ctorm/ctorm/main.py | 39 ++++++++++++++++++++++++-------- 4 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 ctorm/ctorm/cnm_sender.py create mode 100644 ctorm/ctorm/commands/__init__.py diff --git a/ctorm/ctorm/cnm_sender.py b/ctorm/ctorm/cnm_sender.py new file mode 100644 index 0000000..645aa4a --- /dev/null +++ b/ctorm/ctorm/cnm_sender.py @@ -0,0 +1,6 @@ +class CnmSender: + def __init__(self, cfg): + self.cfg = cfg + + def send(self): + pass diff --git a/ctorm/ctorm/commands/__init__.py b/ctorm/ctorm/commands/__init__.py new file mode 100644 index 0000000..3f5b404 --- /dev/null +++ b/ctorm/ctorm/commands/__init__.py @@ -0,0 +1,27 @@ +import argparse +import logging + +from ctorm.config import CtormConfig +from ctorm.prepare import CtormPrepare + +log = logging.getLogger(__name__) + + +def cmd_prepare(args: argparse.Namespace): + cfg = CtormConfig.from_file( + cfg_file=args.cfg_file, + ) + prepare = CtormPrepare(cfg) + prepare.prepare() + + +def cmd_cnm_sender(args: argparse.Namespace): + # TODO: something like: + # + # from ctorm.cnm_sender import CnmSender + # + # cfg = CtormConfig.from_file(cfg_file=args.cfg_file) + # cnm_sender = CnmSender(cfg) + # cnm_sender.run(max_messages=args.max_messages) + # + raise NotImplementedError("cnm_sender command is not implemented yet") diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index e6b84fd..dfe026c 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -1,6 +1,5 @@ -from dataclasses import dataclass, fields - import tomllib +from dataclasses import dataclass, fields try: from typing import Self diff --git a/ctorm/ctorm/main.py b/ctorm/ctorm/main.py index 7f99f54..000ceb5 100644 --- a/ctorm/ctorm/main.py +++ b/ctorm/ctorm/main.py @@ -8,8 +8,7 @@ import sys from typing import Optional -from ctorm.config import CtormConfig -from ctorm.prepare import CtormPrepare +from ctorm.commands import cmd_cnm_sender, cmd_prepare log = logging.getLogger(__name__) @@ -27,13 +26,37 @@ def get_parser() -> argparse.ArgumentParser: parser.add_argument("--profile", help="AWS profile name") parser.add_argument( + "--cfg-file", + help="Config file", + ) + + subparsers = parser.add_subparsers( + dest="command", + required=True, + ) + + prepare_parser = subparsers.add_parser( + "prepare", + help="Fill the granules SQS queue with work for the load test.", + ) + prepare_parser.add_argument( "--source-bucket", help="S3 bucket source granules", ) - parser.add_argument( - "--cfg-file", - help="Config file", + prepare_parser.set_defaults(func=cmd_prepare) + + cnm_sender_parser = subparsers.add_parser( + "cnm_sender", + help="Read queued work and formulate load-test/CNM messages.", + ) + cnm_sender_parser.add_argument( + "--max-messages", + type=int, + default=-1, # -1 is unlimited + help="Maximum number of SQS messages to process in one invocation. Default is unlimited.", ) + cnm_sender_parser.set_defaults(func=cmd_cnm_sender) + return parser @@ -70,11 +93,7 @@ def main(args: Optional[list[str]] = None): root_logger.addHandler(screenlog) try: - cfg = CtormConfig.from_file( - cfg_file=pargs.cfg_file, - ) - prepare = CtormPrepare(cfg) - prepare.prepare() + pargs.func(pargs) except Exception: log.exception("") From 6ae0194dacff222f28d935dc1bf6bb6b2f5a0e24 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 18 Jun 2026 12:45:57 -0800 Subject: [PATCH 10/31] PR-7501 counting granules and exit condition for prepare loop --- ctorm/ctorm/config.py | 3 +++ ctorm/ctorm/prepare.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index dfe026c..63d47e1 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -24,6 +24,9 @@ class CtormConfig: source_buckets: list granules_sqs_queue_url: str + # This is the number of granules that will be prepared for test. + granule_goal: int = 2000000 + @classmethod def from_file( cls, diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 0508485..f0e3bff 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -63,10 +63,10 @@ class CtormPrepare: def __init__(self, cfg: CtormConfig): self.cfg = cfg + self.total_granules = 0 def prepare(self): - goal = 50 - while goal > 0: # TODO: replace with while true and an exit condition + while self.total_granules < self.cfg.granule_goal: # This is the loop that creates a SQS message from multiple objects. sqs_msg = CtormSqsMessage(self.cfg) for b in self.cfg.source_buckets: @@ -86,10 +86,10 @@ def prepare(self): file = self.process_ummg(ummg, b) if file: sqs_msg.add_file(file) - goal -= 1 + self.total_granules += 1 b.next_cont_token = page.get("NextContinuationToken") - log.debug("goal: %d", goal) + log.debug("tot granules: %d/%d", self.total_granules, self.cfg.granule_goal) log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) if sqs_msg.check_message_size(): From ae98c0ccf22eb7eef14bcd1ccceaddb41dbc92f8 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 18 Jun 2026 16:34:30 -0800 Subject: [PATCH 11/31] PR-7501 Support for using multiple access creds to the various prod buckets. --- ctorm/ctorm.cfg.example | 9 ++++++--- ctorm/ctorm/config.py | 6 ++++++ ctorm/ctorm/prepare.py | 38 +++++++++++++++++++++++++------------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/ctorm/ctorm.cfg.example b/ctorm/ctorm.cfg.example index d65282f..fab895b 100644 --- a/ctorm/ctorm.cfg.example +++ b/ctorm/ctorm.cfg.example @@ -4,13 +4,16 @@ granules_sqs_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/ctorm [[ ctorm.source_buckets ]] bucketname = "nisar-bucket-0" -ummg_prefix = "UMMG/FOO_COLLECTION/" share = 50 +ummg_prefix = "UMMG/FOO_COLLECTION/" +keypair_name = "NISAR" [[ctorm.source_buckets]] bucketname = "nisar-bucket-2" -ummg_prefix = "UMMG/BAR_COLLECTION/" +keypair_name = "NISAR" share = 45 +ummg_prefix = "UMMG/BAR_COLLECTION/" [[ctorm.source_buckets]] bucketname = "opera-bucket-0" -ummg_prefix = "UMMG/BAZ_COLLECTION/" +keypair_name = "OPERA" share = 5 +ummg_prefix = "UMMG/BAZ_COLLECTION/" diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index 63d47e1..a59bb9e 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -1,6 +1,9 @@ import tomllib from dataclasses import dataclass, fields +from boto3.session import Session +from mypy_boto3_s3 import S3Client + try: from typing import Self except ImportError: @@ -13,9 +16,12 @@ @dataclass class CtormBucket: bucketname: str + keypair_name: str # Looks for a keypair in the form of `AWS_ACCESS_KEY_ID_[keypair_name]` share: int = "" next_cont_token: str = None + s3_client: S3Client = None + session: Session = None ummg_prefix: str = "UMMG/" diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index f0e3bff..8665dee 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -1,5 +1,6 @@ import hashlib import json +import os import re from enum import StrEnum from functools import cache @@ -14,11 +15,6 @@ log = getLogger(__name__) -@cache -def get_s3_client(): - return boto3.client("s3", region_name=AWS_REGION) - - @cache def get_sqs_client(): return boto3.client("sqs", region_name=AWS_REGION) @@ -65,6 +61,20 @@ def __init__(self, cfg: CtormConfig): self.cfg = cfg self.total_granules = 0 + self._boto_sessions = {} + + # init buckets + for bkt in self.cfg.source_buckets: + bkt.next_cont_token = None + keysuffix = f"_{bkt.keypair_name}" if bkt.keypair_name else "" + if bkt.keypair_name not in self._boto_sessions: + self._boto_sessions[bkt.keypair_name] = boto3.Session( + region_name=AWS_REGION, + aws_access_key_id=os.getenv(f"AWS_ACCESS_KEY_ID{keysuffix}"), + aws_secret_access_key=os.getenv(f"AWS_SECRET_ACCESS_KEY{keysuffix}"), + ) + bkt.s3_client = self._boto_sessions[bkt.keypair_name].client("s3") + def prepare(self): while self.total_granules < self.cfg.granule_goal: # This is the loop that creates a SQS message from multiple objects. @@ -82,7 +92,7 @@ def prepare(self): continue log.debug(" %s", obj["Key"]) # download the json object and load it into a var: - ummg = self.download_ummg(b.bucketname, obj["Key"]) + ummg = self.download_ummg(b, obj["Key"]) file = self.process_ummg(ummg, b) if file: sqs_msg.add_file(file) @@ -91,6 +101,7 @@ def prepare(self): log.debug("tot granules: %d/%d", self.total_granules, self.cfg.granule_goal) log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) + log.debug("sqs message percentage: %d%%", (len(sqs_msg.to_json()) / CtormSqsMessage.MAX_MESSAGE_SIZE) * 100) log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) if sqs_msg.check_message_size(): log.debug("sqs_msg: %s", sqs_msg.to_json()) @@ -110,11 +121,11 @@ def get_ummg_page(self, ct_bukt: CtormBucket) -> ListObjectsV2OutputTypeDef: } if ct_bukt.next_cont_token: kwargs["ContinuationToken"] = ct_bukt.next_cont_token - ret = get_s3_client().list_objects_v2(**kwargs) + ret = ct_bukt.s3_client.list_objects_v2(**kwargs) return ret - def download_ummg(self, bucketname: str, key: str) -> dict: - resp = get_s3_client().get_object(Bucket=bucketname, Key=key) + def download_ummg(self, b: CtormBucket, key: str) -> dict: + resp = b.s3_client.get_object(Bucket=b.bucketname, Key=key) ummgfile = resp["Body"].read() return json.loads(ummgfile) @@ -149,14 +160,14 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: # We must look to S3 for the size and md5 log.debug('getting head for "%s"', objloc) try: - headobj = get_s3_client().head_object(Bucket=bucket, Key=objloc) + headobj = ct_bkt.s3_client.head_object(Bucket=bucket, Key=objloc) log.debug("head_object: %s", headobj) filedict[K.SIZE] = headobj["ContentLength"] filedict[K.MD5] = headobj["ETag"].replace('"', "") if filedict[K.MD5].endswith("-1"): # This was a multipart upload. We'll have to do something clever to get the MD5 of it. log.debug("multipart upload detected for %s", objloc) - filedict[K.MD5] = self.get_real_md5(bucket, objloc) + filedict[K.MD5] = self.get_real_md5(ct_bkt, bucket, objloc) except ClientError as e: log.error("head_object failed: %s", e) # TODO: trash entire message for this granule? @@ -167,11 +178,12 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: def get_real_md5( self, - bucket: str, + b: CtormBucket, + obj_bucket: str, key: str, ) -> str: # The file is small enough we may as well download it to memory and get the MD5 that way. - resp = get_s3_client().get_object(Bucket=bucket, Key=key) + resp = b.s3_client.get_object(Bucket=obj_bucket, Key=key) md5_accumulator = hashlib.md5() for chunk in iter(lambda: resp["Body"].read(self.MD5_DL_CHUNK_MB * 1024 * 1024), b""): md5_accumulator.update(chunk) From cb2173ddb9110a54db7759c242de1232a8f37ac1 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 18 Jun 2026 17:02:18 -0800 Subject: [PATCH 12/31] PR-7501 Terraform resources for EC2 that will runn prepare. --- ctorm/README.md | 33 ++ ctorm/infra/terraform/main.tf | 93 ++++++ ctorm/infra/terraform/outputs.tf | 12 + ctorm/infra/terraform/terraform.tfstate | 416 +++++++++++++++++++++++- ctorm/infra/terraform/variables.tf | 13 + 5 files changed, 550 insertions(+), 17 deletions(-) diff --git a/ctorm/README.md b/ctorm/README.md index 4652c21..d44d3f6 100644 --- a/ctorm/README.md +++ b/ctorm/README.md @@ -2,3 +2,36 @@ Cumulus Throughput... uh... ORM? Load tester tool for Cumulus. + +Operates as two steps: + +* Prepare + * The prepare step goes through the UMMG directory of specified prod buckets and + extracts the metadata necessary to recreate CNM messages. It then creates SQS messages + containing lists of these metadata. +* CNM Sender + * The CNM Sender step reads the SQS messages and sends them to the Cumulus ingest queue. + +## Prepare + +### Cfg file + +See the `ctorm.cfg.example` file. + +#### source_buckets + +| key | value | +|--------------|-----------------------------------------------------------| +| bucketname | Name of bucket in which to find the UMMG files | +| keypair_name | Arbritrary name to suffix to the AWS keypair env var | +| share | Number of granules from this collection to add to the CNM | +| ummg_prefix | Path in bucketname where to find the desired UMMG files | + +Let's say you have the following source buckets set up: + +### Env vars + +You will need to have the following environment variables set: + +* `AWS_ACCESS_KEY_ID_[keypair_name]` where `keypair_name` matches an entry in the ctorm cfg toml file. +* `AWS_SECRET_ACCESS_KEY_[keypair_name]` where `keypair_name` matches an entry in the ctorm cfg toml file. diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 2ed67e9..468ac01 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -186,3 +186,96 @@ resource "aws_lambda_permission" "allow_eventbridge" { principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.every_minute.arn } + + +data "aws_iam_policy_document" "prepare_ec2_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "prepare_ec2" { + name = "${var.name_prefix}-prepare-ec2-role" + assume_role_policy = data.aws_iam_policy_document.prepare_ec2_assume_role.json +} + +data "aws_iam_policy_document" "prepare_ec2" { + statement { + actions = [ + "sqs:SendMessage", + "sqs:GetQueueAttributes" + ] + + resources = [ + aws_sqs_queue.granules.arn + ] + } + + statement { + actions = [ + "s3:ListBucket" + ] + + resources = [ + for bucket_name in var.prepare_source_bucket_names : "arn:aws:s3:::${bucket_name}" + ] + } + + statement { + actions = [ + "s3:GetObject" + ] + + resources = [ + for bucket_name in var.prepare_source_bucket_names : "arn:aws:s3:::${bucket_name}/*" + ] + } +} + +resource "aws_iam_role_policy" "prepare_ec2" { + name = "${var.name_prefix}-prepare-ec2-policy" + role = aws_iam_role.prepare_ec2.id + policy = data.aws_iam_policy_document.prepare_ec2.json +} + +resource "aws_iam_instance_profile" "prepare_ec2" { + name = "${var.name_prefix}-prepare-ec2-profile" + role = aws_iam_role.prepare_ec2.name +} + + + +resource "aws_s3_bucket" "scratch" { + bucket = "${var.name_prefix}-scratch" + + tags = { + Name = "${var.name_prefix}-scratch" + } +} + +data "aws_iam_policy_document" "allow_scratch_bucket_access" { + statement { + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket", + "s3:DeleteObject" + ] + + resources = [ + aws_s3_bucket.scratch.arn, + "${aws_s3_bucket.scratch.arn}/*" + ] + } +} + +resource "aws_iam_role_policy" "scratch_bucket_access" { + name = "${var.name_prefix}-scratch-access" + role = aws_iam_role.cnm-sender.id # assumes this role already exists + policy = data.aws_iam_policy_document.allow_scratch_bucket_access.json +} diff --git a/ctorm/infra/terraform/outputs.tf b/ctorm/infra/terraform/outputs.tf index 4c02dc4..11ca41e 100644 --- a/ctorm/infra/terraform/outputs.tf +++ b/ctorm/infra/terraform/outputs.tf @@ -17,3 +17,15 @@ output "state_table_name" { output "worker_lambda_name" { value = aws_lambda_function.cnm-sender.function_name } + +output "prepare_ec2_instance_profile_name" { + value = aws_iam_instance_profile.prepare_ec2.name +} + +output "prepare_ec2_role_arn" { + value = aws_iam_role.prepare_ec2.arn +} + +output "scratch_bucket" { + value = aws_s3_bucket.scratch.bucket +} diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index f80b91c..bb737dd 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,25 +1,37 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 12, + "serial": 23, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { - "state_table_name": { - "value": "ctorm-dev-state", - "type": "string" - }, - "work_dlq_url": { + "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" }, - "work_queue_arn": { + "granules_queue_arn": { "value": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", "type": "string" }, - "work_queue_url": { + "granules_queue_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "type": "string" }, + "prepare_ec2_instance_profile_name": { + "value": "ctorm-dev-prepare-ec2-profile", + "type": "string" + }, + "prepare_ec2_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-dev-scratch", + "type": "string" + }, + "state_table_name": { + "value": "ctorm-dev-state", + "type": "string" + }, "worker_lambda_name": { "value": "ctorm-dev-cnm-sender", "type": "string" @@ -63,6 +75,50 @@ } ] }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "allow_scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "632361540", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:ListBucket\",\n \"s3:GetObject\",\n \"s3:DeleteObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::ctorm-dev-scratch/*\",\n \"arn:aws:s3:::ctorm-dev-scratch\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch", + "arn:aws:s3:::ctorm-dev-scratch/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -200,6 +256,132 @@ } ] }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1999618104", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sqs:GetQueueAttributes", + "sqs:SendMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::asf-cumulus-dev-opera-products", + "arn:aws:s3:::asf-cumulus-prod-opera-products", + "arn:aws:s3:::asf-cumulus-test-opera-products", + "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", + "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", + "arn:aws:s3:::sds-n-cumulus-test-nisar-products" + ], + "sid": "" + }, + { + "actions": [ + "s3:GetObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::asf-cumulus-dev-opera-products/*", + "arn:aws:s3:::asf-cumulus-prod-opera-products/*", + "arn:aws:s3:::asf-cumulus-test-opera-products/*", + "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", + "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", + "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "prepare_ec2_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2851119427", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "ec2.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "managed", "type": "aws_cloudwatch_event_rule", @@ -221,7 +403,7 @@ "role_arn": "", "schedule_expression": "rate(1 minute)", "state": "DISABLED", - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -295,7 +477,7 @@ "name_prefix": "", "retention_in_days": 14, "skip_destroy": false, - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -351,7 +533,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -367,6 +549,35 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", + "create_date": "2026-06-19T01:00:34Z", + "id": "ctorm-dev-prepare-ec2-profile", + "name": "ctorm-dev-prepare-ec2-profile", + "name_prefix": "", + "path": "/", + "role": "ctorm-dev-prepare-ec2-role", + "tags": null, + "tags_all": {}, + "unique_id": "AIPA4VW62R3RCASOXJGOK" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.prepare_ec2", + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -382,14 +593,23 @@ "description": "", "force_detach_policies": false, "id": "ctorm-dev-cnm-sender-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-dev-cnm-sender-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + }, + { + "name": "ctorm-dev-scratch-access", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-dev-cnm-sender-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROA4VW62R3RPOQ35FULV" }, @@ -401,6 +621,40 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-19T01:00:33Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-prepare-ec2-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-prepare-ec2-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROA4VW62R3RKO3MKUI7D" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -430,6 +684,59 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", + "name": "ctorm-dev-prepare-ec2-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", + "role": "ctorm-dev-prepare-ec2-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.prepare_ec2", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.prepare_ec2", + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cnm-sender-role:ctorm-dev-scratch-access", + "name": "ctorm-dev-scratch-access", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}", + "role": "ctorm-dev-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cnm-sender", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.allow_scratch_bucket_access", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_lambda_function", @@ -471,7 +778,7 @@ "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", "last_modified": "2026-06-10T20:02:16.662+0000", - "layers": null, + "layers": [], "logging_config": [ { "application_log_level": "", @@ -499,7 +806,7 @@ "snap_start": [], "source_code_hash": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", "source_code_size": 339, - "tags": null, + "tags": {}, "tags_all": {}, "timeout": 120, "timeouts": null, @@ -567,6 +874,81 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "scratch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch", + "bucket": "ctorm-dev-scratch", + "bucket_domain_name": "ctorm-dev-scratch.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "ctorm-dev-scratch.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [ + { + "id": "a7e8aeefe5d49ed5062fbc1a1d29f8b05e5150edf37ef1739efda49e5443a5ee", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-dev-scratch", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Name": "ctorm-dev-scratch" + }, + "tags_all": { + "Name": "ctorm-dev-scratch" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", @@ -594,7 +976,7 @@ "redrive_allow_policy": "", "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq\",\"maxReceiveCount\":5}", "sqs_managed_sse_enabled": true, - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", @@ -635,7 +1017,7 @@ "redrive_allow_policy": "", "redrive_policy": "", "sqs_managed_sse_enabled": true, - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", diff --git a/ctorm/infra/terraform/variables.tf b/ctorm/infra/terraform/variables.tf index 155e9c3..e661b9b 100644 --- a/ctorm/infra/terraform/variables.tf +++ b/ctorm/infra/terraform/variables.tf @@ -18,3 +18,16 @@ variable "cumulus_ingest_queue_url" { type = string description = "URL of the target Cumulus ingest SQS queue." } + +variable "prepare_source_bucket_names" { + type = list(string) + description = "S3 bucket names that ctorm prepare is allowed to read from." + default = [ + "asf-cumulus-dev-opera-products", + "asf-cumulus-prod-opera-products", + "asf-cumulus-test-opera-products", + "sds-n-cumulus-dev-nisar-products", + "sds-n-cumulus-prod-nisar-products", + "sds-n-cumulus-test-nisar-products", + ] +} From 470f1fe0a8b7c5dfbec77a5c31408023615ca32f Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 18 Jun 2026 20:08:57 -0800 Subject: [PATCH 13/31] PR-7502 stubby load test lambda --- ctorm/ctorm/lambda_run.py | 21 +++++++++++++ ctorm/ctorm/load_tester.py | 58 +++++++++++++++++++++++++++++++++++ ctorm/infra/terraform/main.tf | 5 +-- 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 ctorm/ctorm/lambda_run.py create mode 100644 ctorm/ctorm/load_tester.py diff --git a/ctorm/ctorm/lambda_run.py b/ctorm/ctorm/lambda_run.py new file mode 100644 index 0000000..6d1dcd5 --- /dev/null +++ b/ctorm/ctorm/lambda_run.py @@ -0,0 +1,21 @@ +""" +Emulates a lambda for local dev. +""" + +import json + +from ctorm.load_tester import lambda_handler + +# mock event data +with open("../../data/lambda_event.json", "r") as f: + mock_event = json.load(f) + + +# Mock context object +class MockContext: + function_name = "local_test" + memory_limit_in_mb = 128 + + +# Run it +print(lambda_handler(mock_event, MockContext())) diff --git a/ctorm/ctorm/load_tester.py b/ctorm/ctorm/load_tester.py new file mode 100644 index 0000000..a805525 --- /dev/null +++ b/ctorm/ctorm/load_tester.py @@ -0,0 +1,58 @@ +import logging +import os + +from cnm_sender import CnmSender + +log = logging.getLogger(__name__) + + +def configure_logging() -> None: + level_name = os.environ.get("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + + root_logger = logging.getLogger() + root_logger.setLevel(level) + + for noisy_logger_name in ("boto3", "botocore", "urllib3", "s3transfer"): + logging.getLogger(noisy_logger_name).setLevel(logging.WARNING) + + # AWS Lambda installs a handler before invoking your code. Reuse it instead + # of adding a new handler on every warm invocation. + for handler in root_logger.handlers: + handler.setLevel(level) + handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d - %(message)s") + ) + + +def load_test(): + c_sender = CnmSender({}) + c_sender.send() + + +def lambda_handler(event, context): + configure_logging() + + log.info( + "Starting CNM sender invocation", + extra={ + "aws_request_id": getattr(context, "aws_request_id", None), + }, + ) + + try: + log.debug("Received event: %s", event) + + load_test() + + log.info("CNM sender invocation completed") + return {"ok": True} + + except Exception: + log.exception("CNM sender invocation failed") + raise + + +if __name__ == "__main__": + configure_logging() + lambda_handler({}, {}) diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 468ac01..ebcc003 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -123,7 +123,7 @@ data "archive_file" "placeholder_lambda" { import json import os -def handler(event, context): +def lambda_handler(event, context): print(json.dumps({ "event": event, "granules_queue_url": os.environ.get("GRANULES_QUEUE_URL"), @@ -142,7 +142,7 @@ resource "aws_lambda_function" "cnm-sender" { function_name = local.lambda_name role = aws_iam_role.cnm-sender.arn runtime = "python3.12" - handler = "index.handler" + handler = "load_tester.lambda_handler" filename = data.archive_file.placeholder_lambda.output_path source_code_hash = data.archive_file.placeholder_lambda.output_base64sha256 @@ -155,6 +155,7 @@ resource "aws_lambda_function" "cnm-sender" { GRANULES_QUEUE_URL = aws_sqs_queue.granules.url TABLE_NAME = aws_dynamodb_table.state.name CUMULUS_INGEST_QUEUE_URL = var.cumulus_ingest_queue_url + LOG_LEVEL = "INFO" } } From 267fb5ba6d47c96b547abd289fb711d10818e537 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 19 Jun 2026 11:06:53 -0800 Subject: [PATCH 14/31] PR-7502 Refactored to accomodate lambda load tester --- ctorm/ctorm.cfg.example | 23 +++++++++++++++++------ ctorm/ctorm/config.py | 24 +++++++++++++++++------- ctorm/ctorm/prepare.py | 22 +++++++++++----------- ctorm/tests/test_config.py | 6 +++--- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/ctorm/ctorm.cfg.example b/ctorm/ctorm.cfg.example index fab895b..e50e4e1 100644 --- a/ctorm/ctorm.cfg.example +++ b/ctorm/ctorm.cfg.example @@ -2,18 +2,29 @@ granules_sqs_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/ctorm-dev-granules" -[[ ctorm.source_buckets ]] +[[ ctorm.pipelines ]] bucketname = "nisar-bucket-0" +ingest_keypair_name = "" +ingest_queue_name = "sds-n-cumulus-test-nisar-workflow-queue" +ingest_sqs_queue_url = "" share = 50 ummg_prefix = "UMMG/FOO_COLLECTION/" -keypair_name = "NISAR" -[[ctorm.source_buckets]] +prepare_keypair_name = "NISAR_P" + +[[ctorm.pipelines]] bucketname = "nisar-bucket-2" -keypair_name = "NISAR" +ingest_keypair_name = "" +ingest_queue_name = "sds-n-cumulus-test-nisar-workflow-queue" +ingest_sqs_queue_url = "" +prepare_keypair_name = "NISAR" share = 45 ummg_prefix = "UMMG/BAR_COLLECTION/" -[[ctorm.source_buckets]] + +[[ctorm.pipelines]] bucketname = "opera-bucket-0" -keypair_name = "OPERA" +ingest_keypair_name = "" +ingest_queue_name = "asf-cumulus-test-opera-workflow-queue" +ingest_sqs_queue_url = "" +prepare_keypair_name = "OPERA" share = 5 ummg_prefix = "UMMG/BAZ_COLLECTION/" diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index a59bb9e..965aa2d 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -1,6 +1,6 @@ -import tomllib from dataclasses import dataclass, fields +import tomllib from boto3.session import Session from mypy_boto3_s3 import S3Client @@ -14,11 +14,21 @@ @dataclass -class CtormBucket: +class CtormPipeline: bucketname: str - keypair_name: str # Looks for a keypair in the form of `AWS_ACCESS_KEY_ID_[keypair_name]` - share: int = "" + # Looks for a keypair in the form of `AWS_ACCESS_KEY_ID_[keypair_name]` + # This should have permissions to send SQS to the maturity we're sending CNM to + ingest_keypair_name: str + ingest_queue_name: str + ingest_sqs_queue_url: str + + # Looks for a keypair in the form of `AWS_ACCESS_KEY_ID_[keypair_name]` + # This should have permissions to read from prod bucket where data resides + prepare_keypair_name: str + share: int + + # These are used at runtime next_cont_token: str = None s3_client: S3Client = None session: Session = None @@ -27,7 +37,7 @@ class CtormBucket: @dataclass class CtormConfig: - source_buckets: list + pipelines: list granules_sqs_queue_url: str # This is the number of granules that will be prepared for test. @@ -43,8 +53,8 @@ def from_file( valid_fields = {f.name for f in fields(cls)} # Filter the input dictionary kwargs = {k: v for k, v in cfg["ctorm"].items() if k in valid_fields} - # create CtormBuckets for each bucket in cfg['source_buckets'] - kwargs["source_buckets"] = [CtormBucket(**b) for b in kwargs["source_buckets"]] + # create CtormPipeline for each bucket in cfg['pipelines'] + kwargs["pipelines"] = [CtormPipeline(**b) for b in kwargs["pipelines"]] obj = cls(**kwargs) return obj diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 8665dee..9a19794 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -10,7 +10,7 @@ from botocore.exceptions import ClientError from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef -from ctorm.config import AWS_REGION, CtormBucket, CtormConfig +from ctorm.config import AWS_REGION, CtormConfig, CtormPipeline log = getLogger(__name__) @@ -64,22 +64,22 @@ def __init__(self, cfg: CtormConfig): self._boto_sessions = {} # init buckets - for bkt in self.cfg.source_buckets: + for bkt in self.cfg.pipelines: bkt.next_cont_token = None - keysuffix = f"_{bkt.keypair_name}" if bkt.keypair_name else "" - if bkt.keypair_name not in self._boto_sessions: - self._boto_sessions[bkt.keypair_name] = boto3.Session( + keysuffix = f"_{bkt.prepare_keypair_name}" if bkt.prepare_keypair_name else "" + if bkt.prepare_keypair_name not in self._boto_sessions: + self._boto_sessions[bkt.prepare_keypair_name] = boto3.Session( region_name=AWS_REGION, aws_access_key_id=os.getenv(f"AWS_ACCESS_KEY_ID{keysuffix}"), aws_secret_access_key=os.getenv(f"AWS_SECRET_ACCESS_KEY{keysuffix}"), ) - bkt.s3_client = self._boto_sessions[bkt.keypair_name].client("s3") + bkt.s3_client = self._boto_sessions[bkt.prepare_keypair_name].client("s3") def prepare(self): while self.total_granules < self.cfg.granule_goal: # This is the loop that creates a SQS message from multiple objects. sqs_msg = CtormSqsMessage(self.cfg) - for b in self.cfg.source_buckets: + for b in self.cfg.pipelines: # This is the loop that goes into each bucket we're interested in. log.debug("getting objects from %s", b.bucketname) @@ -113,7 +113,7 @@ def prepare(self): raise Exception("Message too big") # TODO: deal with this smarter - def get_ummg_page(self, ct_bukt: CtormBucket) -> ListObjectsV2OutputTypeDef: + def get_ummg_page(self, ct_bukt: CtormPipeline) -> ListObjectsV2OutputTypeDef: kwargs = { "Bucket": ct_bukt.bucketname, "MaxKeys": ct_bukt.share, @@ -124,12 +124,12 @@ def get_ummg_page(self, ct_bukt: CtormBucket) -> ListObjectsV2OutputTypeDef: ret = ct_bukt.s3_client.list_objects_v2(**kwargs) return ret - def download_ummg(self, b: CtormBucket, key: str) -> dict: + def download_ummg(self, b: CtormPipeline, key: str) -> dict: resp = b.s3_client.get_object(Bucket=b.bucketname, Key=key) ummgfile = resp["Body"].read() return json.loads(ummgfile) - def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: + def process_ummg(self, ummg: dict, ct_bkt: CtormPipeline) -> dict: outdict = { K.BKT_MAP: {ct_bkt.bucketname: "B1"}, # bucket map K.GRANULE: ummg["GranuleUR"], @@ -178,7 +178,7 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormBucket) -> dict: def get_real_md5( self, - b: CtormBucket, + b: CtormPipeline, obj_bucket: str, key: str, ) -> str: diff --git a/ctorm/tests/test_config.py b/ctorm/tests/test_config.py index cf6b4e5..ac4bebb 100644 --- a/ctorm/tests/test_config.py +++ b/ctorm/tests/test_config.py @@ -1,10 +1,10 @@ from unittest.mock import mock_open, patch -from ctorm.config import ConfigError, CtormBucket, CtormConfig +from ctorm.config import ConfigError, CtormConfig, CtormPipeline def test_ctorm_bucket_initialization(): - bucket = CtormBucket(bucketname="test_bucket", share=50, ummg_prefix="test_prefix/") + bucket = CtormPipeline(bucketname="test_bucket", share=50, ummg_prefix="test_prefix/") assert bucket.bucketname == "test_bucket" assert bucket.share == 50 @@ -29,7 +29,7 @@ def test_ctorm_config_from_file_success(): assert config.granules_sqs_queue_url == "https://sqs.queue.url/" assert len(config.source_buckets) == 1 - assert isinstance(config.source_buckets[0], CtormBucket) + assert isinstance(config.source_buckets[0], CtormPipeline) assert config.source_buckets[0].bucketname == "test_bucket" From d723e04cd96080bc49148ce018d7a712cea2a626 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 19 Jun 2026 15:58:44 -0800 Subject: [PATCH 15/31] PR-7502 Implemented CNM creation --- .github/workflows/lint.yml | 6 +- .gitignore | 2 + ctorm/ctorm.cfg.example | 7 +- ctorm/ctorm/cnm.py | 131 +++++++++++++++++++++++++++++++++++++ ctorm/ctorm/cnm_sender.py | 25 ++++++- ctorm/ctorm/config.py | 22 ++++++- ctorm/ctorm/lambda_run.py | 12 ++++ ctorm/ctorm/load_tester.py | 35 +++++++--- ctorm/ctorm/main.py | 4 +- ctorm/ctorm/prepare.py | 45 +++++++++---- ctorm/pyproject.toml | 4 ++ ctorm/tests/test_cnm.py | 33 ++++++++++ ctorm/tests/test_config.py | 18 +++-- ctorm/uv.lock | 19 +++++- 14 files changed, 326 insertions(+), 37 deletions(-) create mode 100644 ctorm/ctorm/cnm.py create mode 100644 ctorm/tests/test_cnm.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f80b011..d1c62ef 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,9 +14,9 @@ jobs: python-version: 3.11 - uses: TrueBrain/actions-flake8@v2 with: - flake8_version: 6.0.0 + flake8_version: 7.3.0 max_line_length: 120 - plugins: flake8-isort==6.0.0 flake8-quotes==3.3.2 + plugins: flake8-isort==7.0.0, flake8-quotes==3.4.0 ruff-format: runs-on: ubuntu-latest @@ -25,5 +25,5 @@ jobs: - uses: actions/checkout@v4 - uses: astral-sh/ruff-action@v3 with: - version: "~=0.13.3" + version: "~=0.15.13" args: format --check --diff --output-format=github diff --git a/.gitignore b/.gitignore index d447d56..72e2440 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,7 @@ poetry.lock .tox +**.tfstate.backup + # Auto generated by Make tox.ini diff --git a/ctorm/ctorm.cfg.example b/ctorm/ctorm.cfg.example index e50e4e1..0ce05c4 100644 --- a/ctorm/ctorm.cfg.example +++ b/ctorm/ctorm.cfg.example @@ -1,5 +1,7 @@ [ctorm] +calc_md5 = true +granule_goal = 200 granules_sqs_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/ctorm-dev-granules" [[ ctorm.pipelines ]] @@ -7,9 +9,10 @@ bucketname = "nisar-bucket-0" ingest_keypair_name = "" ingest_queue_name = "sds-n-cumulus-test-nisar-workflow-queue" ingest_sqs_queue_url = "" +prepare_keypair_name = "NISAR_P" +provider = "FOO" share = 50 ummg_prefix = "UMMG/FOO_COLLECTION/" -prepare_keypair_name = "NISAR_P" [[ctorm.pipelines]] bucketname = "nisar-bucket-2" @@ -17,6 +20,7 @@ ingest_keypair_name = "" ingest_queue_name = "sds-n-cumulus-test-nisar-workflow-queue" ingest_sqs_queue_url = "" prepare_keypair_name = "NISAR" +provider = "FOO" share = 45 ummg_prefix = "UMMG/BAR_COLLECTION/" @@ -26,5 +30,6 @@ ingest_keypair_name = "" ingest_queue_name = "asf-cumulus-test-opera-workflow-queue" ingest_sqs_queue_url = "" prepare_keypair_name = "OPERA" +provider = "FOO" share = 5 ummg_prefix = "UMMG/BAZ_COLLECTION/" diff --git a/ctorm/ctorm/cnm.py b/ctorm/ctorm/cnm.py new file mode 100644 index 0000000..327bbb4 --- /dev/null +++ b/ctorm/ctorm/cnm.py @@ -0,0 +1,131 @@ +import re +import uuid +from collections.abc import Callable +from datetime import datetime, timezone +from logging import getLogger +from pathlib import Path + +from ctorm.config import ( + MD5_CHECKSUM_PATTERN, + CtormPipeline, + CtormPreparedFile, + CtormPreparedGranule, +) + +log = getLogger(__name__) + +DATA_TYPE_MAP = { + ".context.json": "metadata", + ".dataset.json": "metadata", + ".h5": "data", + ".iso.xml": "metadata", + ".jpg": "browse", + ".log": "metadata", + ".md5": "metadata", + ".met.json": "metadata", + ".nc": "data", + ".pdf": "qa", + ".png": "browse", + ".qa.h5": "qa", + ".rc.yaml": "metadata", + ".tif": "data", + ".xml": "data", +} + +type CtormCnmSGeneratorType = Callable[[CtormPreparedGranule, str], dict] # noqa + + +class CtormCnmSGenerator: + """input dict like: + ``` + { + "bm": { + "sds-n-cumulus-prod-nisar-products": "B1", + "sds-n-cumulus-prod-nisar-jpl-private-data": "B2" + }, + "g": "NISAR_L0_RRST_VC08_20250821T101036_20250821T101041_P00408_J_001", + "c": "NISAR_L0A_RRST_BETA_V1", + "cv": "1", + "f": [ + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin", + "m": "e989430f4c5bfa04eeb26e56f4cd8375", + "s": 1073829760 + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.rc.yaml", + "s": 120597, + "m": "11bcaec780f0d246f3996b3f08680410" + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin.qa", + "s": 1134, + }, + ] + }, + ``` + """ + + def _assemble_file_dict( + self, file_dict: CtormPreparedFile, g_name: str, bucket_map: dict + ): + file_s3uri = file_dict["f"].replace("$G", g_name) + for bucket, replacetoken in bucket_map.items(): + file_s3uri = file_s3uri.replace(f"${replacetoken}", bucket) + filename = Path(file_s3uri).name + return { + "name": filename, + "type": self._get_type(filename), + "uri": file_s3uri, + "size": file_dict["s"], + "checksum": self._get_checksum(file_dict), + "checksumType": "md5", + } + + def __call__( + self, + granule_dict: CtormPreparedGranule, + provider: str, + ) -> dict: + cnm_s = { + "identifier": str(uuid.uuid4()), + "collection": granule_dict["c"], + "version": "1.3", + "submissionTime": datetime.now(tz=timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ), + "product": { + "name": granule_dict["g"], + "dataVersion": granule_dict["cv"], + "files": [ + self._assemble_file_dict( + file, granule_dict["g"], granule_dict["bm"] + ) + for file in granule_dict["f"] + ], + }, + "provider": provider, + } + + return cnm_s + + def _get_type(self, filename: str): + + suffixes = Path(filename).suffixes + while suffixes: + data_type = DATA_TYPE_MAP.get("".join(suffixes)) + + if data_type: + return data_type + + suffixes = suffixes[1:] + + log.debug("suffix from %s is not in map. Using 'data'", filename) + return "data" + + def _get_checksum(self, file_dict: CtormPreparedFile): + m = MD5_CHECKSUM_PATTERN.match(file_dict.get("m", "not md5")) + if m: + return m.group(1) + + return "00000000000000000000000000000000" diff --git a/ctorm/ctorm/cnm_sender.py b/ctorm/ctorm/cnm_sender.py index 645aa4a..07347e1 100644 --- a/ctorm/ctorm/cnm_sender.py +++ b/ctorm/ctorm/cnm_sender.py @@ -1,6 +1,25 @@ +from logging import getLogger +from typing import List + +from ctorm.cnm import CtormCnmSGenerator +from ctorm.config import CtormConfig, CtormPreparedGranule + +log = getLogger(__name__) + + class CnmSender: - def __init__(self, cfg): + def __init__( + self, + cfg: CtormConfig, + granules: List[CtormPreparedGranule], + provider: str = "TODO: PROVIDER", + ): self.cfg = cfg + self.granules = granules + self.provider = provider + self.cnm_s_generator = CtormCnmSGenerator() - def send(self): - pass + def send_all(self): + for granule in self.granules: + cnms = self.cnm_s_generator(granule, self.provider) + log.debug("Sending %s", cnms) diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py index 965aa2d..dba4c7a 100644 --- a/ctorm/ctorm/config.py +++ b/ctorm/ctorm/config.py @@ -1,6 +1,8 @@ +import re +import tomllib from dataclasses import dataclass, fields +from typing import NotRequired, TypedDict -import tomllib from boto3.session import Session from mypy_boto3_s3 import S3Client @@ -11,6 +13,21 @@ DEFAULT = "default" AWS_REGION = "us-west-2" # We will never not want us-west-2 +MD5_CHECKSUM_PATTERN = re.compile(r"^([\da-f]{32})$") + + +class CtormPreparedFile(TypedDict): + f: str + m: NotRequired[str] + s: int + + +class CtormPreparedGranule(TypedDict): + bm: dict[str, str] + g: str + c: str + cv: str + f: list[CtormPreparedFile] @dataclass @@ -33,6 +50,7 @@ class CtormPipeline: s3_client: S3Client = None session: Session = None ummg_prefix: str = "UMMG/" + provider: str = "TODO" @dataclass @@ -40,6 +58,8 @@ class CtormConfig: pipelines: list granules_sqs_queue_url: str + calc_md5: bool = True + # This is the number of granules that will be prepared for test. granule_goal: int = 2000000 diff --git a/ctorm/ctorm/lambda_run.py b/ctorm/ctorm/lambda_run.py index 6d1dcd5..10e26cc 100644 --- a/ctorm/ctorm/lambda_run.py +++ b/ctorm/ctorm/lambda_run.py @@ -3,9 +3,21 @@ """ import json +import logging from ctorm.load_tester import lambda_handler +log = logging.getLogger() + +base_fmt_str = "%(levelname)s: %(message)s (%(filename)s line %(lineno)d/)" +screen_fmt = logging.Formatter( + "%(asctime)s.%(msecs)d " + base_fmt_str, "%Y-%m-%dT%H:%M:%S" +) +screenlog = logging.StreamHandler() +screenlog.setFormatter(screen_fmt) +log.addHandler(screenlog) + + # mock event data with open("../../data/lambda_event.json", "r") as f: mock_event = json.load(f) diff --git a/ctorm/ctorm/load_tester.py b/ctorm/ctorm/load_tester.py index a805525..6effb63 100644 --- a/ctorm/ctorm/load_tester.py +++ b/ctorm/ctorm/load_tester.py @@ -1,11 +1,27 @@ +import json import logging import os +from typing import List +from aws_lambda_typing import context as context_ +from aws_lambda_typing import events from cnm_sender import CnmSender +from ctorm.config import ( + CtormConfig, + CtormPreparedGranule, +) + log = logging.getLogger(__name__) +def configure_cfg(): + cfg = CtormConfig.from_file( + cfg_file=os.getenv("CFG_FILE", "./ctorm.cfg"), + ) + return cfg + + def configure_logging() -> None: level_name = os.environ.get("LOG_LEVEL", "INFO").upper() level = getattr(logging, level_name, logging.INFO) @@ -21,17 +37,20 @@ def configure_logging() -> None: for handler in root_logger.handlers: handler.setLevel(level) handler.setFormatter( - logging.Formatter("%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d - %(message)s") + logging.Formatter( + "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d - %(message)s" + ) ) -def load_test(): - c_sender = CnmSender({}) - c_sender.send() +def load_test(cfg: CtormConfig, granule_list: List[CtormPreparedGranule]): + c_sender = CnmSender(cfg, granule_list) + c_sender.send_all() -def lambda_handler(event, context): +def lambda_handler(event: events.EventBridgeEvent, context: context_.Context): configure_logging() + cfg = configure_cfg() log.info( "Starting CNM sender invocation", @@ -42,8 +61,9 @@ def lambda_handler(event, context): try: log.debug("Received event: %s", event) - - load_test() + g_list = event["Records"].pop().get("body") + g_list = json.loads(g_list).get("granules") + load_test(cfg, g_list) log.info("CNM sender invocation completed") return {"ok": True} @@ -54,5 +74,4 @@ def lambda_handler(event, context): if __name__ == "__main__": - configure_logging() lambda_handler({}, {}) diff --git a/ctorm/ctorm/main.py b/ctorm/ctorm/main.py index 000ceb5..6cdaf4b 100644 --- a/ctorm/ctorm/main.py +++ b/ctorm/ctorm/main.py @@ -87,7 +87,9 @@ def main(args: Optional[list[str]] = None): botocore_logger.setLevel(logging.DEBUG) base_fmt_str = "%(levelname)s: %(message)s (%(filename)s line %(lineno)d/)" - screen_fmt = logging.Formatter("%(asctime)s.%(msecs)d " + base_fmt_str, "%Y-%m-%dT%H:%M:%S") + screen_fmt = logging.Formatter( + "%(asctime)s.%(msecs)d " + base_fmt_str, "%Y-%m-%dT%H:%M:%S" + ) screenlog = logging.StreamHandler() screenlog.setFormatter(screen_fmt) root_logger.addHandler(screenlog) diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 9a19794..6866da6 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -39,13 +39,13 @@ class CtormSqsMessage: def __init__(self, cfg: CtormConfig): self.cfg = cfg - self.files = [] + self.granules = [] def add_file(self, file: dict): - self.files.append(file) + self.granules.append(file) def to_dict(self): - return {"files": self.files} + return {"granules": self.granules} def to_json(self): return json.dumps(self.to_dict()) @@ -66,12 +66,16 @@ def __init__(self, cfg: CtormConfig): # init buckets for bkt in self.cfg.pipelines: bkt.next_cont_token = None - keysuffix = f"_{bkt.prepare_keypair_name}" if bkt.prepare_keypair_name else "" + keysuffix = ( + f"_{bkt.prepare_keypair_name}" if bkt.prepare_keypair_name else "" + ) if bkt.prepare_keypair_name not in self._boto_sessions: self._boto_sessions[bkt.prepare_keypair_name] = boto3.Session( region_name=AWS_REGION, aws_access_key_id=os.getenv(f"AWS_ACCESS_KEY_ID{keysuffix}"), - aws_secret_access_key=os.getenv(f"AWS_SECRET_ACCESS_KEY{keysuffix}"), + aws_secret_access_key=os.getenv( + f"AWS_SECRET_ACCESS_KEY{keysuffix}" + ), ) bkt.s3_client = self._boto_sessions[bkt.prepare_keypair_name].client("s3") @@ -101,7 +105,10 @@ def prepare(self): log.debug("tot granules: %d/%d", self.total_granules, self.cfg.granule_goal) log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) - log.debug("sqs message percentage: %d%%", (len(sqs_msg.to_json()) / CtormSqsMessage.MAX_MESSAGE_SIZE) * 100) + log.debug( + "sqs message percentage: %d%%", + (len(sqs_msg.to_json()) / CtormSqsMessage.MAX_MESSAGE_SIZE) * 100, + ) log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) if sqs_msg.check_message_size(): log.debug("sqs_msg: %s", sqs_msg.to_json()) @@ -147,10 +154,14 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormPipeline) -> dict: # Replace the granulename with a token for compression. Will reconstitute in the lambda fileval = ( - r_urls["URL"].replace(outdict[K.GRANULE], "$G").replace(bucket, f"${outdict[K.BKT_MAP][bucket]}") + r_urls["URL"] + .replace(outdict[K.GRANULE], "$G") + .replace(bucket, f"${outdict[K.BKT_MAP][bucket]}") ) filedict = {"f": fileval} - for distr_file in ummg["DataGranule"]["ArchiveAndDistributionInformation"]: + for distr_file in ummg["DataGranule"][ + "ArchiveAndDistributionInformation" + ]: if r_urls["URL"].endswith(distr_file["Name"]): # We handily have the md5 and size in the ummg filedict[K.MD5] = distr_file["Checksum"]["Value"] @@ -160,14 +171,22 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormPipeline) -> dict: # We must look to S3 for the size and md5 log.debug('getting head for "%s"', objloc) try: - headobj = ct_bkt.s3_client.head_object(Bucket=bucket, Key=objloc) + headobj = ct_bkt.s3_client.head_object( + Bucket=bucket, Key=objloc + ) log.debug("head_object: %s", headobj) filedict[K.SIZE] = headobj["ContentLength"] filedict[K.MD5] = headobj["ETag"].replace('"', "") if filedict[K.MD5].endswith("-1"): # This was a multipart upload. We'll have to do something clever to get the MD5 of it. - log.debug("multipart upload detected for %s", objloc) - filedict[K.MD5] = self.get_real_md5(ct_bkt, bucket, objloc) + + if self.cfg.calc_md5: + log.debug( + "multipart upload detected for %s", objloc + ) + filedict[K.MD5] = self.get_real_md5( + ct_bkt, bucket, objloc + ) except ClientError as e: log.error("head_object failed: %s", e) # TODO: trash entire message for this granule? @@ -185,6 +204,8 @@ def get_real_md5( # The file is small enough we may as well download it to memory and get the MD5 that way. resp = b.s3_client.get_object(Bucket=obj_bucket, Key=key) md5_accumulator = hashlib.md5() - for chunk in iter(lambda: resp["Body"].read(self.MD5_DL_CHUNK_MB * 1024 * 1024), b""): + for chunk in iter( + lambda: resp["Body"].read(self.MD5_DL_CHUNK_MB * 1024 * 1024), b"" + ): md5_accumulator.update(chunk) return md5_accumulator.hexdigest() diff --git a/ctorm/pyproject.toml b/ctorm/pyproject.toml index faea8cd..b7e2e61 100644 --- a/ctorm/pyproject.toml +++ b/ctorm/pyproject.toml @@ -8,6 +8,7 @@ authors = [ ] requires-python = ">=3.12" dependencies = [ + "aws-lambda-typing>=2.20.0", "boto3>=1.43.26", "mypy-boto3-s3>=1.43.14", ] @@ -43,3 +44,6 @@ deps = commands = pytest """ + +[tool.ruff] +target-version = "py312" diff --git a/ctorm/tests/test_cnm.py b/ctorm/tests/test_cnm.py new file mode 100644 index 0000000..96ed241 --- /dev/null +++ b/ctorm/tests/test_cnm.py @@ -0,0 +1,33 @@ +from ctorm.cnm import CtormCnmSGenerator +from ctorm.config import CtormPreparedGranule + + +def test_ctormcnmgenerator(): + test_input: CtormPreparedGranule = { + "bm": { + "sds-n-cumulus-prod-nisar-products": "B1", + "sds-n-cumulus-prod-nisar-jpl-private-data": "B2", + }, + "g": "NISAR_L0_RRST_VC08_20250821T101036_20250821T101041_P00408_J_001", + "c": "NISAR_L0A_RRST_BETA_V1", + "cv": "1", + "f": [ + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin", + "m": "e989430f4c5bfa04eeb26e56f4cd8375", + "s": 1073829760, + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.rc.yaml", + "s": 120597, + "m": "11bcaec780f0d246f3996b3f08680410", + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin.qa", + "s": 1134, + }, + ], + } + ctorm_cnmsgen = CtormCnmSGenerator() + output = ctorm_cnmsgen(test_input, "FOO") + assert output diff --git a/ctorm/tests/test_config.py b/ctorm/tests/test_config.py index ac4bebb..1e76c50 100644 --- a/ctorm/tests/test_config.py +++ b/ctorm/tests/test_config.py @@ -1,10 +1,12 @@ from unittest.mock import mock_open, patch -from ctorm.config import ConfigError, CtormConfig, CtormPipeline +from ctorm.config import CtormConfig, CtormPipeline def test_ctorm_bucket_initialization(): - bucket = CtormPipeline(bucketname="test_bucket", share=50, ummg_prefix="test_prefix/") + bucket = CtormPipeline( + bucketname="test_bucket", share=50, ummg_prefix="test_prefix/" + ) assert bucket.bucketname == "test_bucket" assert bucket.share == 50 @@ -18,7 +20,10 @@ def test_ctorm_config_from_file_success(): source_buckets = [{"bucketname": "test_bucket", "share": 50}] granules_sqs_queue_url = "https://sqs.queue.url/" """ - with patch("builtins.open", mock_open(read_data=mock_data)), patch("tomllib.load") as mock_toml: + with ( + patch("builtins.open", mock_open(read_data=mock_data)), + patch("tomllib.load") as mock_toml, + ): mock_toml.return_value = { "ctorm": { "source_buckets": [{"bucketname": "test_bucket", "share": 50}], @@ -35,9 +40,12 @@ def test_ctorm_config_from_file_success(): def test_ctorm_config_from_file_missing_section(): mock_data = "{}" - with patch("builtins.open", mock_open(read_data=mock_data)), patch("tomllib.load") as mock_toml: + with ( + patch("builtins.open", mock_open(read_data=mock_data)), + patch("tomllib.load") as mock_toml, + ): mock_toml.return_value = {} try: CtormConfig.from_file("test.cfg") - except ConfigError as e: + except KeyError as e: assert str(e) == "No 'ctorm' section in config file" diff --git a/ctorm/uv.lock b/ctorm/uv.lock index 153e96f..ccd00a2 100644 --- a/ctorm/uv.lock +++ b/ctorm/uv.lock @@ -2,6 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.12" +[[package]] +name = "aws-lambda-typing" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/2a/14e7a3dba738265db6eddfd0c9ec1619c0c14e48fcf70627fc75fca83305/aws-lambda-typing-2.20.0.tar.gz", hash = "sha256:78b0d8ebab73b3a6b0da98a7969f4e9c4bb497298ec50f3217da8a8dfba17154", size = 19413, upload-time = "2024-04-02T09:43:03.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/3d/4031f5950d65e89136d20a18b5ac985d5f087c29c902f285b6e8063619e4/aws_lambda_typing-2.20.0-py3-none-any.whl", hash = "sha256:1d44264cabfeab5ac38e67ddd0c874e677b2cbbae77a42d0519df470e6bbb49b", size = 35296, upload-time = "2024-04-02T09:43:05.881Z" }, +] + [[package]] name = "black" version = "26.5.1" @@ -113,26 +122,30 @@ name = "ctorm" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "aws-lambda-typing" }, { name = "boto3" }, + { name = "mypy-boto3-s3" }, ] [package.dev-dependencies] dev = [ { name = "black" }, { name = "boto3-stubs" }, - { name = "mypy-boto3-s3" }, { name = "pytest" }, { name = "ruff" }, ] [package.metadata] -requires-dist = [{ name = "boto3", specifier = ">=1.43.26" }] +requires-dist = [ + { name = "aws-lambda-typing", specifier = ">=2.20.0" }, + { name = "boto3", specifier = ">=1.43.26" }, + { name = "mypy-boto3-s3", specifier = ">=1.43.14" }, +] [package.metadata.requires-dev] dev = [ { name = "black", specifier = ">=26.5.1" }, { name = "boto3-stubs", specifier = ">=1.43.27" }, - { name = "mypy-boto3-s3", specifier = ">=1.43.14" }, { name = "pytest", specifier = ">=8.0.0" }, { name = "ruff", specifier = ">=0.15.13" }, ] From ee46ce936cb4b32202be90114722154d052e029a Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 19 Jun 2026 20:20:05 -0800 Subject: [PATCH 16/31] PR-7502 Refactored to grab SQS from lambda --- ctorm/ctorm/cnm_sender.py | 6 +++- ctorm/ctorm/lambda_run.py | 7 +++-- ctorm/ctorm/load_tester.py | 59 ++++++++++++++++++++++++++++++++++---- ctorm/ctorm/prepare.py | 27 ++++++++--------- 4 files changed, 77 insertions(+), 22 deletions(-) diff --git a/ctorm/ctorm/cnm_sender.py b/ctorm/ctorm/cnm_sender.py index 07347e1..12f1dbf 100644 --- a/ctorm/ctorm/cnm_sender.py +++ b/ctorm/ctorm/cnm_sender.py @@ -19,7 +19,11 @@ def __init__( self.provider = provider self.cnm_s_generator = CtormCnmSGenerator() - def send_all(self): + def send_all(self) -> bool: for granule in self.granules: cnms = self.cnm_s_generator(granule, self.provider) log.debug("Sending %s", cnms) + # TODO: actually send + + # TODO: determine successfulllness + return True diff --git a/ctorm/ctorm/lambda_run.py b/ctorm/ctorm/lambda_run.py index 10e26cc..cf4556f 100644 --- a/ctorm/ctorm/lambda_run.py +++ b/ctorm/ctorm/lambda_run.py @@ -5,6 +5,8 @@ import json import logging +from aws_lambda_typing import context as context_ + from ctorm.load_tester import lambda_handler log = logging.getLogger() @@ -19,14 +21,15 @@ # mock event data -with open("../../data/lambda_event.json", "r") as f: +with open("../../data/lambda-event-eventbridge.json", "r") as f: mock_event = json.load(f) # Mock context object -class MockContext: +class MockContext(context_.Context): function_name = "local_test" memory_limit_in_mb = 128 + aws_request_id = "LOLOCAL" # Run it diff --git a/ctorm/ctorm/load_tester.py b/ctorm/ctorm/load_tester.py index 6effb63..1cd3138 100644 --- a/ctorm/ctorm/load_tester.py +++ b/ctorm/ctorm/load_tester.py @@ -1,13 +1,16 @@ import json import logging import os +from functools import cache from typing import List +import boto3 from aws_lambda_typing import context as context_ from aws_lambda_typing import events from cnm_sender import CnmSender from ctorm.config import ( + AWS_REGION, CtormConfig, CtormPreparedGranule, ) @@ -15,6 +18,12 @@ log = logging.getLogger(__name__) +@cache +def get_sqs_client(): + return boto3.client("sqs", region_name=AWS_REGION) + + +@cache def configure_cfg(): cfg = CtormConfig.from_file( cfg_file=os.getenv("CFG_FILE", "./ctorm.cfg"), @@ -43,9 +52,46 @@ def configure_logging() -> None: ) -def load_test(cfg: CtormConfig, granule_list: List[CtormPreparedGranule]): - c_sender = CnmSender(cfg, granule_list) - c_sender.send_all() +def get_granule_list(event: events.EventBridgeEvent) -> List[CtormPreparedGranule]: + # TODO: Implement + return [] + + +def load_test( + cfg: CtormConfig, + gr_queue_url: str, +): + # fetch 10 messages from SQS + messages = [] + while len(messages) < 10: + # Since we're trying to send x granules per invocation, we will do what we can + # to get a full load of 10 messages. + sqs_resp = get_sqs_client().receive_message( + QueueUrl=gr_queue_url, + MaxNumberOfMessages=min(10 - len(messages), 10), + WaitTimeSeconds=5, + VisibilityTimeout=3600, + ) + + batch = sqs_resp.get("Messages", []) + if not batch: + break + + messages.extend(batch) + + for message in sqs_resp.get("Messages", []): + body = json.loads(message["Body"]) + receipt_handle = message["ReceiptHandle"] + granule_list = body.get("granules", []) + + c_sender = CnmSender(cfg, granule_list) + success = c_sender.send_all() + if success: + log.debug("Deleting message %s", receipt_handle) + # TODO: uncomment after some dev'ing + # get_sqs_client().delete_message( + # QueueUrl=gr_queue_url, ReceiptHandle=receipt_handle + # ) def lambda_handler(event: events.EventBridgeEvent, context: context_.Context): @@ -61,9 +107,10 @@ def lambda_handler(event: events.EventBridgeEvent, context: context_.Context): try: log.debug("Received event: %s", event) - g_list = event["Records"].pop().get("body") - g_list = json.loads(g_list).get("granules") - load_test(cfg, g_list) + # g_list = event["Records"].pop().get("body") + # g_list = json.loads(g_list).get("granules") + + load_test(cfg, event["granules_queue_url"]) log.info("CNM sender invocation completed") return {"ok": True} diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py index 6866da6..7cbd6f6 100644 --- a/ctorm/ctorm/prepare.py +++ b/ctorm/ctorm/prepare.py @@ -10,7 +10,7 @@ from botocore.exceptions import ClientError from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef -from ctorm.config import AWS_REGION, CtormConfig, CtormPipeline +from ctorm.config import AWS_REGION, MD5_CHECKSUM_PATTERN, CtormConfig, CtormPipeline log = getLogger(__name__) @@ -164,8 +164,9 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormPipeline) -> dict: ]: if r_urls["URL"].endswith(distr_file["Name"]): # We handily have the md5 and size in the ummg - filedict[K.MD5] = distr_file["Checksum"]["Value"] + # TODO: double-check this test is correct and we're not unnecessarily HEADing too many files. filedict[K.SIZE] = distr_file["SizeInBytes"] + filedict[K.MD5] = distr_file["Checksum"]["Value"] break else: # We must look to S3 for the size and md5 @@ -176,17 +177,17 @@ def process_ummg(self, ummg: dict, ct_bkt: CtormPipeline) -> dict: ) log.debug("head_object: %s", headobj) filedict[K.SIZE] = headobj["ContentLength"] - filedict[K.MD5] = headobj["ETag"].replace('"', "") - if filedict[K.MD5].endswith("-1"): - # This was a multipart upload. We'll have to do something clever to get the MD5 of it. - - if self.cfg.calc_md5: - log.debug( - "multipart upload detected for %s", objloc - ) - filedict[K.MD5] = self.get_real_md5( - ct_bkt, bucket, objloc - ) + + md5 = headobj["ETag"].replace('"', "") + if MD5_CHECKSUM_PATTERN.fullmatch(md5): + filedict[K.MD5] = md5 + elif self.cfg.calc_md5: + log.debug("multipart upload detected for %s", objloc) + filedict[K.MD5] = self.get_real_md5( + ct_bkt, bucket, objloc + ) + else: + log.debug("no need to calculate md5 for %s", objloc) except ClientError as e: log.error("head_object failed: %s", e) # TODO: trash entire message for this granule? From d4b674a2b7077d6497afcd03fbe292f32376415d Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 23 Jun 2026 18:03:02 -0800 Subject: [PATCH 17/31] PR-7509 method for pulling all metadata out of the cumulus RDS --- ctorm/cumulus-db-md-extract/README.md | 74 +++++++++++++++++++ .../export-granules-daily.sh | 47 ++++++++++++ .../export-granules-slice-daily.sql | 67 +++++++++++++++++ .../generate-slices-daily.sh | 31 ++++++++ 4 files changed, 219 insertions(+) create mode 100644 ctorm/cumulus-db-md-extract/README.md create mode 100644 ctorm/cumulus-db-md-extract/export-granules-daily.sh create mode 100644 ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql create mode 100644 ctorm/cumulus-db-md-extract/generate-slices-daily.sh diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md new file mode 100644 index 0000000..3b04473 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/README.md @@ -0,0 +1,74 @@ +# Cumulus DB MD Extractor + +The purpose of this is to get enough medata from the prod cumulus RDS db and into a Dynamo DB to be able to recreate +CNM-S messages. + +# How to use: + +## From the web console + +* Open a web console from Kion + * Open a tab with the secrets manager + * Find the secret with `db_login` in the name. This is where you find the host, user/pass, etc. + * Open a tab with the RDS + * Go to the query editor + * Open a CloudShell in anohter tab. + * Pick the VPC + * Pick the same subnet as the RDS + * Pick the default security group and the one(s) with `_rds_` in the name. + * TODO: see if we can do this with EC2 instead of cloudshell + +## From the shell + +### prepare the environment + +Create AWS credentials to upload to the CTORM bucket. + +```bash +mkdir .aws +vi .aws/credentials + +[ctorm] +aws_access_key_id = AKYEAH... +aws_secret_access_key = foofoofooofooo +``` + +Make some env vars: + +```bash + +read -s PGPASSWORD +export PGPASSWORD +export PGHOST='from secrets mgr' +export PGUSER='from secrets mgr' +export PGDATABASE='from secrets mgr' + +# this is the ctorm bucket and path info where the db dumps are stored +export CTORM_BUCKET='ctorm-dev-scratch' +export DUMP_SUBDIR='nisar' + +``` + +Get these files into the instance one way or another. + +* ctorm/cumulus-db-md-extract/export-granules-daily.sh +* ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql +* ctorm/cumulus-db-md-extract/generate-slices-daily.sh + +## Run the scripts + +```bash +bash generate-slices-daily.sh +``` + +This creates a `slices_daily.tsv` file. Basically a list of chunked work for export-granules-daily.sh to use. + +The following step will query the database and create gzipped ljson files and upload them to S3: + +```bash +bash export-granules-daily.sh +``` + +If running in cloudshell, you need to put some sort of input into the console every 10 minutes or so, or AWS will kill +the session. +Really should try to create a EC2 next time. diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh new file mode 100644 index 0000000..ec07cc7 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -0,0 +1,47 @@ +set -euo pipefail + +S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" + +while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do + if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then + echo "Skipping malformed slice row: collection=${COLLECTION:-} yyyymmdd=${YYYYMMDD:-}" >&2 + continue + fi + + YYYYMM="${YYYYMMDD:0:6}" + SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') + S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + + echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" + echo "Destination: ${S3_URI}" + + psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --set=ON_ERROR_STOP=1 \ + --set=collection="$COLLECTION" \ + --set=yyyymmdd="$YYYYMMDD" \ + --file=export-granules-slice-daily.sql \ + | awk -v label="${COLLECTION} ${YYYYMMDD}" ' + { + print; + count++; + if (count % 1000 == 0) { + printf("[%s] exported %d JSONL rows\n", label, count) > "/dev/stderr"; + fflush("/dev/stderr"); + } + } + END { + printf("[%s] export complete: %d JSONL rows\n", label, count) > "/dev/stderr"; + } + ' \ + | gzip -c \ + | aws --profile=ctorm s3 cp - "$S3_URI" + + echo "Uploaded ${S3_URI}" + +done < slices_daily.tsv diff --git a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql new file mode 100644 index 0000000..b23af06 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql @@ -0,0 +1,67 @@ +\pset tuples_only on +\pset format unaligned +\pset pager off + +WITH export_rows AS ( + SELECT + g.granule_id, + c.name AS collection, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMM') AS yyyymm, + to_char( + g.beginning_date_time AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) AS beginning_date_time, + jsonb_agg( + jsonb_build_object( + 'name', f.file_name, + 'type', f.type, + 'uri', 's3://' || f.bucket || '/' || f.key, + 'size', f.file_size, + 'checksum', f.checksum_value, + 'checksumType', 'md5' + ) + ORDER BY f.file_name + ) AS files + FROM public.granules g + JOIN public.collections c + ON c.cumulus_id = g.collection_cumulus_id + JOIN public.files f + ON f.granule_cumulus_id = g.cumulus_id + AND f.checksum_value IS NOT NULL + WHERE g.status = 'completed' + AND c.name = :'collection' + AND g.beginning_date_time >= to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') AT TIME ZONE 'UTC' + AND g.beginning_date_time < ( + to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') + interval '1 month' + ) AT TIME ZONE 'UTC' + GROUP BY + g.granule_id, + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMM'), + to_char( + g.beginning_date_time AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) +) +SELECT jsonb_strip_nulls( + jsonb_build_object( + 'pk', collection || '#' || yyyymm, + 'sk', beginning_date_time || '#' || granule_id, + + 'gsi1pk', granule_id, + 'gsi1sk', collection || '#' || yyyymm, + + 'collection', collection, + 'yyyymm', yyyymm, + 'beginning_date_time', beginning_date_time, + 'granule_id', granule_id, + 'load_test_count', 0, + + 'n', granule_id, + 'c', collection, + 'cv', '1', + 'f', files + ) +) +FROM export_rows +ORDER BY beginning_date_time, granule_id; \ No newline at end of file diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh new file mode 100644 index 0000000..60b9407 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -0,0 +1,31 @@ +psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --tuples-only \ + --no-align \ + --field-separator $'\t' \ + --set=ON_ERROR_STOP=1 \ + --command " + SELECT + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') AS yyyymmdd, + count(DISTINCT g.cumulus_id) AS granule_count, + count(f.cumulus_id) AS file_count + FROM public.granules g + JOIN public.collections c + ON c.cumulus_id = g.collection_cumulus_id + JOIN public.files f + ON f.granule_cumulus_id = g.cumulus_id + AND f.checksum_value IS NOT NULL + WHERE g.status = 'completed' + GROUP BY + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') + ORDER BY + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); + " > slices_daily.tsv From a32bd6c935465780d686e86086796dc25445152b Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 24 Jun 2026 11:53:25 -0800 Subject: [PATCH 18/31] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/README.md | 33 + .../terraform/.terraform.lock.hcl | 45 + .../terraform/codebuild-db-md-extract.tf | 296 ++++ .../terraform/nisar-prod.tfvars | 22 + .../terraform/opera-prod.tfvars | 22 + .../terraform/opera-uat.tfvars | 23 + .../terraform/terraform.tfstate | 360 +++++ .../terraform/terraform.tfvars | 1 + ctorm/infra/terraform/main.tf | 87 ++ ctorm/infra/terraform/outputs.tf | 4 + ctorm/infra/terraform/placeholder-lambda.zip | Bin 339 -> 344 bytes ctorm/infra/terraform/terraform.tfstate | 238 +++- .../infra/terraform/terraform.tfstate.backup | 1226 +++++++++++++++++ ctorm/infra/terraform/variables.tf | 5 + 14 files changed, 2344 insertions(+), 18 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl create mode 100644 ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf create mode 100644 ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars create mode 100644 ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars create mode 100644 ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform.tfstate create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform.tfvars create mode 100644 ctorm/infra/terraform/terraform.tfstate.backup diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index 3b04473..65c4d70 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -72,3 +72,36 @@ bash export-granules-daily.sh If running in cloudshell, you need to put some sort of input into the console every 10 minutes or so, or AWS will kill the session. Really should try to create a EC2 next time. + +## Using Codebuild + +### Terraform + +```bash +```bash +export AWS_PROFILE="cumulus-uat-6921" +export VARFILE=opera-uat.tfvars +terraform init + +terraform plan \ + -var-file="${VARFILE}" + +terraform apply \ + -var-file="${VARFILE}" + +terraform destroy + + +``` + +``` + +```bash +aws codebuild create-project \ + --name "OneTimeDataPull" \ + --source '{"type": "S3", "location": "ctorm-dev-scratch/codebuild/codebuild-noop.zip"}' \ + --environment '{"type": "LINUX_CONTAINER", "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", "computeType": "BUILD_GENERAL1_SMALL"}' \ + --artifacts '{"type": "NO_ARTIFACTS"}' \ + --service-role "arn:aws:iam::123456789012:role/service-role/YourCodeBuildRole" \ + --timeout-in-minutes 240 +``` diff --git a/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl b/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..8e5ba2e --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl @@ -0,0 +1,45 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.0" + hashes = [ + "h1:WB6H5ksIZiyq1lQlD/PWeh+tn4FLsbSjVnRW3+4xe2Y=", + "zh:0d14713fdc259fb377d0b899ad3c650a34194bd52194c863303ef22a65a580e2", + "zh:369b56040c7a8085d04e7e8ffac1e2b321a3170e502f788819bc34b868ec016f", + "zh:4d1a3b983ed6af5a52bfe12794674ae55cbadfa6021b37106ade68b433ad216a", + "zh:5c547549e26e083573c78a966ca68ce6d7df6bb8f3948f66a575f07da46b74ea", + "zh:6de093e62a975eb19a5e3017ce38e6e3cb639c17b79648d2000e0a8348f0e997", + "zh:7267936c2cdbc448efeb594d73e6b56a53d6a7ae14fe88cdd2a4133adc3302f0", + "zh:7482f023050ed426b4b45116e1761643bc33b1fd4ce4a6fab207ae2571f35940", + "zh:76bbd93b234e5a2927d98b511d86565700f549b570871a194c35f944b96cefb7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:c6afc4bc1f002bac9c173007dd4da05fde788cd14c2916089f958c33fedb0dfa", + "zh:d3ba40bd806a3a08e9237dece679193c99afb2085de6b45d7f5d1f673cfcd368", + "zh:e1ad7ded53ecd6f0e5b473a3b44eae2b2e885653a56050ab583d387332be02e4", + "zh:e93e78575ce82be6084cc153c24ba8f385dc8d6880888ee66e918460c870953d", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.52.0" + hashes = [ + "h1:lpXqosKH8yAahK3SA1P5Pdy1ziXJcY+blUidY0q9yGk=", + "zh:1ab1d78f2336fed42b4e13fa0077a0be9d86a7899897cda5b9f1a60051ec2e93", + "zh:1df11f5f252030803939a1c778931dbdcee1b1070b38b98d9a8cbfc26c2aeb8e", + "zh:20ec9af03c0c1f2f8582a8805d43a4cd1a0a65082308e00f025d87605715a88f", + "zh:2d5562ed0e7cb0892fb537c7989d25fdde1d0ac1f3a768ba15fa087985f2a0f5", + "zh:40d64f668961a172355c3d11d258e19172ffafb421c40d89266b129ed8f92a5d", + "zh:497792bccc33001247473bc32a148c067982cb3d245b8f3610afb920635d2235", + "zh:8011f9167082af74ed9257582a83d54e889388656597721b2691797f1dd6fb58", + "zh:8b10d1bbca51b0da1e1be0967ce75b039f2d5d86f2ca7339994a92d35cdf47a8", + "zh:9549647dbf7c913512c26fed6badd7bf24a29a36c2bd308536849303a85427da", + "zh:97c4b726cdbe48166f4b9e6a1230312a7933dc19dd139d941ca6aca86706eb33", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a484bc8166278b546bfe53698fa9e0a919dffe3bfda3c8bf8a0fc6811b5b9e66", + "zh:aa96e76bd9f93e5395a553fccec485554a74acae46b3834b1296374eba4f010f", + "zh:cf290ee3d0dfe91b596445f860440d9f18826f7395ff785f83f70d36cedadd1c", + "zh:d56b6a79207663673f66d9ed378d705c1bd1b4d117eeb5e2be3938cf1b75b7be", + "zh:febef068317f8e49bd438ccdf2f54345fc3bc4b80a2699d9ab3c449d7e521d8e", + ] +} diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf new file mode 100644 index 0000000..61c2f10 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -0,0 +1,296 @@ +# codebuild-db-md-extract.tf + +variable "name_prefix" { + type = string + default = "ctorm" +} + +variable "aws_region" { + type = string + default = "us-west-2" +} + +variable "db_md_extract_secret_arn" { + type = string + description = "ARN of the Secrets Manager secret containing host, username, password, and database." +} + +variable "db_md_extract_vpc_id" { + type = string + description = "VPC where CodeBuild should run to reach the RDS instance." +} + +variable "db_md_extract_subnet_ids" { + type = list(string) + description = "Private subnet IDs that can reach the RDS instance." +} + +variable "db_md_extract_security_group_ids" { + type = list(string) + description = "Security group IDs for CodeBuild. These must allow egress to the RDS port and be allowed by the RDS SG." +} + +variable "db_md_extract_dump_subdir" { + type = string + default = "nisar" +} + +variable "db_md_extract_target_bucket" { + type = string + default = "" + description = "Destination S3 bucket for exported dump files." +} + +variable "db_md_extract_target_role_arn" { + type = string + description = "Role ARN in the target account that CodeBuild can assume to write exported files." +} + +locals { + db_md_extract_project_name = "${var.name_prefix}-cumulus-db-md-extract" + db_md_extract_source_key = "codebuild/cumulus-db-md-extract.zip" +} + +data "archive_file" "cumulus_db_md_extract" { + type = "zip" + source_dir = "${path.module}/../../cumulus-db-md-extract" + output_path = "${path.module}/cumulus-db-md-extract.zip" +} + +resource "aws_s3_object" "cumulus_db_md_extract_source" { + bucket = aws_s3_bucket.scratch.bucket + key = local.db_md_extract_source_key + source = data.archive_file.cumulus_db_md_extract.output_path + source_hash = data.archive_file.cumulus_db_md_extract.output_base64sha256 +} + +data "aws_iam_policy_document" "codebuild_db_md_extract_assume_role" { + statement { + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "Service" + + identifiers = [ + "codebuild.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role" "codebuild_db_md_extract" { + name = "${local.db_md_extract_project_name}-role" + assume_role_policy = data.aws_iam_policy_document.codebuild_db_md_extract_assume_role.json +} + +resource "aws_cloudwatch_log_group" "codebuild_db_md_extract" { + name = "/aws/codebuild/${local.db_md_extract_project_name}" + retention_in_days = 14 +} + +data "aws_iam_policy_document" "codebuild_db_md_extract" { + statement { + sid = "WriteCodeBuildLogs" + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = [ + "${aws_cloudwatch_log_group.codebuild_db_md_extract.arn}:*", + ] + } + + statement { + sid = "ReadBuildSourceFromScratchBucket" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + ] + + resources = [ + "${aws_s3_bucket.scratch.arn}/${local.db_md_extract_source_key}", + ] + } + + statement { + sid = "ListScratchBucketForBuildSource" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.scratch.arn, + ] + + condition { + test = "StringLike" + variable = "s3:prefix" + + values = [ + "codebuild/*", + ] + } + } + + statement { + sid = "ReadDatabaseSecret" + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ] + + resources = [ + var.db_md_extract_secret_arn, + ] + } + + statement { + sid = "AssumeTargetAccountUploadRole" + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + resources = [ + var.db_md_extract_target_role_arn, + ] + } + + statement { + sid = "DescribeNetworkForVpcBuild" + effect = "Allow" + + actions = [ + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:CreateNetworkInterface", + ] + + resources = [ + "*", + ] + } + + statement { + sid = "CreateCodeBuildNetworkInterfacePermission" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterfacePermission", + ] + + resources = [ + "arn:aws:ec2:${var.aws_region}:*:network-interface/*", + ] + + condition { + test = "StringEquals" + variable = "ec2:AuthorizedService" + + values = [ + "codebuild.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role_policy" "codebuild_db_md_extract" { + name = "${local.db_md_extract_project_name}-policy" + role = aws_iam_role.codebuild_db_md_extract.id + policy = data.aws_iam_policy_document.codebuild_db_md_extract.json +} + +resource "aws_codebuild_project" "cumulus_db_md_extract" { + name = local.db_md_extract_project_name + description = "One-time Cumulus RDS metadata export to S3" + service_role = aws_iam_role.codebuild_db_md_extract.arn + build_timeout = 480 + + artifacts { + type = "NO_ARTIFACTS" + } + + source { + type = "S3" + location = "${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + buildspec = "buildspec.yaml" + } + + environment { + type = "LINUX_CONTAINER" + image = "aws/codebuild/amazonlinux2-x86_64-standard:5.0" + compute_type = "BUILD_GENERAL1_SMALL" + + environment_variable { + name = "AWS_DEFAULT_REGION" + value = var.aws_region + } + + environment_variable { + name = "CTORM_BUCKET" + value = var.db_md_extract_target_bucket + } + + environment_variable { + name = "DUMP_SUBDIR" + value = var.db_md_extract_dump_subdir + } + + environment_variable { + name = "DB_SECRET_ARN" + value = var.db_md_extract_secret_arn + } + + environment_variable { + name = "TARGET_ROLE_ARN" + value = var.db_md_extract_target_role_arn + } + } + + vpc_config { + vpc_id = var.db_md_extract_vpc_id + subnets = var.db_md_extract_subnet_ids + security_group_ids = var.db_md_extract_security_group_ids + } + + logs_config { + cloudwatch_logs { + group_name = aws_cloudwatch_log_group.codebuild_db_md_extract.name + status = "ENABLED" + } + } + + depends_on = [ + aws_iam_role_policy.codebuild_db_md_extract, + aws_s3_object.cumulus_db_md_extract_source, + ] +} + +output "cumulus_db_md_extract_codebuild_project_name" { + value = aws_codebuild_project.cumulus_db_md_extract.name +} + +output "cumulus_db_md_extract_codebuild_role_arn" { + value = aws_iam_role.codebuild_db_md_extract.arn +} + +output "cumulus_db_md_extract_source_s3_uri" { + value = "s3://${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" +} diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars new file mode 100644 index 0000000..de3cc0a --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -0,0 +1,22 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" + +db_md_extract_vpc_id = "vpc-..." + +db_md_extract_subnet_ids = [ + "subnet-...", + "subnet-...", +] + +db_md_extract_security_group_ids = [ + "sg-...", +] + +db_md_extract_source_bucket = "" + +db_md_extract_target_bucket = "ctorm-dev-scratch" + +db_md_extract_target_role_arn = "arn:aws:iam:::role/" + +db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars new file mode 100644 index 0000000..de3cc0a --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -0,0 +1,22 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" + +db_md_extract_vpc_id = "vpc-..." + +db_md_extract_subnet_ids = [ + "subnet-...", + "subnet-...", +] + +db_md_extract_security_group_ids = [ + "sg-...", +] + +db_md_extract_source_bucket = "" + +db_md_extract_target_bucket = "ctorm-dev-scratch" + +db_md_extract_target_role_arn = "arn:aws:iam:::role/" + +db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars new file mode 100644 index 0000000..b27da44 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars @@ -0,0 +1,23 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + +db_md_extract_vpc_id = "vpc-080322e163a0e7390" + +db_md_extract_subnet_ids = [ + "subnet-0ac04d1e36d0a50ea", +] + +db_md_extract_security_group_ids = [ + "sg-03beee748d2a43cc9", + "sg-0afdcb46b1a38db46", + "sg-06255aca3a87784a0", +] + +db_md_extract_source_bucket = "ctorm-dev-scratch" + +db_md_extract_target_bucket = "ctorm-dev-scratch" + +db_md_extract_target_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + +db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate new file mode 100644 index 0000000..a6e1dad --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate @@ -0,0 +1,360 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 6, + "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "baed98d70a081d5ddd95b21dade6d5b45112af30", + "output_base64sha256": "bS+Xh1vcj006QvtACc+o8twj7kCPntn13M4MAkySTvw=", + "output_base64sha512": "bB6iw4IJxB+G1ieprcxj96PdqyZbKT6ahc6BbPDrK5ONIk43JvXrz0xbCXud8vubsLC5o46yqXIhXQUqBtMYjw==", + "output_file_mode": null, + "output_md5": "c5beacdcad3c55176fedde20acb8616f", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "baed98d70a081d5ddd95b21dade6d5b45112af30", + "output_sha256": "6d2f97875bdc8f4d3a42fb4009cfa8f2dc23ee408f9ed9f5dcce0c024c924efc", + "output_sha512": "6c1ea2c38209c41f86d627a9adcc63f7a3ddab265b293e9a85ce816cf0eb2b938d224e3726f5ebcf4c5b097b9df2fb9bb0b0b9a38eb2a972215d052a06d3188f", + "output_size": 5074, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3670884144", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeNetworkInterfaces\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"ManageCodeBuildNetworkInterfaces\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Resource\":\"*\"},{\"Sid\":\"ManageCodeBuildNetworkInterfaces\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormUploadRole" + }, + { + "actions": [ + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "ManageCodeBuildNetworkInterfaces" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T19:45:21Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGEV76IAVBEI" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"ManageCodeBuildNetworkInterfaces\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars b/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars new file mode 100644 index 0000000..d078194 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars @@ -0,0 +1 @@ +aws_region = "us-west-2" diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index ebcc003..c84e99c 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -280,3 +280,90 @@ resource "aws_iam_role_policy" "scratch_bucket_access" { role = aws_iam_role.cnm-sender.id # assumes this role already exists policy = data.aws_iam_policy_document.allow_scratch_bucket_access.json } + + +data "aws_iam_policy_document" "cumulus_db_md_extract_upload_assume_role" { + statement { + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "AWS" + + identifiers = [ + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::082931748743:root", + "arn:aws:iam::510296831643:root" + ] + } + + condition { + test = "ArnEquals" + variable = "aws:PrincipalArn" + + values = [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + ] + } + } +} + +resource "aws_iam_role" "cumulus_db_md_extract_upload" { + name = "${var.name_prefix}-cumulus-db-md-extract-upload" + assume_role_policy = data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role.json +} + +data "aws_iam_policy_document" "cumulus_db_md_extract_upload" { + statement { + sid = "UploadExtractedGranules" + effect = "Allow" + + actions = [ + "s3:PutObject", + "s3:AbortMultipartUpload", + ] + + resources = [ + "${aws_s3_bucket.scratch.arn}/cumulus-granules/*", + ] + } + + statement { + sid = "ListDestinationPrefix" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.scratch.arn, + ] + + condition { + test = "StringLike" + variable = "s3:prefix" + + values = [ + "cumulus-granules/*", + ] + } + } +} + +resource "aws_iam_role_policy" "cumulus_db_md_extract_upload" { + name = "${var.name_prefix}-cumulus-db-md-extract-upload" + role = aws_iam_role.cumulus_db_md_extract_upload.id + policy = data.aws_iam_policy_document.cumulus_db_md_extract_upload.json +} diff --git a/ctorm/infra/terraform/outputs.tf b/ctorm/infra/terraform/outputs.tf index 11ca41e..6e734a2 100644 --- a/ctorm/infra/terraform/outputs.tf +++ b/ctorm/infra/terraform/outputs.tf @@ -29,3 +29,7 @@ output "prepare_ec2_role_arn" { output "scratch_bucket" { value = aws_s3_bucket.scratch.bucket } + +output "cumulus_db_md_extract_upload_role_arn" { + value = aws_iam_role.cumulus_db_md_extract_upload.arn +} diff --git a/ctorm/infra/terraform/placeholder-lambda.zip b/ctorm/infra/terraform/placeholder-lambda.zip index 767a6863d86904218024568cb08edb144365f8de..e71ce41ec254e01f994788da16c3a3105f7b80fd 100644 GIT binary patch delta 248 zcmcc2bc1PvL;a@P8^b1h`WkgB?Ax6azG0EYa(VgBXXoG4S4^w$YdjLp=h-1Be=xFM zW^Lsr6RRH9c@sbJ&S6TeIi0G@z%b1sSWmCabW?oL$9<2yPliq~jX9cc=j#2&U}m_R z+u@5RrJPnxI9JRQ;yZ(Tr?tu4ywH5PEtgKcaMnJ3wkM~bk2f-{zkY{v@olZkrqy2$ zTW(B0e)5b-;;H|6`!84DeRc0nx%E~4{rNvMyf?OW%v3)3?DYqwYzd{uTLWv@7#RNl t5AbH^*t=P9-ZcgWhD1gNh5&CyCJ_b^1`Z$u0l3^`1x8&q9-s;a1^|x)X7K<3 delta 243 zcmcb?beU;_L;b6=HCIc5CeL&gxF5SsJ0h^m&))9YZ2QZ5T26eJ#I*9g%%mV@yMx94 z&Btdv|ORE=n@940v>9OrR{^TAD>s*)h z-W)3(tb7dv43*NW%2!B5&%S0Gk=(nvJ?i;v$vgH{C0EY>=)WDBxP5D(?^g5ua~7X@ zq`I@y&8MdH_uRX;Z*6;7!?~4o!3{i{>3<2JZOd<>-3>-iR0&uy>s*Jj9>_8O^3;-(=Y?%N6 diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index bb737dd..9137406 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,9 +1,13 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 23, + "serial": 28, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { + "cumulus_db_md_extract_upload_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "type": "string" + }, "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" @@ -49,19 +53,19 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "f09bd3c52a0ccf18e91ad5fb208898ddd415b903", - "output_base64sha256": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", - "output_base64sha512": "YA5YdhMOE4f3g+N2rFuq2vpYa8KXnFSH+C7bkq9ixIvMxZqDxZBI/MR5E9vrLJ8hLk/mV0kMKS1wQ269QdyIFg==", + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", "output_file_mode": null, - "output_md5": "de3b86299704b85f7d5621f69bdb2914", + "output_md5": "94564c5dd9caf22baaf170b00693c497", "output_path": "./placeholder-lambda.zip", - "output_sha": "f09bd3c52a0ccf18e91ad5fb208898ddd415b903", - "output_sha256": "bbf01937538c639828cf7e2bd3fb95d4cd105dd13563565b0ced59b2ef90e291", - "output_sha512": "600e5876130e1387f783e376ac5baadafa586bc2979c5487f82edb92af62c48bccc59a83c59048fcc47913dbeb2c9f212e4fe657490c292d70436ebd41dc8816", - "output_size": 339, + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, "source": [ { - "content": "import json\nimport os\n\ndef handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", "filename": "index.py" } ], @@ -211,6 +215,133 @@ } ] }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "923578907", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:AbortMultipartUpload", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*" + ], + "sid": "UploadExtractedGranules" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "cumulus-granules/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch" + ], + "sid": "ListDestinationPrefix" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2147741451", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnEquals", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -565,7 +696,7 @@ "name_prefix": "", "path": "/", "role": "ctorm-dev-prepare-ec2-role", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AIPA4VW62R3RCASOXJGOK" }, @@ -621,6 +752,45 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T19:29:26Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cumulus-db-md-extract-upload", + "inline_policy": [ + { + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RJW53W2D5X" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -636,14 +806,19 @@ "description": "", "force_detach_policies": false, "id": "ctorm-dev-prepare-ec2-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-dev-prepare-ec2-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-dev-prepare-ec2-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROA4VW62R3RKO3MKUI7D" }, @@ -684,6 +859,32 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "role": "ctorm-dev-cumulus-db-md-extract-upload" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cumulus_db_md_extract_upload", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -750,7 +951,7 @@ "x86_64" ], "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", - "code_sha256": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", + "code_sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", "code_signing_config_arn": "", "dead_letter_config": [], "description": "", @@ -759,6 +960,7 @@ "variables": { "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "LOG_LEVEL": "INFO", "TABLE_NAME": "ctorm-dev-state" } } @@ -771,13 +973,13 @@ "file_system_config": [], "filename": "./placeholder-lambda.zip", "function_name": "ctorm-dev-cnm-sender", - "handler": "index.handler", + "handler": "load_tester.lambda_handler", "id": "ctorm-dev-cnm-sender", "image_config": [], "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", - "last_modified": "2026-06-10T20:02:16.662+0000", + "last_modified": "2026-06-24T19:29:31.000+0000", "layers": [], "logging_config": [ { @@ -804,8 +1006,8 @@ "signing_profile_version_arn": "", "skip_destroy": false, "snap_start": [], - "source_code_hash": "u/AZN1OMY5goz34r0/uV1M0QXdE1Y1ZbDO1Zsu+Q4pE=", - "source_code_size": 339, + "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "source_code_size": 344, "tags": {}, "tags_all": {}, "timeout": 120, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup new file mode 100644 index 0000000..0f77000 --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -0,0 +1,1226 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 27, + "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", + "outputs": { + "granules_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "type": "string" + }, + "granules_queue_arn": { + "value": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "type": "string" + }, + "granules_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "type": "string" + }, + "prepare_ec2_instance_profile_name": { + "value": "ctorm-dev-prepare-ec2-profile", + "type": "string" + }, + "prepare_ec2_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-dev-scratch", + "type": "string" + }, + "state_table_name": { + "value": "ctorm-dev-state", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-dev-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", + "output_file_mode": null, + "output_md5": "94564c5dd9caf22baaf170b00693c497", + "output_path": "./placeholder-lambda.zip", + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, + "source": [ + { + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "allow_scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "632361540", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:ListBucket\",\n \"s3:GetObject\",\n \"s3:DeleteObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::ctorm-dev-scratch/*\",\n \"arn:aws:s3:::ctorm-dev-scratch\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch", + "arn:aws:s3:::ctorm-dev-scratch/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2455266930", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "923578907", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:AbortMultipartUpload", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*" + ], + "sid": "UploadExtractedGranules" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "cumulus-granules/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-dev-scratch" + ], + "sid": "ListDestinationPrefix" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2147741451", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnEquals", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1999618104", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sqs:GetQueueAttributes", + "sqs:SendMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::asf-cumulus-dev-opera-products", + "arn:aws:s3:::asf-cumulus-prod-opera-products", + "arn:aws:s3:::asf-cumulus-test-opera-products", + "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", + "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", + "arn:aws:s3:::sds-n-cumulus-test-nisar-products" + ], + "sid": "" + }, + { + "actions": [ + "s3:GetObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::asf-cumulus-dev-opera-products/*", + "arn:aws:s3:::asf-cumulus-prod-opera-products/*", + "arn:aws:s3:::asf-cumulus-test-opera-products/*", + "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", + "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", + "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "prepare_ec2_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2851119427", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "ec2.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-dev-every-minute", + "is_enabled": false, + "name": "ctorm-dev-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "appsync_target": [], + "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [], + "event_bus_name": "default", + "force_destroy": false, + "http_target": [], + "id": "ctorm-dev-every-minute-terraform-20260610200222615700000001", + "input": "{\"source\":\"eventbridge\"}", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "", + "rule": "ctorm-dev-every-minute", + "run_command_targets": [], + "sagemaker_pipeline_target": [], + "sqs_target": [], + "target_id": "terraform-20260610200222615700000001" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender", + "id": "/aws/lambda/ctorm-dev-cnm-sender", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/lambda/ctorm-dev-cnm-sender", + "name_prefix": "", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "state", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", + "attribute": [ + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [], + "hash_key": "pk", + "id": "ctorm-dev-state", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-dev-state", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", + "create_date": "2026-06-19T01:00:34Z", + "id": "ctorm-dev-prepare-ec2-profile", + "name": "ctorm-dev-prepare-ec2-profile", + "name_prefix": "", + "path": "/", + "role": "ctorm-dev-prepare-ec2-role", + "tags": {}, + "tags_all": {}, + "unique_id": "AIPA4VW62R3RCASOXJGOK" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.prepare_ec2", + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cnm-sender-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-10T20:01:24Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cnm-sender-role", + "inline_policy": [ + { + "name": "ctorm-dev-cnm-sender-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + }, + { + "name": "ctorm-dev-scratch-access", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cnm-sender-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RPOQ35FULV" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T19:29:26Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-cumulus-db-md-extract-upload", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROA4VW62R3RJW53W2D5X" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-19T01:00:33Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-prepare-ec2-role", + "inline_policy": [ + { + "name": "ctorm-dev-prepare-ec2-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-prepare-ec2-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RKO3MKUI7D" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", + "name": "ctorm-dev-cnm-sender-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "role": "ctorm-dev-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "role": "ctorm-dev-cumulus-db-md-extract-upload" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cumulus_db_md_extract_upload", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "prepare_ec2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", + "name": "ctorm-dev-prepare-ec2-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", + "role": "ctorm-dev-prepare-ec2-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.prepare_ec2", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.prepare_ec2", + "data.aws_iam_policy_document.prepare_ec2_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-cnm-sender-role:ctorm-dev-scratch-access", + "name": "ctorm-dev-scratch-access", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch/*\",\"arn:aws:s3:::ctorm-dev-scratch\"]}]}", + "role": "ctorm-dev-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cnm-sender", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.allow_scratch_bucket_access", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender", + "code_sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [ + { + "variables": { + "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", + "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "LOG_LEVEL": "INFO", + "TABLE_NAME": "ctorm-dev-state" + } + } + ], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "./placeholder-lambda.zip", + "function_name": "ctorm-dev-cnm-sender", + "handler": "load_tester.lambda_handler", + "id": "ctorm-dev-cnm-sender", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", + "kms_key_arn": "", + "last_modified": "2026-06-24T19:29:31.000+0000", + "layers": [], + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/ctorm-dev-cnm-sender", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::871271927522:role/ctorm-dev-cnm-sender-role", + "runtime": "python3.12", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "source_code_size": 344, + "tags": {}, + "tags_all": {}, + "timeout": 120, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_eventbridge", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "ctorm-dev-cnm-sender", + "function_url_auth_type": null, + "id": "AllowExecutionFromEventBridge", + "principal": "events.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "statement_id": "AllowExecutionFromEventBridge", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.state", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "scratch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch", + "bucket": "ctorm-dev-scratch", + "bucket_domain_name": "ctorm-dev-scratch.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "ctorm-dev-scratch.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [ + { + "id": "a7e8aeefe5d49ed5062fbc1a1d29f8b05e5150edf37ef1739efda49e5443a5ee", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-dev-scratch", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Name": "ctorm-dev-scratch" + }, + "tags_all": { + "Name": "ctorm-dev-scratch" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": {}, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": {}, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/terraform/variables.tf b/ctorm/infra/terraform/variables.tf index e661b9b..030971c 100644 --- a/ctorm/infra/terraform/variables.tf +++ b/ctorm/infra/terraform/variables.tf @@ -31,3 +31,8 @@ variable "prepare_source_bucket_names" { "sds-n-cumulus-test-nisar-products", ] } + +# variable "cumulus_db_md_extract_remote_codebuild_role_arn" { +# type = string +# description = "CodeBuild service role ARN from the remote/RDS account." +# } From c23bb8a28e7c767f759c08aabdbbe200012e4db4 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 24 Jun 2026 17:53:24 -0800 Subject: [PATCH 19/31] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/README.md | 47 +- ctorm/cumulus-db-md-extract/buildspec.yaml | 78 +++ .../export-granules-daily.sh | 44 +- .../export-granules-slice-daily.sql | 7 +- .../generate-slices-daily.sh | 66 +++ .../terraform/codebuild-db-md-extract.tf | 86 ++- .../terraform/nisar-prod.tfvars | 6 +- .../terraform/opera-prod.tfvars | 8 +- .../terraform/opera-uat.tfvars | 10 +- .../terraform/terraform.tfstate | 493 ++++++++++++++++-- ctorm/infra/terraform/main.tf | 15 +- ctorm/infra/terraform/terraform.tfstate | 22 +- 12 files changed, 753 insertions(+), 129 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/buildspec.yaml diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index 65c4d70..f3556bd 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -77,7 +77,6 @@ Really should try to create a EC2 next time. ### Terraform -```bash ```bash export AWS_PROFILE="cumulus-uat-6921" export VARFILE=opera-uat.tfvars @@ -94,14 +93,44 @@ terraform destroy ``` -``` +### Codebuild ```bash -aws codebuild create-project \ - --name "OneTimeDataPull" \ - --source '{"type": "S3", "location": "ctorm-dev-scratch/codebuild/codebuild-noop.zip"}' \ - --environment '{"type": "LINUX_CONTAINER", "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", "computeType": "BUILD_GENERAL1_SMALL"}' \ - --artifacts '{"type": "NO_ARTIFACTS"}' \ - --service-role "arn:aws:iam::123456789012:role/service-role/YourCodeBuildRole" \ - --timeout-in-minutes 240 +export AWS_PROFILE="cumulus-uat-6921" +export AWS_REGION="us-west-2" +export CODEBUILD_PROJECT="$(terraform output -raw cumulus_db_md_extract_codebuild_project_name)" + +# start build: +BUILD_ID="$( + aws codebuild start-build \ + --project-name "${CODEBUILD_PROJECT}" \ + --region "${AWS_REGION}" \ + --query 'build.id' \ + --output text +)" + +echo "${BUILD_ID}" + +# get status: +aws codebuild batch-get-builds \ + --ids "${BUILD_ID}" \ + --region "${AWS_REGION}" \ + --query 'builds[0].{status:buildStatus,phase:currentPhase,start:startTime,end:endTime,logs:logs.deepLink}' \ + --output table + +# get build logs: +aws codebuild batch-get-builds \ + --ids "${BUILD_ID}" \ + --region "${AWS_REGION}" \ + --query 'builds[0].logs.deepLink' \ + --output text + +LOG_GROUP="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.groupName' --output text)" +LOG_STREAM="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.streamName' --output text)" + +aws logs tail "${LOG_GROUP}" \ + --log-stream-names "${LOG_STREAM}" \ + --follow \ + --region "${AWS_REGION}" + ``` diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml new file mode 100644 index 0000000..e5da973 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -0,0 +1,78 @@ +version: 0.2 +env: + variables: + AWS_DEFAULT_REGION: "us-west-2" + secrets-manager: + PGHOST: "${DB_SECRET_ARN}:host" + PGUSER: "${DB_SECRET_ARN}:username" + PGPASSWORD: "${DB_SECRET_ARN}:password" + PGDATABASE: "${DB_SECRET_ARN}:database" + +phases: + install: + commands: + - echo "Inspecting build image..." + - cat /etc/os-release || true + - aws --version || true + - jq --version || true + - unzip -v | head -n 2 || true + - psql --version || true + - echo "Installing required packages if missing..." + - | + if ! command -v jq >/dev/null 2>&1; then + yum install -y jq + fi + - | + if ! command -v unzip >/dev/null 2>&1; then + yum install -y unzip + fi + - | + if ! command -v psql >/dev/null 2>&1; then + echo "psql not found; trying PostgreSQL client package installs..." + + yum install -y postgresql15 \ + || yum install -y postgresql14 \ + || yum install -y postgresql13 \ + || yum install -y postgresql12 \ + || yum install -y postgresql \ + || { + echo "Unable to install psql from enabled yum repositories." + echo "Available postgres-related packages:" + yum search postgresql || true + exit 1 + } + fi + - psql --version + - aws --version + + pre_build: + commands: + - ls -lah + - chmod +x generate-slices-daily.sh export-granules-daily.sh + - | + echo 'Before assuming role: ' + - aws sts get-caller-identity + - | + echo "Assuming ctorm upload role..." + - | + CREDS_JSON="$(aws sts assume-role \ + --role-arn "${CTORM_S3_ROLE_ARN}" \ + --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER}")" + + export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" + export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" + export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + - | + echo 'After assuming role: ' + - aws sts get-caller-identity + + build: + commands: + + - aws sts get-caller-identity + - ./generate-slices-daily.sh + - ./export-granules-daily.sh + post_build: + commands: + - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + - echo "Build complete." diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index ec07cc7..8e5a9e8 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -1,6 +1,20 @@ set -euo pipefail S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" +FAILED_SLICES_FILE="failed_slices_daily.tsv" + +rm -f "$FAILED_SLICES_FILE" + +echo "Statement timeout: " +psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --no-psqlrc \ + -c "SHOW statement_timeout;" + + while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then @@ -15,7 +29,7 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" echo "Destination: ${S3_URI}" - psql \ + if ! PGOPTIONS="-c statement_timeout=${PG_STATEMENT_TIMEOUT:-0}" psql \ --host="$PGHOST" \ --port="${PGPORT:-5432}" \ --username="$PGUSER" \ @@ -26,22 +40,22 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do --set=collection="$COLLECTION" \ --set=yyyymmdd="$YYYYMMDD" \ --file=export-granules-slice-daily.sql \ - | awk -v label="${COLLECTION} ${YYYYMMDD}" ' - { - print; - count++; - if (count % 1000 == 0) { - printf("[%s] exported %d JSONL rows\n", label, count) > "/dev/stderr"; - fflush("/dev/stderr"); - } - } - END { - printf("[%s] export complete: %d JSONL rows\n", label, count) > "/dev/stderr"; - } - ' \ | gzip -c \ - | aws --profile=ctorm s3 cp - "$S3_URI" + | aws s3 cp - "$S3_URI"; then + + echo "FAILED collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + continue + fi echo "Uploaded ${S3_URI}" done < slices_daily.tsv + +if [[ -s "$FAILED_SLICES_FILE" ]]; then + FAILED_SLICES_S3_URI="${S3_PREFIX}/failed_slices_daily.tsv" + aws s3 cp "$FAILED_SLICES_FILE" "$FAILED_SLICES_S3_URI" + echo "Some slices failed. Uploaded failure list to ${FAILED_SLICES_S3_URI}" >&2 +else + echo "All slices exported successfully." +fi diff --git a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql index b23af06..61a8f1f 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql +++ b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql @@ -30,9 +30,10 @@ WITH export_rows AS ( AND f.checksum_value IS NOT NULL WHERE g.status = 'completed' AND c.name = :'collection' - AND g.beginning_date_time >= to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') AT TIME ZONE 'UTC' + + AND g.beginning_date_time >= to_date(:'yyyymmdd', 'YYYYMMDD') AND g.beginning_date_time < ( - to_timestamp(:'yyyymm' || '01', 'YYYYMMDD') + interval '1 month' + to_date(:'yyyymmdd', 'YYYYMMDD') + interval '1 day' ) AT TIME ZONE 'UTC' GROUP BY g.granule_id, @@ -64,4 +65,4 @@ SELECT jsonb_strip_nulls( ) ) FROM export_rows -ORDER BY beginning_date_time, granule_id; \ No newline at end of file +ORDER BY beginning_date_time, granule_id; diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh index 60b9407..3651d74 100644 --- a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -1,3 +1,9 @@ + +set -euo pipefail + + +S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" + psql \ --host="$PGHOST" \ --port="${PGPORT:-5432}" \ @@ -22,6 +28,65 @@ psql \ ON f.granule_cumulus_id = g.cumulus_id AND f.checksum_value IS NOT NULL WHERE g.status = 'completed' + AND c.name in ( + 'ANTPAT', + 'BFPQ', + 'CORNER_REFL', + 'DCOP', + 'DC_RADAR', + 'DSG_STATIC', + 'EA_L0B_L_CRSD', + 'EA_L0B_L_RRSD', + 'EA_L1_L_RIFG', + 'EA_L1_L_ROFF', + 'EA_L1_L_RSLC', + 'EA_L1_L_RUNW', + 'EA_L2_L_GCOV', + 'EA_L2_L_GOFF', + 'EA_L2_L_GSLC', + 'EA_L2_L_GUNW', + 'EA_L3_L_SME2', + 'FOE', + 'FRP', + 'FT_PARAM', + 'FT_WAVEFORM', + 'LRCLK_UTC', + 'L_CHAN_DATA', + 'MOE', + 'NISAR_DEM_TIF', + 'NISAR_DEM_VRT', + 'NISAR_L0A_RRST_BETA_V1', + 'NISAR_L0B_CRSD_BETA_V1', + 'NISAR_L0B_RRSD_BETA_V1', + 'NISAR_L1_RIFG_BETA_V1', + 'NISAR_L1_ROFF_BETA_V1', + 'NISAR_L1_RSLC_BETA_V1', + 'NISAR_L1_RUNW_BETA_V1', + 'NISAR_L2_GCOV_BETA_V1', + 'NISAR_L2_GOFF_BETA_V1', + 'NISAR_L2_GSLC_BETA_V1', + 'NISAR_L2_GUNW_BETA_V1', + 'NISAR_L3_SME2_BETA_V1', + 'NISAR_VWC', + 'NISAR_WATERMASK_TIF', + 'NISAR_WATERMASK_VRT', + 'NOE', + 'NRP', + 'OROST', + 'PA_L0B_L_RRSD', + 'POE', + 'PRP', + 'STUF', + 'TEC', + 'TFDB', + 'UR_L0B_L_RRSD', + + 'OPERA_L2_CSLC-S1_V1', + 'OPERA_L2_RTC-S1_V1', + 'OPERA_L3_DISP-S1_V1', + 'OPERA_L3_DIST-ALERT-S1_V1', + 'OPERA_L4_TROPO-ZENITH_V1' + ) GROUP BY c.name, to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') @@ -29,3 +94,4 @@ psql \ c.name, to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); " > slices_daily.tsv + aws s3 cp slices_daily.tsv "$S3_URI/" diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf index 61c2f10..c2acbcb 100644 --- a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -12,7 +12,7 @@ variable "aws_region" { variable "db_md_extract_secret_arn" { type = string - description = "ARN of the Secrets Manager secret containing host, username, password, and database." + description = "ARN of the Secrets Manager secret containing host, username, password, and database in the RDS account." } variable "db_md_extract_vpc_id" { @@ -35,15 +35,15 @@ variable "db_md_extract_dump_subdir" { default = "nisar" } -variable "db_md_extract_target_bucket" { +variable "db_md_extract_ctorm_bucket" { type = string - default = "" - description = "Destination S3 bucket for exported dump files." + default = "ctorm-dev-scratch" + description = "Destination CTORM S3 bucket for exported dump files." } -variable "db_md_extract_target_role_arn" { +variable "db_md_extract_ctorm_s3_role_arn" { type = string - description = "Role ARN in the target account that CodeBuild can assume to write exported files." + description = "Role ARN in the CTORM account that CodeBuild can assume to write exported files." } locals { @@ -51,14 +51,56 @@ locals { db_md_extract_source_key = "codebuild/cumulus-db-md-extract.zip" } +resource "aws_s3_bucket" "codebuild_source" { + bucket = "${local.db_md_extract_project_name}-source-${substr(data.aws_caller_identity.current.account_id, 8, 4)}" + force_destroy = true + +} + +resource "aws_s3_bucket_ownership_controls" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +data "aws_caller_identity" "current" {} + data "archive_file" "cumulus_db_md_extract" { type = "zip" - source_dir = "${path.module}/../../cumulus-db-md-extract" - output_path = "${path.module}/cumulus-db-md-extract.zip" + source_dir = "${path.module}/.." + output_path = "${path.module}/.terraform/cumulus-db-md-extract.zip" + + excludes = [ + "terraform", + "terraform/*", + ".terraform", + ".terraform/*", + ] } resource "aws_s3_object" "cumulus_db_md_extract_source" { - bucket = aws_s3_bucket.scratch.bucket + bucket = aws_s3_bucket.codebuild_source.bucket key = local.db_md_extract_source_key source = data.archive_file.cumulus_db_md_extract.output_path source_hash = data.archive_file.cumulus_db_md_extract.output_base64sha256 @@ -108,7 +150,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { } statement { - sid = "ReadBuildSourceFromScratchBucket" + sid = "ReadBuildSource" effect = "Allow" actions = [ @@ -117,12 +159,12 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { ] resources = [ - "${aws_s3_bucket.scratch.arn}/${local.db_md_extract_source_key}", + "${aws_s3_bucket.codebuild_source.arn}/${local.db_md_extract_source_key}", ] } statement { - sid = "ListScratchBucketForBuildSource" + sid = "ListBuildSourceBucket" effect = "Allow" actions = [ @@ -130,7 +172,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { ] resources = [ - aws_s3_bucket.scratch.arn, + aws_s3_bucket.codebuild_source.arn, ] condition { @@ -158,7 +200,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { } statement { - sid = "AssumeTargetAccountUploadRole" + sid = "AssumeCtormAccountUploadRole" effect = "Allow" actions = [ @@ -166,7 +208,7 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { ] resources = [ - var.db_md_extract_target_role_arn, + var.db_md_extract_ctorm_s3_role_arn, ] } @@ -175,6 +217,8 @@ data "aws_iam_policy_document" "codebuild_db_md_extract" { effect = "Allow" actions = [ + "ec2:DescribeDhcpOptions", + "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs", @@ -219,7 +263,7 @@ resource "aws_iam_role_policy" "codebuild_db_md_extract" { resource "aws_codebuild_project" "cumulus_db_md_extract" { name = local.db_md_extract_project_name - description = "One-time Cumulus RDS metadata export to S3" + description = "One-time Cumulus RDS metadata export to CTORM S3" service_role = aws_iam_role.codebuild_db_md_extract.arn build_timeout = 480 @@ -229,7 +273,7 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { source { type = "S3" - location = "${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + location = "${aws_s3_bucket.codebuild_source.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" buildspec = "buildspec.yaml" } @@ -245,7 +289,7 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { environment_variable { name = "CTORM_BUCKET" - value = var.db_md_extract_target_bucket + value = var.db_md_extract_ctorm_bucket } environment_variable { @@ -259,8 +303,8 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { } environment_variable { - name = "TARGET_ROLE_ARN" - value = var.db_md_extract_target_role_arn + name = "CTORM_S3_ROLE_ARN" + value = var.db_md_extract_ctorm_s3_role_arn } } @@ -292,5 +336,5 @@ output "cumulus_db_md_extract_codebuild_role_arn" { } output "cumulus_db_md_extract_source_s3_uri" { - value = "s3://${aws_s3_bucket.scratch.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + value = "s3://${aws_s3_bucket.codebuild_source.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" } diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars index de3cc0a..66ff300 100644 --- a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -13,10 +13,6 @@ db_md_extract_security_group_ids = [ "sg-...", ] -db_md_extract_source_bucket = "" - -db_md_extract_target_bucket = "ctorm-dev-scratch" - -db_md_extract_target_role_arn = "arn:aws:iam:::role/" +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index de3cc0a..588e880 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -13,10 +13,6 @@ db_md_extract_security_group_ids = [ "sg-...", ] -db_md_extract_source_bucket = "" +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." -db_md_extract_target_bucket = "ctorm-dev-scratch" - -db_md_extract_target_role_arn = "arn:aws:iam:::role/" - -db_md_extract_dump_subdir = "nisar" +db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars index b27da44..ac198b3 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars @@ -1,4 +1,4 @@ -name_prefix = "ctorm" +name_prefix = "ctorm-opera-uat" db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" @@ -14,10 +14,6 @@ db_md_extract_security_group_ids = [ "sg-06255aca3a87784a0", ] -db_md_extract_source_bucket = "ctorm-dev-scratch" +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" -db_md_extract_target_bucket = "ctorm-dev-scratch" - -db_md_extract_target_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" - -db_md_extract_dump_subdir = "opera" +db_md_extract_dump_subdir = "opera-uat" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate index a6e1dad..de16529 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate @@ -1,19 +1,19 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 6, + "serial": 51, "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { - "value": "ctorm-cumulus-db-md-extract", + "value": "ctorm-opera-uat-cumulus-db-md-extract", "type": "string" }, "cumulus_db_md_extract_codebuild_role_arn": { - "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "value": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", "type": "string" }, "cumulus_db_md_extract_source_s3_uri": { - "value": "s3://ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip", + "value": "s3://ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", "type": "string" } }, @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "baed98d70a081d5ddd95b21dade6d5b45112af30", - "output_base64sha256": "bS+Xh1vcj006QvtACc+o8twj7kCPntn13M4MAkySTvw=", - "output_base64sha512": "bB6iw4IJxB+G1ieprcxj96PdqyZbKT6ahc6BbPDrK5ONIk43JvXrz0xbCXud8vubsLC5o46yqXIhXQUqBtMYjw==", + "id": "c9cfb643537aef04775bf2b4289a4952f88fe75c", + "output_base64sha256": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", + "output_base64sha512": "dJ4dl6jPOMuVC7pkaqZ3IbGEwbzDfqI7gCJfVi4BUVV4jZH66Ks3YolX82N88kJFuwErzgcAk/DxCJGHoPvcRg==", "output_file_mode": null, - "output_md5": "c5beacdcad3c55176fedde20acb8616f", + "output_md5": "c5761ab3cd6148053f00bd7b09e6571c", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "baed98d70a081d5ddd95b21dade6d5b45112af30", - "output_sha256": "6d2f97875bdc8f4d3a42fb4009cfa8f2dc23ee408f9ed9f5dcce0c024c924efc", - "output_sha512": "6c1ea2c38209c41f86d627a9adcc63f7a3ddab265b293e9a85ce816cf0eb2b938d224e3726f5ebcf4c5b097b9df2fb9bb0b0b9a38eb2a972215d052a06d3188f", - "output_size": 5074, + "output_sha": "c9cfb643537aef04775bf2b4289a4952f88fe75c", + "output_sha256": "d5071c6d85da396d685a4606832345349755303b707a8ddea5f856655b86fb20", + "output_sha512": "749e1d97a8cf38cb950bba646aa67721b184c1bcc37ea23b80225f562e015155788d91fae8ab37628957f3637cf24245bb012bce070093f0f1089187a0fbdc46", + "output_size": 5457, "source": [], "source_content": null, "source_content_filename": null, @@ -55,6 +55,24 @@ } ] }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "097260566921", + "arn": "arn:aws:iam::097260566921:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "097260566921", + "user_id": "AIDARNJJOPGERLGUKQ553" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -64,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "3670884144", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeNetworkInterfaces\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"ManageCodeBuildNetworkInterfaces\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Resource\":\"*\"},{\"Sid\":\"ManageCodeBuildNetworkInterfaces\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "2729057441", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -85,7 +103,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" ], "sid": "WriteCodeBuildLogs" }, @@ -101,7 +119,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip" + "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip" ], "sid": "ReadBuildSource" }, @@ -124,7 +142,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:s3:::ctorm-dev-scratch" + "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921" ], "sid": "ListBuildSourceBucket" }, @@ -157,11 +175,15 @@ "resources": [ "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" ], - "sid": "AssumeCtormUploadRole" + "sid": "AssumeCtormAccountUploadRole" }, { "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs" @@ -177,22 +199,6 @@ ], "sid": "DescribeNetworkForVpcBuild" }, - { - "actions": [ - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "*" - ], - "sid": "ManageCodeBuildNetworkInterfaces" - }, { "actions": [ "ec2:CreateNetworkInterfacePermission" @@ -277,17 +283,17 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", "deletion_protection_enabled": false, - "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "id": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", "kms_key_id": "", "log_group_class": "STANDARD", - "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", "name_prefix": "", "region": "us-west-2", "retention_in_days": 14, "skip_destroy": false, - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -295,6 +301,163 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-opera-uat-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:097260566921:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "opera-uat" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-opera-uat-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-opera-uat-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-03beee748d2a43cc9", + "sg-06255aca3a87784a0", + "sg-0afdcb46b1a38db46" + ], + "subnets": [ + "subnet-0ac04d1e36d0a50ea" + ], + "vpc_id": "vpc-080322e163a0e7390" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -304,22 +467,27 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", - "create_date": "2026-06-24T19:45:21Z", + "create_date": "2026-06-24T23:12:29Z", "description": "", "force_detach_policies": false, - "id": "ctorm-cumulus-db-md-extract-role", - "inline_policy": [], + "id": "ctorm-opera-uat-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, - "name": "ctorm-cumulus-db-md-extract-role", + "name": "ctorm-opera-uat-cumulus-db-md-extract-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, - "unique_id": "AROARNJJOPGEV76IAVBEI" + "unique_id": "AROARNJJOPGE2BF3AA7RB" }, "sensitive_attributes": [], "private": "bnVsbA==", @@ -338,22 +506,247 @@ { "schema_version": 0, "attributes": { - "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", - "name": "ctorm-cumulus-db-md-extract-policy", + "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", + "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeNetworkInterfaces\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":[\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"ManageCodeBuildNetworkInterfaces\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", - "role": "ctorm-cumulus-db-md-extract-role" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-opera-uat-cumulus-db-md-extract-role" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_cloudwatch_log_group.codebuild_db_md_extract", "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", "data.aws_iam_policy_document.codebuild_db_md_extract", "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" ] } ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket_domain_name": "ctorm-opera-uat-cumulus-db-md-extract-source-6921.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-opera-uat-cumulus-db-md-extract-source-6921.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "expected_bucket_owner": "", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "c5761ab3cd6148053f00bd7b09e6571c", + "force_destroy": false, + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] } ], "check_results": null diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index c84e99c..f1e6ce5 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -304,16 +304,27 @@ data "aws_iam_policy_document" "cumulus_db_md_extract_upload_assume_role" { } condition { - test = "ArnEquals" + test = "ArnLike" variable = "aws:PrincipalArn" values = [ "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" ] } } diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 9137406..7426ae8 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 28, + "serial": 30, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -288,9 +288,9 @@ { "schema_version": 0, "attributes": { - "id": "2147741451", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "id": "4124935366", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -305,12 +305,12 @@ { "test": "ArnEquals", "values": [ - "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + "arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role" ], "variable": "aws:PrincipalArn" } @@ -762,7 +762,7 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", "create_date": "2026-06-24T19:29:26Z", "description": "", "force_detach_policies": false, From e77628914b2d65560ddac1edf6bbc17adb04d33f Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Wed, 24 Jun 2026 20:17:47 -0800 Subject: [PATCH 20/31] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/README.md | 10 +- .../terraform/opera-prod.tfvars | 13 +- .../terraform/terraform-opera-prod.tfstate | 752 +++++++++++++ ...rm.tfstate => terraform-opera-uat.tfstate} | 32 +- .../opera-prod/terraform.tfstate | 563 ++++++++++ ctorm/infra/README.md | 9 + ctorm/infra/terraform/main.tf | 71 +- ctorm/infra/terraform/outputs.tf | 12 +- ctorm/infra/terraform/terraform.tfstate | 299 +----- .../infra/terraform/terraform.tfstate.backup | 428 +------- .../ctorm-dev/terraform.tfstate | 294 ++++++ .../ctorm-prod/terraform.tfstate | 998 ++++++++++++++++++ 12 files changed, 2722 insertions(+), 759 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate rename ctorm/cumulus-db-md-extract/terraform/{terraform.tfstate => terraform-opera-uat.tfstate} (82%) create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate create mode 100644 ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate create mode 100644 ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index f3556bd..b8b3293 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -80,12 +80,20 @@ Really should try to create a EC2 next time. ```bash export AWS_PROFILE="cumulus-uat-6921" export VARFILE=opera-uat.tfvars +export STATEFILE=terraform-opera-uat.tfstate + +terraform workspace new opera-uat + +export TF_WORKSPACE=opera-uat + terraform init terraform plan \ + -state="${STATEFILE}" \ -var-file="${VARFILE}" terraform apply \ + -state="${STATEFILE}" \ -var-file="${VARFILE}" terraform destroy @@ -98,7 +106,7 @@ terraform destroy ```bash export AWS_PROFILE="cumulus-uat-6921" export AWS_REGION="us-west-2" -export CODEBUILD_PROJECT="$(terraform output -raw cumulus_db_md_extract_codebuild_project_name)" +export CODEBUILD_PROJECT="$(terraform output -state="${STATEFILE}" -raw cumulus_db_md_extract_codebuild_project_name)" # start build: BUILD_ID="$( diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index 588e880..b7cb678 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -1,18 +1,19 @@ name_prefix = "ctorm" -db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" -db_md_extract_vpc_id = "vpc-..." +db_md_extract_vpc_id = "vpc-0e36a2fca2e2a73d5" db_md_extract_subnet_ids = [ - "subnet-...", - "subnet-...", + "subnet-02dda0af17b4a0d19", ] db_md_extract_security_group_ids = [ - "sg-...", + "sg-0249655cac679df2c", + "sg-0ed18cd0fb6e79f58", + "sg-0ed18cd0fb6e79f58", ] -db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate new file mode 100644 index 0000000..691cf7d --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -0,0 +1,752 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 41, + "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", + "output_base64sha256": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", + "output_base64sha512": "J8ZtLQzzPeFUcNb4IW4DV3ar9M0/uFonIhPlfZDv6Z1cGcr+nSGpuJCcASX4HlLX6dJcILMBEtdwGbEuB/3rvQ==", + "output_file_mode": null, + "output_md5": "692e40b861d2f40dcc12eb78cbe57de4", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", + "output_sha256": "0a3e592d9256955ef7682a2b64dff82c1d25096c1ef8b6eaf2eb817fc1850723", + "output_sha512": "27c66d2d0cf33de15470d6f8216e035776abf4cd3fb85a272213e57d90efe99d5c19cafe9d21a9b8909c0125f81e52d7e9d25c20b30112d77019b12e07fdebbd", + "output_size": 5350, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "510296831643", + "arn": "arn:aws:iam::510296831643:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "510296831643", + "user_id": "AIDAXNUAPW2NZFPXV6JKY" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1104794953", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:510296831643:project/ctorm-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:510296831643:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "opera" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:510296831643:project/ctorm-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-0249655cac679df2c", + "sg-0ed18cd0fb6e79f58" + ], + "subnets": [ + "subnet-02dda0af17b4a0d19" + ], + "vpc_id": "vpc-0e36a2fca2e2a73d5" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T03:25:11Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROAXNUAPW2N44OBYZNKD" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643", + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-1643.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-1643.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "a055848272cdca6935f41c8d486234d66c39ff78b8f9ef89db0d949987311ce3", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "692e40b861d2f40dcc12eb78cbe57de4", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate similarity index 82% rename from ctorm/cumulus-db-md-extract/terraform/terraform.tfstate rename to ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate index de16529..71af6a5 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 51, + "serial": 62, "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "c9cfb643537aef04775bf2b4289a4952f88fe75c", - "output_base64sha256": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", - "output_base64sha512": "dJ4dl6jPOMuVC7pkaqZ3IbGEwbzDfqI7gCJfVi4BUVV4jZH66Ks3YolX82N88kJFuwErzgcAk/DxCJGHoPvcRg==", + "id": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", + "output_base64sha256": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", + "output_base64sha512": "PZDUjfN60Uim+bdem7zB2ilfEUKqP4pE0UADkxHIkR4YTYGcP+Txtuj+Rm7zSpLQ2fZk+rgOjhJKIWLy0lyWYQ==", "output_file_mode": null, - "output_md5": "c5761ab3cd6148053f00bd7b09e6571c", + "output_md5": "63b93d1d0d04eb5a9595f406ba9653cc", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "c9cfb643537aef04775bf2b4289a4952f88fe75c", - "output_sha256": "d5071c6d85da396d685a4606832345349755303b707a8ddea5f856655b86fb20", - "output_sha512": "749e1d97a8cf38cb950bba646aa67721b184c1bcc37ea23b80225f562e015155788d91fae8ab37628957f3637cf24245bb012bce070093f0f1089187a0fbdc46", - "output_size": 5457, + "output_sha": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", + "output_sha256": "b7ddad71c2668d0d4a2d31668752520c9e0b4e9dee0d8188eb3d3f9a89a79f4c", + "output_sha512": "3d90d48df37ad148a6f9b75e9bbcc1da295f1142aa3f8a44d140039311c8911e184d819c3fe4f1b6e8fe466ef34a92d0d9f664fab80e8e124a2162f2d25c9661", + "output_size": 5347, "source": [], "source_content": null, "source_content_filename": null, @@ -82,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "2729057441", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "1034869219", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -103,7 +103,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract" ], "sid": "WriteCodeBuildLogs" }, @@ -509,7 +509,7 @@ "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", "role": "ctorm-opera-uat-cumulus-db-md-extract-role" }, "sensitive_attributes": [], @@ -718,7 +718,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "c5761ab3cd6148053f00bd7b09e6571c", + "etag": "63b93d1d0d04eb5a9595f406ba9653cc", "force_destroy": false, "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -731,7 +731,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "1QccbYXaOW1oWkYGgyNFNJdVMDtweo3epfhWZVuG+yA=", + "source_hash": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate new file mode 100644 index 0000000..b7216a0 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate @@ -0,0 +1,563 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 10, + "lineage": "9578e429-f029-9119-8845-752daa480ee7", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "77650f5f87b9d429f0ae3ee4141a2d4408515477", + "output_base64sha256": "W1Cg6btOTGKbYpBaANU0oW3rd3EKlsQKsMP1/ARFPnY=", + "output_base64sha512": "TqEppCWs6YEod3UPDPcniu7MgiOD3UWx1JXpUkKFr476zHH0D6UVkqJOXuR8seBnRlq9HgydFHLTEcTYFps4Sw==", + "output_file_mode": null, + "output_md5": "34b9f1764b49aa11cd3c0f0cd3db5538", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "77650f5f87b9d429f0ae3ee4141a2d4408515477", + "output_sha256": "5b50a0e9bb4e4c629b62905a00d534a16deb77710a96c40ab0c3f5fc04453e76", + "output_sha512": "4ea129a425ace9812877750f0cf7278aeecc822383dd45b1d495e9524285af8efacc71f40fa51592a24e5ee47cb1e067465abd1e0c9d1472d311c4d8169b384b", + "output_size": 5323, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "097260566921", + "arn": "arn:aws:iam::097260566921:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "097260566921", + "user_id": "AIDARNJJOPGERLGUKQ553" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3830342824", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:\\u003cremote-account-id\\u003e:secret:\\u003csecret-name\\u003e\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::...\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:\\u003cremote-account-id\\u003e:secret:\\u003csecret-name\\u003e\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::...\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:\u003cremote-account-id\u003e:secret:\u003csecret-name\u003e" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::..." + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:48:50Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGEVXWJQGTNS" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921", + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-6921.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-6921.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": null, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "34b9f1764b49aa11cd3c0f0cd3db5538", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": null, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "W1Cg6btOTGKbYpBaANU0oW3rd3EKlsQKsMP1/ARFPnY=", + "storage_class": "STANDARD", + "tags": null, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 67aaa58..06d8a93 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -1,7 +1,16 @@ # CTORM infrastructure ```bash +export AWS_PROFILE="cumulus-sbx-7522" export VARFILE=dev.tfvars + +# create workspaces and select the one to use +terraform workspace new ctorm-dev + +terraform workspace new ctorm-prod + +export TF_WORKSPACE=ctorm-dev + terraform init terraform plan \ diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index f1e6ce5..2ba024e 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -17,8 +17,8 @@ resource "aws_sqs_queue" "granules" { }) } -resource "aws_dynamodb_table" "state" { - name = "${var.name_prefix}-state" +resource "aws_dynamodb_table" "granules" { + name = "${var.name_prefix}-granules" billing_mode = "PAY_PER_REQUEST" hash_key = "pk" @@ -101,7 +101,7 @@ data "aws_iam_policy_document" "cnm-sender" { ] resources = [ - aws_dynamodb_table.state.arn + aws_dynamodb_table.granules.arn ] } } @@ -153,7 +153,7 @@ resource "aws_lambda_function" "cnm-sender" { environment { variables = { GRANULES_QUEUE_URL = aws_sqs_queue.granules.url - TABLE_NAME = aws_dynamodb_table.state.name + TABLE_NAME = aws_dynamodb_table.granules.name CUMULUS_INGEST_QUEUE_URL = var.cumulus_ingest_queue_url LOG_LEVEL = "INFO" } @@ -188,69 +188,6 @@ resource "aws_lambda_permission" "allow_eventbridge" { source_arn = aws_cloudwatch_event_rule.every_minute.arn } - -data "aws_iam_policy_document" "prepare_ec2_assume_role" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } -} - -resource "aws_iam_role" "prepare_ec2" { - name = "${var.name_prefix}-prepare-ec2-role" - assume_role_policy = data.aws_iam_policy_document.prepare_ec2_assume_role.json -} - -data "aws_iam_policy_document" "prepare_ec2" { - statement { - actions = [ - "sqs:SendMessage", - "sqs:GetQueueAttributes" - ] - - resources = [ - aws_sqs_queue.granules.arn - ] - } - - statement { - actions = [ - "s3:ListBucket" - ] - - resources = [ - for bucket_name in var.prepare_source_bucket_names : "arn:aws:s3:::${bucket_name}" - ] - } - - statement { - actions = [ - "s3:GetObject" - ] - - resources = [ - for bucket_name in var.prepare_source_bucket_names : "arn:aws:s3:::${bucket_name}/*" - ] - } -} - -resource "aws_iam_role_policy" "prepare_ec2" { - name = "${var.name_prefix}-prepare-ec2-policy" - role = aws_iam_role.prepare_ec2.id - policy = data.aws_iam_policy_document.prepare_ec2.json -} - -resource "aws_iam_instance_profile" "prepare_ec2" { - name = "${var.name_prefix}-prepare-ec2-profile" - role = aws_iam_role.prepare_ec2.name -} - - - resource "aws_s3_bucket" "scratch" { bucket = "${var.name_prefix}-scratch" diff --git a/ctorm/infra/terraform/outputs.tf b/ctorm/infra/terraform/outputs.tf index 6e734a2..4dc5e83 100644 --- a/ctorm/infra/terraform/outputs.tf +++ b/ctorm/infra/terraform/outputs.tf @@ -10,22 +10,14 @@ output "granules_dlq_url" { value = aws_sqs_queue.granules_dlq.url } -output "state_table_name" { - value = aws_dynamodb_table.state.name +output "granule_table_name" { + value = aws_dynamodb_table.granules.name } output "worker_lambda_name" { value = aws_lambda_function.cnm-sender.function_name } -output "prepare_ec2_instance_profile_name" { - value = aws_iam_instance_profile.prepare_ec2.name -} - -output "prepare_ec2_role_arn" { - value = aws_iam_role.prepare_ec2.arn -} - output "scratch_bucket" { value = aws_s3_bucket.scratch.bucket } diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 7426ae8..c50a504 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,13 +1,17 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 30, + "serial": 45, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", "type": "string" }, + "granule_table_name": { + "value": "ctorm-dev-granules", + "type": "string" + }, "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" @@ -20,22 +24,10 @@ "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "type": "string" }, - "prepare_ec2_instance_profile_name": { - "value": "ctorm-dev-prepare-ec2-profile", - "type": "string" - }, - "prepare_ec2_role_arn": { - "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "type": "string" - }, "scratch_bucket": { "value": "ctorm-dev-scratch", "type": "string" }, - "state_table_name": { - "value": "ctorm-dev-state", - "type": "string" - }, "worker_lambda_name": { "value": "ctorm-dev-cnm-sender", "type": "string" @@ -132,9 +124,9 @@ { "schema_version": 0, "attributes": { - "id": "2455266930", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "id": "3548715077", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -204,7 +196,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules" ], "sid": "" } @@ -288,9 +280,9 @@ { "schema_version": 0, "attributes": { - "id": "4124935366", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}}}]}", + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -303,14 +295,20 @@ ], "condition": [ { - "test": "ArnEquals", + "test": "ArnLike", "values": [ - "arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role" + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" ], "variable": "aws:PrincipalArn" } @@ -387,132 +385,6 @@ } ] }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "1999618104", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sqs:GetQueueAttributes", - "sqs:SendMessage" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" - ], - "sid": "" - }, - { - "actions": [ - "s3:ListBucket" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products", - "arn:aws:s3:::asf-cumulus-prod-opera-products", - "arn:aws:s3:::asf-cumulus-test-opera-products", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products" - ], - "sid": "" - }, - { - "actions": [ - "s3:GetObject" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products/*", - "arn:aws:s3:::asf-cumulus-prod-opera-products/*", - "arn:aws:s3:::asf-cumulus-test-opera-products/*", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" - ], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2_assume_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "2851119427", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sts:AssumeRole" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [ - { - "identifiers": [ - "ec2.amazonaws.com" - ], - "type": "Service" - } - ], - "resources": [], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, { "mode": "managed", "type": "aws_cloudwatch_event_rule", @@ -578,7 +450,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", @@ -619,13 +491,13 @@ { "mode": "managed", "type": "aws_dynamodb_table", - "name": "state", + "name": "granules", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 1, "attributes": { - "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", "attribute": [ { "name": "pk", @@ -640,10 +512,10 @@ "deletion_protection_enabled": false, "global_secondary_index": [], "hash_key": "pk", - "id": "ctorm-dev-state", + "id": "ctorm-dev-granules", "import_table": [], "local_secondary_index": [], - "name": "ctorm-dev-state", + "name": "ctorm-dev-granules", "on_demand_throughput": [], "point_in_time_recovery": [ { @@ -664,7 +536,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -680,35 +552,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", - "create_date": "2026-06-19T01:00:34Z", - "id": "ctorm-dev-prepare-ec2-profile", - "name": "ctorm-dev-prepare-ec2-profile", - "name_prefix": "", - "path": "/", - "role": "ctorm-dev-prepare-ec2-role", - "tags": {}, - "tags_all": {}, - "unique_id": "AIPA4VW62R3RCASOXJGOK" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role", @@ -762,7 +605,7 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-opera-uat-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-opera-uat-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", "create_date": "2026-06-24T19:29:26Z", "description": "", "force_detach_policies": false, @@ -791,45 +634,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_role", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", - "create_date": "2026-06-19T01:00:33Z", - "description": "", - "force_detach_policies": false, - "id": "ctorm-dev-prepare-ec2-role", - "inline_policy": [ - { - "name": "ctorm-dev-prepare-ec2-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" - } - ], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "ctorm-dev-prepare-ec2-role", - "name_prefix": "", - "path": "/", - "permissions_boundary": "", - "tags": {}, - "tags_all": {}, - "unique_id": "AROA4VW62R3RKO3MKUI7D" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -842,14 +646,14 @@ "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", "name": "ctorm-dev-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", "role": "ctorm-dev-cnm-sender-role" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_sqs_queue.granules", "aws_sqs_queue.granules_dlq", @@ -885,33 +689,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_role_policy", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", - "name": "ctorm-dev-prepare-ec2-policy", - "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "role": "ctorm-dev-prepare-ec2-role" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "aws_sqs_queue.granules", - "aws_sqs_queue.granules_dlq", - "data.aws_iam_policy_document.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -961,7 +738,7 @@ "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "LOG_LEVEL": "INFO", - "TABLE_NAME": "ctorm-dev-state" + "TABLE_NAME": "ctorm-dev-granules" } } ], @@ -979,7 +756,7 @@ "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", - "last_modified": "2026-06-24T19:29:31.000+0000", + "last_modified": "2026-06-25T02:29:36.000+0000", "layers": [], "logging_config": [ { @@ -1024,7 +801,7 @@ "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_sqs_queue.granules", @@ -1063,7 +840,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 0f77000..270feb7 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,9 +1,17 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 27, + "serial": 40, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { + "cumulus_db_md_extract_upload_role_arn": { + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", + "type": "string" + }, + "granule_table_name": { + "value": "ctorm-dev-granules", + "type": "string" + }, "granules_dlq_url": { "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", "type": "string" @@ -16,22 +24,10 @@ "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "type": "string" }, - "prepare_ec2_instance_profile_name": { - "value": "ctorm-dev-prepare-ec2-profile", - "type": "string" - }, - "prepare_ec2_role_arn": { - "value": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "type": "string" - }, "scratch_bucket": { "value": "ctorm-dev-scratch", "type": "string" }, - "state_table_name": { - "value": "ctorm-dev-state", - "type": "string" - }, "worker_lambda_name": { "value": "ctorm-dev-cnm-sender", "type": "string" @@ -119,98 +115,6 @@ } ] }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "cnm-sender", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "2455266930", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" - ], - "sid": "" - }, - { - "actions": [ - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:ReceiveMessage" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" - ], - "sid": "" - }, - { - "actions": [ - "sqs:SendMessage", - "sqs:SendMessageBatch" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" - ], - "sid": "" - }, - { - "actions": [ - "dynamodb:GetItem", - "dynamodb:PutItem", - "dynamodb:Query", - "dynamodb:UpdateItem" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state" - ], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, { "mode": "data", "type": "aws_iam_policy_document", @@ -284,9 +188,9 @@ { "schema_version": 0, "attributes": { - "id": "2147741451", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnEquals\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}}}]}", + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -299,14 +203,20 @@ ], "condition": [ { - "test": "ArnEquals", + "test": "ArnLike", "values": [ "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", - "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role" + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" ], "variable": "aws:PrincipalArn" } @@ -383,132 +293,6 @@ } ] }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "1999618104", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessage\",\n \"sqs:GetQueueAttributes\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": \"s3:GetObject\",\n \"Resource\": [\n \"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\n \"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\n \"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\n \"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"\n ]\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Effect\":\"Allow\",\"Action\":\"s3:GetObject\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sqs:GetQueueAttributes", - "sqs:SendMessage" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" - ], - "sid": "" - }, - { - "actions": [ - "s3:ListBucket" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products", - "arn:aws:s3:::asf-cumulus-prod-opera-products", - "arn:aws:s3:::asf-cumulus-test-opera-products", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products" - ], - "sid": "" - }, - { - "actions": [ - "s3:GetObject" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [], - "resources": [ - "arn:aws:s3:::asf-cumulus-dev-opera-products/*", - "arn:aws:s3:::asf-cumulus-prod-opera-products/*", - "arn:aws:s3:::asf-cumulus-test-opera-products/*", - "arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*", - "arn:aws:s3:::sds-n-cumulus-test-nisar-products/*" - ], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, - { - "mode": "data", - "type": "aws_iam_policy_document", - "name": "prepare_ec2_assume_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "2851119427", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"ec2.amazonaws.com\"\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}]}", - "override_json": null, - "override_policy_documents": null, - "policy_id": null, - "source_json": null, - "source_policy_documents": null, - "statement": [ - { - "actions": [ - "sts:AssumeRole" - ], - "condition": [], - "effect": "Allow", - "not_actions": [], - "not_principals": [], - "not_resources": [], - "principals": [ - { - "identifiers": [ - "ec2.amazonaws.com" - ], - "type": "Service" - } - ], - "resources": [], - "sid": "" - } - ], - "version": "2012-10-17" - }, - "sensitive_attributes": [] - } - ] - }, { "mode": "managed", "type": "aws_cloudwatch_event_rule", @@ -574,7 +358,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", @@ -612,99 +396,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_dynamodb_table", - "name": "state", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state", - "attribute": [ - { - "name": "pk", - "type": "S" - }, - { - "name": "sk", - "type": "S" - } - ], - "billing_mode": "PAY_PER_REQUEST", - "deletion_protection_enabled": false, - "global_secondary_index": [], - "hash_key": "pk", - "id": "ctorm-dev-state", - "import_table": [], - "local_secondary_index": [], - "name": "ctorm-dev-state", - "on_demand_throughput": [], - "point_in_time_recovery": [ - { - "enabled": false, - "recovery_period_in_days": 0 - } - ], - "range_key": "sk", - "read_capacity": 0, - "replica": [], - "restore_date_time": null, - "restore_source_name": null, - "restore_source_table_arn": null, - "restore_to_latest_time": null, - "server_side_encryption": [], - "stream_arn": "", - "stream_enabled": false, - "stream_label": "", - "stream_view_type": "", - "table_class": "STANDARD", - "tags": {}, - "tags_all": {}, - "timeouts": null, - "ttl": [ - { - "attribute_name": "", - "enabled": false - } - ], - "write_capacity": 0 - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:instance-profile/ctorm-dev-prepare-ec2-profile", - "create_date": "2026-06-19T01:00:34Z", - "id": "ctorm-dev-prepare-ec2-profile", - "name": "ctorm-dev-prepare-ec2-profile", - "name_prefix": "", - "path": "/", - "role": "ctorm-dev-prepare-ec2-role", - "tags": {}, - "tags_all": {}, - "unique_id": "AIPA4VW62R3RCASOXJGOK" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role", @@ -758,65 +449,31 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnEquals\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", "create_date": "2026-06-24T19:29:26Z", "description": "", "force_detach_policies": false, "id": "ctorm-dev-cumulus-db-md-extract-upload", - "inline_policy": [], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "ctorm-dev-cumulus-db-md-extract-upload", - "name_prefix": "", - "path": "/", - "permissions_boundary": "", - "tags": null, - "tags_all": {}, - "unique_id": "AROA4VW62R3RJW53W2D5X" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_iam_role", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::871271927522:role/ctorm-dev-prepare-ec2-role", - "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", - "create_date": "2026-06-19T01:00:33Z", - "description": "", - "force_detach_policies": false, - "id": "ctorm-dev-prepare-ec2-role", "inline_policy": [ { - "name": "ctorm-dev-prepare-ec2-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}" + "name": "ctorm-dev-cumulus-db-md-extract-upload", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], "max_session_duration": 3600, - "name": "ctorm-dev-prepare-ec2-role", + "name": "ctorm-dev-cumulus-db-md-extract-upload", "name_prefix": "", "path": "/", "permissions_boundary": "", "tags": {}, "tags_all": {}, - "unique_id": "AROA4VW62R3RKO3MKUI7D" + "unique_id": "AROA4VW62R3RJW53W2D5X" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ - "data.aws_iam_policy_document.prepare_ec2_assume_role" + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" ] } ] @@ -840,6 +497,7 @@ "private": "bnVsbA==", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_sqs_queue.granules", @@ -876,33 +534,6 @@ } ] }, - { - "mode": "managed", - "type": "aws_iam_role_policy", - "name": "prepare_ec2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "id": "ctorm-dev-prepare-ec2-role:ctorm-dev-prepare-ec2-policy", - "name": "ctorm-dev-prepare-ec2-policy", - "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"sqs:SendMessage\",\"sqs:GetQueueAttributes\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":\"s3:ListBucket\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products\",\"arn:aws:s3:::asf-cumulus-test-opera-products\",\"arn:aws:s3:::asf-cumulus-prod-opera-products\",\"arn:aws:s3:::asf-cumulus-dev-opera-products\"]},{\"Action\":\"s3:GetObject\",\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::sds-n-cumulus-test-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-prod-nisar-products/*\",\"arn:aws:s3:::sds-n-cumulus-dev-nisar-products/*\",\"arn:aws:s3:::asf-cumulus-test-opera-products/*\",\"arn:aws:s3:::asf-cumulus-prod-opera-products/*\",\"arn:aws:s3:::asf-cumulus-dev-opera-products/*\"]}]}", - "role": "ctorm-dev-prepare-ec2-role" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_iam_role.prepare_ec2", - "aws_sqs_queue.granules", - "aws_sqs_queue.granules_dlq", - "data.aws_iam_policy_document.prepare_ec2", - "data.aws_iam_policy_document.prepare_ec2_assume_role" - ] - } - ] - }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -1015,6 +646,7 @@ "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", @@ -1054,7 +686,7 @@ "dependencies": [ "aws_cloudwatch_event_rule.every_minute", "aws_cloudwatch_log_group.cnm-sender", - "aws_dynamodb_table.state", + "aws_dynamodb_table.granules", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_lambda_function.cnm-sender", diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate new file mode 100644 index 0000000..e4ddffc --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-dev/terraform.tfstate @@ -0,0 +1,294 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 9, + "lineage": "848003f0-637c-6830-7392-8ca48da894a1", + "outputs": { + "granule_table_name": { + "value": "ctorm-dev-granules", + "type": "string" + }, + "granules_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "type": "string" + }, + "granules_queue_arn": { + "value": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "type": "string" + }, + "granules_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-dev-scratch", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-dev-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", + "output_file_mode": null, + "output_md5": "94564c5dd9caf22baaf170b00693c497", + "output_path": "./placeholder-lambda.zip", + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, + "source": [ + { + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnLike", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:871271927522:rule/ctorm-dev-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-dev-every-minute", + "is_enabled": false, + "name": "ctorm-dev-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-dev-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate new file mode 100644 index 0000000..278e8a2 --- /dev/null +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -0,0 +1,998 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 18, + "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", + "outputs": { + "cumulus_db_md_extract_upload_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload", + "type": "string" + }, + "granule_table_name": { + "value": "ctorm-granules", + "type": "string" + }, + "granules_dlq_url": { + "value": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", + "type": "string" + }, + "granules_queue_arn": { + "value": "arn:aws:sqs:us-west-2:097260566921:ctorm-granules", + "type": "string" + }, + "granules_queue_url": { + "value": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "type": "string" + }, + "scratch_bucket": { + "value": "ctorm-scratch", + "type": "string" + }, + "worker_lambda_name": { + "value": "ctorm-cnm-sender", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "placeholder_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_base64sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "output_base64sha512": "Kf1anE514t1db0WYLXSHASh7M23HxVS30uaMvh8f1CIpaAZXUvQtOBUrPgjvsxbko7IVN0lQ3U5t+ssVEs5A0Q==", + "output_file_mode": null, + "output_md5": "94564c5dd9caf22baaf170b00693c497", + "output_path": "./placeholder-lambda.zip", + "output_sha": "7451212364088fb05c08860c9d80f78e29fa76b5", + "output_sha256": "dcb94745f089385ac47525ee4894bbc80b433cbe6308648849ff45db813acc56", + "output_sha512": "29fd5a9c4e75e2dd5d6f45982d748701287b336dc7c554b7d2e68cbe1f1fd4222968065752f42d38152b3e08efb316e4a3b215374950dd4e6dfacb1512ce40d1", + "output_size": 344, + "source": [ + { + "content": "import json\nimport os\n\ndef lambda_handler(event, context):\n print(json.dumps({\n \"event\": event,\n \"granules_queue_url\": os.environ.get(\"GRANULES_QUEUE_URL\"),\n \"table_name\": os.environ.get(\"TABLE_NAME\"),\n \"cumulus_ingest_queue_url\": os.environ.get(\"CUMULUS_INGEST_QUEUE_URL\"),\n }))\n\n return {\n \"ok\": True\n }\n", + "filename": "index.py" + } + ], + "source_content": null, + "source_content_filename": null, + "source_dir": null, + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "allow_scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3026183601", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:ListBucket\",\n \"s3:GetObject\",\n \"s3:DeleteObject\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::ctorm-scratch/*\",\n \"arn:aws:s3:::ctorm-scratch\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Resource\":[\"arn:aws:s3:::ctorm-scratch/*\",\"arn:aws:s3:::ctorm-scratch\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-scratch", + "arn:aws:s3:::ctorm-scratch/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2687090556", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:097260566921:ctorm-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2685467495", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "s3:AbortMultipartUpload", + "s3:PutObject" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-scratch/cumulus-granules/*" + ], + "sid": "UploadExtractedGranules" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "cumulus-granules/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-scratch" + ], + "sid": "ListDestinationPrefix" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cumulus_db_md_extract_upload_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2827657917", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"AWS\": [\n \"arn:aws:iam::871271927522:root\",\n \"arn:aws:iam::725875338589:root\",\n \"arn:aws:iam::510296831643:root\",\n \"arn:aws:iam::372059463218:root\",\n \"arn:aws:iam::097260566921:root\",\n \"arn:aws:iam::082931748743:root\"\n ]\n },\n \"Condition\": {\n \"ArnLike\": {\n \"aws:PrincipalArn\": [\n \"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\n \"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"\n ]\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]},\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [ + { + "test": "ArnLike", + "values": [ + "arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role" + ], + "variable": "aws:PrincipalArn" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "arn:aws:iam::082931748743:root", + "arn:aws:iam::097260566921:root", + "arn:aws:iam::372059463218:root", + "arn:aws:iam::510296831643:root", + "arn:aws:iam::725875338589:root", + "arn:aws:iam::871271927522:root" + ], + "type": "AWS" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "lambda_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2690255455", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"lambda.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "lambda.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_rule", + "name": "every_minute", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:events:us-west-2:097260566921:rule/ctorm-every-minute", + "description": "", + "event_bus_name": "default", + "event_pattern": null, + "force_destroy": false, + "id": "ctorm-every-minute", + "is_enabled": false, + "name": "ctorm-every-minute", + "name_prefix": "", + "role_arn": "", + "schedule_expression": "rate(1 minute)", + "state": "DISABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_event_target", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "appsync_target": [], + "arn": "arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender", + "batch_target": [], + "dead_letter_config": [], + "ecs_target": [], + "event_bus_name": "default", + "force_destroy": false, + "http_target": [], + "id": "ctorm-every-minute-terraform-20260625024146730800000001", + "input": "{\"source\":\"eventbridge\"}", + "input_path": "", + "input_transformer": [], + "kinesis_target": [], + "redshift_target": [], + "retry_policy": [], + "role_arn": "", + "rule": "ctorm-every-minute", + "run_command_targets": [], + "sagemaker_pipeline_target": [], + "sqs_target": [], + "target_id": "terraform-20260625024146730800000001" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender", + "id": "/aws/lambda/ctorm-cnm-sender", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/lambda/ctorm-cnm-sender", + "name_prefix": "", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules", + "attribute": [ + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [], + "hash_key": "pk", + "id": "ctorm-granules", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-granules", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": null, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cnm-sender-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:40:48Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cnm-sender-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cnm-sender-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGE3PVM46W7W" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Condition\":{\"ArnLike\":{\"aws:PrincipalArn\":[\"arn:aws:iam::725875338589:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::725875338589:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::871271927522:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::372059463218:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::097260566921:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::082931748743:role/ctorm-*-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role\",\"arn:aws:iam::510296831643:role/ctorm-*-cumulus-db-md-extract-role\"]}},\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam::871271927522:root\",\"arn:aws:iam::725875338589:root\",\"arn:aws:iam::510296831643:root\",\"arn:aws:iam::372059463218:root\",\"arn:aws:iam::097260566921:root\",\"arn:aws:iam::082931748743:root\"]}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:40:48Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-upload", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-upload", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGE4XUWPCFMM" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cnm-sender-role:ctorm-cnm-sender-policy", + "name": "ctorm-cnm-sender-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "role": "ctorm-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "cumulus_db_md_extract_upload", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-upload:ctorm-cumulus-db-md-extract-upload", + "name": "ctorm-cumulus-db-md-extract-upload", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "role": "ctorm-cumulus-db-md-extract-upload" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cumulus_db_md_extract_upload", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload", + "data.aws_iam_policy_document.cumulus_db_md_extract_upload_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "scratch_bucket_access", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cnm-sender-role:ctorm-scratch-access", + "name": "ctorm-scratch-access", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch/*\",\"arn:aws:s3:::ctorm-scratch\"]}]}", + "role": "ctorm-cnm-sender-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_iam_role.cnm-sender", + "aws_s3_bucket.scratch", + "data.aws_iam_policy_document.allow_scratch_bucket_access", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender", + "code_sha256": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [ + { + "variables": { + "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", + "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "LOG_LEVEL": "INFO", + "TABLE_NAME": "ctorm-granules" + } + } + ], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "./placeholder-lambda.zip", + "function_name": "ctorm-cnm-sender", + "handler": "load_tester.lambda_handler", + "id": "ctorm-cnm-sender", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender/invocations", + "kms_key_arn": "", + "last_modified": "2026-06-25T02:41:40.841+0000", + "layers": null, + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/ctorm-cnm-sender", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::097260566921:role/ctorm-cnm-sender-role", + "runtime": "python3.12", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", + "source_code_size": 344, + "tags": null, + "tags_all": {}, + "timeout": 120, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_eventbridge", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "ctorm-cnm-sender", + "function_url_auth_type": null, + "id": "AllowExecutionFromEventBridge", + "principal": "events.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:events:us-west-2:097260566921:rule/ctorm-every-minute", + "statement_id": "AllowExecutionFromEventBridge", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_event_rule.every_minute", + "aws_cloudwatch_log_group.cnm-sender", + "aws_dynamodb_table.granules", + "aws_iam_role.cnm-sender", + "aws_iam_role_policy.cnm-sender", + "aws_lambda_function.cnm-sender", + "aws_sqs_queue.granules", + "aws_sqs_queue.granules_dlq", + "data.archive_file.placeholder_lambda", + "data.aws_iam_policy_document.cnm-sender", + "data.aws_iam_policy_document.lambda_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "scratch", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-scratch", + "bucket": "ctorm-scratch", + "bucket_domain_name": "ctorm-scratch.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "ctorm-scratch.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-scratch", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Name": "ctorm-scratch" + }, + "tags_all": { + "Name": "ctorm-scratch" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:097260566921:ctorm-granules", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-granules", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granule-dlq\",\"maxReceiveCount\":5}", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", + "visibility_timeout_seconds": 180 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19", + "dependencies": [ + "aws_sqs_queue.granules_dlq" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "granules_dlq", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:sqs:us-west-2:097260566921:ctorm-granule-dlq", + "content_based_deduplication": false, + "deduplication_scope": "", + "delay_seconds": 0, + "fifo_queue": false, + "fifo_throughput_limit": "", + "id": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", + "kms_data_key_reuse_period_seconds": 300, + "kms_master_key_id": "", + "max_message_size": 262144, + "message_retention_seconds": 345600, + "name": "ctorm-granule-dlq", + "name_prefix": "", + "policy": "", + "receive_wait_time_seconds": 0, + "redrive_allow_policy": "", + "redrive_policy": "", + "sqs_managed_sse_enabled": true, + "tags": null, + "tags_all": {}, + "timeouts": null, + "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", + "visibility_timeout_seconds": 30 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsImRlbGV0ZSI6MTgwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} From 0ee5897b61c64a09d7e9aed73dd8ab969855df06 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 11:24:58 -0800 Subject: [PATCH 21/31] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/buildspec.yaml | 7 ++++ .../terraform/nisar-prod.tfvars | 13 +++---- .../terraform/opera-prod.tfvars | 3 +- .../terraform/terraform-opera-prod.tfstate | 36 +++++++++---------- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index e5da973..665d8f7 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -65,6 +65,13 @@ phases: - | echo 'After assuming role: ' - aws sts get-caller-identity + - | + id=$(aws sts get-caller-identity) + if [ -z "$id" ]; then + echo "Error: `aws sts get-caller-identity` returned no output." >&2 + exit 1 + fi + build: commands: diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars index 66ff300..474dc28 100644 --- a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -1,18 +1,19 @@ name_prefix = "ctorm" -db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2::secret:" +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" -db_md_extract_vpc_id = "vpc-..." +db_md_extract_vpc_id = "vpc-05bb1a5fa3d38fddd" db_md_extract_subnet_ids = [ - "subnet-...", - "subnet-...", + "subnet-028b8ab53b5232fa5", ] db_md_extract_security_group_ids = [ - "sg-...", + "sg-01b8118dc0c9ea180", + "sg-0e56e1310879eaef3", + "sg-0a4f1b9d764ef7b38", ] -db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::..." +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "nisar" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index b7cb678..7701459 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -14,6 +14,7 @@ db_md_extract_security_group_ids = [ "sg-0ed18cd0fb6e79f58", ] -db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." +# Look at the ctorm tf output for `cumulus_db_md_extract_upload_role_arn` +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "opera" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 691cf7d..2c75342 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 41, + "serial": 55, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", - "output_base64sha256": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", - "output_base64sha512": "J8ZtLQzzPeFUcNb4IW4DV3ar9M0/uFonIhPlfZDv6Z1cGcr+nSGpuJCcASX4HlLX6dJcILMBEtdwGbEuB/3rvQ==", + "id": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", + "output_base64sha256": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", + "output_base64sha512": "bbAx+FqrN2kSRGI8VXAMr9dj+nGcUHwVSq0X7JE9pjAeuF11V4Xqscovy/5i7a3dXnIK9EQB1zZL7sAh3lY8og==", "output_file_mode": null, - "output_md5": "692e40b861d2f40dcc12eb78cbe57de4", + "output_md5": "f565d14b76dda95c7f2e971e583ce9d7", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "0a08aaa9fc5c84173f02b04b0f738ad6fbc611ee", - "output_sha256": "0a3e592d9256955ef7682a2b64dff82c1d25096c1ef8b6eaf2eb817fc1850723", - "output_sha512": "27c66d2d0cf33de15470d6f8216e035776abf4cd3fb85a272213e57d90efe99d5c19cafe9d21a9b8909c0125f81e52d7e9d25c20b30112d77019b12e07fdebbd", - "output_size": 5350, + "output_sha": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", + "output_sha256": "78de28700a9ba06d35d8caa2edb812266d418165dcaecd30c128188f36b12fff", + "output_sha512": "6db031f85aab37691244623c55700cafd763fa719c507c154aad17ec913da6301eb85d755785eab1ca2fcbfe62edaddd5e720af44401d7364beec021de563ca2", + "output_size": 5410, "source": [], "source_content": null, "source_content_filename": null, @@ -82,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "1104794953", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "3636791408", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -173,7 +173,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" ], "sid": "AssumeCtormAccountUploadRole" }, @@ -370,7 +370,7 @@ { "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", - "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload." + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" } ], "fleet": [], @@ -475,7 +475,7 @@ "inline_policy": [ { "name": "ctorm-cumulus-db-md-extract-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" } ], "managed_policy_arns": [], @@ -508,7 +508,7 @@ "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", "name": "ctorm-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", "role": "ctorm-cumulus-db-md-extract-role" }, "sensitive_attributes": [], @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "692e40b861d2f40dcc12eb78cbe57de4", + "etag": "f565d14b76dda95c7f2e971e583ce9d7", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "Cj5ZLZJWlV73aCorZN/4LB0lCWwe+Lbq8uuBf8GFByM=", + "source_hash": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From 1582c025878d86267893f6d9cec2cfaea90db356 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 13:13:31 -0800 Subject: [PATCH 22/31] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/buildspec.yaml | 11 +- .../terraform/nisar-prod.tfvars | 2 + .../terraform/opera-prod.tfvars | 2 + .../terraform/terraform-nisar-prod.tfstate | 753 ++++++++++++++++++ .../terraform/terraform-opera-prod.tfstate | 26 +- ctorm/cumulus-db-md-extract/test-s3-cp.sh | 7 + 6 files changed, 780 insertions(+), 21 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate create mode 100644 ctorm/cumulus-db-md-extract/test-s3-cp.sh diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index 665d8f7..33b0998 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -48,7 +48,7 @@ phases: pre_build: commands: - ls -lah - - chmod +x generate-slices-daily.sh export-granules-daily.sh + - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh - | echo 'Before assuming role: ' - aws sts get-caller-identity @@ -65,18 +65,13 @@ phases: - | echo 'After assuming role: ' - aws sts get-caller-identity - - | - id=$(aws sts get-caller-identity) - if [ -z "$id" ]; then - echo "Error: `aws sts get-caller-identity` returned no output." >&2 - exit 1 - fi build: commands: + # fail fast if we can't actually copy to S3: + - ./test-s3-cp.sh - - aws sts get-caller-identity - ./generate-slices-daily.sh - ./export-granules-daily.sh post_build: diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars index 474dc28..ac07320 100644 --- a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -17,3 +17,5 @@ db_md_extract_security_group_ids = [ db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "nisar" + +db_md_extract_ctorm_bucket = "ctorm-scratch" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars index 7701459..353e7c9 100644 --- a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -18,3 +18,5 @@ db_md_extract_security_group_ids = [ db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" db_md_extract_dump_subdir = "opera" + +db_md_extract_ctorm_bucket = "ctorm-scratch" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate new file mode 100644 index 0000000..71c1cb1 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate @@ -0,0 +1,753 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 17, + "lineage": "2aab81d2-395f-d5fa-becf-2772136eaa03", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", + "output_file_mode": null, + "output_md5": "8d91e1621f990906971ffa67900b6743", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", + "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", + "output_size": 5692, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "372059463218", + "arn": "arn:aws:iam::372059463218:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "372059463218", + "user_id": "AIDAVNIDY3YZHQAL4QVMF" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "202837701", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:372059463218:project/ctorm-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:372059463218:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "nisar" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:372059463218:project/ctorm-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-01b8118dc0c9ea180", + "sg-0a4f1b9d764ef7b38", + "sg-0e56e1310879eaef3" + ], + "subnets": [ + "subnet-028b8ab53b5232fa5" + ], + "vpc_id": "vpc-05bb1a5fa3d38fddd" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T19:31:14Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROAVNIDY3YZPQXYNWXSP" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218", + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-3218.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-3218.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "d6f242167244d0b7b820212432e41dbdcbde816cf4992ec7f7967eab0154eab5", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "8d91e1621f990906971ffa67900b6743", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 2c75342..238ecee 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 55, + "serial": 78, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", - "output_base64sha256": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", - "output_base64sha512": "bbAx+FqrN2kSRGI8VXAMr9dj+nGcUHwVSq0X7JE9pjAeuF11V4Xqscovy/5i7a3dXnIK9EQB1zZL7sAh3lY8og==", + "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", "output_file_mode": null, - "output_md5": "f565d14b76dda95c7f2e971e583ce9d7", + "output_md5": "8d91e1621f990906971ffa67900b6743", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "7cfab52aaf85e09f11c731591f62d65b6f4afe3c", - "output_sha256": "78de28700a9ba06d35d8caa2edb812266d418165dcaecd30c128188f36b12fff", - "output_sha512": "6db031f85aab37691244623c55700cafd763fa719c507c154aad17ec913da6301eb85d755785eab1ca2fcbfe62edaddd5e720af44401d7364beec021de563ca2", - "output_size": 5410, + "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", + "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", + "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", + "output_size": 5692, "source": [], "source_content": null, "source_content_filename": null, @@ -355,7 +355,7 @@ { "name": "CTORM_BUCKET", "type": "PLAINTEXT", - "value": "ctorm-dev-scratch" + "value": "ctorm-scratch" }, { "name": "DUMP_SUBDIR", @@ -475,7 +475,7 @@ "inline_policy": [ { "name": "ctorm-cumulus-db-md-extract-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload.\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" } ], "managed_policy_arns": [], @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "f565d14b76dda95c7f2e971e583ce9d7", + "etag": "8d91e1621f990906971ffa67900b6743", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "eN4ocAqboG012Mqi7bgSJm1BgWXcrs0wwSgYjzaxL/8=", + "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/test-s3-cp.sh b/ctorm/cumulus-db-md-extract/test-s3-cp.sh new file mode 100644 index 0000000..7b32da5 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/test-s3-cp.sh @@ -0,0 +1,7 @@ +set -euo pipefail + +S3_URI_COPYTEST="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" +echo "Failing fast if we can't write to ${S3_URI_COPYTEST}..." +touch ./test-s3-write.txt +aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write.txt" +rm ./test-s3-write.txt From 994e7df01313e595c6b1c52695f986fe5233e441 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 15:35:26 -0800 Subject: [PATCH 23/31] RP-7509 handle short assume role and long queries --- ctorm/cumulus-db-md-extract/buildspec.yaml | 2 +- .../export-granules-daily.sh | 30 ++++++++++--------- .../generate-slices-daily.sh | 10 ++++--- .../refresh-ctorm-upload-role.sh | 19 ++++++++++++ .../terraform/terraform-opera-prod.tfstate | 22 +++++++------- 5 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index 33b0998..a766071 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -48,7 +48,7 @@ phases: pre_build: commands: - ls -lah - - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh + - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh refresh-ctorm-upload-role.sh - | echo 'Before assuming role: ' - aws sts get-caller-identity diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index 8e5a9e8..b95b13f 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -5,17 +5,6 @@ FAILED_SLICES_FILE="failed_slices_daily.tsv" rm -f "$FAILED_SLICES_FILE" -echo "Statement timeout: " -psql \ - --host="$PGHOST" \ - --port="${PGPORT:-5432}" \ - --username="$PGUSER" \ - --dbname="$PGDATABASE" \ - --no-psqlrc \ - -c "SHOW statement_timeout;" - - - while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then echo "Skipping malformed slice row: collection=${COLLECTION:-} yyyymmdd=${YYYYMMDD:-}" >&2 @@ -25,9 +14,11 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do YYYYMM="${YYYYMMDD:0:6}" SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + TMP_OUTPUT="$(mktemp "${SAFE_COLLECTION}.${YYYYMMDD}.XXXXXX.jsonl.gz")" echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" echo "Destination: ${S3_URI}" + echo "Temporary output: ${TMP_OUTPUT}" if ! PGOPTIONS="-c statement_timeout=${PG_STATEMENT_TIMEOUT:-0}" psql \ --host="$PGHOST" \ @@ -40,19 +31,30 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do --set=collection="$COLLECTION" \ --set=yyyymmdd="$YYYYMMDD" \ --file=export-granules-slice-daily.sql \ - | gzip -c \ - | aws s3 cp - "$S3_URI"; then + | gzip -c > "$TMP_OUTPUT"; then + + echo "FAILED query/gzip collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + rm -f "$TMP_OUTPUT" + continue + fi + + source ./refresh-ctorm-upload-role.sh - echo "FAILED collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + if ! aws s3 cp "$TMP_OUTPUT" "$S3_URI"; then + echo "FAILED upload collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + rm -f "$TMP_OUTPUT" continue fi + rm -f "$TMP_OUTPUT" echo "Uploaded ${S3_URI}" done < slices_daily.tsv if [[ -s "$FAILED_SLICES_FILE" ]]; then + source ./refresh-ctorm-upload-role.sh FAILED_SLICES_S3_URI="${S3_PREFIX}/failed_slices_daily.tsv" aws s3 cp "$FAILED_SLICES_FILE" "$FAILED_SLICES_S3_URI" echo "Some slices failed. Uploaded failure list to ${FAILED_SLICES_S3_URI}" >&2 diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh index 3651d74..b506300 100644 --- a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -1,9 +1,6 @@ set -euo pipefail - -S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" - psql \ --host="$PGHOST" \ --port="${PGPORT:-5432}" \ @@ -94,4 +91,9 @@ psql \ c.name, to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); " > slices_daily.tsv - aws s3 cp slices_daily.tsv "$S3_URI/" + +source ./refresh-ctorm-upload-role.sh + +SLICES_S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily.tsv" +aws s3 cp slices_daily.tsv "${SLICES_S3_URI}" +echo "Uploaded ${SLICES_S3_URI}" diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh new file mode 100644 index 0000000..28ca391 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${CTORM_S3_ROLE_ARN:-}" ]]; then + echo "CTORM_S3_ROLE_ARN is required" >&2 + exit 1 +fi + +echo "Refreshing CTORM upload role credentials..." >&2 + +CREDS_JSON="$(aws sts assume-role \ + --role-arn "${CTORM_S3_ROLE_ARN}" \ + --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER:-manual}-$(date +%s)")" + +export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" +export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" +export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + +aws sts get-caller-identity >&2 diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 238ecee..7ac4399 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 78, + "serial": 83, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", - "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", + "id": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", + "output_base64sha256": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", + "output_base64sha512": "dyN57NtzXihAKHxJioCsORk+6seeqMPqmOjRjZ/MGtoqISkL/zQBVIy+bMmMzSWV9TWAQjgrVrNlyR/n32V7fA==", "output_file_mode": null, - "output_md5": "8d91e1621f990906971ffa67900b6743", + "output_md5": "b1956744784cbb7f29513ed64d125aa8", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", - "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", - "output_size": 5692, + "output_sha": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", + "output_sha256": "ec5523d82308e38fce8172f5035607b2fa19ff702d80ecfadd9ca22a83993773", + "output_sha512": "772379ecdb735e2840287c498a80ac39193eeac79ea8c3ea98e8d18d9fcc1ada2a21290bff3401548cbe6cc98ccd2595f5358042382b56b365c91fe7df657b7c", + "output_size": 6385, "source": [], "source_content": null, "source_content_filename": null, @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "8d91e1621f990906971ffa67900b6743", + "etag": "b1956744784cbb7f29513ed64d125aa8", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "source_hash": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From 814fd1274e9fdbd95a5f768fbc69a8b4a80a0d0e Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Thu, 25 Jun 2026 17:22:49 -0800 Subject: [PATCH 24/31] PR-7509 progress w/ tf for codebuild stuff --- ctorm/cumulus-db-md-extract/buildspec.yaml | 11 ++-------- .../refresh-ctorm-upload-role.sh | 1 + .../terraform/terraform-opera-prod.tfstate | 22 +++++++++---------- ctorm/cumulus-db-md-extract/test-s3-cp.sh | 5 ++++- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml index a766071..7702062 100644 --- a/ctorm/cumulus-db-md-extract/buildspec.yaml +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -54,17 +54,10 @@ phases: - aws sts get-caller-identity - | echo "Assuming ctorm upload role..." - - | - CREDS_JSON="$(aws sts assume-role \ - --role-arn "${CTORM_S3_ROLE_ARN}" \ - --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER}")" - - export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" - export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" - export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + source ./refresh-ctorm-upload-role.sh - | echo 'After assuming role: ' - - aws sts get-caller-identity + aws sts get-caller-identity build: diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh index 28ca391..e28fc6a 100644 --- a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -8,6 +8,7 @@ fi echo "Refreshing CTORM upload role credentials..." >&2 +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CREDS_JSON="$(aws sts assume-role \ --role-arn "${CTORM_S3_ROLE_ARN}" \ --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER:-manual}-$(date +%s)")" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 7ac4399..50f232d 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 83, + "serial": 99, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", - "output_base64sha256": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", - "output_base64sha512": "dyN57NtzXihAKHxJioCsORk+6seeqMPqmOjRjZ/MGtoqISkL/zQBVIy+bMmMzSWV9TWAQjgrVrNlyR/n32V7fA==", + "id": "9fbb26adcb1cd2335401493f40f3f51313a97f96", + "output_base64sha256": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", + "output_base64sha512": "/0V83kNA5dBpxhDHHVYvvxhHTSdeerh1OAfI0c0IAXSyIp1NF2g0uFabWQ8AezYL/wARFo6uyGKw0qa8acR6iw==", "output_file_mode": null, - "output_md5": "b1956744784cbb7f29513ed64d125aa8", + "output_md5": "b3291b9c9adcab32d68819222e5d76f8", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "4fcb5b972a11469fb57186a2e6efd6b5891c2f41", - "output_sha256": "ec5523d82308e38fce8172f5035607b2fa19ff702d80ecfadd9ca22a83993773", - "output_sha512": "772379ecdb735e2840287c498a80ac39193eeac79ea8c3ea98e8d18d9fcc1ada2a21290bff3401548cbe6cc98ccd2595f5358042382b56b365c91fe7df657b7c", - "output_size": 6385, + "output_sha": "9fbb26adcb1cd2335401493f40f3f51313a97f96", + "output_sha256": "6d3a2810ff5968aeae413f9189e0596f11fe57a1d5b6ac6afa871cce11c34bc0", + "output_sha512": "ff457cde4340e5d069c610c71d562fbf18474d275e7ab8753807c8d1cd080174b2229d4d176834b8569b590f007b360bff0011168eaec862b0d2a6bc69c47a8b", + "output_size": 6320, "source": [], "source_content": null, "source_content_filename": null, @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "b1956744784cbb7f29513ed64d125aa8", + "etag": "b3291b9c9adcab32d68819222e5d76f8", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "7FUj2CMI44/OgXL1A1YHsvoZ/3AtgOz63ZyiKoOZN3M=", + "source_hash": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/test-s3-cp.sh b/ctorm/cumulus-db-md-extract/test-s3-cp.sh index 7b32da5..1f3c596 100644 --- a/ctorm/cumulus-db-md-extract/test-s3-cp.sh +++ b/ctorm/cumulus-db-md-extract/test-s3-cp.sh @@ -1,7 +1,10 @@ set -euo pipefail + + S3_URI_COPYTEST="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" echo "Failing fast if we can't write to ${S3_URI_COPYTEST}..." touch ./test-s3-write.txt -aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write.txt" +source ./refresh-ctorm-upload-role.sh +aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write-${CODEBUILD_BUILD_NUMBER}.txt" rm ./test-s3-write.txt From 3e19e8303495fe9fdcedcea126a93e222ee0aacc Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 26 Jun 2026 15:40:26 -0800 Subject: [PATCH 25/31] PR-7510 addition of Dynamo DB loader --- ctorm/README.md | 40 +- .../terraform/terraform-opera-prod.tfstate | 22 +- ctorm/granule-md-db-loader/README.md | 20 + ctorm/granule-md-db-loader/buildspec.yaml | 16 + .../granule_md_db_loader.py | 99 ++++ ctorm/infra/terraform/main.tf | 130 ++++- ctorm/infra/terraform/terraform.tfstate | 312 +++++++++++- .../infra/terraform/terraform.tfstate.backup | 457 +++++++++++++++++- .../ctorm-prod/terraform.tfstate | 38 +- 9 files changed, 1063 insertions(+), 71 deletions(-) create mode 100644 ctorm/granule-md-db-loader/README.md create mode 100644 ctorm/granule-md-db-loader/buildspec.yaml create mode 100644 ctorm/granule-md-db-loader/granule_md_db_loader.py diff --git a/ctorm/README.md b/ctorm/README.md index d44d3f6..eeb4e20 100644 --- a/ctorm/README.md +++ b/ctorm/README.md @@ -1,37 +1,15 @@ # CTORM + Cumulus Throughput... uh... ORM? Load tester tool for Cumulus. -Operates as two steps: - -* Prepare - * The prepare step goes through the UMMG directory of specified prod buckets and - extracts the metadata necessary to recreate CNM messages. It then creates SQS messages - containing lists of these metadata. -* CNM Sender - * The CNM Sender step reads the SQS messages and sends them to the Cumulus ingest queue. - -## Prepare - -### Cfg file - -See the `ctorm.cfg.example` file. - -#### source_buckets - -| key | value | -|--------------|-----------------------------------------------------------| -| bucketname | Name of bucket in which to find the UMMG files | -| keypair_name | Arbritrary name to suffix to the AWS keypair env var | -| share | Number of granules from this collection to add to the CNM | -| ummg_prefix | Path in bucketname where to find the desired UMMG files | - -Let's say you have the following source buckets set up: - -### Env vars - -You will need to have the following environment variables set: +Operates as x steps: -* `AWS_ACCESS_KEY_ID_[keypair_name]` where `keypair_name` matches an entry in the ctorm cfg toml file. -* `AWS_SECRET_ACCESS_KEY_[keypair_name]` where `keypair_name` matches an entry in the ctorm cfg toml file. +* cumulus-db-md-extract + * Operates in the prod account. + * Gets the metadata from the cumulus database and writes it to a bunch of .jsonl files in a S3 bucket in the + operational account. +* granule-md-db-loader + * Operates in the account in which the load test will operate. + * Reads the .jsonl files from the S3 bucket and loads them into the CTORM granules dynamo db. diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index 50f232d..be02ac5 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 99, + "serial": 112, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "9fbb26adcb1cd2335401493f40f3f51313a97f96", - "output_base64sha256": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", - "output_base64sha512": "/0V83kNA5dBpxhDHHVYvvxhHTSdeerh1OAfI0c0IAXSyIp1NF2g0uFabWQ8AezYL/wARFo6uyGKw0qa8acR6iw==", + "id": "15e0432811ac45b77440e902f20b433130b97e70", + "output_base64sha256": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", + "output_base64sha512": "loaVRKTjqeDeRXYLTynxgJIcoeFMuaH5+Yx1GZPZymzfK4GSIarTAvusSWNs7DHEH+baqi6JJ8D5SjrMvSmT4Q==", "output_file_mode": null, - "output_md5": "b3291b9c9adcab32d68819222e5d76f8", + "output_md5": "cdd0aab259d06df74b4ae3ddbbd7615e", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "9fbb26adcb1cd2335401493f40f3f51313a97f96", - "output_sha256": "6d3a2810ff5968aeae413f9189e0596f11fe57a1d5b6ac6afa871cce11c34bc0", - "output_sha512": "ff457cde4340e5d069c610c71d562fbf18474d275e7ab8753807c8d1cd080174b2229d4d176834b8569b590f007b360bff0011168eaec862b0d2a6bc69c47a8b", - "output_size": 6320, + "output_sha": "15e0432811ac45b77440e902f20b433130b97e70", + "output_sha256": "f37531a4a276be2b5853d2383331e343604a81c2b63926c24f83c318e3960d9f", + "output_sha512": "96869544a4e3a9e0de45760b4f29f180921ca1e14cb9a1f9f98c751993d9ca6cdf2b819221aad302fbac49636cec31c41fe6daaa2e8927c0f94a3accbd2993e1", + "output_size": 6341, "source": [], "source_content": null, "source_content_filename": null, @@ -717,7 +717,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "b3291b9c9adcab32d68819222e5d76f8", + "etag": "cdd0aab259d06df74b4ae3ddbbd7615e", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +730,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "bTooEP9ZaK6uQT+RieBZbxH+V6HVtqxq+occzhHDS8A=", + "source_hash": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md new file mode 100644 index 0000000..8a75dcb --- /dev/null +++ b/ctorm/granule-md-db-loader/README.md @@ -0,0 +1,20 @@ +# Granule MD DB Loader + +## Deploying + +The infrastructure for this is provided by the [CTORM terraform](../infra/README.md) + +## Running + +```bash +# for dev: +export AWS_PROFILE="cumulus-sbx-7522" +export CODEBUILD_PROJECT="ctorm-dev-granule-md-db-loader" + +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--query 'build.id' \ +--output text + +``` diff --git a/ctorm/granule-md-db-loader/buildspec.yaml b/ctorm/granule-md-db-loader/buildspec.yaml new file mode 100644 index 0000000..d40dd5b --- /dev/null +++ b/ctorm/granule-md-db-loader/buildspec.yaml @@ -0,0 +1,16 @@ +version: 0.2 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - echo "Installing dependencies..." + - pip install boto3 + build: + commands: + - echo "Starting DynamoDB Import script..." + - python granule_md_db_loader.py + post_build: + commands: + - echo "Import finished." diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py new file mode 100644 index 0000000..b7ebc5b --- /dev/null +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -0,0 +1,99 @@ +import os +import sys +import gzip +import json +import logging +import datetime +from decimal import Decimal +import boto3 + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger(__name__) + + +def main(): + bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") + table_name = os.environ.get("TABLE_NAME") + prefix = os.environ.get("PREFIX", "cumulus-granules/") + + if not table_name: + logger.error("TABLE_NAME environment variable is required.") + sys.exit(1) + + s3 = boto3.client("s3") + dynamodb = boto3.resource("dynamodb") + table = dynamodb.Table(table_name) + + logger.info(f"Starting import from s3://{bucket_name}/{prefix} into DynamoDB table {table_name}") + + paginator = s3.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + total_files = 0 + total_records = 0 + + # Walk through the bucket objects + for page in pages: + if "Contents" not in page: + continue + + for obj in page["Contents"]: + key = obj["Key"] + if not key.endswith(".jsonl.gz"): + continue + + logger.info(f"Processing s3://{bucket_name}/{key}") + total_files += 1 + + # Download and decompress the file + local_path = "/tmp/temp.jsonl.gz" + try: + s3.download_file(bucket_name, key, local_path) + except Exception as e: + logger.error(f"Failed to download s3://{bucket_name}/{key}: {e}") + continue + + # Open, decompress and parse + records_in_file = 0 + try: + with gzip.open(local_path, "rt", encoding="utf-8") as f: + with table.batch_writer() as batch: + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + logger.error(f"Failed to parse JSON line: {e}") + continue + + # Add/modify fields if needed + # Example additional fields: + item["imported_at"] = datetime.datetime.utcnow().isoformat() + "Z" + + # Ensure partition and sort keys are present + if "pk" not in item or "sk" not in item: + logger.warning(f"Skipping record missing pk/sk: {item.get('granule_id')}") + continue + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 + + logger.info(f"Successfully imported {records_in_file} records from {key}") + except Exception as e: + logger.error(f"Error processing file {key}: {e}") + finally: + if os.path.exists(local_path): + os.remove(local_path) + + logger.info(f"Import complete. Processed {total_files} files, imported {total_records} total records.") + + +if __name__ == "__main__": + main() diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 2ba024e..ab1a595 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -33,6 +33,23 @@ resource "aws_dynamodb_table" "granules" { name = "sk" type = "S" } + + attribute { + name = "gsi1pk" + type = "S" + } + + attribute { + name = "gsi1sk" + type = "S" + } + + global_secondary_index { + name = "gsi1" + hash_key = "gsi1pk" + range_key = "gsi1sk" + projection_type = "ALL" + } } @@ -101,7 +118,8 @@ data "aws_iam_policy_document" "cnm-sender" { ] resources = [ - aws_dynamodb_table.granules.arn + aws_dynamodb_table.granules.arn, + "${aws_dynamodb_table.granules.arn}/index/*" ] } } @@ -315,3 +333,113 @@ resource "aws_iam_role_policy" "cumulus_db_md_extract_upload" { role = aws_iam_role.cumulus_db_md_extract_upload.id policy = data.aws_iam_policy_document.cumulus_db_md_extract_upload.json } + + +# Granule DynamoDB loader. +data "archive_file" "granule_md_db_loader" { + type = "zip" + source_dir = "${path.module}/../../granule-md-db-loader" + output_path = "${path.module}/.terraform/granule-md-db-loader.zip" +} + +resource "aws_s3_object" "granule_md_db_loader_source" { + bucket = aws_s3_bucket.scratch.bucket + key = "codebuild/granule-md-db-loader.zip" + source = data.archive_file.granule_md_db_loader.output_path + source_hash = data.archive_file.granule_md_db_loader.output_base64sha256 +} + +resource "aws_iam_role" "granule_md_db_loader_role" { + name = "${var.name_prefix}-codebuild-granule_md_db_loader-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "codebuild.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy" "codebuild_granule_md_db_loader_role_policy" { + name = "${var.name_prefix}-codebuild-granule-md-db-loader-role-policy" + role = aws_iam_role.granule_md_db_loader_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:*:*:*" + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:ListBucket" + ] + Resource = [ + aws_s3_bucket.scratch.arn, + "${aws_s3_bucket.scratch.arn}/*" + ] + }, + { + Effect = "Allow" + Action = [ + "dynamodb:PutItem", + "dynamodb:BatchWriteItem" + ] + Resource = [ + aws_dynamodb_table.granules.arn + ] + } + ] + }) +} + +resource "aws_codebuild_project" "granule_md_db_loader" { + name = "${var.name_prefix}-granule-md-db-loader" + description = "Imports granules jsonl.gz files from S3 to DynamoDB" + service_role = aws_iam_role.granule_md_db_loader_role.arn + build_timeout = 480 + + artifacts { + type = "NO_ARTIFACTS" + } + + source { + type = "S3" + location = "${aws_s3_bucket.scratch.bucket}/${aws_s3_object.granule_md_db_loader_source.key}" + } + + environment { + type = "LINUX_CONTAINER" + image = "aws/codebuild/amazonlinux2-x86_64-standard:5.0" + compute_type = "BUILD_GENERAL1_SMALL" + + environment_variable { + name = "CTORM_BUCKET" + value = aws_s3_bucket.scratch.bucket + } + + environment_variable { + name = "TABLE_NAME" + value = aws_dynamodb_table.granules.name + } + + environment_variable { + name = "PREFIX" + value = "cumulus-granules/" + } + } +} diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index c50a504..99bc4f2 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 45, + "serial": 55, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -34,6 +34,38 @@ } }, "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", + "output_base64sha256": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", + "output_base64sha512": "p79O0G8V4Rv5GWXKpy6XA/SPVqK8Ubrs3aXSg4WbWcOxqk9n0cIV3xDEI9QufnQ6LABj00SEA9y4SIGQ5h+NsA==", + "output_file_mode": null, + "output_md5": "efc9aa787b5b0e96b00fb4bab67e84d2", + "output_path": "./.terraform/granule-md-db-loader.zip", + "output_sha": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", + "output_sha256": "4ddb1329757729a39d4ed38faa7221a0aae698515e00a1c271733c9f7989ac4c", + "output_sha512": "a7bf4ed06f15e11bf91965caa72e9703f48f56a2bc51baecdda5d283859b59c3b1aa4f67d1c215df10c423d42e7e743a2c0063d3448403dcb8488190e61f8db0", + "output_size": 2135, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./../../granule-md-db-loader", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "archive_file", @@ -124,9 +156,9 @@ { "schema_version": 0, "attributes": { - "id": "3548715077", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", + "id": "2517841586", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": [\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -196,7 +228,8 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules" + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*" ], "sid": "" } @@ -488,6 +521,133 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "Imports granules jsonl.gz files from S3 to DynamoDB", + "encryption_key": "arn:aws:kms:us-west-2:871271927522:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [ + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "TABLE_NAME", + "type": "PLAINTEXT", + "value": "ctorm-dev-granules" + }, + { + "name": "PREFIX", + "type": "PLAINTEXT", + "value": "cumulus-granules/" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-dev-granule-md-db-loader", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch", + "aws_s3_object.granule_md_db_loader_source", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_dynamodb_table", @@ -499,6 +659,14 @@ "attributes": { "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", "attribute": [ + { + "name": "gsi1pk", + "type": "S" + }, + { + "name": "gsi1sk", + "type": "S" + }, { "name": "pk", "type": "S" @@ -510,7 +678,18 @@ ], "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, - "global_secondary_index": [], + "global_secondary_index": [ + { + "hash_key": "gsi1pk", + "name": "gsi1", + "non_key_attributes": [], + "on_demand_throughput": [], + "projection_type": "ALL", + "range_key": "gsi1sk", + "read_capacity": 0, + "write_capacity": 0 + } + ], "hash_key": "pk", "id": "ctorm-dev-granules", "import_table": [], @@ -536,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -570,7 +749,7 @@ "inline_policy": [ { "name": "ctorm-dev-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" }, { "name": "ctorm-dev-scratch-access", @@ -634,6 +813,42 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "granule_md_db_loader_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-26T21:54:18Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-codebuild-granule_md_db_loader-role", + "inline_policy": [ + { + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-codebuild-granule_md_db_loader-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROA4VW62R3RISLTCQYHZ" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -646,7 +861,7 @@ "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", "name": "ctorm-dev-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", "role": "ctorm-dev-cnm-sender-role" }, "sensitive_attributes": [], @@ -663,6 +878,31 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_granule_md_db_loader_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-codebuild-granule_md_db_loader-role:ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", + "role": "ctorm-dev-codebuild-granule_md_db_loader-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -928,6 +1168,60 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "granule_md_db_loader_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "bucket": "ctorm-dev-scratch", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "efc9aa787b5b0e96b00fb4bab67e84d2", + "force_destroy": false, + "id": "codebuild/granule-md-db-loader.zip", + "key": "codebuild/granule-md-db-loader.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "server_side_encryption": "AES256", + "source": "./.terraform/granule-md-db-loader.zip", + "source_hash": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.scratch", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 270feb7..378680f 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 40, + "serial": 53, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -34,6 +34,38 @@ } }, "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "95710df96fcf68aad0084e50ecd58c0f928f7862", + "output_base64sha256": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", + "output_base64sha512": "6bcsla7ow6YaZjJnx8kt7WausRTy/XS/3H9A2fuy56ueg+uXlNVN8jwaY45wS2CEbsZhPPop64S8Vw7JJ3Uk3A==", + "output_file_mode": null, + "output_md5": "19d483f8f2202fea76c210c12ce8ccb2", + "output_path": "./.terraform/granule-md-db-loader.zip", + "output_sha": "95710df96fcf68aad0084e50ecd58c0f928f7862", + "output_sha256": "d1f89ebdac2a779f3fcc3d32a31f518418d9868a77e7ed2c05e7ee864318bc78", + "output_sha512": "e9b72c95aee8c3a61a663267c7c92ded66aeb114f2fd74bfdc7f40d9fbb2e7ab9e83eb9794d54df23c1a638e704b60846ec6613cfa29eb84bc570ec9277524dc", + "output_size": 1719, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./../../granule-md-db-loader", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "archive_file", @@ -115,6 +147,99 @@ } ] }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "cnm-sender", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2517841586", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": [\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\n \"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*" + ], + "sid": "" + }, + { + "actions": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules" + ], + "sid": "" + }, + { + "actions": [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" + ], + "sid": "" + }, + { + "actions": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", + "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*" + ], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "aws_iam_policy_document", @@ -396,6 +521,216 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "Imports granules jsonl.gz files from S3 to DynamoDB", + "encryption_key": "arn:aws:kms:us-west-2:871271927522:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [ + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "TABLE_NAME", + "type": "PLAINTEXT", + "value": "ctorm-dev-granules" + }, + { + "name": "PREFIX", + "type": "PLAINTEXT", + "value": "cumulus-granules/" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:871271927522:project/ctorm-dev-granule-md-db-loader", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-dev-granule-md-db-loader", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": null, + "tags_all": {}, + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch", + "aws_s3_object.granule_md_db_loader_source", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_dynamodb_table", + "name": "granules", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules", + "attribute": [ + { + "name": "gsi1pk", + "type": "S" + }, + { + "name": "gsi1sk", + "type": "S" + }, + { + "name": "pk", + "type": "S" + }, + { + "name": "sk", + "type": "S" + } + ], + "billing_mode": "PAY_PER_REQUEST", + "deletion_protection_enabled": false, + "global_secondary_index": [ + { + "hash_key": "gsi1pk", + "name": "gsi1", + "non_key_attributes": [], + "on_demand_throughput": [], + "projection_type": "ALL", + "range_key": "gsi1sk", + "read_capacity": 0, + "write_capacity": 0 + } + ], + "hash_key": "pk", + "id": "ctorm-dev-granules", + "import_table": [], + "local_secondary_index": [], + "name": "ctorm-dev-granules", + "on_demand_throughput": [], + "point_in_time_recovery": [ + { + "enabled": false, + "recovery_period_in_days": 0 + } + ], + "range_key": "sk", + "read_capacity": 0, + "replica": [], + "restore_date_time": null, + "restore_source_name": null, + "restore_source_table_arn": null, + "restore_to_latest_time": null, + "server_side_encryption": [], + "stream_arn": "", + "stream_enabled": false, + "stream_label": "", + "stream_view_type": "", + "table_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "timeouts": null, + "ttl": [ + { + "attribute_name": "", + "enabled": false + } + ], + "write_capacity": 0 + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, { "mode": "managed", "type": "aws_iam_role", @@ -414,7 +749,7 @@ "inline_policy": [ { "name": "ctorm-dev-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}" }, { "name": "ctorm-dev-scratch-access", @@ -478,6 +813,37 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "granule_md_db_loader_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::871271927522:role/ctorm-dev-codebuild-granule_md_db_loader-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-26T21:54:18Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-dev-codebuild-granule_md_db_loader-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-dev-codebuild-granule_md_db_loader-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROA4VW62R3RISLTCQYHZ" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -490,7 +856,7 @@ "id": "ctorm-dev-cnm-sender-role:ctorm-dev-cnm-sender-policy", "name": "ctorm-dev-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-state\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", "role": "ctorm-dev-cnm-sender-role" }, "sensitive_attributes": [], @@ -498,7 +864,6 @@ "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", "aws_dynamodb_table.granules", - "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_sqs_queue.granules", "aws_sqs_queue.granules_dlq", @@ -508,6 +873,31 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_granule_md_db_loader_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-dev-codebuild-granule_md_db_loader-role:ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}", + "role": "ctorm-dev-codebuild-granule_md_db_loader-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -583,7 +973,7 @@ "CUMULUS_INGEST_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest", "GRANULES_QUEUE_URL": "https://sqs.us-west-2.amazonaws.com/871271927522/ctorm-dev-granules", "LOG_LEVEL": "INFO", - "TABLE_NAME": "ctorm-dev-state" + "TABLE_NAME": "ctorm-dev-granules" } } ], @@ -601,7 +991,7 @@ "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:871271927522:function:ctorm-dev-cnm-sender/invocations", "kms_key_arn": "", - "last_modified": "2026-06-24T19:29:31.000+0000", + "last_modified": "2026-06-25T02:29:36.000+0000", "layers": [], "logging_config": [ { @@ -647,7 +1037,6 @@ "dependencies": [ "aws_cloudwatch_log_group.cnm-sender", "aws_dynamodb_table.granules", - "aws_dynamodb_table.state", "aws_iam_role.cnm-sender", "aws_iam_role_policy.cnm-sender", "aws_sqs_queue.granules", @@ -774,6 +1163,60 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "granule_md_db_loader_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-dev-scratch/codebuild/granule-md-db-loader.zip", + "bucket": "ctorm-dev-scratch", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "19d483f8f2202fea76c210c12ce8ccb2", + "force_destroy": false, + "id": "codebuild/granule-md-db-loader.zip", + "key": "codebuild/granule-md-db-loader.zip", + "kms_key_id": null, + "metadata": null, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "server_side_encryption": "AES256", + "source": "./.terraform/granule-md-db-loader.zip", + "source_hash": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", + "storage_class": "STANDARD", + "tags": null, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.scratch", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate index 278e8a2..8c6c0eb 100644 --- a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 18, + "serial": 25, "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -406,7 +406,7 @@ "role_arn": "", "schedule_expression": "rate(1 minute)", "state": "DISABLED", - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -480,7 +480,7 @@ "name_prefix": "", "retention_in_days": 14, "skip_destroy": false, - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], @@ -536,7 +536,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -567,14 +567,23 @@ "description": "", "force_detach_policies": false, "id": "ctorm-cnm-sender-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-cnm-sender-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}" + }, + { + "name": "ctorm-scratch-access", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:ListBucket\",\"s3:GetObject\",\"s3:DeleteObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch/*\",\"arn:aws:s3:::ctorm-scratch\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-cnm-sender-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROARNJJOPGE3PVM46W7W" }, @@ -601,14 +610,19 @@ "description": "", "force_detach_policies": false, "id": "ctorm-cumulus-db-md-extract-upload", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-upload", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-cumulus-db-md-extract-upload", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROARNJJOPGE4XUWPCFMM" }, @@ -743,7 +757,7 @@ "invoke_arn": "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:097260566921:function:ctorm-cnm-sender/invocations", "kms_key_arn": "", "last_modified": "2026-06-25T02:41:40.841+0000", - "layers": null, + "layers": [], "logging_config": [ { "application_log_level": "", @@ -771,7 +785,7 @@ "snap_start": [], "source_code_hash": "3LlHRfCJOFrEdSXuSJS7yAtDPL5jCGSISf9F24E6zFY=", "source_code_size": 344, - "tags": null, + "tags": {}, "tags_all": {}, "timeout": 120, "timeouts": null, @@ -941,7 +955,7 @@ "redrive_allow_policy": "", "redrive_policy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granule-dlq\",\"maxReceiveCount\":5}", "sqs_managed_sse_enabled": true, - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granules", @@ -982,7 +996,7 @@ "redrive_allow_policy": "", "redrive_policy": "", "sqs_managed_sse_enabled": true, - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "url": "https://sqs.us-west-2.amazonaws.com/097260566921/ctorm-granule-dlq", From 93bf745fdee082364ba79285c225916962dac37f Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Fri, 26 Jun 2026 17:04:57 -0800 Subject: [PATCH 26/31] PR-7510 Added ability to use provisioned dynamodb for loading granules --- .../export-granules-daily.sh | 3 +- ctorm/granule-md-db-loader/README.md | 72 +++++++++++++++++++ .../granule_md_db_loader.py | 59 +++++++++++++-- ctorm/infra/README.md | 12 ++++ ctorm/infra/terraform/main.tf | 15 ++-- ctorm/infra/terraform/terraform.tfstate | 34 ++++----- .../infra/terraform/terraform.tfstate.backup | 41 ++++++----- ctorm/infra/terraform/variables.tf | 12 ++++ 8 files changed, 202 insertions(+), 46 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index b95b13f..c497664 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -11,9 +11,10 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do continue fi + YYYY="${YYYYMMDD:0:4)" YYYYMM="${YYYYMMDD:0:6}" SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') - S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYY}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" TMP_OUTPUT="$(mktemp "${SAFE_COLLECTION}.${YYYYMMDD}.XXXXXX.jsonl.gz")" echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md index 8a75dcb..5289584 100644 --- a/ctorm/granule-md-db-loader/README.md +++ b/ctorm/granule-md-db-loader/README.md @@ -4,17 +4,89 @@ The infrastructure for this is provided by the [CTORM terraform](../infra/README.md) +## Before running + +AWS throttles writes to dynamo DB if we insert repeatedly into the same partion, so when we can run +multiple collections at once, each going into their own partition. + +Additionally, running as PAY_PER_REQUEST will get us throttled no matter what. So for the loading of the granules, we +will switch the tf to use PROVISIONED. See the [CTORM infrastruture README](../infra/README.md) for the command to do +this. + ## Running ```bash # for dev: export AWS_PROFILE="cumulus-sbx-7522" export CODEBUILD_PROJECT="ctorm-dev-granule-md-db-loader" +export CTORM_BUCKET="ctorm-dev-scratch" + + +# get list of collections +aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/nisar/ |grep PRE +aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/opera/ |grep PRE + +# If you want do them all sequentially, not recommended and not going to work with all the data: +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--query 'build.id' \ +--output text + + +# Running a specific collection in parallel (by overriding the PREFIX): +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L2_CSLC-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L2_RTC-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DISP-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DIST-ALERT-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L0B_CRSD_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text aws codebuild start-build --profile="${AWS_PROFILE}" \ --project-name "${CODEBUILD_PROJECT}" \ --region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L0B_RRSD_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L2/,type=PLAINTEXT \ --query 'build.id' \ --output text ``` + +## After running + +Restore the main.tf to its original state with billing_mode set to PAY_PER_REQUEST, etc. diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index b7ebc5b..6ef48c6 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -1,17 +1,49 @@ -import os -import sys +import datetime import gzip import json import logging -import datetime +import os +import sys +import time from decimal import Decimal + import boto3 +from botocore.config import Config # Configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) +class PartitionRateLimiter: + """ + Limits the number of write requests to any single partition key (pk) + to prevent exceeding DynamoDB's physical limit of 1000 operations/sec per partition. + """ + + def __init__(self, max_rate_per_sec: int = 890): + self.max_rate = max_rate_per_sec + self.history = {} # pk -> list of timestamps + + def limit(self, pk): + now = time.time() + if pk not in self.history: + self.history[pk] = [] + + # Retain only timestamps from the last 1.0 second + self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] + + if len(self.history[pk]) >= self.max_rate: + # Calculate sleep duration to let the oldest request roll off the 1-second window + sleep_time = 1.0 - (now - self.history[pk][0]) + if sleep_time > 0: + time.sleep(sleep_time) + now = time.time() + self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] + + self.history[pk].append(now) + + def main(): bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") table_name = os.environ.get("TABLE_NAME") @@ -22,7 +54,15 @@ def main(): sys.exit(1) s3 = boto3.client("s3") - dynamodb = boto3.resource("dynamodb") + + # Configure boto3 with more aggressive retries to gracefully handle scale peaks + retry_config = Config( + retries={ + "max_attempts": 10, + "mode": "standard" + } + ) + dynamodb = boto3.resource("dynamodb", config=retry_config) table = dynamodb.Table(table_name) logger.info(f"Starting import from s3://{bucket_name}/{prefix} into DynamoDB table {table_name}") @@ -33,6 +73,9 @@ def main(): total_files = 0 total_records = 0 + # Initialize partition-level rate limiter set to a safe threshold (850 writes/sec per pk) + rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) + # Walk through the bucket objects for page in pages: if "Contents" not in page: @@ -72,14 +115,18 @@ def main(): continue # Add/modify fields if needed - # Example additional fields: item["imported_at"] = datetime.datetime.utcnow().isoformat() + "Z" # Ensure partition and sort keys are present - if "pk" not in item or "sk" not in item: + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: logger.warning(f"Skipping record missing pk/sk: {item.get('granule_id')}") continue + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + # Batch insert into DynamoDB batch.put_item(Item=item) records_in_file += 1 diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 06d8a93..5ae884c 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -19,6 +19,18 @@ terraform plan \ terraform apply \ -var-file="${VARFILE}" +# For temporary use while loading granule metadata: +terraform plan \ + -var-file="${VARFILE}" \ + -var="granule_table_billing_mode=PROVISIONED" \ + -var="granule_table_write_capacity=10000" + +terraform apply \ + -var-file="${VARFILE}" \ + -var="granule_table_billing_mode=PROVISIONED" \ + -var="granule_table_write_capacity=10000" + + terraform destroy diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index ab1a595..53ccb0d 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -18,11 +18,14 @@ resource "aws_sqs_queue" "granules" { } resource "aws_dynamodb_table" "granules" { - name = "${var.name_prefix}-granules" - billing_mode = "PAY_PER_REQUEST" + name = "${var.name_prefix}-granules" + # billing_mode = "PROVISIONED" # Temporarily switch to PROVISIONED for bulk load + billing_mode = var.granule_table_billing_mode - hash_key = "pk" - range_key = "sk" + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + hash_key = "pk" + range_key = "sk" attribute { name = "pk" @@ -49,10 +52,14 @@ resource "aws_dynamodb_table" "granules" { hash_key = "gsi1pk" range_key = "gsi1sk" projection_type = "ALL" + + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null } } + resource "aws_cloudwatch_log_group" "cnm-sender" { name = "/aws/lambda/${local.lambda_name}" retention_in_days = 14 diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 99bc4f2..f2714d5 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 55, + "serial": 63, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", - "output_base64sha256": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", - "output_base64sha512": "p79O0G8V4Rv5GWXKpy6XA/SPVqK8Ubrs3aXSg4WbWcOxqk9n0cIV3xDEI9QufnQ6LABj00SEA9y4SIGQ5h+NsA==", + "id": "15541977bfb6559cdcb3dce4cb03186ba760a70c", + "output_base64sha256": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", + "output_base64sha512": "5JDpSb4/VU/oaT7BC6qYuY+/ljGpjHWd5SY8B/8SFH98/DPmit3/MLGGHvjgZzhbsrOghg4nq8IQ168xlz/mpg==", "output_file_mode": null, - "output_md5": "efc9aa787b5b0e96b00fb4bab67e84d2", + "output_md5": "8198f06853d8821f035afa1477cc3d3c", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "a30642c511ec09bd5c5e8045ce0fabf946a7e1d4", - "output_sha256": "4ddb1329757729a39d4ed38faa7221a0aae698515e00a1c271733c9f7989ac4c", - "output_sha512": "a7bf4ed06f15e11bf91965caa72e9703f48f56a2bc51baecdda5d283859b59c3b1aa4f67d1c215df10c423d42e7e743a2c0063d3448403dcb8488190e61f8db0", - "output_size": 2135, + "output_sha": "15541977bfb6559cdcb3dce4cb03186ba760a70c", + "output_sha256": "ed4bcd8457abb06013f1d7949c3363de3d829ae4778ccbbf36f9d69e80807a28", + "output_sha512": "e490e949be3f554fe8693ec10baa98b98fbf9631a98c759de5263c07ff12147f7cfc33e68addff30b1861ef8e067385bb2b3a0860e27abc210d7af31973fe6a6", + "output_size": 3346, "source": [], "source_content": null, "source_content_filename": null, @@ -676,7 +676,7 @@ "type": "S" } ], - "billing_mode": "PAY_PER_REQUEST", + "billing_mode": "PROVISIONED", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -686,8 +686,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 0, - "write_capacity": 0 + "read_capacity": 5, + "write_capacity": 10000 } ], "hash_key": "pk", @@ -703,7 +703,7 @@ } ], "range_key": "sk", - "read_capacity": 0, + "read_capacity": 5, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -715,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -724,7 +724,7 @@ "enabled": false } ], - "write_capacity": 0 + "write_capacity": 10000 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -1194,7 +1194,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "efc9aa787b5b0e96b00fb4bab67e84d2", + "etag": "8198f06853d8821f035afa1477cc3d3c", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1206,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "TdsTKXV3KaOdTtOPqnIhoKrmmFFeAKHCcXM8n3mJrEw=", + "source_hash": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 378680f..0bd8316 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 53, + "serial": 59, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "95710df96fcf68aad0084e50ecd58c0f928f7862", - "output_base64sha256": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", - "output_base64sha512": "6bcsla7ow6YaZjJnx8kt7WausRTy/XS/3H9A2fuy56ueg+uXlNVN8jwaY45wS2CEbsZhPPop64S8Vw7JJ3Uk3A==", + "id": "5def5225e70ce457c014d699b2bd3e2b93a9788e", + "output_base64sha256": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", + "output_base64sha512": "yt3czhsZeaAaC4zMOD5W5zjajGbY/Z09AXYEwpFDW2hmbfqTepKrLT5xoDg5zTEU4CnE2iQyza54OPqIOOZb+w==", "output_file_mode": null, - "output_md5": "19d483f8f2202fea76c210c12ce8ccb2", + "output_md5": "b6108bd6d78e6ebfc4b059f9b0e2f033", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "95710df96fcf68aad0084e50ecd58c0f928f7862", - "output_sha256": "d1f89ebdac2a779f3fcc3d32a31f518418d9868a77e7ed2c05e7ee864318bc78", - "output_sha512": "e9b72c95aee8c3a61a663267c7c92ded66aeb114f2fd74bfdc7f40d9fbb2e7ab9e83eb9794d54df23c1a638e704b60846ec6613cfa29eb84bc570ec9277524dc", - "output_size": 1719, + "output_sha": "5def5225e70ce457c014d699b2bd3e2b93a9788e", + "output_sha256": "da5f4df5ac6f5c647bd90006b2d8ac57ea305bbe74620e7efd6aa95a5244aace", + "output_sha512": "cadddcce1b1979a01a0b8ccc383e56e738da8c66d8fd9d3d017604c291435b68666dfa937a92ab2d3e71a03839cd3114e029c4da2432cdae7838fa8838e65bfb", + "output_size": 3128, "source": [], "source_content": null, "source_content_filename": null, @@ -632,7 +632,7 @@ } ], "source_version": "", - "tags": null, + "tags": {}, "tags_all": {}, "vpc_config": [] }, @@ -715,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -749,7 +749,7 @@ "inline_policy": [ { "name": "ctorm-dev-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:871271927522:log-group:/aws/lambda/ctorm-dev-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:871271927522:ctorm-dev-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules/index/*\",\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" }, { "name": "ctorm-dev-scratch-access", @@ -828,14 +828,19 @@ "description": "", "force_detach_policies": false, "id": "ctorm-dev-codebuild-granule_md_db_loader-role", - "inline_policy": [], + "inline_policy": [ + { + "name": "ctorm-dev-codebuild-granule-md-db-loader-role-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-dev-scratch\",\"arn:aws:s3:::ctorm-dev-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:871271927522:table/ctorm-dev-granules\"]}]}" + } + ], "managed_policy_arns": [], "max_session_duration": 3600, "name": "ctorm-dev-codebuild-granule_md_db_loader-role", "name_prefix": "", "path": "/", "permissions_boundary": "", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROA4VW62R3RISLTCQYHZ" }, @@ -1189,21 +1194,21 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "19d483f8f2202fea76c210c12ce8ccb2", + "etag": "b6108bd6d78e6ebfc4b059f9b0e2f033", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", "kms_key_id": null, - "metadata": null, + "metadata": {}, "object_lock_legal_hold_status": "", "object_lock_mode": "", "object_lock_retain_until_date": "", "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "0fievawqd58/zD0yox9RhBjZhop35+0sBefuhkMYvHg=", + "source_hash": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", "storage_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "version_id": "", "website_redirect": "" diff --git a/ctorm/infra/terraform/variables.tf b/ctorm/infra/terraform/variables.tf index 030971c..ef9903b 100644 --- a/ctorm/infra/terraform/variables.tf +++ b/ctorm/infra/terraform/variables.tf @@ -32,6 +32,18 @@ variable "prepare_source_bucket_names" { ] } +variable "granule_table_billing_mode" { + type = string + default = "PAY_PER_REQUEST" + description = "Must be either PROVISIONED or PAY_PER_REQUEST" +} + +variable "granule_table_write_capacity" { + type = number + default = 10000 + description = "When billing_mode is PROVISIONED, forces write_capacity/10 physical partition splits immediately. Set to 1000*number of parallel granule_md_db_loader runs." +} + # variable "cumulus_db_md_extract_remote_codebuild_role_arn" { # type = string # description = "CodeBuild service role ARN from the remote/RDS account." From 03dcbf6884930a8cd3b1a7d8717088e6f0c8c53a Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Sun, 28 Jun 2026 17:33:29 -0800 Subject: [PATCH 27/31] Fixing bugs relating to specifying a slices tsv file --- ctorm/cumulus-db-md-extract/README.md | 18 + .../export-granules-daily.sh | 2 +- .../generate-slices-daily.sh | 182 +++++----- .../generate-slices-daily.sql | 21 ++ .../refresh-ctorm-upload-role.sh | 2 +- .../terraform/codebuild-db-md-extract.tf | 10 + .../terraform/terraform-opera-prod.tfstate | 32 +- .../terraform/terraform-opera-uat.tfstate | 44 ++- ctorm/infra/terraform/main.tf | 3 +- ctorm/infra/terraform/terraform.tfstate | 43 +-- .../infra/terraform/terraform.tfstate.backup | 24 +- .../ctorm-prod/terraform.tfstate | 321 +++++++++++++++++- 12 files changed, 532 insertions(+), 170 deletions(-) create mode 100644 ctorm/cumulus-db-md-extract/generate-slices-daily.sql diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index b8b3293..1fe8459 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -108,17 +108,35 @@ export AWS_PROFILE="cumulus-uat-6921" export AWS_REGION="us-west-2" export CODEBUILD_PROJECT="$(terraform output -state="${STATEFILE}" -raw cumulus_db_md_extract_codebuild_project_name)" +# You can do one collection at a time, many in parallel if necessary: +export COLLECTION="OPERA_L2_RTC-S1_V1" + # start build: BUILD_ID="$( aws codebuild start-build \ --project-name "${CODEBUILD_PROJECT}" \ --region "${AWS_REGION}" \ + --environment-variables-override name=COLLECTION,value="${COLLECTION}",type=PLAINTEXT \ --query 'build.id' \ --output text )" echo "${BUILD_ID}" +# Or start build specifying a custom slices.tsv file from S3: +# BUILD_ID="$( + aws codebuild start-build \ + --project-name "${CODEBUILD_PROJECT}" \ + --region "${AWS_REGION}" \ + --environment-variables-override '[ + {"name": "SLICES_S3_URI", "value": "s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_RTC-S1_V1-20170910-20181203.tsv", "type": "PLAINTEXT"}, + {"name": "COLLECTION", "value": "OPERA_L2_RTC-S1_V1", "type": "PLAINTEXT"} + ]' \ + --query 'build.id' \ + --output text +# )" + + # get status: aws codebuild batch-get-builds \ --ids "${BUILD_ID}" \ diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index c497664..c28d176 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -11,7 +11,7 @@ while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do continue fi - YYYY="${YYYYMMDD:0:4)" + YYYY="${YYYYMMDD:0:4}" YYYYMM="${YYYYMMDD:0:6}" SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYY}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh index b506300..3599117 100644 --- a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -1,99 +1,95 @@ set -euo pipefail -psql \ - --host="$PGHOST" \ - --port="${PGPORT:-5432}" \ - --username="$PGUSER" \ - --dbname="$PGDATABASE" \ - --quiet \ - --no-psqlrc \ - --tuples-only \ - --no-align \ - --field-separator $'\t' \ - --set=ON_ERROR_STOP=1 \ - --command " - SELECT - c.name, - to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') AS yyyymmdd, - count(DISTINCT g.cumulus_id) AS granule_count, - count(f.cumulus_id) AS file_count - FROM public.granules g - JOIN public.collections c - ON c.cumulus_id = g.collection_cumulus_id - JOIN public.files f - ON f.granule_cumulus_id = g.cumulus_id - AND f.checksum_value IS NOT NULL - WHERE g.status = 'completed' - AND c.name in ( - 'ANTPAT', - 'BFPQ', - 'CORNER_REFL', - 'DCOP', - 'DC_RADAR', - 'DSG_STATIC', - 'EA_L0B_L_CRSD', - 'EA_L0B_L_RRSD', - 'EA_L1_L_RIFG', - 'EA_L1_L_ROFF', - 'EA_L1_L_RSLC', - 'EA_L1_L_RUNW', - 'EA_L2_L_GCOV', - 'EA_L2_L_GOFF', - 'EA_L2_L_GSLC', - 'EA_L2_L_GUNW', - 'EA_L3_L_SME2', - 'FOE', - 'FRP', - 'FT_PARAM', - 'FT_WAVEFORM', - 'LRCLK_UTC', - 'L_CHAN_DATA', - 'MOE', - 'NISAR_DEM_TIF', - 'NISAR_DEM_VRT', - 'NISAR_L0A_RRST_BETA_V1', - 'NISAR_L0B_CRSD_BETA_V1', - 'NISAR_L0B_RRSD_BETA_V1', - 'NISAR_L1_RIFG_BETA_V1', - 'NISAR_L1_ROFF_BETA_V1', - 'NISAR_L1_RSLC_BETA_V1', - 'NISAR_L1_RUNW_BETA_V1', - 'NISAR_L2_GCOV_BETA_V1', - 'NISAR_L2_GOFF_BETA_V1', - 'NISAR_L2_GSLC_BETA_V1', - 'NISAR_L2_GUNW_BETA_V1', - 'NISAR_L3_SME2_BETA_V1', - 'NISAR_VWC', - 'NISAR_WATERMASK_TIF', - 'NISAR_WATERMASK_VRT', - 'NOE', - 'NRP', - 'OROST', - 'PA_L0B_L_RRSD', - 'POE', - 'PRP', - 'STUF', - 'TEC', - 'TFDB', - 'UR_L0B_L_RRSD', - 'OPERA_L2_CSLC-S1_V1', - 'OPERA_L2_RTC-S1_V1', - 'OPERA_L3_DISP-S1_V1', - 'OPERA_L3_DIST-ALERT-S1_V1', - 'OPERA_L4_TROPO-ZENITH_V1' - ) - GROUP BY - c.name, - to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD') - ORDER BY - c.name, - to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMMDD'); - " > slices_daily.tsv +#ANTPAT', +#BFPQ', +#CORNER_REFL', +#DCOP', +#DC_RADAR', +#DSG_STATIC', +#EA_L0B_L_CRSD', +#EA_L0B_L_RRSD', +#EA_L1_L_RIFG', +#EA_L1_L_ROFF', +#EA_L1_L_RSLC', +#EA_L1_L_RUNW',-- +#EA_L2_L_GCOV', +#EA_L2_L_GOFF', +#EA_L2_L_GSLC', +#EA_L2_L_GUNW', +#EA_L3_L_SME2', +#FOE', +#FRP', +#FT_PARAM', +#FT_WAVEFORM', +#LRCLK_UTC', +#L_CHAN_DATA', +#MOE', +#NISAR_DEM_TIF', +#NISAR_DEM_VRT', +#NISAR_L0A_RRST_BETA_V1', +#NISAR_L0B_CRSD_BETA_V1', +#NISAR_L0B_RRSD_BETA_V1', +#NISAR_L1_RIFG_BETA_V1', +#NISAR_L1_ROFF_BETA_V1', +#NISAR_L1_RSLC_BETA_V1', +#NISAR_L1_RUNW_BETA_V1', +#NISAR_L2_GCOV_BETA_V1', +#NISAR_L2_GOFF_BETA_V1', +#NISAR_L2_GSLC_BETA_V1', +#NISAR_L2_GUNW_BETA_V1', +#NISAR_L3_SME2_BETA_V1', +#NISAR_VWC', +#NISAR_WATERMASK_TIF', +#NISAR_WATERMASK_VRT', +#NOE', +#NRP', +#OROST', +#PA_L0B_L_RRSD', +#POE', +#PRP', +#STUF', +#TEC', +#TFDB', +#UR_L0B_L_RRSD', +# Opera collections: +#COLLECTION="OPERA_L2_CSLC-S1_V1" +#COLLECTION="OPERA_L2_RTC-S1_V1" +#COLLECTION="OPERA_L3_DISP-S1_V1" +#COLLECTION="OPERA_L3_DIST-ALERT-S1_V1" +#COLLECTION="OPERA_L4_TROPO-ZENITH_V1" -source ./refresh-ctorm-upload-role.sh +echo "SLICES_S3_URI: ${SLICES_S3_URI}" +echo "COLLECTION: ${COLLECTION}" -SLICES_S3_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily.tsv" -aws s3 cp slices_daily.tsv "${SLICES_S3_URI}" -echo "Uploaded ${SLICES_S3_URI}" +if [[ -n "${SLICES_S3_URI:-}" ]]; then + echo "SLICES_S3_URI is specified: ${SLICES_S3_URI}" + echo "Downloading specified slices file instead of querying database..." + source ./refresh-ctorm-upload-role.sh + aws s3 cp "${SLICES_S3_URI}" slices_daily.tsv +else + # Generate slices TSV file + echo "SLICES_S3_URI not set. Going to run a big query now..." + psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --tuples-only \ + --no-align \ + --field-separator $'\t' \ + --set=ON_ERROR_STOP=1 \ + --set=collection="$COLLECTION" \ + --file=generate-slices-daily.sql \ + > slices_daily.tsv + + source ./refresh-ctorm-upload-role.sh + + # Upload slices TSV file to S3 for archive purposes. + SLICES_S3_UPLOAD_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily-${COLLECTION}-${CODEBUILD_BUILD_NUMBER}.tsv" + aws s3 cp slices_daily.tsv "${SLICES_S3_UPLOAD_URI}" + echo "Uploaded ${SLICES_S3_UPLOAD_URI}" +fi diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sql b/ctorm/cumulus-db-md-extract/generate-slices-daily.sql new file mode 100644 index 0000000..ae81935 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sql @@ -0,0 +1,21 @@ +SELECT + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') AS YYYYMMDD, + COUNT(DISTINCT G.CUMULUS_ID) AS GRANULE_COUNT, + COUNT(F.CUMULUS_ID) AS FILE_COUNT +FROM + PUBLIC.GRANULES G + JOIN PUBLIC.COLLECTIONS C + ON C.CUMULUS_ID = G.COLLECTION_CUMULUS_ID + JOIN PUBLIC.FILES F + ON F.GRANULE_CUMULUS_ID = G.CUMULUS_ID + AND F.CHECKSUM_VALUE IS NOT NULL +WHERE + G.STATUS = 'completed' + AND C.NAME = :'collection' +GROUP BY + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') +ORDER BY + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD'); diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh index e28fc6a..ced1a23 100644 --- a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -17,4 +17,4 @@ export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKey export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" -aws sts get-caller-identity >&2 +# aws sts get-caller-identity >&2 diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf index c2acbcb..aeb8efe 100644 --- a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -306,6 +306,16 @@ resource "aws_codebuild_project" "cumulus_db_md_extract" { name = "CTORM_S3_ROLE_ARN" value = var.db_md_extract_ctorm_s3_role_arn } + + environment_variable { + name = "COLLECTION" + value = "" + } + + environment_variable { + name = "SLICES_S3_URI" + value = "" + } } vpc_config { diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate index be02ac5..84f099f 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 112, + "serial": 142, "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "15e0432811ac45b77440e902f20b433130b97e70", - "output_base64sha256": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", - "output_base64sha512": "loaVRKTjqeDeRXYLTynxgJIcoeFMuaH5+Yx1GZPZymzfK4GSIarTAvusSWNs7DHEH+baqi6JJ8D5SjrMvSmT4Q==", + "id": "a96216dceaf9e6e503bf02981b53784e03ef13b8", + "output_base64sha256": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=", + "output_base64sha512": "RLp6HJBUgfaHuTiqD2qyQKwBJyHtvZ+/VDK/urPEE58Gf5Md2MDIJeCVoQIux45wacZU5HiEe/eoFtx28IitLw==", "output_file_mode": null, - "output_md5": "cdd0aab259d06df74b4ae3ddbbd7615e", + "output_md5": "5a4c52cf01416b48350304df429135a5", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "15e0432811ac45b77440e902f20b433130b97e70", - "output_sha256": "f37531a4a276be2b5853d2383331e343604a81c2b63926c24f83c318e3960d9f", - "output_sha512": "96869544a4e3a9e0de45760b4f29f180921ca1e14cb9a1f9f98c751993d9ca6cdf2b819221aad302fbac49636cec31c41fe6daaa2e8927c0f94a3accbd2993e1", - "output_size": 6341, + "output_sha": "a96216dceaf9e6e503bf02981b53784e03ef13b8", + "output_sha256": "8882f14372977b33e1aa89c08d14ad49217280b96c1c984d91c0f02f7042b750", + "output_sha512": "44ba7a1c905481f687b938aa0f6ab240ac012721edbd9fbf5432bfbab3c4139f067f931dd8c0c825e095a1022ec78e7069c654e478847bf7a816dc76f088ad2f", + "output_size": 7000, "source": [], "source_content": null, "source_content_filename": null, @@ -371,6 +371,16 @@ "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_URI", + "type": "PLAINTEXT", + "value": "" } ], "fleet": [], @@ -717,7 +727,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "cdd0aab259d06df74b4ae3ddbbd7615e", + "etag": "5a4c52cf01416b48350304df429135a5", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -730,7 +740,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "83UxpKJ2vitYU9I4MzHjQ2BKgcK2OSbCT4PDGOOWDZ8=", + "source_hash": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate index 71af6a5..bdc7311 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 62, + "serial": 69, "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", - "output_base64sha256": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", - "output_base64sha512": "PZDUjfN60Uim+bdem7zB2ilfEUKqP4pE0UADkxHIkR4YTYGcP+Txtuj+Rm7zSpLQ2fZk+rgOjhJKIWLy0lyWYQ==", + "id": "2ea86e5ad0e886c9ead4d329ab2a5a1e5025bc8f", + "output_base64sha256": "PvweHBk8zXkKLm/Sf7cg9BQCZO4d5G7XeJiAdyjkEBs=", + "output_base64sha512": "PxlBXxsKqs9FBH1t/uUkUhmh7jm22Jluqzp7HD9CzqGMktLgIoGwqYQiZwHgo9zBcjFaTo1AIvEUmj2OTQUe5g==", "output_file_mode": null, - "output_md5": "63b93d1d0d04eb5a9595f406ba9653cc", + "output_md5": "72ff56f7afe5d2511991649ddca2a839", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "b4b6d0d38eae27847ab3ef6bd28ce6fa9eac61c1", - "output_sha256": "b7ddad71c2668d0d4a2d31668752520c9e0b4e9dee0d8188eb3d3f9a89a79f4c", - "output_sha512": "3d90d48df37ad148a6f9b75e9bbcc1da295f1142aa3f8a44d140039311c8911e184d819c3fe4f1b6e8fe466ef34a92d0d9f664fab80e8e124a2162f2d25c9661", - "output_size": 5347, + "output_sha": "2ea86e5ad0e886c9ead4d329ab2a5a1e5025bc8f", + "output_sha256": "3efc1e1c193ccd790a2e6fd27fb720f4140264ee1de46ed77898807728e4101b", + "output_sha512": "3f19415f1b0aaacf45047d6dfee5245219a1ee39b6d8996eab3a7b1c3f42cea18c92d2e02281b0a984226701e0a3dcc172315a4e8d4022f1149a3d8e4d051ee6", + "output_size": 6923, "source": [], "source_content": null, "source_content_filename": null, @@ -82,9 +82,9 @@ { "schema_version": 0, "attributes": { - "id": "1034869219", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "id": "2729057441", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -103,7 +103,7 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract" + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" ], "sid": "WriteCodeBuildLogs" }, @@ -371,6 +371,16 @@ "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_PATH", + "type": "PLAINTEXT", + "value": "" } ], "fleet": [], @@ -476,7 +486,7 @@ "inline_policy": [ { "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" } ], "managed_policy_arns": [], @@ -509,7 +519,7 @@ "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", "role": "ctorm-opera-uat-cumulus-db-md-extract-role" }, "sensitive_attributes": [], @@ -718,7 +728,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "63b93d1d0d04eb5a9595f406ba9653cc", + "etag": "72ff56f7afe5d2511991649ddca2a839", "force_destroy": false, "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -731,7 +741,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "t92tccJmjQ1KLTFmh1JSDJ4LTp3uDYGI6z0/momnn0w=", + "source_hash": "PvweHBk8zXkKLm/Sf7cg9BQCZO4d5G7XeJiAdyjkEBs=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 53ccb0d..4f0d2fb 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -303,8 +303,9 @@ data "aws_iam_policy_document" "cumulus_db_md_extract_upload" { effect = "Allow" actions = [ - "s3:PutObject", "s3:AbortMultipartUpload", + "s3:GetObject", + "s3:PutObject", ] resources = [ diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index f2714d5..8c00f44 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 63, + "serial": 69, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "15541977bfb6559cdcb3dce4cb03186ba760a70c", - "output_base64sha256": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", - "output_base64sha512": "5JDpSb4/VU/oaT7BC6qYuY+/ljGpjHWd5SY8B/8SFH98/DPmit3/MLGGHvjgZzhbsrOghg4nq8IQ168xlz/mpg==", + "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", "output_file_mode": null, - "output_md5": "8198f06853d8821f035afa1477cc3d3c", + "output_md5": "10648403b2277ad95dd6324dd58521c9", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "15541977bfb6559cdcb3dce4cb03186ba760a70c", - "output_sha256": "ed4bcd8457abb06013f1d7949c3363de3d829ae4778ccbbf36f9d69e80807a28", - "output_sha512": "e490e949be3f554fe8693ec10baa98b98fbf9631a98c759de5263c07ff12147f7cfc33e68addff30b1861ef8e067385bb2b3a0860e27abc210d7af31973fe6a6", - "output_size": 3346, + "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", + "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", + "output_size": 3296, "source": [], "source_content": null, "source_content_filename": null, @@ -249,9 +249,9 @@ { "schema_version": 0, "attributes": { - "id": "923578907", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "id": "3026717548", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -261,6 +261,7 @@ { "actions": [ "s3:AbortMultipartUpload", + "s3:GetObject", "s3:PutObject" ], "condition": [], @@ -676,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PROVISIONED", + "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -686,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 5, - "write_capacity": 10000 + "read_capacity": 0, + "write_capacity": 0 } ], "hash_key": "pk", @@ -703,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 5, + "read_capacity": 0, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -715,7 +716,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -724,7 +725,7 @@ "enabled": false } ], - "write_capacity": 10000 + "write_capacity": 0 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -915,7 +916,7 @@ "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", "name": "ctorm-dev-cumulus-db-md-extract-upload", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", "role": "ctorm-dev-cumulus-db-md-extract-upload" }, "sensitive_attributes": [], @@ -1194,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "8198f06853d8821f035afa1477cc3d3c", + "etag": "10648403b2277ad95dd6324dd58521c9", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "7UvNhFersGAT8deUnDNj3j2CmuR3jMu/NvnWnoCAeig=", + "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 0bd8316..23ab993 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 59, + "serial": 67, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "5def5225e70ce457c014d699b2bd3e2b93a9788e", - "output_base64sha256": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", - "output_base64sha512": "yt3czhsZeaAaC4zMOD5W5zjajGbY/Z09AXYEwpFDW2hmbfqTepKrLT5xoDg5zTEU4CnE2iQyza54OPqIOOZb+w==", + "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", "output_file_mode": null, - "output_md5": "b6108bd6d78e6ebfc4b059f9b0e2f033", + "output_md5": "10648403b2277ad95dd6324dd58521c9", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "5def5225e70ce457c014d699b2bd3e2b93a9788e", - "output_sha256": "da5f4df5ac6f5c647bd90006b2d8ac57ea305bbe74620e7efd6aa95a5244aace", - "output_sha512": "cadddcce1b1979a01a0b8ccc383e56e738da8c66d8fd9d3d017604c291435b68666dfa937a92ab2d3e71a03839cd3114e029c4da2432cdae7838fa8838e65bfb", - "output_size": 3128, + "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", + "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", + "output_size": 3296, "source": [], "source_content": null, "source_content_filename": null, @@ -715,7 +715,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -1194,7 +1194,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "b6108bd6d78e6ebfc4b059f9b0e2f033", + "etag": "10648403b2277ad95dd6324dd58521c9", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1206,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "2l9N9axvXGR72QAGstisV+owW750Yg5+/WqpWlJEqs4=", + "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate index 8c6c0eb..6fae66e 100644 --- a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 25, + "serial": 48, "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -34,6 +34,38 @@ } }, "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": null, + "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "output_file_mode": null, + "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_path": "./.terraform/granule-md-db-loader.zip", + "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", + "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", + "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", + "output_size": 3296, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./../../granule-md-db-loader", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, { "mode": "data", "type": "archive_file", @@ -124,9 +156,9 @@ { "schema_version": 0, "attributes": { - "id": "2687090556", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "id": "1964661201", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:ReceiveMessage\",\n \"sqs:GetQueueAttributes\",\n \"sqs:DeleteMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"sqs:SendMessageBatch\",\n \"sqs:SendMessage\"\n ],\n \"Resource\": \"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"dynamodb:UpdateItem\",\n \"dynamodb:Query\",\n \"dynamodb:PutItem\",\n \"dynamodb:GetItem\"\n ],\n \"Resource\": [\n \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\n \"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"\n ]\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Effect\":\"Allow\",\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Effect\":\"Allow\",\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -196,7 +228,8 @@ "not_resources": [], "principals": [], "resources": [ - "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules" + "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules", + "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*" ], "sid": "" } @@ -216,9 +249,9 @@ { "schema_version": 0, "attributes": { - "id": "2685467495", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "id": "1452847858", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -228,6 +261,7 @@ { "actions": [ "s3:AbortMultipartUpload", + "s3:GetObject", "s3:PutObject" ], "condition": [], @@ -488,6 +522,133 @@ } ] }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "granule_md_db_loader", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-granule-md-db-loader", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "Imports granules jsonl.gz files from S3 to DynamoDB", + "encryption_key": "arn:aws:kms:us-west-2:097260566921:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [ + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-scratch" + }, + { + "name": "TABLE_NAME", + "type": "PLAINTEXT", + "value": "ctorm-granules" + }, + { + "name": "PREFIX", + "type": "PLAINTEXT", + "value": "cumulus-granules/" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-granule-md-db-loader", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-granule-md-db-loader", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::097260566921:role/ctorm-codebuild-granule_md_db_loader-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-scratch/codebuild/granule-md-db-loader.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch", + "aws_s3_object.granule_md_db_loader_source", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_dynamodb_table", @@ -499,6 +660,14 @@ "attributes": { "arn": "arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules", "attribute": [ + { + "name": "gsi1pk", + "type": "S" + }, + { + "name": "gsi1sk", + "type": "S" + }, { "name": "pk", "type": "S" @@ -510,7 +679,18 @@ ], "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, - "global_secondary_index": [], + "global_secondary_index": [ + { + "hash_key": "gsi1pk", + "name": "gsi1", + "non_key_attributes": [], + "on_demand_throughput": [], + "projection_type": "ALL", + "range_key": "gsi1sk", + "read_capacity": 0, + "write_capacity": 0 + } + ], "hash_key": "pk", "id": "ctorm-granules", "import_table": [], @@ -570,7 +750,7 @@ "inline_policy": [ { "name": "ctorm-cnm-sender-policy", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}" }, { "name": "ctorm-scratch-access", @@ -613,7 +793,7 @@ "inline_policy": [ { "name": "ctorm-cumulus-db-md-extract-upload", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], @@ -634,6 +814,42 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "granule_md_db_loader_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-codebuild-granule_md_db_loader-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-28T22:55:49Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-codebuild-granule_md_db_loader-role", + "inline_policy": [ + { + "name": "ctorm-codebuild-granule-md-db-loader-role-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch\",\"arn:aws:s3:::ctorm-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-codebuild-granule_md_db_loader-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROARNJJOPGEWW276Y44U" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -646,7 +862,7 @@ "id": "ctorm-cnm-sender-role:ctorm-cnm-sender-policy", "name": "ctorm-cnm-sender-policy", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/lambda/ctorm-cnm-sender:*\"},{\"Action\":[\"sqs:ReceiveMessage\",\"sqs:GetQueueAttributes\",\"sqs:DeleteMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:097260566921:ctorm-granules\"},{\"Action\":[\"sqs:SendMessageBatch\",\"sqs:SendMessage\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:sqs:us-west-2:123456789012:cumulus-ingest\"},{\"Action\":[\"dynamodb:UpdateItem\",\"dynamodb:Query\",\"dynamodb:PutItem\",\"dynamodb:GetItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules/index/*\",\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}", "role": "ctorm-cnm-sender-role" }, "sensitive_attributes": [], @@ -663,6 +879,31 @@ } ] }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_granule_md_db_loader_role_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-codebuild-granule_md_db_loader-role:ctorm-codebuild-granule-md-db-loader-role-policy", + "name": "ctorm-codebuild-granule-md-db-loader-role-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:CreateLogGroup\",\"logs:CreateLogStream\",\"logs:PutLogEvents\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:*:*:*\"},{\"Action\":[\"s3:GetObject\",\"s3:ListBucket\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::ctorm-scratch\",\"arn:aws:s3:::ctorm-scratch/*\"]},{\"Action\":[\"dynamodb:PutItem\",\"dynamodb:BatchWriteItem\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:dynamodb:us-west-2:097260566921:table/ctorm-granules\"]}]}", + "role": "ctorm-codebuild-granule_md_db_loader-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_dynamodb_table.granules", + "aws_iam_role.granule_md_db_loader_role", + "aws_s3_bucket.scratch" + ] + } + ] + }, { "mode": "managed", "type": "aws_iam_role_policy", @@ -675,7 +916,7 @@ "id": "ctorm-cumulus-db-md-extract-upload:ctorm-cumulus-db-md-extract-upload", "name": "ctorm-cumulus-db-md-extract-upload", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", "role": "ctorm-cumulus-db-md-extract-upload" }, "sensitive_attributes": [], @@ -928,6 +1169,60 @@ } ] }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "granule_md_db_loader_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-scratch/codebuild/granule-md-db-loader.zip", + "bucket": "ctorm-scratch", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "10648403b2277ad95dd6324dd58521c9", + "force_destroy": false, + "id": "codebuild/granule-md-db-loader.zip", + "key": "codebuild/granule-md-db-loader.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "server_side_encryption": "AES256", + "source": "./.terraform/granule-md-db-loader.zip", + "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.scratch", + "data.archive_file.granule_md_db_loader" + ] + } + ] + }, { "mode": "managed", "type": "aws_sqs_queue", From ebf4eea9aeec87962acc6e70c7a0300066cbfde2 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Sun, 28 Jun 2026 17:50:09 -0800 Subject: [PATCH 28/31] linting, sigh --- .../granule_md_db_loader.py | 132 ++++++++++++------ 1 file changed, 89 insertions(+), 43 deletions(-) diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 6ef48c6..3a33dcf 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -11,7 +11,8 @@ from botocore.config import Config # Configure logging -logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +fmt = "%(asctime)s [%(levelname)s] %(message)s" +logging.basicConfig(level=logging.INFO, format=fmt) logger = logging.getLogger(__name__) @@ -34,7 +35,8 @@ def limit(self, pk): self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] if len(self.history[pk]) >= self.max_rate: - # Calculate sleep duration to let the oldest request roll off the 1-second window + # Calculate sleep duration to let the oldest request roll + # off the 1-second window sleep_time = 1.0 - (now - self.history[pk][0]) if sleep_time > 0: time.sleep(sleep_time) @@ -44,6 +46,53 @@ def limit(self, pk): self.history[pk].append(now) +def handle_file( + local_path, + table, + rate_limiter: PartitionRateLimiter, + records_in_file, + total_records, +): + with gzip.open(local_path, "rt", encoding="utf-8") as f: + with table.batch_writer() as batch: + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for + # DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + logger.error(f"Failed to parse JSON line: {e}") + continue + + # Add/modify fields if needed + newdate = datetime.datetime.utcnow().isoformat() + "Z" + item["imported_at"] = newdate + + # Ensure partition and sort keys are present + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: + logger.warning( + "Skipping record missing pk/sk: %s", + item.get('granule_id'), + ) + + continue + + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 + return records_in_file, total_records + + def main(): bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") table_name = os.environ.get("TABLE_NAME") @@ -55,17 +104,23 @@ def main(): s3 = boto3.client("s3") - # Configure boto3 with more aggressive retries to gracefully handle scale peaks + # Configure boto3 with more aggressive retries to gracefully + # handle scale peaks retry_config = Config( retries={ "max_attempts": 10, - "mode": "standard" + "mode": "standard", } ) dynamodb = boto3.resource("dynamodb", config=retry_config) - table = dynamodb.Table(table_name) + dyndb_table = dynamodb.Table(table_name) - logger.info(f"Starting import from s3://{bucket_name}/{prefix} into DynamoDB table {table_name}") + logger.info( + "Starting import from s3://%s/%s into DynamoDB table %s", + bucket_name, + prefix, + table_name, + ) paginator = s3.get_paginator("list_objects_v2") pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) @@ -73,7 +128,8 @@ def main(): total_files = 0 total_records = 0 - # Initialize partition-level rate limiter set to a safe threshold (850 writes/sec per pk) + # Initialize partition-level rate limiter set to a safe + # threshold (850 writes/sec per pk) rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) # Walk through the bucket objects @@ -94,52 +150,42 @@ def main(): try: s3.download_file(bucket_name, key, local_path) except Exception as e: - logger.error(f"Failed to download s3://{bucket_name}/{key}: {e}") + logger.error( + "Failed to download s3://{%s}/%s: %s", + bucket_name, + key, + e, + ) continue # Open, decompress and parse records_in_file = 0 try: - with gzip.open(local_path, "rt", encoding="utf-8") as f: - with table.batch_writer() as batch: - for line in f: - line = line.strip() - if not line: - continue - - try: - # Convert floats/doubles to Decimal for DynamoDB compatibility - item = json.loads(line, parse_float=Decimal) - except Exception as e: - logger.error(f"Failed to parse JSON line: {e}") - continue - - # Add/modify fields if needed - item["imported_at"] = datetime.datetime.utcnow().isoformat() + "Z" - - # Ensure partition and sort keys are present - pk = item.get("pk") - sk = item.get("sk") - if not pk or not sk: - logger.warning(f"Skipping record missing pk/sk: {item.get('granule_id')}") - continue - - # Apply dynamic rate limit based on the partition key - rate_limiter.limit(pk) - - # Batch insert into DynamoDB - batch.put_item(Item=item) - records_in_file += 1 - total_records += 1 - - logger.info(f"Successfully imported {records_in_file} records from {key}") + records_in_file, total_records = handle_file( + local_path, + dyndb_table, + rate_limiter, + records_in_file, + total_records + ) + + logger.info( + "Successfully imported %s records from %s", + records_in_file, + key, + + ) except Exception as e: - logger.error(f"Error processing file {key}: {e}") + logger.error("Error processing file %s: %s", key, e) finally: if os.path.exists(local_path): os.remove(local_path) - logger.info(f"Import complete. Processed {total_files} files, imported {total_records} total records.") + logger.info( + "Import complete. Processed %s files, imported %s total records.", + total_files, + total_records, + ) if __name__ == "__main__": From 05ea0d9d0a33c00f842e09ff074b11b2fbb4b2c8 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Mon, 29 Jun 2026 13:30:12 -0800 Subject: [PATCH 29/31] PR-7510 refinements to documentation, etc --- ctorm/cumulus-db-md-extract/README.md | 5 +- .../export-granules-daily.sh | 2 + ctorm/granule-md-db-loader/README.md | 3 +- .../granule_md_db_loader.py | 91 ++++++++++--------- ctorm/infra/README.md | 19 +++- ctorm/infra/terraform/terraform.tfstate | 24 ++--- .../infra/terraform/terraform.tfstate.backup | 45 ++++----- 7 files changed, 100 insertions(+), 89 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md index 1fe8459..17e7950 100644 --- a/ctorm/cumulus-db-md-extract/README.md +++ b/ctorm/cumulus-db-md-extract/README.md @@ -128,10 +128,7 @@ echo "${BUILD_ID}" aws codebuild start-build \ --project-name "${CODEBUILD_PROJECT}" \ --region "${AWS_REGION}" \ - --environment-variables-override '[ - {"name": "SLICES_S3_URI", "value": "s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_RTC-S1_V1-20170910-20181203.tsv", "type": "PLAINTEXT"}, - {"name": "COLLECTION", "value": "OPERA_L2_RTC-S1_V1", "type": "PLAINTEXT"} - ]' \ + --environment-variables-override name=SLICES_S3_URI,value="s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_CSLC-S1_V1-20180707-20200522.tsv",type=PLAINTEXT \ --query 'build.id' \ --output text # )" diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh index c28d176..d5796a9 100644 --- a/ctorm/cumulus-db-md-extract/export-granules-daily.sh +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -1,5 +1,7 @@ set -euo pipefail +# Takes slice TSV files and creates .jsonl files out of the cumulus DB + S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" FAILED_SLICES_FILE="failed_slices_daily.tsv" diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md index 5289584..d1812f4 100644 --- a/ctorm/granule-md-db-loader/README.md +++ b/ctorm/granule-md-db-loader/README.md @@ -26,7 +26,7 @@ export CTORM_BUCKET="ctorm-dev-scratch" aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/nisar/ |grep PRE aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/opera/ |grep PRE -# If you want do them all sequentially, not recommended and not going to work with all the data: +# If you want do them all sequentially, not recommended and not going to work with all the data because throttling:: aws codebuild start-build --profile="${AWS_PROFILE}" \ --project-name "${CODEBUILD_PROJECT}" \ --region "us-west-2" \ @@ -60,6 +60,7 @@ aws codebuild start-build --profile="${AWS_PROFILE}" \ --environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DIST-ALERT-S1_V1_/,type=PLAINTEXT \ --query 'build.id' \ --output text +# nisar aws codebuild start-build --profile="${AWS_PROFILE}" \ --project-name "${CODEBUILD_PROJECT}" \ --region "us-west-2" \ diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 3a33dcf..27e020a 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -47,49 +47,51 @@ def limit(self, pk): def handle_file( - local_path, - table, - rate_limiter: PartitionRateLimiter, - records_in_file, - total_records, + local_path: str, + table, + rate_limiter: PartitionRateLimiter, + records_in_file: int, + total_records: int, ): - with gzip.open(local_path, "rt", encoding="utf-8") as f: - with table.batch_writer() as batch: - for line in f: - line = line.strip() - if not line: - continue - - try: - # Convert floats/doubles to Decimal for - # DynamoDB compatibility - item = json.loads(line, parse_float=Decimal) - except Exception as e: - logger.error(f"Failed to parse JSON line: {e}") - continue - - # Add/modify fields if needed - newdate = datetime.datetime.utcnow().isoformat() + "Z" - item["imported_at"] = newdate - - # Ensure partition and sort keys are present - pk = item.get("pk") - sk = item.get("sk") - if not pk or not sk: - logger.warning( - "Skipping record missing pk/sk: %s", - item.get('granule_id'), - ) - - continue - - # Apply dynamic rate limit based on the partition key - rate_limiter.limit(pk) - - # Batch insert into DynamoDB - batch.put_item(Item=item) - records_in_file += 1 - total_records += 1 + with ( + gzip.open(local_path, "rt", encoding="utf-8") as f, + table.batch_writer() as batch, + ): + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for + # DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + logger.error("Failed to parse JSON line: %s", e) + continue + + # Add/modify fields if needed + newdate = datetime.datetime.utcnow().isoformat() + "Z" + item["imported_at"] = newdate + + # Ensure partition and sort keys are present + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: + logger.warning( + "Skipping record missing pk/sk: %s", + item.get("granule_id"), + ) + + continue + + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 return records_in_file, total_records @@ -142,7 +144,7 @@ def main(): if not key.endswith(".jsonl.gz"): continue - logger.info(f"Processing s3://{bucket_name}/{key}") + logger.info("Processing s3://%s/%s", bucket_name, key) total_files += 1 # Download and decompress the file @@ -166,14 +168,13 @@ def main(): dyndb_table, rate_limiter, records_in_file, - total_records + total_records, ) logger.info( "Successfully imported %s records from %s", records_in_file, key, - ) except Exception as e: logger.error("Error processing file %s: %s", key, e) diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md index 5ae884c..e9c6d92 100644 --- a/ctorm/infra/README.md +++ b/ctorm/infra/README.md @@ -19,16 +19,25 @@ terraform plan \ terraform apply \ -var-file="${VARFILE}" +terraform destroy +``` + +## Increasing DynamoDB table capacity + +For loading the DynamoDB table with the .jsonl files, we'll need to increase the capacity of the table. +AWS will throttle drastically if we don't. There is some limitation on how often we can switch between +PROVISIONED and PAY_PER_REQUEST, so don't do it willy-nilly. + +```bash # For temporary use while loading granule metadata: terraform plan \ -var-file="${VARFILE}" \ -var="granule_table_billing_mode=PROVISIONED" \ - -var="granule_table_write_capacity=10000" + -var="granule_table_write_capacity=5000" -terraform apply \ - -var-file="${VARFILE}" \ - -var="granule_table_billing_mode=PROVISIONED" \ - -var="granule_table_write_capacity=10000" +# To set it back to the default: +terraform plan \ + -var-file="${VARFILE}" terraform destroy diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index 8c00f44..f09bb23 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 69, + "serial": 80, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", - "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "id": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", + "output_base64sha256": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", + "output_base64sha512": "cYqWFwuEi6LTUM2lmqkXnWuYX+bf0oG86hslOpldD33Zq+jezzEQpTt9V3MGu8xmBy+knCvhX457ql2fPXo2qw==", "output_file_mode": null, - "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_md5": "c7918cf0c08ea6e16f8828296cd3d06a", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", - "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", - "output_size": 3296, + "output_sha": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", + "output_sha256": "b09c8d8bc03812252df7ed5e19fc6a7afea4871f07c97c4bde8db96c4ae22f96", + "output_sha512": "718a96170b848ba2d350cda59aa9179d6b985fe6dfd281bcea1b253a995d0f7dd9abe8decf3110a53b7d577306bbcc66072fa49c2be15f8e7baa5d9f3d7a36ab", + "output_size": 3481, "source": [], "source_content": null, "source_content_filename": null, @@ -793,7 +793,7 @@ "inline_policy": [ { "name": "ctorm-dev-cumulus-db-md-extract-upload", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "10648403b2277ad95dd6324dd58521c9", + "etag": "c7918cf0c08ea6e16f8828296cd3d06a", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "source_hash": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 23ab993..6c2a0ab 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 67, + "serial": 76, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", - "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "id": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", + "output_base64sha256": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", + "output_base64sha512": "/tl0guEuEuyrvuj+stzcDIneyXnGYZxXS+GHq3+Od3b3Rgyni0trdrvK2XAEb/SD0ccNAACcZ9AJxPkDFGv/EA==", "output_file_mode": null, - "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_md5": "66541e163cc3aaff54beddc162ad3aca", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", - "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", - "output_size": 3296, + "output_sha": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", + "output_sha256": "935f7853bdb8561ea66efb62eea3b49ab1f8b837c552aa0e020494b58f33996d", + "output_sha512": "fed97482e12e12ecabbee8feb2dcdc0c89dec979c6619c574be187ab7f8e7776f7460ca78b4b6b76bbcad970046ff483d1c70d00009c67d009c4f903146bff10", + "output_size": 3469, "source": [], "source_content": null, "source_content_filename": null, @@ -249,9 +249,9 @@ { "schema_version": 0, "attributes": { - "id": "923578907", - "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", - "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", + "id": "3026717548", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"UploadExtractedGranules\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:AbortMultipartUpload\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"\n },\n {\n \"Sid\": \"ListDestinationPrefix\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-dev-scratch\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"cumulus-granules/*\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"UploadExtractedGranules\",\"Effect\":\"Allow\",\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\"},{\"Sid\":\"ListDestinationPrefix\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}}}]}", "override_json": null, "override_policy_documents": null, "policy_id": null, @@ -261,6 +261,7 @@ { "actions": [ "s3:AbortMultipartUpload", + "s3:GetObject", "s3:PutObject" ], "condition": [], @@ -676,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PAY_PER_REQUEST", + "billing_mode": "PROVISIONED", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -686,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 0, - "write_capacity": 0 + "read_capacity": 5000, + "write_capacity": 5 } ], "hash_key": "pk", @@ -703,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 0, + "read_capacity": 5000, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -715,7 +716,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": {}, + "tags": null, "tags_all": {}, "timeouts": null, "ttl": [ @@ -724,7 +725,7 @@ "enabled": false } ], - "write_capacity": 0 + "write_capacity": 5 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -792,7 +793,7 @@ "inline_policy": [ { "name": "ctorm-dev-cumulus-db-md-extract-upload", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}" } ], "managed_policy_arns": [], @@ -915,7 +916,7 @@ "id": "ctorm-dev-cumulus-db-md-extract-upload:ctorm-dev-cumulus-db-md-extract-upload", "name": "ctorm-dev-cumulus-db-md-extract-upload", "name_prefix": "", - "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"s3:PutObject\",\"s3:GetObject\",\"s3:AbortMultipartUpload\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch/cumulus-granules/*\",\"Sid\":\"UploadExtractedGranules\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"cumulus-granules/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-dev-scratch\",\"Sid\":\"ListDestinationPrefix\"}]}", "role": "ctorm-dev-cumulus-db-md-extract-upload" }, "sensitive_attributes": [], @@ -1194,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "10648403b2277ad95dd6324dd58521c9", + "etag": "66541e163cc3aaff54beddc162ad3aca", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1206,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "source_hash": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From e6bc6f33a7141cb65fa7995eed7a39b136cdb307 Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 30 Jun 2026 11:42:25 -0800 Subject: [PATCH 30/31] PR-7510 Bug fixes to prevent hot partition --- .../terraform/terraform-nisar-prod.tfstate | 32 +++--- .../granule_md_db_loader.py | 97 ++++++++++--------- ctorm/infra/terraform/main.tf | 8 +- ctorm/infra/terraform/terraform.tfstate | 22 ++--- .../infra/terraform/terraform.tfstate.backup | 34 +++---- .../ctorm-prod/terraform.tfstate | 32 +++--- 6 files changed, 122 insertions(+), 103 deletions(-) diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate index 71c1cb1..6ffa2ce 100644 --- a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 17, + "serial": 23, "lineage": "2aab81d2-395f-d5fa-becf-2772136eaa03", "outputs": { "cumulus_db_md_extract_codebuild_project_name": { @@ -34,16 +34,16 @@ "terraform", "terraform/*" ], - "id": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_base64sha256": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", - "output_base64sha512": "PXdKET55u58x5KakCf2qqzNNWRJn/dwWLCSkwFeiIFKas9Hs1UYNpmyH5T02/iZXH9qiqn+aDOds7NlKkhNacQ==", + "id": "41bd43108980d112c9deba5135abfc48dc0f4e63", + "output_base64sha256": "X7mJWPMPJp0AvEIb4z20oix4IxI8yBgrZjrukcaMtQY=", + "output_base64sha512": "nc7b+wBbmYgZU4fEDXvkye+F6a7NM51R2dqfTRyn6CZSy9YVWE3XogIuIQIguytQs1PnDCsGyEqTAY7qxHpMgg==", "output_file_mode": null, - "output_md5": "8d91e1621f990906971ffa67900b6743", + "output_md5": "d3becd0cd2b647b5c5542370840070f1", "output_path": "./.terraform/cumulus-db-md-extract.zip", - "output_sha": "14b2ad4c67060ab3ed406af94a5db3f435f3c891", - "output_sha256": "1d4fa082bbb61fc04f54ef3be1b7f11423ab4f959619601a4466b7111b5c6c48", - "output_sha512": "3d774a113e79bb9f31e4a6a409fdaaab334d591267fddc162c24a4c057a220529ab3d1ecd5460da66c87e53d36fe26571fdaa2aa7f9a0ce76cecd94a92135a71", - "output_size": 5692, + "output_sha": "41bd43108980d112c9deba5135abfc48dc0f4e63", + "output_sha256": "5fb98958f30f269d00bc421be33db4a22c7823123cc8182b663aee91c68cb506", + "output_sha512": "9dcedbfb005b9988195387c40d7be4c9ef85e9aecd339d51d9da9f4d1ca7e82652cbd615584dd7a2022e210220bb2b50b353e70c2b06c84a93018eeac47a4c82", + "output_size": 7005, "source": [], "source_content": null, "source_content_filename": null, @@ -371,6 +371,16 @@ "name": "CTORM_S3_ROLE_ARN", "type": "PLAINTEXT", "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_URI", + "type": "PLAINTEXT", + "value": "" } ], "fleet": [], @@ -718,7 +728,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "8d91e1621f990906971ffa67900b6743", + "etag": "d3becd0cd2b647b5c5542370840070f1", "force_destroy": false, "id": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", "key": "codebuild/cumulus-db-md-extract.zip", @@ -731,7 +741,7 @@ "region": "us-west-2", "server_side_encryption": "AES256", "source": "./.terraform/cumulus-db-md-extract.zip", - "source_hash": "HU+ggru2H8BPVO874bfxFCOrT5WWGWAaRGa3ERtcbEg=", + "source_hash": "X7mJWPMPJp0AvEIb4z20oix4IxI8yBgrZjrukcaMtQY=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 27e020a..99deb24 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -6,6 +6,7 @@ import sys import time from decimal import Decimal +import random import boto3 from botocore.config import Config @@ -13,7 +14,7 @@ # Configure logging fmt = "%(asctime)s [%(levelname)s] %(message)s" logging.basicConfig(level=logging.INFO, format=fmt) -logger = logging.getLogger(__name__) +log = logging.getLogger(__name__) class PartitionRateLimiter: @@ -67,18 +68,18 @@ def handle_file( # DynamoDB compatibility item = json.loads(line, parse_float=Decimal) except Exception as e: - logger.error("Failed to parse JSON line: %s", e) + log.error("Failed to parse JSON line: %s", e) continue # Add/modify fields if needed - newdate = datetime.datetime.utcnow().isoformat() + "Z" + newdate = datetime.datetime.now(datetime.timezone.utc).isoformat() + "Z" item["imported_at"] = newdate # Ensure partition and sort keys are present pk = item.get("pk") sk = item.get("sk") if not pk or not sk: - logger.warning( + log.warning( "Skipping record missing pk/sk: %s", item.get("granule_id"), ) @@ -101,7 +102,7 @@ def main(): prefix = os.environ.get("PREFIX", "cumulus-granules/") if not table_name: - logger.error("TABLE_NAME environment variable is required.") + log.error("TABLE_NAME environment variable is required.") sys.exit(1) s3 = boto3.client("s3") @@ -117,7 +118,7 @@ def main(): dynamodb = boto3.resource("dynamodb", config=retry_config) dyndb_table = dynamodb.Table(table_name) - logger.info( + log.info( "Starting import from s3://%s/%s into DynamoDB table %s", bucket_name, prefix, @@ -134,55 +135,63 @@ def main(): # threshold (850 writes/sec per pk) rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) - # Walk through the bucket objects + # create random list of .json.gz objects: + # This will alleviate writing to the same partition for several files in a row. + jsongz_list = [] for page in pages: if "Contents" not in page: continue - for obj in page["Contents"]: key = obj["Key"] if not key.endswith(".jsonl.gz"): continue - - logger.info("Processing s3://%s/%s", bucket_name, key) + jsongz_list.append(key) total_files += 1 - # Download and decompress the file - local_path = "/tmp/temp.jsonl.gz" - try: - s3.download_file(bucket_name, key, local_path) - except Exception as e: - logger.error( - "Failed to download s3://{%s}/%s: %s", - bucket_name, - key, - e, - ) - continue - - # Open, decompress and parse - records_in_file = 0 - try: - records_in_file, total_records = handle_file( - local_path, - dyndb_table, - rate_limiter, - records_in_file, - total_records, - ) + total_files = len(jsongz_list) + random.shuffle(jsongz_list) - logger.info( - "Successfully imported %s records from %s", - records_in_file, - key, - ) - except Exception as e: - logger.error("Error processing file %s: %s", key, e) - finally: - if os.path.exists(local_path): - os.remove(local_path) + # Walk through the bucket objects + for key in jsongz_list: + + log.info("Processing s3://%s/%s", bucket_name, key) + + # Download and decompress the file + local_path = "/tmp/temp.jsonl.gz" + try: + s3.download_file(bucket_name, key, local_path) + except Exception as e: + log.error( + "Failed to download s3://{%s}/%s: %s", + bucket_name, + key, + e, + ) + continue - logger.info( + # Open, decompress and parse + records_in_file = 0 + try: + records_in_file, total_records = handle_file( + local_path, + dyndb_table, + rate_limiter, + records_in_file, + total_records, + ) + + log.info( + "Successfully imported %s records from %s", + records_in_file, + key, + ) + except Exception as e: + log.error("Error processing file %s: %s", key, e) + finally: + if os.path.exists(local_path): + os.remove(local_path) + + log.info( "Import complete. Processed %s files, imported %s total records.", total_files, total_records, diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf index 4f0d2fb..c0945fb 100644 --- a/ctorm/infra/terraform/main.tf +++ b/ctorm/infra/terraform/main.tf @@ -22,8 +22,8 @@ resource "aws_dynamodb_table" "granules" { # billing_mode = "PROVISIONED" # Temporarily switch to PROVISIONED for bulk load billing_mode = var.granule_table_billing_mode - read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null - write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null hash_key = "pk" range_key = "sk" @@ -53,8 +53,8 @@ resource "aws_dynamodb_table" "granules" { range_key = "gsi1sk" projection_type = "ALL" - read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null - write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null } } diff --git a/ctorm/infra/terraform/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate index f09bb23..1ac42bd 100644 --- a/ctorm/infra/terraform/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 80, + "serial": 84, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", - "output_base64sha256": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", - "output_base64sha512": "cYqWFwuEi6LTUM2lmqkXnWuYX+bf0oG86hslOpldD33Zq+jezzEQpTt9V3MGu8xmBy+knCvhX457ql2fPXo2qw==", + "id": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_base64sha256": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", + "output_base64sha512": "WFh/qul/y/sdiVqJFSXxIez+r8Imr3JHAZ+r9joPIaKF/1zPmZ7tgvM6cAVP583pRXyyevCCOvB1VdL8g9oliQ==", "output_file_mode": null, - "output_md5": "c7918cf0c08ea6e16f8828296cd3d06a", + "output_md5": "fdc46333571602f7f4296ffd55d381d8", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "52e34db2cc95ccb0b9a4b2c00d666b0fa966c7d6", - "output_sha256": "b09c8d8bc03812252df7ed5e19fc6a7afea4871f07c97c4bde8db96c4ae22f96", - "output_sha512": "718a96170b848ba2d350cda59aa9179d6b985fe6dfd281bcea1b253a995d0f7dd9abe8decf3110a53b7d577306bbcc66072fa49c2be15f8e7baa5d9f3d7a36ab", - "output_size": 3481, + "output_sha": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_sha256": "8859bacbf9a50d8139553bfdcf2ca9be2d6619f433fb4e545d4ac1e3e9b88dc7", + "output_sha512": "58587faae97fcbfb1d895a891525f121ecfeafc226af7247019fabf63a0f21a285ff5ccf999eed82f33a70054fe7cde9457cb27af0823af07555d2fc83da2589", + "output_size": 3578, "source": [], "source_content": null, "source_content_filename": null, @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "c7918cf0c08ea6e16f8828296cd3d06a", + "etag": "fdc46333571602f7f4296ffd55d381d8", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "sJyNi8A4EiUt9+1eGfxqev6khx8HyXxL3o25bEriL5Y=", + "source_hash": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.backup b/ctorm/infra/terraform/terraform.tfstate.backup index 6c2a0ab..48845ac 100644 --- a/ctorm/infra/terraform/terraform.tfstate.backup +++ b/ctorm/infra/terraform/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 76, + "serial": 82, "lineage": "b892c75c-f0f0-4b0a-da62-f1b3dc822c9c", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", - "output_base64sha256": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", - "output_base64sha512": "/tl0guEuEuyrvuj+stzcDIneyXnGYZxXS+GHq3+Od3b3Rgyni0trdrvK2XAEb/SD0ccNAACcZ9AJxPkDFGv/EA==", + "id": "51a8bb37eb8de1dc17e0d4eb7128915bbd774f3a", + "output_base64sha256": "DHqC2CGBZ9xOueY2LbY0MjhPAdqzGLGCfdX1PGJprc0=", + "output_base64sha512": "Kl12QLhrC3g5mI7yLmsNaUMBPbKyZUBJDL9eYbDB57ejDd2qK1LQXW4I5XXtwklte6oBgEjh92ADEvdd+cjGHg==", "output_file_mode": null, - "output_md5": "66541e163cc3aaff54beddc162ad3aca", + "output_md5": "4b0eca1c7acc876b5b03c0aba84ec786", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "d4be9c10fb356c2942476bb405fe6e86236ca9d9", - "output_sha256": "935f7853bdb8561ea66efb62eea3b49ab1f8b837c552aa0e020494b58f33996d", - "output_sha512": "fed97482e12e12ecabbee8feb2dcdc0c89dec979c6619c574be187ab7f8e7776f7460ca78b4b6b76bbcad970046ff483d1c70d00009c67d009c4f903146bff10", - "output_size": 3469, + "output_sha": "51a8bb37eb8de1dc17e0d4eb7128915bbd774f3a", + "output_sha256": "0c7a82d8218167dc4eb9e6362db63432384f01dab318b1827dd5f53c6269adcd", + "output_sha512": "2a5d7640b86b0b7839988ef22e6b0d6943013db2b26540490cbf5e61b0c1e7b7a30dddaa2b52d05d6e08e575edc2496d7baa018048e1f7600312f75df9c8c61e", + "output_size": 3570, "source": [], "source_content": null, "source_content_filename": null, @@ -677,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PROVISIONED", + "billing_mode": "PAY_PER_REQUEST", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -687,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 5000, - "write_capacity": 5 + "read_capacity": 0, + "write_capacity": 0 } ], "hash_key": "pk", @@ -704,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 5000, + "read_capacity": 0, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -716,7 +716,7 @@ "stream_label": "", "stream_view_type": "", "table_class": "STANDARD", - "tags": null, + "tags": {}, "tags_all": {}, "timeouts": null, "ttl": [ @@ -725,7 +725,7 @@ "enabled": false } ], - "write_capacity": 5 + "write_capacity": 0 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "66541e163cc3aaff54beddc162ad3aca", + "etag": "4b0eca1c7acc876b5b03c0aba84ec786", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "k194U724Vh6mbvti7qO0mrH4uDfFUqoOAgSUtY8zmW0=", + "source_hash": "DHqC2CGBZ9xOueY2LbY0MjhPAdqzGLGCfdX1PGJprc0=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, diff --git a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate index 6fae66e..9abab4f 100644 --- a/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate +++ b/ctorm/infra/terraform/terraform.tfstate.d/ctorm-prod/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.9.2", - "serial": 48, + "serial": 88, "lineage": "c41741c8-bc95-3af9-8022-5fe4475f750a", "outputs": { "cumulus_db_md_extract_upload_role_arn": { @@ -45,16 +45,16 @@ "attributes": { "exclude_symlink_directories": null, "excludes": null, - "id": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_base64sha256": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", - "output_base64sha512": "6zrwnliKIyNwiEayn3ouS6dmWHY7Ft777al1qI7xKBzFgVo4+kmCKy1gSqHsXrxGI6i7HnMtSxFsMdQiovpI2g==", + "id": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_base64sha256": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", + "output_base64sha512": "WFh/qul/y/sdiVqJFSXxIez+r8Imr3JHAZ+r9joPIaKF/1zPmZ7tgvM6cAVP583pRXyyevCCOvB1VdL8g9oliQ==", "output_file_mode": null, - "output_md5": "10648403b2277ad95dd6324dd58521c9", + "output_md5": "fdc46333571602f7f4296ffd55d381d8", "output_path": "./.terraform/granule-md-db-loader.zip", - "output_sha": "8aba5fc615e31fb8e702c9792731579710ec9e7f", - "output_sha256": "a8d24a5a9edf4b801282b8f959d7e0a53c8b12b9c09bb5b1e4adb2acf0f49b0c", - "output_sha512": "eb3af09e588a2323708846b29f7a2e4ba76658763b16defbeda975a88ef1281cc5815a38fa49822b2d604aa1ec5ebc4623a8bb1e732d4b116c31d422a2fa48da", - "output_size": 3296, + "output_sha": "a6cba0c602a893246d7bd58c418fb8d9b105cd3e", + "output_sha256": "8859bacbf9a50d8139553bfdcf2ca9be2d6619f433fb4e545d4ac1e3e9b88dc7", + "output_sha512": "58587faae97fcbfb1d895a891525f121ecfeafc226af7247019fabf63a0f21a285ff5ccf999eed82f33a70054fe7cde9457cb27af0823af07555d2fc83da2589", + "output_size": 3578, "source": [], "source_content": null, "source_content_filename": null, @@ -677,7 +677,7 @@ "type": "S" } ], - "billing_mode": "PAY_PER_REQUEST", + "billing_mode": "PROVISIONED", "deletion_protection_enabled": false, "global_secondary_index": [ { @@ -687,8 +687,8 @@ "on_demand_throughput": [], "projection_type": "ALL", "range_key": "gsi1sk", - "read_capacity": 0, - "write_capacity": 0 + "read_capacity": 5, + "write_capacity": 25000 } ], "hash_key": "pk", @@ -704,7 +704,7 @@ } ], "range_key": "sk", - "read_capacity": 0, + "read_capacity": 5, "replica": [], "restore_date_time": null, "restore_source_name": null, @@ -725,7 +725,7 @@ "enabled": false } ], - "write_capacity": 0 + "write_capacity": 25000 }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjYwMDAwMDAwMDAwMCwidXBkYXRlIjozNjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" @@ -1195,7 +1195,7 @@ "content_encoding": "", "content_language": "", "content_type": "application/octet-stream", - "etag": "10648403b2277ad95dd6324dd58521c9", + "etag": "fdc46333571602f7f4296ffd55d381d8", "force_destroy": false, "id": "codebuild/granule-md-db-loader.zip", "key": "codebuild/granule-md-db-loader.zip", @@ -1207,7 +1207,7 @@ "override_provider": [], "server_side_encryption": "AES256", "source": "./.terraform/granule-md-db-loader.zip", - "source_hash": "qNJKWp7fS4ASgrj5WdfgpTyLErnAm7Wx5K2yrPD0mww=", + "source_hash": "iFm6y/mlDYE5VTv9zyypvi1mGfQz+05UXUrB4+m4jcc=", "storage_class": "STANDARD", "tags": {}, "tags_all": {}, From 613bb79b377be81a0ac23bc024a61c941ce60bdc Mon Sep 17 00:00:00 2001 From: Ben Barton Date: Tue, 30 Jun 2026 14:43:27 -0800 Subject: [PATCH 31/31] linting, bah --- ctorm/granule-md-db-loader/granule_md_db_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py index 99deb24..daf1c7f 100644 --- a/ctorm/granule-md-db-loader/granule_md_db_loader.py +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -3,10 +3,10 @@ import json import logging import os +import random import sys import time from decimal import Decimal -import random import boto3 from botocore.config import Config @@ -153,7 +153,6 @@ def main(): # Walk through the bucket objects for key in jsongz_list: - log.info("Processing s3://%s/%s", bucket_name, key) # Download and decompress the file