diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f80b011..d1c62ef 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,9 +14,9 @@ jobs: python-version: 3.11 - uses: TrueBrain/actions-flake8@v2 with: - flake8_version: 6.0.0 + flake8_version: 7.3.0 max_line_length: 120 - plugins: flake8-isort==6.0.0 flake8-quotes==3.3.2 + plugins: flake8-isort==7.0.0, flake8-quotes==3.4.0 ruff-format: runs-on: ubuntu-latest @@ -25,5 +25,5 @@ jobs: - uses: actions/checkout@v4 - uses: astral-sh/ruff-action@v3 with: - version: "~=0.13.3" + version: "~=0.15.13" args: format --check --diff --output-format=github diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 30de2f6..5809c84 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,17 +8,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: [ '3.9', '3.10', '3.11', '3.12' ] fail-fast: false steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install poetry "tox<4.0.0" tox-poetry tox-gh-actions - - run: make tests + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry "tox<4.0.0" tox-poetry tox-gh-actions + - name: Run tests + run: | + make tests + if [ "${{ matrix.python-version }}" = "3.12" ]; then + # ctorm supports py312 only. + make ctorm-test + fi diff --git a/.gitignore b/.gitignore index d447d56..72e2440 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,7 @@ poetry.lock .tox +**.tfstate.backup + # Auto generated by Make tox.ini diff --git a/Makefile b/Makefile index 92b819c..d562f55 100644 --- a/Makefile +++ b/Makefile @@ -13,3 +13,7 @@ TOX_INIS := $(PACKAGES:%=%/tox.ini) .PHONY: tests tests: $(TOX_INIS) ss=0; $(foreach package,$(PACKAGES),cd $(package) && tox || ss=1;cd ..;) exit $$ss + +.PHONY: ctorm-test +ctorm-test: + cd ctorm && tox -c pyproject.toml diff --git a/ctorm/.python-version b/ctorm/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/ctorm/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/ctorm/README.md b/ctorm/README.md new file mode 100644 index 0000000..eeb4e20 --- /dev/null +++ b/ctorm/README.md @@ -0,0 +1,15 @@ +# CTORM + +Cumulus Throughput... uh... ORM? + +Load tester tool for Cumulus. + +Operates as x steps: + +* cumulus-db-md-extract + * Operates in the prod account. + * Gets the metadata from the cumulus database and writes it to a bunch of .jsonl files in a S3 bucket in the + operational account. +* granule-md-db-loader + * Operates in the account in which the load test will operate. + * Reads the .jsonl files from the S3 bucket and loads them into the CTORM granules dynamo db. diff --git a/ctorm/ctorm.cfg.example b/ctorm/ctorm.cfg.example new file mode 100644 index 0000000..0ce05c4 --- /dev/null +++ b/ctorm/ctorm.cfg.example @@ -0,0 +1,35 @@ +[ctorm] + +calc_md5 = true +granule_goal = 200 +granules_sqs_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/ctorm-dev-granules" + +[[ ctorm.pipelines ]] +bucketname = "nisar-bucket-0" +ingest_keypair_name = "" +ingest_queue_name = "sds-n-cumulus-test-nisar-workflow-queue" +ingest_sqs_queue_url = "" +prepare_keypair_name = "NISAR_P" +provider = "FOO" +share = 50 +ummg_prefix = "UMMG/FOO_COLLECTION/" + +[[ctorm.pipelines]] +bucketname = "nisar-bucket-2" +ingest_keypair_name = "" +ingest_queue_name = "sds-n-cumulus-test-nisar-workflow-queue" +ingest_sqs_queue_url = "" +prepare_keypair_name = "NISAR" +provider = "FOO" +share = 45 +ummg_prefix = "UMMG/BAR_COLLECTION/" + +[[ctorm.pipelines]] +bucketname = "opera-bucket-0" +ingest_keypair_name = "" +ingest_queue_name = "asf-cumulus-test-opera-workflow-queue" +ingest_sqs_queue_url = "" +prepare_keypair_name = "OPERA" +provider = "FOO" +share = 5 +ummg_prefix = "UMMG/BAZ_COLLECTION/" diff --git a/ctorm/ctorm/__init__.py b/ctorm/ctorm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/ctorm/__main__.py b/ctorm/ctorm/__main__.py new file mode 100644 index 0000000..bb0820d --- /dev/null +++ b/ctorm/ctorm/__main__.py @@ -0,0 +1,3 @@ +from ctorm.main import main + +main() diff --git a/ctorm/ctorm/cnm.py b/ctorm/ctorm/cnm.py new file mode 100644 index 0000000..327bbb4 --- /dev/null +++ b/ctorm/ctorm/cnm.py @@ -0,0 +1,131 @@ +import re +import uuid +from collections.abc import Callable +from datetime import datetime, timezone +from logging import getLogger +from pathlib import Path + +from ctorm.config import ( + MD5_CHECKSUM_PATTERN, + CtormPipeline, + CtormPreparedFile, + CtormPreparedGranule, +) + +log = getLogger(__name__) + +DATA_TYPE_MAP = { + ".context.json": "metadata", + ".dataset.json": "metadata", + ".h5": "data", + ".iso.xml": "metadata", + ".jpg": "browse", + ".log": "metadata", + ".md5": "metadata", + ".met.json": "metadata", + ".nc": "data", + ".pdf": "qa", + ".png": "browse", + ".qa.h5": "qa", + ".rc.yaml": "metadata", + ".tif": "data", + ".xml": "data", +} + +type CtormCnmSGeneratorType = Callable[[CtormPreparedGranule, str], dict] # noqa + + +class CtormCnmSGenerator: + """input dict like: + ``` + { + "bm": { + "sds-n-cumulus-prod-nisar-products": "B1", + "sds-n-cumulus-prod-nisar-jpl-private-data": "B2" + }, + "g": "NISAR_L0_RRST_VC08_20250821T101036_20250821T101041_P00408_J_001", + "c": "NISAR_L0A_RRST_BETA_V1", + "cv": "1", + "f": [ + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin", + "m": "e989430f4c5bfa04eeb26e56f4cd8375", + "s": 1073829760 + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.rc.yaml", + "s": 120597, + "m": "11bcaec780f0d246f3996b3f08680410" + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin.qa", + "s": 1134, + }, + ] + }, + ``` + """ + + def _assemble_file_dict( + self, file_dict: CtormPreparedFile, g_name: str, bucket_map: dict + ): + file_s3uri = file_dict["f"].replace("$G", g_name) + for bucket, replacetoken in bucket_map.items(): + file_s3uri = file_s3uri.replace(f"${replacetoken}", bucket) + filename = Path(file_s3uri).name + return { + "name": filename, + "type": self._get_type(filename), + "uri": file_s3uri, + "size": file_dict["s"], + "checksum": self._get_checksum(file_dict), + "checksumType": "md5", + } + + def __call__( + self, + granule_dict: CtormPreparedGranule, + provider: str, + ) -> dict: + cnm_s = { + "identifier": str(uuid.uuid4()), + "collection": granule_dict["c"], + "version": "1.3", + "submissionTime": datetime.now(tz=timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ), + "product": { + "name": granule_dict["g"], + "dataVersion": granule_dict["cv"], + "files": [ + self._assemble_file_dict( + file, granule_dict["g"], granule_dict["bm"] + ) + for file in granule_dict["f"] + ], + }, + "provider": provider, + } + + return cnm_s + + def _get_type(self, filename: str): + + suffixes = Path(filename).suffixes + while suffixes: + data_type = DATA_TYPE_MAP.get("".join(suffixes)) + + if data_type: + return data_type + + suffixes = suffixes[1:] + + log.debug("suffix from %s is not in map. Using 'data'", filename) + return "data" + + def _get_checksum(self, file_dict: CtormPreparedFile): + m = MD5_CHECKSUM_PATTERN.match(file_dict.get("m", "not md5")) + if m: + return m.group(1) + + return "00000000000000000000000000000000" diff --git a/ctorm/ctorm/cnm_sender.py b/ctorm/ctorm/cnm_sender.py new file mode 100644 index 0000000..12f1dbf --- /dev/null +++ b/ctorm/ctorm/cnm_sender.py @@ -0,0 +1,29 @@ +from logging import getLogger +from typing import List + +from ctorm.cnm import CtormCnmSGenerator +from ctorm.config import CtormConfig, CtormPreparedGranule + +log = getLogger(__name__) + + +class CnmSender: + def __init__( + self, + cfg: CtormConfig, + granules: List[CtormPreparedGranule], + provider: str = "TODO: PROVIDER", + ): + self.cfg = cfg + self.granules = granules + self.provider = provider + self.cnm_s_generator = CtormCnmSGenerator() + + def send_all(self) -> bool: + for granule in self.granules: + cnms = self.cnm_s_generator(granule, self.provider) + log.debug("Sending %s", cnms) + # TODO: actually send + + # TODO: determine successfulllness + return True diff --git a/ctorm/ctorm/commands/__init__.py b/ctorm/ctorm/commands/__init__.py new file mode 100644 index 0000000..3f5b404 --- /dev/null +++ b/ctorm/ctorm/commands/__init__.py @@ -0,0 +1,27 @@ +import argparse +import logging + +from ctorm.config import CtormConfig +from ctorm.prepare import CtormPrepare + +log = logging.getLogger(__name__) + + +def cmd_prepare(args: argparse.Namespace): + cfg = CtormConfig.from_file( + cfg_file=args.cfg_file, + ) + prepare = CtormPrepare(cfg) + prepare.prepare() + + +def cmd_cnm_sender(args: argparse.Namespace): + # TODO: something like: + # + # from ctorm.cnm_sender import CnmSender + # + # cfg = CtormConfig.from_file(cfg_file=args.cfg_file) + # cnm_sender = CnmSender(cfg) + # cnm_sender.run(max_messages=args.max_messages) + # + raise NotImplementedError("cnm_sender command is not implemented yet") diff --git a/ctorm/ctorm/config.py b/ctorm/ctorm/config.py new file mode 100644 index 0000000..dba4c7a --- /dev/null +++ b/ctorm/ctorm/config.py @@ -0,0 +1,80 @@ +import re +import tomllib +from dataclasses import dataclass, fields +from typing import NotRequired, TypedDict + +from boto3.session import Session +from mypy_boto3_s3 import S3Client + +try: + from typing import Self +except ImportError: + Self = "CtormConfig" + +DEFAULT = "default" +AWS_REGION = "us-west-2" # We will never not want us-west-2 +MD5_CHECKSUM_PATTERN = re.compile(r"^([\da-f]{32})$") + + +class CtormPreparedFile(TypedDict): + f: str + m: NotRequired[str] + s: int + + +class CtormPreparedGranule(TypedDict): + bm: dict[str, str] + g: str + c: str + cv: str + f: list[CtormPreparedFile] + + +@dataclass +class CtormPipeline: + bucketname: str + + # Looks for a keypair in the form of `AWS_ACCESS_KEY_ID_[keypair_name]` + # This should have permissions to send SQS to the maturity we're sending CNM to + ingest_keypair_name: str + ingest_queue_name: str + ingest_sqs_queue_url: str + + # Looks for a keypair in the form of `AWS_ACCESS_KEY_ID_[keypair_name]` + # This should have permissions to read from prod bucket where data resides + prepare_keypair_name: str + share: int + + # These are used at runtime + next_cont_token: str = None + s3_client: S3Client = None + session: Session = None + ummg_prefix: str = "UMMG/" + provider: str = "TODO" + + +@dataclass +class CtormConfig: + pipelines: list + granules_sqs_queue_url: str + + calc_md5: bool = True + + # This is the number of granules that will be prepared for test. + granule_goal: int = 2000000 + + @classmethod + def from_file( + cls, + cfg_file: str, + ) -> Self: + with open(cfg_file, "rb") as f: + cfg = tomllib.load(f) + valid_fields = {f.name for f in fields(cls)} + # Filter the input dictionary + kwargs = {k: v for k, v in cfg["ctorm"].items() if k in valid_fields} + # create CtormPipeline for each bucket in cfg['pipelines'] + kwargs["pipelines"] = [CtormPipeline(**b) for b in kwargs["pipelines"]] + obj = cls(**kwargs) + + return obj diff --git a/ctorm/ctorm/lambda_run.py b/ctorm/ctorm/lambda_run.py new file mode 100644 index 0000000..cf4556f --- /dev/null +++ b/ctorm/ctorm/lambda_run.py @@ -0,0 +1,36 @@ +""" +Emulates a lambda for local dev. +""" + +import json +import logging + +from aws_lambda_typing import context as context_ + +from ctorm.load_tester import lambda_handler + +log = logging.getLogger() + +base_fmt_str = "%(levelname)s: %(message)s (%(filename)s line %(lineno)d/)" +screen_fmt = logging.Formatter( + "%(asctime)s.%(msecs)d " + base_fmt_str, "%Y-%m-%dT%H:%M:%S" +) +screenlog = logging.StreamHandler() +screenlog.setFormatter(screen_fmt) +log.addHandler(screenlog) + + +# mock event data +with open("../../data/lambda-event-eventbridge.json", "r") as f: + mock_event = json.load(f) + + +# Mock context object +class MockContext(context_.Context): + function_name = "local_test" + memory_limit_in_mb = 128 + aws_request_id = "LOLOCAL" + + +# Run it +print(lambda_handler(mock_event, MockContext())) diff --git a/ctorm/ctorm/load_tester.py b/ctorm/ctorm/load_tester.py new file mode 100644 index 0000000..1cd3138 --- /dev/null +++ b/ctorm/ctorm/load_tester.py @@ -0,0 +1,124 @@ +import json +import logging +import os +from functools import cache +from typing import List + +import boto3 +from aws_lambda_typing import context as context_ +from aws_lambda_typing import events +from cnm_sender import CnmSender + +from ctorm.config import ( + AWS_REGION, + CtormConfig, + CtormPreparedGranule, +) + +log = logging.getLogger(__name__) + + +@cache +def get_sqs_client(): + return boto3.client("sqs", region_name=AWS_REGION) + + +@cache +def configure_cfg(): + cfg = CtormConfig.from_file( + cfg_file=os.getenv("CFG_FILE", "./ctorm.cfg"), + ) + return cfg + + +def configure_logging() -> None: + level_name = os.environ.get("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + + root_logger = logging.getLogger() + root_logger.setLevel(level) + + for noisy_logger_name in ("boto3", "botocore", "urllib3", "s3transfer"): + logging.getLogger(noisy_logger_name).setLevel(logging.WARNING) + + # AWS Lambda installs a handler before invoking your code. Reuse it instead + # of adding a new handler on every warm invocation. + for handler in root_logger.handlers: + handler.setLevel(level) + handler.setFormatter( + logging.Formatter( + "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d - %(message)s" + ) + ) + + +def get_granule_list(event: events.EventBridgeEvent) -> List[CtormPreparedGranule]: + # TODO: Implement + return [] + + +def load_test( + cfg: CtormConfig, + gr_queue_url: str, +): + # fetch 10 messages from SQS + messages = [] + while len(messages) < 10: + # Since we're trying to send x granules per invocation, we will do what we can + # to get a full load of 10 messages. + sqs_resp = get_sqs_client().receive_message( + QueueUrl=gr_queue_url, + MaxNumberOfMessages=min(10 - len(messages), 10), + WaitTimeSeconds=5, + VisibilityTimeout=3600, + ) + + batch = sqs_resp.get("Messages", []) + if not batch: + break + + messages.extend(batch) + + for message in sqs_resp.get("Messages", []): + body = json.loads(message["Body"]) + receipt_handle = message["ReceiptHandle"] + granule_list = body.get("granules", []) + + c_sender = CnmSender(cfg, granule_list) + success = c_sender.send_all() + if success: + log.debug("Deleting message %s", receipt_handle) + # TODO: uncomment after some dev'ing + # get_sqs_client().delete_message( + # QueueUrl=gr_queue_url, ReceiptHandle=receipt_handle + # ) + + +def lambda_handler(event: events.EventBridgeEvent, context: context_.Context): + configure_logging() + cfg = configure_cfg() + + log.info( + "Starting CNM sender invocation", + extra={ + "aws_request_id": getattr(context, "aws_request_id", None), + }, + ) + + try: + log.debug("Received event: %s", event) + # g_list = event["Records"].pop().get("body") + # g_list = json.loads(g_list).get("granules") + + load_test(cfg, event["granules_queue_url"]) + + log.info("CNM sender invocation completed") + return {"ok": True} + + except Exception: + log.exception("CNM sender invocation failed") + raise + + +if __name__ == "__main__": + lambda_handler({}, {}) diff --git a/ctorm/ctorm/main.py b/ctorm/ctorm/main.py new file mode 100644 index 0000000..6cdaf4b --- /dev/null +++ b/ctorm/ctorm/main.py @@ -0,0 +1,102 @@ +""" +A script for load testing cumulus. + +""" + +import argparse +import logging +import sys +from typing import Optional + +from ctorm.commands import cmd_cnm_sender, cmd_prepare + +log = logging.getLogger(__name__) + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + + parser.add_argument( + "--verbose", + "-v", + help="Increase verbosity level. Can be passed multiple times.", + action="count", + default=0, + ) + + parser.add_argument("--profile", help="AWS profile name") + parser.add_argument( + "--cfg-file", + help="Config file", + ) + + subparsers = parser.add_subparsers( + dest="command", + required=True, + ) + + prepare_parser = subparsers.add_parser( + "prepare", + help="Fill the granules SQS queue with work for the load test.", + ) + prepare_parser.add_argument( + "--source-bucket", + help="S3 bucket source granules", + ) + prepare_parser.set_defaults(func=cmd_prepare) + + cnm_sender_parser = subparsers.add_parser( + "cnm_sender", + help="Read queued work and formulate load-test/CNM messages.", + ) + cnm_sender_parser.add_argument( + "--max-messages", + type=int, + default=-1, # -1 is unlimited + help="Maximum number of SQS messages to process in one invocation. Default is unlimited.", + ) + cnm_sender_parser.set_defaults(func=cmd_cnm_sender) + + return parser + + +def main(args: Optional[list[str]] = None): + parser = get_parser() + pargs = parser.parse_args(args=args) + + root_logger = logging.getLogger() + boto3_logger = logging.getLogger("boto3") + botocore_logger = logging.getLogger("botocore") + urllib3_logger = logging.getLogger("urllib3") + s3transfer_logger = logging.getLogger("s3transfer") + + root_logger.setLevel(logging.INFO) + boto3_logger.setLevel(logging.WARNING) + botocore_logger.setLevel(logging.WARNING) + urllib3_logger.setLevel(logging.WARNING) + s3transfer_logger.setLevel(logging.WARNING) + + if pargs.verbose >= 1: + root_logger.setLevel(logging.DEBUG) + if pargs.verbose >= 2: + boto3_logger.setLevel(logging.DEBUG) + botocore_logger.setLevel(logging.INFO) + urllib3_logger.setLevel(logging.DEBUG) + s3transfer_logger.setLevel(logging.DEBUG) + if pargs.verbose >= 3: + botocore_logger.setLevel(logging.DEBUG) + + base_fmt_str = "%(levelname)s: %(message)s (%(filename)s line %(lineno)d/)" + screen_fmt = logging.Formatter( + "%(asctime)s.%(msecs)d " + base_fmt_str, "%Y-%m-%dT%H:%M:%S" + ) + screenlog = logging.StreamHandler() + screenlog.setFormatter(screen_fmt) + root_logger.addHandler(screenlog) + + try: + pargs.func(pargs) + + except Exception: + log.exception("") + sys.exit(-1) diff --git a/ctorm/ctorm/prepare.py b/ctorm/ctorm/prepare.py new file mode 100644 index 0000000..7cbd6f6 --- /dev/null +++ b/ctorm/ctorm/prepare.py @@ -0,0 +1,212 @@ +import hashlib +import json +import os +import re +from enum import StrEnum +from functools import cache +from logging import getLogger + +import boto3 +from botocore.exceptions import ClientError +from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef + +from ctorm.config import AWS_REGION, MD5_CHECKSUM_PATTERN, CtormConfig, CtormPipeline + +log = getLogger(__name__) + + +@cache +def get_sqs_client(): + return boto3.client("sqs", region_name=AWS_REGION) + + +class K(StrEnum): + """Since we want to keep the message json as light as possible, we'll keep + the keys for the message sent to the CTORM SQS queue here. + """ + + BKT_MAP = "bm" + GRANULE = "g" + COLLECTION = "c" + COLLECTION_VERSION = "cv" + FILES = "f" + MD5 = "m" + SIZE = "s" + + +class CtormSqsMessage: + MAX_MESSAGE_SIZE = 262144 + + def __init__(self, cfg: CtormConfig): + self.cfg = cfg + self.granules = [] + + def add_file(self, file: dict): + self.granules.append(file) + + def to_dict(self): + return {"granules": self.granules} + + def to_json(self): + return json.dumps(self.to_dict()) + + def check_message_size(self): + return len(self.to_json()) < self.MAX_MESSAGE_SIZE + + +class CtormPrepare: + MD5_DL_CHUNK_MB = 8 + + def __init__(self, cfg: CtormConfig): + self.cfg = cfg + self.total_granules = 0 + + self._boto_sessions = {} + + # init buckets + for bkt in self.cfg.pipelines: + bkt.next_cont_token = None + keysuffix = ( + f"_{bkt.prepare_keypair_name}" if bkt.prepare_keypair_name else "" + ) + if bkt.prepare_keypair_name not in self._boto_sessions: + self._boto_sessions[bkt.prepare_keypair_name] = boto3.Session( + region_name=AWS_REGION, + aws_access_key_id=os.getenv(f"AWS_ACCESS_KEY_ID{keysuffix}"), + aws_secret_access_key=os.getenv( + f"AWS_SECRET_ACCESS_KEY{keysuffix}" + ), + ) + bkt.s3_client = self._boto_sessions[bkt.prepare_keypair_name].client("s3") + + def prepare(self): + while self.total_granules < self.cfg.granule_goal: + # This is the loop that creates a SQS message from multiple objects. + sqs_msg = CtormSqsMessage(self.cfg) + for b in self.cfg.pipelines: + # This is the loop that goes into each bucket we're interested in. + log.debug("getting objects from %s", b.bucketname) + + # get a page from obj_iterator + # We want only one page because we want to get only b.share files for this go. + page = self.get_ummg_page(b) + for obj in page.get("Contents", []): + if not obj["Key"].endswith(".cmr.json"): + log.debug("skipping %s", obj["Key"]) + continue + log.debug(" %s", obj["Key"]) + # download the json object and load it into a var: + ummg = self.download_ummg(b, obj["Key"]) + file = self.process_ummg(ummg, b) + if file: + sqs_msg.add_file(file) + self.total_granules += 1 + b.next_cont_token = page.get("NextContinuationToken") + + log.debug("tot granules: %d/%d", self.total_granules, self.cfg.granule_goal) + log.debug("sqs_msg size: %d", len(sqs_msg.to_json())) + log.debug( + "sqs message percentage: %d%%", + (len(sqs_msg.to_json()) / CtormSqsMessage.MAX_MESSAGE_SIZE) * 100, + ) + log.debug("sqs_msg size OK?: %d", sqs_msg.check_message_size()) + if sqs_msg.check_message_size(): + log.debug("sqs_msg: %s", sqs_msg.to_json()) + get_sqs_client().send_message( + QueueUrl=self.cfg.granules_sqs_queue_url, + MessageBody=sqs_msg.to_json(), + ) + else: + raise Exception("Message too big") + # TODO: deal with this smarter + + def get_ummg_page(self, ct_bukt: CtormPipeline) -> ListObjectsV2OutputTypeDef: + kwargs = { + "Bucket": ct_bukt.bucketname, + "MaxKeys": ct_bukt.share, + "Prefix": ct_bukt.ummg_prefix, + } + if ct_bukt.next_cont_token: + kwargs["ContinuationToken"] = ct_bukt.next_cont_token + ret = ct_bukt.s3_client.list_objects_v2(**kwargs) + return ret + + def download_ummg(self, b: CtormPipeline, key: str) -> dict: + resp = b.s3_client.get_object(Bucket=b.bucketname, Key=key) + ummgfile = resp["Body"].read() + return json.loads(ummgfile) + + def process_ummg(self, ummg: dict, ct_bkt: CtormPipeline) -> dict: + outdict = { + K.BKT_MAP: {ct_bkt.bucketname: "B1"}, # bucket map + K.GRANULE: ummg["GranuleUR"], + K.COLLECTION: ummg["CollectionReference"]["ShortName"], + K.COLLECTION_VERSION: ummg["CollectionReference"]["Version"], + K.FILES: [], # list of files + } + + for r_urls in ummg["RelatedUrls"]: + if r_urls["URL"].startswith("s3://"): + bucket = re.sub(r"^s3://([^/]+).*$", r"\1", r_urls["URL"]) + if bucket != ct_bkt.bucketname: + outdict[K.BKT_MAP][bucket] = f"B{len(outdict[K.BKT_MAP])}" + objloc = r_urls["URL"].removeprefix(f"s3://{bucket}/") + + # Replace the granulename with a token for compression. Will reconstitute in the lambda + fileval = ( + r_urls["URL"] + .replace(outdict[K.GRANULE], "$G") + .replace(bucket, f"${outdict[K.BKT_MAP][bucket]}") + ) + filedict = {"f": fileval} + for distr_file in ummg["DataGranule"][ + "ArchiveAndDistributionInformation" + ]: + if r_urls["URL"].endswith(distr_file["Name"]): + # We handily have the md5 and size in the ummg + # TODO: double-check this test is correct and we're not unnecessarily HEADing too many files. + filedict[K.SIZE] = distr_file["SizeInBytes"] + filedict[K.MD5] = distr_file["Checksum"]["Value"] + break + else: + # We must look to S3 for the size and md5 + log.debug('getting head for "%s"', objloc) + try: + headobj = ct_bkt.s3_client.head_object( + Bucket=bucket, Key=objloc + ) + log.debug("head_object: %s", headobj) + filedict[K.SIZE] = headobj["ContentLength"] + + md5 = headobj["ETag"].replace('"', "") + if MD5_CHECKSUM_PATTERN.fullmatch(md5): + filedict[K.MD5] = md5 + elif self.cfg.calc_md5: + log.debug("multipart upload detected for %s", objloc) + filedict[K.MD5] = self.get_real_md5( + ct_bkt, bucket, objloc + ) + else: + log.debug("no need to calculate md5 for %s", objloc) + except ClientError as e: + log.error("head_object failed: %s", e) + # TODO: trash entire message for this granule? + + outdict[K.FILES].append(filedict) + + return outdict + + def get_real_md5( + self, + b: CtormPipeline, + obj_bucket: str, + key: str, + ) -> str: + # The file is small enough we may as well download it to memory and get the MD5 that way. + resp = b.s3_client.get_object(Bucket=obj_bucket, Key=key) + md5_accumulator = hashlib.md5() + for chunk in iter( + lambda: resp["Body"].read(self.MD5_DL_CHUNK_MB * 1024 * 1024), b"" + ): + md5_accumulator.update(chunk) + return md5_accumulator.hexdigest() diff --git a/ctorm/cumulus-db-md-extract/README.md b/ctorm/cumulus-db-md-extract/README.md new file mode 100644 index 0000000..17e7950 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/README.md @@ -0,0 +1,159 @@ +# Cumulus DB MD Extractor + +The purpose of this is to get enough medata from the prod cumulus RDS db and into a Dynamo DB to be able to recreate +CNM-S messages. + +# How to use: + +## From the web console + +* Open a web console from Kion + * Open a tab with the secrets manager + * Find the secret with `db_login` in the name. This is where you find the host, user/pass, etc. + * Open a tab with the RDS + * Go to the query editor + * Open a CloudShell in anohter tab. + * Pick the VPC + * Pick the same subnet as the RDS + * Pick the default security group and the one(s) with `_rds_` in the name. + * TODO: see if we can do this with EC2 instead of cloudshell + +## From the shell + +### prepare the environment + +Create AWS credentials to upload to the CTORM bucket. + +```bash +mkdir .aws +vi .aws/credentials + +[ctorm] +aws_access_key_id = AKYEAH... +aws_secret_access_key = foofoofooofooo +``` + +Make some env vars: + +```bash + +read -s PGPASSWORD +export PGPASSWORD +export PGHOST='from secrets mgr' +export PGUSER='from secrets mgr' +export PGDATABASE='from secrets mgr' + +# this is the ctorm bucket and path info where the db dumps are stored +export CTORM_BUCKET='ctorm-dev-scratch' +export DUMP_SUBDIR='nisar' + +``` + +Get these files into the instance one way or another. + +* ctorm/cumulus-db-md-extract/export-granules-daily.sh +* ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql +* ctorm/cumulus-db-md-extract/generate-slices-daily.sh + +## Run the scripts + +```bash +bash generate-slices-daily.sh +``` + +This creates a `slices_daily.tsv` file. Basically a list of chunked work for export-granules-daily.sh to use. + +The following step will query the database and create gzipped ljson files and upload them to S3: + +```bash +bash export-granules-daily.sh +``` + +If running in cloudshell, you need to put some sort of input into the console every 10 minutes or so, or AWS will kill +the session. +Really should try to create a EC2 next time. + +## Using Codebuild + +### Terraform + +```bash +export AWS_PROFILE="cumulus-uat-6921" +export VARFILE=opera-uat.tfvars +export STATEFILE=terraform-opera-uat.tfstate + +terraform workspace new opera-uat + +export TF_WORKSPACE=opera-uat + +terraform init + +terraform plan \ + -state="${STATEFILE}" \ + -var-file="${VARFILE}" + +terraform apply \ + -state="${STATEFILE}" \ + -var-file="${VARFILE}" + +terraform destroy + + +``` + +### Codebuild + +```bash +export AWS_PROFILE="cumulus-uat-6921" +export AWS_REGION="us-west-2" +export CODEBUILD_PROJECT="$(terraform output -state="${STATEFILE}" -raw cumulus_db_md_extract_codebuild_project_name)" + +# You can do one collection at a time, many in parallel if necessary: +export COLLECTION="OPERA_L2_RTC-S1_V1" + +# start build: +BUILD_ID="$( + aws codebuild start-build \ + --project-name "${CODEBUILD_PROJECT}" \ + --region "${AWS_REGION}" \ + --environment-variables-override name=COLLECTION,value="${COLLECTION}",type=PLAINTEXT \ + --query 'build.id' \ + --output text +)" + +echo "${BUILD_ID}" + +# Or start build specifying a custom slices.tsv file from S3: +# BUILD_ID="$( + aws codebuild start-build \ + --project-name "${CODEBUILD_PROJECT}" \ + --region "${AWS_REGION}" \ + --environment-variables-override name=SLICES_S3_URI,value="s3://ctorm-scratch/cumulus-granules/opera/slices_daily-OPERA_L2_CSLC-S1_V1-20180707-20200522.tsv",type=PLAINTEXT \ + --query 'build.id' \ + --output text +# )" + + +# get status: +aws codebuild batch-get-builds \ + --ids "${BUILD_ID}" \ + --region "${AWS_REGION}" \ + --query 'builds[0].{status:buildStatus,phase:currentPhase,start:startTime,end:endTime,logs:logs.deepLink}' \ + --output table + +# get build logs: +aws codebuild batch-get-builds \ + --ids "${BUILD_ID}" \ + --region "${AWS_REGION}" \ + --query 'builds[0].logs.deepLink' \ + --output text + +LOG_GROUP="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.groupName' --output text)" +LOG_STREAM="$(aws codebuild batch-get-builds --ids "${BUILD_ID}" --region "${AWS_REGION}" --query 'builds[0].logs.streamName' --output text)" + +aws logs tail "${LOG_GROUP}" \ + --log-stream-names "${LOG_STREAM}" \ + --follow \ + --region "${AWS_REGION}" + +``` diff --git a/ctorm/cumulus-db-md-extract/buildspec.yaml b/ctorm/cumulus-db-md-extract/buildspec.yaml new file mode 100644 index 0000000..7702062 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/buildspec.yaml @@ -0,0 +1,73 @@ +version: 0.2 +env: + variables: + AWS_DEFAULT_REGION: "us-west-2" + secrets-manager: + PGHOST: "${DB_SECRET_ARN}:host" + PGUSER: "${DB_SECRET_ARN}:username" + PGPASSWORD: "${DB_SECRET_ARN}:password" + PGDATABASE: "${DB_SECRET_ARN}:database" + +phases: + install: + commands: + - echo "Inspecting build image..." + - cat /etc/os-release || true + - aws --version || true + - jq --version || true + - unzip -v | head -n 2 || true + - psql --version || true + - echo "Installing required packages if missing..." + - | + if ! command -v jq >/dev/null 2>&1; then + yum install -y jq + fi + - | + if ! command -v unzip >/dev/null 2>&1; then + yum install -y unzip + fi + - | + if ! command -v psql >/dev/null 2>&1; then + echo "psql not found; trying PostgreSQL client package installs..." + + yum install -y postgresql15 \ + || yum install -y postgresql14 \ + || yum install -y postgresql13 \ + || yum install -y postgresql12 \ + || yum install -y postgresql \ + || { + echo "Unable to install psql from enabled yum repositories." + echo "Available postgres-related packages:" + yum search postgresql || true + exit 1 + } + fi + - psql --version + - aws --version + + pre_build: + commands: + - ls -lah + - chmod +x generate-slices-daily.sh export-granules-daily.sh test-s3-cp.sh refresh-ctorm-upload-role.sh + - | + echo 'Before assuming role: ' + - aws sts get-caller-identity + - | + echo "Assuming ctorm upload role..." + source ./refresh-ctorm-upload-role.sh + - | + echo 'After assuming role: ' + aws sts get-caller-identity + + + build: + commands: + # fail fast if we can't actually copy to S3: + - ./test-s3-cp.sh + + - ./generate-slices-daily.sh + - ./export-granules-daily.sh + post_build: + commands: + - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + - echo "Build complete." diff --git a/ctorm/cumulus-db-md-extract/export-granules-daily.sh b/ctorm/cumulus-db-md-extract/export-granules-daily.sh new file mode 100644 index 0000000..d5796a9 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/export-granules-daily.sh @@ -0,0 +1,66 @@ +set -euo pipefail + +# Takes slice TSV files and creates .jsonl files out of the cumulus DB + +S3_PREFIX="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" +FAILED_SLICES_FILE="failed_slices_daily.tsv" + +rm -f "$FAILED_SLICES_FILE" + +while IFS=$'\t' read -r COLLECTION YYYYMMDD GRANULE_COUNT FILE_COUNT; do + if [[ -z "${COLLECTION:-}" || ! "${YYYYMMDD:-}" =~ ^[0-9]{8}$ ]]; then + echo "Skipping malformed slice row: collection=${COLLECTION:-} yyyymmdd=${YYYYMMDD:-}" >&2 + continue + fi + + YYYY="${YYYYMMDD:0:4}" + YYYYMM="${YYYYMMDD:0:6}" + SAFE_COLLECTION=$(echo "$COLLECTION" | tr -c 'A-Za-z0-9._-' '_') + S3_URI="${S3_PREFIX}/${SAFE_COLLECTION}/${YYYY}/${YYYYMM}/${YYYYMMDD}.jsonl.gz" + TMP_OUTPUT="$(mktemp "${SAFE_COLLECTION}.${YYYYMMDD}.XXXXXX.jsonl.gz")" + + echo "Exporting collection=${COLLECTION} yyyymmdd=${YYYYMMDD} expected_granules=${GRANULE_COUNT} expected_files=${FILE_COUNT}" + echo "Destination: ${S3_URI}" + echo "Temporary output: ${TMP_OUTPUT}" + + if ! PGOPTIONS="-c statement_timeout=${PG_STATEMENT_TIMEOUT:-0}" psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --set=ON_ERROR_STOP=1 \ + --set=collection="$COLLECTION" \ + --set=yyyymmdd="$YYYYMMDD" \ + --file=export-granules-slice-daily.sql \ + | gzip -c > "$TMP_OUTPUT"; then + + echo "FAILED query/gzip collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + rm -f "$TMP_OUTPUT" + continue + fi + + source ./refresh-ctorm-upload-role.sh + + if ! aws s3 cp "$TMP_OUTPUT" "$S3_URI"; then + echo "FAILED upload collection=${COLLECTION} yyyymmdd=${YYYYMMDD}; continuing with next slice" >&2 + printf '%s\t%s\t%s\t%s\n' "$COLLECTION" "$YYYYMMDD" "$GRANULE_COUNT" "$FILE_COUNT" >> "$FAILED_SLICES_FILE" + rm -f "$TMP_OUTPUT" + continue + fi + + rm -f "$TMP_OUTPUT" + echo "Uploaded ${S3_URI}" + +done < slices_daily.tsv + +if [[ -s "$FAILED_SLICES_FILE" ]]; then + source ./refresh-ctorm-upload-role.sh + FAILED_SLICES_S3_URI="${S3_PREFIX}/failed_slices_daily.tsv" + aws s3 cp "$FAILED_SLICES_FILE" "$FAILED_SLICES_S3_URI" + echo "Some slices failed. Uploaded failure list to ${FAILED_SLICES_S3_URI}" >&2 +else + echo "All slices exported successfully." +fi diff --git a/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql new file mode 100644 index 0000000..61a8f1f --- /dev/null +++ b/ctorm/cumulus-db-md-extract/export-granules-slice-daily.sql @@ -0,0 +1,68 @@ +\pset tuples_only on +\pset format unaligned +\pset pager off + +WITH export_rows AS ( + SELECT + g.granule_id, + c.name AS collection, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMM') AS yyyymm, + to_char( + g.beginning_date_time AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) AS beginning_date_time, + jsonb_agg( + jsonb_build_object( + 'name', f.file_name, + 'type', f.type, + 'uri', 's3://' || f.bucket || '/' || f.key, + 'size', f.file_size, + 'checksum', f.checksum_value, + 'checksumType', 'md5' + ) + ORDER BY f.file_name + ) AS files + FROM public.granules g + JOIN public.collections c + ON c.cumulus_id = g.collection_cumulus_id + JOIN public.files f + ON f.granule_cumulus_id = g.cumulus_id + AND f.checksum_value IS NOT NULL + WHERE g.status = 'completed' + AND c.name = :'collection' + + AND g.beginning_date_time >= to_date(:'yyyymmdd', 'YYYYMMDD') + AND g.beginning_date_time < ( + to_date(:'yyyymmdd', 'YYYYMMDD') + interval '1 day' + ) AT TIME ZONE 'UTC' + GROUP BY + g.granule_id, + c.name, + to_char(g.beginning_date_time AT TIME ZONE 'UTC', 'YYYYMM'), + to_char( + g.beginning_date_time AT TIME ZONE 'UTC', + 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ) +) +SELECT jsonb_strip_nulls( + jsonb_build_object( + 'pk', collection || '#' || yyyymm, + 'sk', beginning_date_time || '#' || granule_id, + + 'gsi1pk', granule_id, + 'gsi1sk', collection || '#' || yyyymm, + + 'collection', collection, + 'yyyymm', yyyymm, + 'beginning_date_time', beginning_date_time, + 'granule_id', granule_id, + 'load_test_count', 0, + + 'n', granule_id, + 'c', collection, + 'cv', '1', + 'f', files + ) +) +FROM export_rows +ORDER BY beginning_date_time, granule_id; diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sh b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh new file mode 100644 index 0000000..3599117 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sh @@ -0,0 +1,95 @@ + +set -euo pipefail + + +#ANTPAT', +#BFPQ', +#CORNER_REFL', +#DCOP', +#DC_RADAR', +#DSG_STATIC', +#EA_L0B_L_CRSD', +#EA_L0B_L_RRSD', +#EA_L1_L_RIFG', +#EA_L1_L_ROFF', +#EA_L1_L_RSLC', +#EA_L1_L_RUNW',-- +#EA_L2_L_GCOV', +#EA_L2_L_GOFF', +#EA_L2_L_GSLC', +#EA_L2_L_GUNW', +#EA_L3_L_SME2', +#FOE', +#FRP', +#FT_PARAM', +#FT_WAVEFORM', +#LRCLK_UTC', +#L_CHAN_DATA', +#MOE', +#NISAR_DEM_TIF', +#NISAR_DEM_VRT', +#NISAR_L0A_RRST_BETA_V1', +#NISAR_L0B_CRSD_BETA_V1', +#NISAR_L0B_RRSD_BETA_V1', +#NISAR_L1_RIFG_BETA_V1', +#NISAR_L1_ROFF_BETA_V1', +#NISAR_L1_RSLC_BETA_V1', +#NISAR_L1_RUNW_BETA_V1', +#NISAR_L2_GCOV_BETA_V1', +#NISAR_L2_GOFF_BETA_V1', +#NISAR_L2_GSLC_BETA_V1', +#NISAR_L2_GUNW_BETA_V1', +#NISAR_L3_SME2_BETA_V1', +#NISAR_VWC', +#NISAR_WATERMASK_TIF', +#NISAR_WATERMASK_VRT', +#NOE', +#NRP', +#OROST', +#PA_L0B_L_RRSD', +#POE', +#PRP', +#STUF', +#TEC', +#TFDB', +#UR_L0B_L_RRSD', +# Opera collections: +#COLLECTION="OPERA_L2_CSLC-S1_V1" +#COLLECTION="OPERA_L2_RTC-S1_V1" +#COLLECTION="OPERA_L3_DISP-S1_V1" +#COLLECTION="OPERA_L3_DIST-ALERT-S1_V1" +#COLLECTION="OPERA_L4_TROPO-ZENITH_V1" + +echo "SLICES_S3_URI: ${SLICES_S3_URI}" +echo "COLLECTION: ${COLLECTION}" + +if [[ -n "${SLICES_S3_URI:-}" ]]; then + echo "SLICES_S3_URI is specified: ${SLICES_S3_URI}" + echo "Downloading specified slices file instead of querying database..." + source ./refresh-ctorm-upload-role.sh + aws s3 cp "${SLICES_S3_URI}" slices_daily.tsv +else + # Generate slices TSV file + echo "SLICES_S3_URI not set. Going to run a big query now..." + psql \ + --host="$PGHOST" \ + --port="${PGPORT:-5432}" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --quiet \ + --no-psqlrc \ + --tuples-only \ + --no-align \ + --field-separator $'\t' \ + --set=ON_ERROR_STOP=1 \ + --set=collection="$COLLECTION" \ + --file=generate-slices-daily.sql \ + > slices_daily.tsv + + source ./refresh-ctorm-upload-role.sh + + # Upload slices TSV file to S3 for archive purposes. + SLICES_S3_UPLOAD_URI="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}/slices_daily-${COLLECTION}-${CODEBUILD_BUILD_NUMBER}.tsv" + aws s3 cp slices_daily.tsv "${SLICES_S3_UPLOAD_URI}" + echo "Uploaded ${SLICES_S3_UPLOAD_URI}" +fi diff --git a/ctorm/cumulus-db-md-extract/generate-slices-daily.sql b/ctorm/cumulus-db-md-extract/generate-slices-daily.sql new file mode 100644 index 0000000..ae81935 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/generate-slices-daily.sql @@ -0,0 +1,21 @@ +SELECT + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') AS YYYYMMDD, + COUNT(DISTINCT G.CUMULUS_ID) AS GRANULE_COUNT, + COUNT(F.CUMULUS_ID) AS FILE_COUNT +FROM + PUBLIC.GRANULES G + JOIN PUBLIC.COLLECTIONS C + ON C.CUMULUS_ID = G.COLLECTION_CUMULUS_ID + JOIN PUBLIC.FILES F + ON F.GRANULE_CUMULUS_ID = G.CUMULUS_ID + AND F.CHECKSUM_VALUE IS NOT NULL +WHERE + G.STATUS = 'completed' + AND C.NAME = :'collection' +GROUP BY + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD') +ORDER BY + C.NAME, + TO_CHAR(G.BEGINNING_DATE_TIME AT TIME ZONE 'UTC', 'YYYYMMDD'); diff --git a/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh new file mode 100644 index 0000000..ced1a23 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/refresh-ctorm-upload-role.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${CTORM_S3_ROLE_ARN:-}" ]]; then + echo "CTORM_S3_ROLE_ARN is required" >&2 + exit 1 +fi + +echo "Refreshing CTORM upload role credentials..." >&2 + +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +CREDS_JSON="$(aws sts assume-role \ + --role-arn "${CTORM_S3_ROLE_ARN}" \ + --role-session-name "cumulus-db-md-extract-${CODEBUILD_BUILD_NUMBER:-manual}-$(date +%s)")" + +export AWS_ACCESS_KEY_ID="$(echo "${CREDS_JSON}" | jq -r '.Credentials.AccessKeyId')" +export AWS_SECRET_ACCESS_KEY="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SecretAccessKey')" +export AWS_SESSION_TOKEN="$(echo "${CREDS_JSON}" | jq -r '.Credentials.SessionToken')" + +# aws sts get-caller-identity >&2 diff --git a/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl b/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..8e5ba2e --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/.terraform.lock.hcl @@ -0,0 +1,45 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.0" + hashes = [ + "h1:WB6H5ksIZiyq1lQlD/PWeh+tn4FLsbSjVnRW3+4xe2Y=", + "zh:0d14713fdc259fb377d0b899ad3c650a34194bd52194c863303ef22a65a580e2", + "zh:369b56040c7a8085d04e7e8ffac1e2b321a3170e502f788819bc34b868ec016f", + "zh:4d1a3b983ed6af5a52bfe12794674ae55cbadfa6021b37106ade68b433ad216a", + "zh:5c547549e26e083573c78a966ca68ce6d7df6bb8f3948f66a575f07da46b74ea", + "zh:6de093e62a975eb19a5e3017ce38e6e3cb639c17b79648d2000e0a8348f0e997", + "zh:7267936c2cdbc448efeb594d73e6b56a53d6a7ae14fe88cdd2a4133adc3302f0", + "zh:7482f023050ed426b4b45116e1761643bc33b1fd4ce4a6fab207ae2571f35940", + "zh:76bbd93b234e5a2927d98b511d86565700f549b570871a194c35f944b96cefb7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:c6afc4bc1f002bac9c173007dd4da05fde788cd14c2916089f958c33fedb0dfa", + "zh:d3ba40bd806a3a08e9237dece679193c99afb2085de6b45d7f5d1f673cfcd368", + "zh:e1ad7ded53ecd6f0e5b473a3b44eae2b2e885653a56050ab583d387332be02e4", + "zh:e93e78575ce82be6084cc153c24ba8f385dc8d6880888ee66e918460c870953d", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.52.0" + hashes = [ + "h1:lpXqosKH8yAahK3SA1P5Pdy1ziXJcY+blUidY0q9yGk=", + "zh:1ab1d78f2336fed42b4e13fa0077a0be9d86a7899897cda5b9f1a60051ec2e93", + "zh:1df11f5f252030803939a1c778931dbdcee1b1070b38b98d9a8cbfc26c2aeb8e", + "zh:20ec9af03c0c1f2f8582a8805d43a4cd1a0a65082308e00f025d87605715a88f", + "zh:2d5562ed0e7cb0892fb537c7989d25fdde1d0ac1f3a768ba15fa087985f2a0f5", + "zh:40d64f668961a172355c3d11d258e19172ffafb421c40d89266b129ed8f92a5d", + "zh:497792bccc33001247473bc32a148c067982cb3d245b8f3610afb920635d2235", + "zh:8011f9167082af74ed9257582a83d54e889388656597721b2691797f1dd6fb58", + "zh:8b10d1bbca51b0da1e1be0967ce75b039f2d5d86f2ca7339994a92d35cdf47a8", + "zh:9549647dbf7c913512c26fed6badd7bf24a29a36c2bd308536849303a85427da", + "zh:97c4b726cdbe48166f4b9e6a1230312a7933dc19dd139d941ca6aca86706eb33", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a484bc8166278b546bfe53698fa9e0a919dffe3bfda3c8bf8a0fc6811b5b9e66", + "zh:aa96e76bd9f93e5395a553fccec485554a74acae46b3834b1296374eba4f010f", + "zh:cf290ee3d0dfe91b596445f860440d9f18826f7395ff785f83f70d36cedadd1c", + "zh:d56b6a79207663673f66d9ed378d705c1bd1b4d117eeb5e2be3938cf1b75b7be", + "zh:febef068317f8e49bd438ccdf2f54345fc3bc4b80a2699d9ab3c449d7e521d8e", + ] +} diff --git a/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf new file mode 100644 index 0000000..aeb8efe --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/codebuild-db-md-extract.tf @@ -0,0 +1,350 @@ +# codebuild-db-md-extract.tf + +variable "name_prefix" { + type = string + default = "ctorm" +} + +variable "aws_region" { + type = string + default = "us-west-2" +} + +variable "db_md_extract_secret_arn" { + type = string + description = "ARN of the Secrets Manager secret containing host, username, password, and database in the RDS account." +} + +variable "db_md_extract_vpc_id" { + type = string + description = "VPC where CodeBuild should run to reach the RDS instance." +} + +variable "db_md_extract_subnet_ids" { + type = list(string) + description = "Private subnet IDs that can reach the RDS instance." +} + +variable "db_md_extract_security_group_ids" { + type = list(string) + description = "Security group IDs for CodeBuild. These must allow egress to the RDS port and be allowed by the RDS SG." +} + +variable "db_md_extract_dump_subdir" { + type = string + default = "nisar" +} + +variable "db_md_extract_ctorm_bucket" { + type = string + default = "ctorm-dev-scratch" + description = "Destination CTORM S3 bucket for exported dump files." +} + +variable "db_md_extract_ctorm_s3_role_arn" { + type = string + description = "Role ARN in the CTORM account that CodeBuild can assume to write exported files." +} + +locals { + db_md_extract_project_name = "${var.name_prefix}-cumulus-db-md-extract" + db_md_extract_source_key = "codebuild/cumulus-db-md-extract.zip" +} + +resource "aws_s3_bucket" "codebuild_source" { + bucket = "${local.db_md_extract_project_name}-source-${substr(data.aws_caller_identity.current.account_id, 8, 4)}" + force_destroy = true + +} + +resource "aws_s3_bucket_ownership_controls" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "codebuild_source" { + bucket = aws_s3_bucket.codebuild_source.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +data "aws_caller_identity" "current" {} + +data "archive_file" "cumulus_db_md_extract" { + type = "zip" + source_dir = "${path.module}/.." + output_path = "${path.module}/.terraform/cumulus-db-md-extract.zip" + + excludes = [ + "terraform", + "terraform/*", + ".terraform", + ".terraform/*", + ] +} + +resource "aws_s3_object" "cumulus_db_md_extract_source" { + bucket = aws_s3_bucket.codebuild_source.bucket + key = local.db_md_extract_source_key + source = data.archive_file.cumulus_db_md_extract.output_path + source_hash = data.archive_file.cumulus_db_md_extract.output_base64sha256 +} + +data "aws_iam_policy_document" "codebuild_db_md_extract_assume_role" { + statement { + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "Service" + + identifiers = [ + "codebuild.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role" "codebuild_db_md_extract" { + name = "${local.db_md_extract_project_name}-role" + assume_role_policy = data.aws_iam_policy_document.codebuild_db_md_extract_assume_role.json +} + +resource "aws_cloudwatch_log_group" "codebuild_db_md_extract" { + name = "/aws/codebuild/${local.db_md_extract_project_name}" + retention_in_days = 14 +} + +data "aws_iam_policy_document" "codebuild_db_md_extract" { + statement { + sid = "WriteCodeBuildLogs" + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = [ + "${aws_cloudwatch_log_group.codebuild_db_md_extract.arn}:*", + ] + } + + statement { + sid = "ReadBuildSource" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + ] + + resources = [ + "${aws_s3_bucket.codebuild_source.arn}/${local.db_md_extract_source_key}", + ] + } + + statement { + sid = "ListBuildSourceBucket" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.codebuild_source.arn, + ] + + condition { + test = "StringLike" + variable = "s3:prefix" + + values = [ + "codebuild/*", + ] + } + } + + statement { + sid = "ReadDatabaseSecret" + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ] + + resources = [ + var.db_md_extract_secret_arn, + ] + } + + statement { + sid = "AssumeCtormAccountUploadRole" + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + resources = [ + var.db_md_extract_ctorm_s3_role_arn, + ] + } + + statement { + sid = "DescribeNetworkForVpcBuild" + effect = "Allow" + + actions = [ + "ec2:DescribeDhcpOptions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:CreateNetworkInterface", + ] + + resources = [ + "*", + ] + } + + statement { + sid = "CreateCodeBuildNetworkInterfacePermission" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterfacePermission", + ] + + resources = [ + "arn:aws:ec2:${var.aws_region}:*:network-interface/*", + ] + + condition { + test = "StringEquals" + variable = "ec2:AuthorizedService" + + values = [ + "codebuild.amazonaws.com", + ] + } + } +} + +resource "aws_iam_role_policy" "codebuild_db_md_extract" { + name = "${local.db_md_extract_project_name}-policy" + role = aws_iam_role.codebuild_db_md_extract.id + policy = data.aws_iam_policy_document.codebuild_db_md_extract.json +} + +resource "aws_codebuild_project" "cumulus_db_md_extract" { + name = local.db_md_extract_project_name + description = "One-time Cumulus RDS metadata export to CTORM S3" + service_role = aws_iam_role.codebuild_db_md_extract.arn + build_timeout = 480 + + artifacts { + type = "NO_ARTIFACTS" + } + + source { + type = "S3" + location = "${aws_s3_bucket.codebuild_source.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" + buildspec = "buildspec.yaml" + } + + environment { + type = "LINUX_CONTAINER" + image = "aws/codebuild/amazonlinux2-x86_64-standard:5.0" + compute_type = "BUILD_GENERAL1_SMALL" + + environment_variable { + name = "AWS_DEFAULT_REGION" + value = var.aws_region + } + + environment_variable { + name = "CTORM_BUCKET" + value = var.db_md_extract_ctorm_bucket + } + + environment_variable { + name = "DUMP_SUBDIR" + value = var.db_md_extract_dump_subdir + } + + environment_variable { + name = "DB_SECRET_ARN" + value = var.db_md_extract_secret_arn + } + + environment_variable { + name = "CTORM_S3_ROLE_ARN" + value = var.db_md_extract_ctorm_s3_role_arn + } + + environment_variable { + name = "COLLECTION" + value = "" + } + + environment_variable { + name = "SLICES_S3_URI" + value = "" + } + } + + vpc_config { + vpc_id = var.db_md_extract_vpc_id + subnets = var.db_md_extract_subnet_ids + security_group_ids = var.db_md_extract_security_group_ids + } + + logs_config { + cloudwatch_logs { + group_name = aws_cloudwatch_log_group.codebuild_db_md_extract.name + status = "ENABLED" + } + } + + depends_on = [ + aws_iam_role_policy.codebuild_db_md_extract, + aws_s3_object.cumulus_db_md_extract_source, + ] +} + +output "cumulus_db_md_extract_codebuild_project_name" { + value = aws_codebuild_project.cumulus_db_md_extract.name +} + +output "cumulus_db_md_extract_codebuild_role_arn" { + value = aws_iam_role.codebuild_db_md_extract.arn +} + +output "cumulus_db_md_extract_source_s3_uri" { + value = "s3://${aws_s3_bucket.codebuild_source.bucket}/${aws_s3_object.cumulus_db_md_extract_source.key}" +} diff --git a/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars new file mode 100644 index 0000000..ac07320 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/nisar-prod.tfvars @@ -0,0 +1,21 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + +db_md_extract_vpc_id = "vpc-05bb1a5fa3d38fddd" + +db_md_extract_subnet_ids = [ + "subnet-028b8ab53b5232fa5", +] + +db_md_extract_security_group_ids = [ + "sg-01b8118dc0c9ea180", + "sg-0e56e1310879eaef3", + "sg-0a4f1b9d764ef7b38", +] + +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + +db_md_extract_dump_subdir = "nisar" + +db_md_extract_ctorm_bucket = "ctorm-scratch" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars new file mode 100644 index 0000000..353e7c9 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/opera-prod.tfvars @@ -0,0 +1,22 @@ +name_prefix = "ctorm" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + +db_md_extract_vpc_id = "vpc-0e36a2fca2e2a73d5" + +db_md_extract_subnet_ids = [ + "subnet-02dda0af17b4a0d19", +] + +db_md_extract_security_group_ids = [ + "sg-0249655cac679df2c", + "sg-0ed18cd0fb6e79f58", + "sg-0ed18cd0fb6e79f58", +] + +# Look at the ctorm tf output for `cumulus_db_md_extract_upload_role_arn` +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + +db_md_extract_dump_subdir = "opera" + +db_md_extract_ctorm_bucket = "ctorm-scratch" diff --git a/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars new file mode 100644 index 0000000..ac198b3 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/opera-uat.tfvars @@ -0,0 +1,19 @@ +name_prefix = "ctorm-opera-uat" + +db_md_extract_secret_arn = "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + +db_md_extract_vpc_id = "vpc-080322e163a0e7390" + +db_md_extract_subnet_ids = [ + "subnet-0ac04d1e36d0a50ea", +] + +db_md_extract_security_group_ids = [ + "sg-03beee748d2a43cc9", + "sg-0afdcb46b1a38db46", + "sg-06255aca3a87784a0", +] + +db_md_extract_ctorm_s3_role_arn = "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + +db_md_extract_dump_subdir = "opera-uat" diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate new file mode 100644 index 0000000..6ffa2ce --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-nisar-prod.tfstate @@ -0,0 +1,763 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 23, + "lineage": "2aab81d2-395f-d5fa-becf-2772136eaa03", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "41bd43108980d112c9deba5135abfc48dc0f4e63", + "output_base64sha256": "X7mJWPMPJp0AvEIb4z20oix4IxI8yBgrZjrukcaMtQY=", + "output_base64sha512": "nc7b+wBbmYgZU4fEDXvkye+F6a7NM51R2dqfTRyn6CZSy9YVWE3XogIuIQIguytQs1PnDCsGyEqTAY7qxHpMgg==", + "output_file_mode": null, + "output_md5": "d3becd0cd2b647b5c5542370840070f1", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "41bd43108980d112c9deba5135abfc48dc0f4e63", + "output_sha256": "5fb98958f30f269d00bc421be33db4a22c7823123cc8182b663aee91c68cb506", + "output_sha512": "9dcedbfb005b9988195387c40d7be4c9ef85e9aecd339d51d9da9f4d1ca7e82652cbd615584dd7a2022e210220bb2b50b353e70c2b06c84a93018eeac47a4c82", + "output_size": 7005, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "372059463218", + "arn": "arn:aws:iam::372059463218:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "372059463218", + "user_id": "AIDAVNIDY3YZHQAL4QVMF" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "202837701", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:372059463218:project/ctorm-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:372059463218:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "nisar" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_URI", + "type": "PLAINTEXT", + "value": "" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:372059463218:project/ctorm-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-01b8118dc0c9ea180", + "sg-0a4f1b9d764ef7b38", + "sg-0e56e1310879eaef3" + ], + "subnets": [ + "subnet-028b8ab53b5232fa5" + ], + "vpc_id": "vpc-05bb1a5fa3d38fddd" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::372059463218:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T19:31:14Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROAVNIDY3YZPQXYNWXSP" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:372059463218:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:372059463218:secret:sds-n-cumulus-prod_db_login20220310213550922100000002-IqFhGT\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218", + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-3218.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-3218.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "d6f242167244d0b7b820212432e41dbdcbde816cf4992ec7f7967eab0154eab5", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-3218", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-3218", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "d3becd0cd2b647b5c5542370840070f1", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-3218/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "X7mJWPMPJp0AvEIb4z20oix4IxI8yBgrZjrukcaMtQY=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate new file mode 100644 index 0000000..84f099f --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-prod.tfstate @@ -0,0 +1,762 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 142, + "lineage": "a3a663d5-6583-392e-e2a7-75d38c4c2886", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "a96216dceaf9e6e503bf02981b53784e03ef13b8", + "output_base64sha256": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=", + "output_base64sha512": "RLp6HJBUgfaHuTiqD2qyQKwBJyHtvZ+/VDK/urPEE58Gf5Md2MDIJeCVoQIux45wacZU5HiEe/eoFtx28IitLw==", + "output_file_mode": null, + "output_md5": "5a4c52cf01416b48350304df429135a5", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "a96216dceaf9e6e503bf02981b53784e03ef13b8", + "output_sha256": "8882f14372977b33e1aa89c08d14ad49217280b96c1c984d91c0f02f7042b750", + "output_sha512": "44ba7a1c905481f687b938aa0f6ab240ac012721edbd9fbf5432bfbab3c4139f067f931dd8c0c825e095a1022ec78e7069c654e478847bf7a816dc76f088ad2f", + "output_size": 7000, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "510296831643", + "arn": "arn:aws:iam::510296831643:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "510296831643", + "user_id": "AIDAXNUAPW2NZFPXV6JKY" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3636791408", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:510296831643:project/ctorm-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:510296831643:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "opera" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_URI", + "type": "PLAINTEXT", + "value": "" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:510296831643:project/ctorm-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-0249655cac679df2c", + "sg-0ed18cd0fb6e79f58" + ], + "subnets": [ + "subnet-02dda0af17b4a0d19" + ], + "vpc_id": "vpc-0e36a2fca2e2a73d5" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::510296831643:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T03:25:11Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROAXNUAPW2N44OBYZNKD" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-cumulus-db-md-extract-role:ctorm-cumulus-db-md-extract-policy", + "name": "ctorm-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:510296831643:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:510296831643:secret:asf-cumulus-prod_db_login20210805193117427100000001-PKZnmc\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643", + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-1643.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-1643.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "a055848272cdca6935f41c8d486234d66c39ff78b8f9ef89db0d949987311ce3", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-1643", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-1643", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "5a4c52cf01416b48350304df429135a5", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-1643/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "iILxQ3KXezPhqonAjRStSSFygLlsHJhNkcDwL3BCt1A=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate new file mode 100644 index 0000000..bdc7311 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform-opera-uat.tfstate @@ -0,0 +1,763 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 69, + "lineage": "f677a0d8-b435-1d95-bf19-4cb403076d4e", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-opera-uat-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "2ea86e5ad0e886c9ead4d329ab2a5a1e5025bc8f", + "output_base64sha256": "PvweHBk8zXkKLm/Sf7cg9BQCZO4d5G7XeJiAdyjkEBs=", + "output_base64sha512": "PxlBXxsKqs9FBH1t/uUkUhmh7jm22Jluqzp7HD9CzqGMktLgIoGwqYQiZwHgo9zBcjFaTo1AIvEUmj2OTQUe5g==", + "output_file_mode": null, + "output_md5": "72ff56f7afe5d2511991649ddca2a839", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "2ea86e5ad0e886c9ead4d329ab2a5a1e5025bc8f", + "output_sha256": "3efc1e1c193ccd790a2e6fd27fb720f4140264ee1de46ed77898807728e4101b", + "output_sha512": "3f19415f1b0aaacf45047d6dfee5245219a1ee39b6d8996eab3a7b1c3f42cea18c92d2e02281b0a984226701e0a3dcc172315a4e8d4022f1149a3d8e4d051ee6", + "output_size": 6923, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "097260566921", + "arn": "arn:aws:iam::097260566921:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "097260566921", + "user_id": "AIDARNJJOPGERLGUKQ553" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "2729057441", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": {}, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_codebuild_project", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-opera-uat-cumulus-db-md-extract", + "artifacts": [ + { + "artifact_identifier": "", + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "name": "", + "namespace_type": "", + "override_artifact_name": false, + "packaging": "", + "path": "", + "type": "NO_ARTIFACTS" + } + ], + "auto_retry_limit": 0, + "badge_enabled": false, + "badge_url": "", + "build_batch_config": [], + "build_timeout": 480, + "cache": [ + { + "cache_namespace": "", + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "concurrent_build_limit": 0, + "description": "One-time Cumulus RDS metadata export to CTORM S3", + "encryption_key": "arn:aws:kms:us-west-2:097260566921:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "docker_server": [], + "environment_variable": [ + { + "name": "AWS_DEFAULT_REGION", + "type": "PLAINTEXT", + "value": "us-west-2" + }, + { + "name": "CTORM_BUCKET", + "type": "PLAINTEXT", + "value": "ctorm-dev-scratch" + }, + { + "name": "DUMP_SUBDIR", + "type": "PLAINTEXT", + "value": "opera-uat" + }, + { + "name": "DB_SECRET_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385" + }, + { + "name": "CTORM_S3_ROLE_ARN", + "type": "PLAINTEXT", + "value": "arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload" + }, + { + "name": "COLLECTION", + "type": "PLAINTEXT", + "value": "" + }, + { + "name": "SLICES_S3_PATH", + "type": "PLAINTEXT", + "value": "" + } + ], + "fleet": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "file_system_locations": [], + "id": "arn:aws:codebuild:us-west-2:097260566921:project/ctorm-opera-uat-cumulus-db-md-extract", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "bucket_owner_access": "", + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "ctorm-opera-uat-cumulus-db-md-extract", + "project_visibility": "PRIVATE", + "public_project_alias": "", + "queued_timeout": 480, + "region": "us-west-2", + "resource_access_role": "", + "secondary_artifacts": [], + "secondary_source_version": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "source": [ + { + "auth": [], + "build_status_config": [], + "buildspec": "buildspec.yaml", + "git_clone_depth": 0, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "report_build_status": false, + "type": "S3" + } + ], + "source_version": "", + "tags": {}, + "tags_all": {}, + "vpc_config": [ + { + "security_group_ids": [ + "sg-03beee748d2a43cc9", + "sg-06255aca3a87784a0", + "sg-0afdcb46b1a38db46" + ], + "subnets": [ + "subnet-0ac04d1e36d0a50ea" + ], + "vpc_id": "vpc-080322e163a0e7390" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_iam_role_policy.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "aws_s3_object.cumulus_db_md_extract_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-opera-uat-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-24T23:12:29Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-opera-uat-cumulus-db-md-extract-role", + "inline_policy": [ + { + "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-opera-uat-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": {}, + "tags_all": {}, + "unique_id": "AROARNJJOPGE2BF3AA7RB" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "ctorm-opera-uat-cumulus-db-md-extract-role:ctorm-opera-uat-cumulus-db-md-extract-policy", + "name": "ctorm-opera-uat-cumulus-db-md-extract-policy", + "name_prefix": "", + "policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-opera-uat-cumulus-db-md-extract:*\",\"Sid\":\"WriteCodeBuildLogs\"},{\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\",\"Sid\":\"ReadBuildSource\"},{\"Action\":\"s3:ListBucket\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921\",\"Sid\":\"ListBuildSourceBucket\"},{\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Effect\":\"Allow\",\"Resource\":\"arn:aws:secretsmanager:us-west-2:097260566921:secret:asf-cumulus-test_db_login20210811234727262600000001-Tks385\",\"Sid\":\"ReadDatabaseSecret\"},{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:iam::871271927522:role/ctorm-dev-cumulus-db-md-extract-upload\",\"Sid\":\"AssumeCtormAccountUploadRole\"},{\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Effect\":\"Allow\",\"Resource\":\"*\",\"Sid\":\"DescribeNetworkForVpcBuild\"},{\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}},\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\"}]}", + "role": "ctorm-opera-uat-cumulus-db-md-extract-role" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_cloudwatch_log_group.codebuild_db_md_extract", + "aws_iam_role.codebuild_db_md_extract", + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current", + "data.aws_iam_policy_document.codebuild_db_md_extract", + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket_domain_name": "ctorm-opera-uat-cumulus-db-md-extract-source-6921.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-opera-uat-cumulus-db-md-extract-source-6921.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "expected_bucket_owner": "", + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-opera-uat-cumulus-db-md-extract-source-6921", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "72ff56f7afe5d2511991649ddca2a839", + "force_destroy": false, + "id": "ctorm-opera-uat-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": {}, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "PvweHBk8zXkKLm/Sf7cg9BQCZO4d5G7XeJiAdyjkEBs=", + "storage_class": "STANDARD", + "tags": {}, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate new file mode 100644 index 0000000..b7216a0 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfstate.d/opera-prod/terraform.tfstate @@ -0,0 +1,563 @@ +{ + "version": 4, + "terraform_version": "1.9.2", + "serial": 10, + "lineage": "9578e429-f029-9119-8845-752daa480ee7", + "outputs": { + "cumulus_db_md_extract_codebuild_project_name": { + "value": "ctorm-cumulus-db-md-extract", + "type": "string" + }, + "cumulus_db_md_extract_codebuild_role_arn": { + "value": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "type": "string" + }, + "cumulus_db_md_extract_source_s3_uri": { + "value": "s3://ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "type": "string" + } + }, + "resources": [ + { + "mode": "data", + "type": "archive_file", + "name": "cumulus_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/archive\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "exclude_symlink_directories": null, + "excludes": [ + ".terraform", + ".terraform/*", + "terraform", + "terraform/*" + ], + "id": "77650f5f87b9d429f0ae3ee4141a2d4408515477", + "output_base64sha256": "W1Cg6btOTGKbYpBaANU0oW3rd3EKlsQKsMP1/ARFPnY=", + "output_base64sha512": "TqEppCWs6YEod3UPDPcniu7MgiOD3UWx1JXpUkKFr476zHH0D6UVkqJOXuR8seBnRlq9HgydFHLTEcTYFps4Sw==", + "output_file_mode": null, + "output_md5": "34b9f1764b49aa11cd3c0f0cd3db5538", + "output_path": "./.terraform/cumulus-db-md-extract.zip", + "output_sha": "77650f5f87b9d429f0ae3ee4141a2d4408515477", + "output_sha256": "5b50a0e9bb4e4c629b62905a00d534a16deb77710a96c40ab0c3f5fc04453e76", + "output_sha512": "4ea129a425ace9812877750f0cf7278aeecc822383dd45b1d495e9524285af8efacc71f40fa51592a24e5ee47cb1e067465abd1e0c9d1472d311c4d8169b384b", + "output_size": 5323, + "source": [], + "source_content": null, + "source_content_filename": null, + "source_dir": "./..", + "source_file": null, + "type": "zip" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "account_id": "097260566921", + "arn": "arn:aws:iam::097260566921:user/NGAPShApplicationDeveloper-bbarton1-941", + "id": "097260566921", + "user_id": "AIDARNJJOPGERLGUKQ553" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "3830342824", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"WriteCodeBuildLogs\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"logs:PutLogEvents\",\n \"logs:CreateLogStream\"\n ],\n \"Resource\": \"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"\n },\n {\n \"Sid\": \"ReadBuildSource\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:GetObjectVersion\",\n \"s3:GetObject\"\n ],\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"\n },\n {\n \"Sid\": \"ListBuildSourceBucket\",\n \"Effect\": \"Allow\",\n \"Action\": \"s3:ListBucket\",\n \"Resource\": \"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921\",\n \"Condition\": {\n \"StringLike\": {\n \"s3:prefix\": \"codebuild/*\"\n }\n }\n },\n {\n \"Sid\": \"ReadDatabaseSecret\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"secretsmanager:GetSecretValue\",\n \"secretsmanager:DescribeSecret\"\n ],\n \"Resource\": \"arn:aws:secretsmanager:us-west-2:\\u003cremote-account-id\\u003e:secret:\\u003csecret-name\\u003e\"\n },\n {\n \"Sid\": \"AssumeCtormAccountUploadRole\",\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::...\"\n },\n {\n \"Sid\": \"DescribeNetworkForVpcBuild\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:DescribeVpcs\",\n \"ec2:DescribeSubnets\",\n \"ec2:DescribeSecurityGroups\",\n \"ec2:DescribeRouteTables\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeDhcpOptions\",\n \"ec2:DeleteNetworkInterface\",\n \"ec2:CreateNetworkInterface\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Sid\": \"CreateCodeBuildNetworkInterfacePermission\",\n \"Effect\": \"Allow\",\n \"Action\": \"ec2:CreateNetworkInterfacePermission\",\n \"Resource\": \"arn:aws:ec2:us-west-2:*:network-interface/*\",\n \"Condition\": {\n \"StringEquals\": {\n \"ec2:AuthorizedService\": \"codebuild.amazonaws.com\"\n }\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"WriteCodeBuildLogs\",\"Effect\":\"Allow\",\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*\"},{\"Sid\":\"ReadBuildSource\",\"Effect\":\"Allow\",\"Action\":[\"s3:GetObjectVersion\",\"s3:GetObject\"],\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip\"},{\"Sid\":\"ListBuildSourceBucket\",\"Effect\":\"Allow\",\"Action\":\"s3:ListBucket\",\"Resource\":\"arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921\",\"Condition\":{\"StringLike\":{\"s3:prefix\":\"codebuild/*\"}}},{\"Sid\":\"ReadDatabaseSecret\",\"Effect\":\"Allow\",\"Action\":[\"secretsmanager:GetSecretValue\",\"secretsmanager:DescribeSecret\"],\"Resource\":\"arn:aws:secretsmanager:us-west-2:\\u003cremote-account-id\\u003e:secret:\\u003csecret-name\\u003e\"},{\"Sid\":\"AssumeCtormAccountUploadRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Resource\":\"arn:aws:iam::...\"},{\"Sid\":\"DescribeNetworkForVpcBuild\",\"Effect\":\"Allow\",\"Action\":[\"ec2:DescribeVpcs\",\"ec2:DescribeSubnets\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeRouteTables\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DescribeDhcpOptions\",\"ec2:DeleteNetworkInterface\",\"ec2:CreateNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"CreateCodeBuildNetworkInterfacePermission\",\"Effect\":\"Allow\",\"Action\":\"ec2:CreateNetworkInterfacePermission\",\"Resource\":\"arn:aws:ec2:us-west-2:*:network-interface/*\",\"Condition\":{\"StringEquals\":{\"ec2:AuthorizedService\":\"codebuild.amazonaws.com\"}}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract:*" + ], + "sid": "WriteCodeBuildLogs" + }, + { + "actions": [ + "s3:GetObject", + "s3:GetObjectVersion" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip" + ], + "sid": "ReadBuildSource" + }, + { + "actions": [ + "s3:ListBucket" + ], + "condition": [ + { + "test": "StringLike", + "values": [ + "codebuild/*" + ], + "variable": "s3:prefix" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921" + ], + "sid": "ListBuildSourceBucket" + }, + { + "actions": [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:secretsmanager:us-west-2:\u003cremote-account-id\u003e:secret:\u003csecret-name\u003e" + ], + "sid": "ReadDatabaseSecret" + }, + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:iam::..." + ], + "sid": "AssumeCtormAccountUploadRole" + }, + { + "actions": [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeDhcpOptions", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "*" + ], + "sid": "DescribeNetworkForVpcBuild" + }, + { + "actions": [ + "ec2:CreateNetworkInterfacePermission" + ], + "condition": [ + { + "test": "StringEquals", + "values": [ + "codebuild.amazonaws.com" + ], + "variable": "ec2:AuthorizedService" + } + ], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [], + "resources": [ + "arn:aws:ec2:us-west-2:*:network-interface/*" + ], + "sid": "CreateCodeBuildNetworkInterfacePermission" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "data", + "type": "aws_iam_policy_document", + "name": "codebuild_db_md_extract_assume_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1229436035", + "json": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n }\n }\n ]\n}", + "minified_json": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}]}", + "override_json": null, + "override_policy_documents": null, + "policy_id": null, + "source_json": null, + "source_policy_documents": null, + "statement": [ + { + "actions": [ + "sts:AssumeRole" + ], + "condition": [], + "effect": "Allow", + "not_actions": [], + "not_principals": [], + "not_resources": [], + "principals": [ + { + "identifiers": [ + "codebuild.amazonaws.com" + ], + "type": "Service" + } + ], + "resources": [], + "sid": "" + } + ], + "version": "2012-10-17" + }, + "sensitive_attributes": [] + } + ] + }, + { + "mode": "managed", + "type": "aws_cloudwatch_log_group", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:logs:us-west-2:097260566921:log-group:/aws/codebuild/ctorm-cumulus-db-md-extract", + "deletion_protection_enabled": false, + "id": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "kms_key_id": "", + "log_group_class": "STANDARD", + "name": "/aws/codebuild/ctorm-cumulus-db-md-extract", + "name_prefix": "", + "region": "us-west-2", + "retention_in_days": 14, + "skip_destroy": false, + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "codebuild_db_md_extract", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::097260566921:role/ctorm-cumulus-db-md-extract-role", + "assume_role_policy": "{\"Statement\":[{\"Action\":\"sts:AssumeRole\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"}}],\"Version\":\"2012-10-17\"}", + "create_date": "2026-06-25T02:48:50Z", + "description": "", + "force_detach_policies": false, + "id": "ctorm-cumulus-db-md-extract-role", + "inline_policy": [], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "ctorm-cumulus-db-md-extract-role", + "name_prefix": "", + "path": "/", + "permissions_boundary": "", + "tags": null, + "tags_all": {}, + "unique_id": "AROARNJJOPGEVXWJQGTNS" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "data.aws_iam_policy_document.codebuild_db_md_extract_assume_role" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921", + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "bucket_domain_name": "ctorm-cumulus-db-md-extract-source-6921.s3.amazonaws.com", + "bucket_namespace": "global", + "bucket_prefix": "", + "bucket_region": "us-west-2", + "bucket_regional_domain_name": "ctorm-cumulus-db-md-extract-source-6921.s3.us-west-2.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "acf107f3e4152ef966bdafe272bf3bb6feacb31359eb02368d07dd590c64837a", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3BJ6K6RIION7M", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-west-2", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": null, + "tags_all": {}, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19", + "dependencies": [ + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_ownership_controls", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "object_ownership": "BucketOwnerEnforced" + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "block_public_acls": true, + "block_public_policy": true, + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "ignore_public_acls": true, + "region": "us-west-2", + "restrict_public_buckets": true, + "skip_destroy": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_server_side_encryption_configuration", + "name": "codebuild_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "expected_bucket_owner": "", + "id": "ctorm-cumulus-db-md-extract-source-6921", + "region": "us-west-2", + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "blocked_encryption_types": [ + "SSE-C" + ], + "bucket_key_enabled": false + } + ] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.aws_caller_identity.current" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_object", + "name": "cumulus_db_md_extract_source", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acl": null, + "arn": "arn:aws:s3:::ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "bucket": "ctorm-cumulus-db-md-extract-source-6921", + "bucket_key_enabled": false, + "cache_control": "", + "checksum_algorithm": null, + "checksum_crc32": "", + "checksum_crc32c": "", + "checksum_crc64nvme": "", + "checksum_sha1": "", + "checksum_sha256": "", + "content": null, + "content_base64": null, + "content_disposition": "", + "content_encoding": "", + "content_language": "", + "content_type": "application/octet-stream", + "etag": "34b9f1764b49aa11cd3c0f0cd3db5538", + "force_destroy": false, + "id": "ctorm-cumulus-db-md-extract-source-6921/codebuild/cumulus-db-md-extract.zip", + "key": "codebuild/cumulus-db-md-extract.zip", + "kms_key_id": null, + "metadata": null, + "object_lock_legal_hold_status": "", + "object_lock_mode": "", + "object_lock_retain_until_date": "", + "override_provider": [], + "region": "us-west-2", + "server_side_encryption": "AES256", + "source": "./.terraform/cumulus-db-md-extract.zip", + "source_hash": "W1Cg6btOTGKbYpBaANU0oW3rd3EKlsQKsMP1/ARFPnY=", + "storage_class": "STANDARD", + "tags": null, + "tags_all": {}, + "version_id": "", + "website_redirect": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.codebuild_source", + "data.archive_file.cumulus_db_md_extract", + "data.aws_caller_identity.current" + ] + } + ] + } + ], + "check_results": null +} diff --git a/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars b/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars new file mode 100644 index 0000000..d078194 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/terraform/terraform.tfvars @@ -0,0 +1 @@ +aws_region = "us-west-2" diff --git a/ctorm/cumulus-db-md-extract/test-s3-cp.sh b/ctorm/cumulus-db-md-extract/test-s3-cp.sh new file mode 100644 index 0000000..1f3c596 --- /dev/null +++ b/ctorm/cumulus-db-md-extract/test-s3-cp.sh @@ -0,0 +1,10 @@ +set -euo pipefail + + + +S3_URI_COPYTEST="s3://${CTORM_BUCKET}/cumulus-granules/${DUMP_SUBDIR}" +echo "Failing fast if we can't write to ${S3_URI_COPYTEST}..." +touch ./test-s3-write.txt +source ./refresh-ctorm-upload-role.sh +aws s3 cp ./test-s3-write.txt "${S3_URI_COPYTEST}/test-s3-write-${CODEBUILD_BUILD_NUMBER}.txt" +rm ./test-s3-write.txt diff --git a/ctorm/granule-md-db-loader/README.md b/ctorm/granule-md-db-loader/README.md new file mode 100644 index 0000000..d1812f4 --- /dev/null +++ b/ctorm/granule-md-db-loader/README.md @@ -0,0 +1,93 @@ +# Granule MD DB Loader + +## Deploying + +The infrastructure for this is provided by the [CTORM terraform](../infra/README.md) + +## Before running + +AWS throttles writes to dynamo DB if we insert repeatedly into the same partion, so when we can run +multiple collections at once, each going into their own partition. + +Additionally, running as PAY_PER_REQUEST will get us throttled no matter what. So for the loading of the granules, we +will switch the tf to use PROVISIONED. See the [CTORM infrastruture README](../infra/README.md) for the command to do +this. + +## Running + +```bash +# for dev: +export AWS_PROFILE="cumulus-sbx-7522" +export CODEBUILD_PROJECT="ctorm-dev-granule-md-db-loader" +export CTORM_BUCKET="ctorm-dev-scratch" + + +# get list of collections +aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/nisar/ |grep PRE +aws s3 ls s3://${CTORM_BUCKET}/cumulus-granules/opera/ |grep PRE + +# If you want do them all sequentially, not recommended and not going to work with all the data because throttling:: +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--query 'build.id' \ +--output text + + + +# Running a specific collection in parallel (by overriding the PREFIX): +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L2_CSLC-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L2_RTC-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DISP-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/opera/OPERA_L3_DIST-ALERT-S1_V1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +# nisar +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L0B_CRSD_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L0B_RRSD_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L1_/,type=PLAINTEXT \ +--query 'build.id' \ +--output text +aws codebuild start-build --profile="${AWS_PROFILE}" \ +--project-name "${CODEBUILD_PROJECT}" \ +--region "us-west-2" \ +--environment-variables-override name=PREFIX,value=cumulus-granules/nisar/NISAR_EA_L2/,type=PLAINTEXT \ +--query 'build.id' \ +--output text + +``` + +## After running + +Restore the main.tf to its original state with billing_mode set to PAY_PER_REQUEST, etc. diff --git a/ctorm/granule-md-db-loader/buildspec.yaml b/ctorm/granule-md-db-loader/buildspec.yaml new file mode 100644 index 0000000..d40dd5b --- /dev/null +++ b/ctorm/granule-md-db-loader/buildspec.yaml @@ -0,0 +1,16 @@ +version: 0.2 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - echo "Installing dependencies..." + - pip install boto3 + build: + commands: + - echo "Starting DynamoDB Import script..." + - python granule_md_db_loader.py + post_build: + commands: + - echo "Import finished." diff --git a/ctorm/granule-md-db-loader/granule_md_db_loader.py b/ctorm/granule-md-db-loader/granule_md_db_loader.py new file mode 100644 index 0000000..daf1c7f --- /dev/null +++ b/ctorm/granule-md-db-loader/granule_md_db_loader.py @@ -0,0 +1,201 @@ +import datetime +import gzip +import json +import logging +import os +import random +import sys +import time +from decimal import Decimal + +import boto3 +from botocore.config import Config + +# Configure logging +fmt = "%(asctime)s [%(levelname)s] %(message)s" +logging.basicConfig(level=logging.INFO, format=fmt) +log = logging.getLogger(__name__) + + +class PartitionRateLimiter: + """ + Limits the number of write requests to any single partition key (pk) + to prevent exceeding DynamoDB's physical limit of 1000 operations/sec per partition. + """ + + def __init__(self, max_rate_per_sec: int = 890): + self.max_rate = max_rate_per_sec + self.history = {} # pk -> list of timestamps + + def limit(self, pk): + now = time.time() + if pk not in self.history: + self.history[pk] = [] + + # Retain only timestamps from the last 1.0 second + self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] + + if len(self.history[pk]) >= self.max_rate: + # Calculate sleep duration to let the oldest request roll + # off the 1-second window + sleep_time = 1.0 - (now - self.history[pk][0]) + if sleep_time > 0: + time.sleep(sleep_time) + now = time.time() + self.history[pk] = [t for t in self.history[pk] if now - t < 1.0] + + self.history[pk].append(now) + + +def handle_file( + local_path: str, + table, + rate_limiter: PartitionRateLimiter, + records_in_file: int, + total_records: int, +): + with ( + gzip.open(local_path, "rt", encoding="utf-8") as f, + table.batch_writer() as batch, + ): + for line in f: + line = line.strip() + if not line: + continue + + try: + # Convert floats/doubles to Decimal for + # DynamoDB compatibility + item = json.loads(line, parse_float=Decimal) + except Exception as e: + log.error("Failed to parse JSON line: %s", e) + continue + + # Add/modify fields if needed + newdate = datetime.datetime.now(datetime.timezone.utc).isoformat() + "Z" + item["imported_at"] = newdate + + # Ensure partition and sort keys are present + pk = item.get("pk") + sk = item.get("sk") + if not pk or not sk: + log.warning( + "Skipping record missing pk/sk: %s", + item.get("granule_id"), + ) + + continue + + # Apply dynamic rate limit based on the partition key + rate_limiter.limit(pk) + + # Batch insert into DynamoDB + batch.put_item(Item=item) + records_in_file += 1 + total_records += 1 + return records_in_file, total_records + + +def main(): + bucket_name = os.environ.get("CTORM_BUCKET", "ctorm-scratch") + table_name = os.environ.get("TABLE_NAME") + prefix = os.environ.get("PREFIX", "cumulus-granules/") + + if not table_name: + log.error("TABLE_NAME environment variable is required.") + sys.exit(1) + + s3 = boto3.client("s3") + + # Configure boto3 with more aggressive retries to gracefully + # handle scale peaks + retry_config = Config( + retries={ + "max_attempts": 10, + "mode": "standard", + } + ) + dynamodb = boto3.resource("dynamodb", config=retry_config) + dyndb_table = dynamodb.Table(table_name) + + log.info( + "Starting import from s3://%s/%s into DynamoDB table %s", + bucket_name, + prefix, + table_name, + ) + + paginator = s3.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + total_files = 0 + total_records = 0 + + # Initialize partition-level rate limiter set to a safe + # threshold (850 writes/sec per pk) + rate_limiter = PartitionRateLimiter(max_rate_per_sec=850) + + # create random list of .json.gz objects: + # This will alleviate writing to the same partition for several files in a row. + jsongz_list = [] + for page in pages: + if "Contents" not in page: + continue + for obj in page["Contents"]: + key = obj["Key"] + if not key.endswith(".jsonl.gz"): + continue + jsongz_list.append(key) + total_files += 1 + + total_files = len(jsongz_list) + random.shuffle(jsongz_list) + + # Walk through the bucket objects + for key in jsongz_list: + log.info("Processing s3://%s/%s", bucket_name, key) + + # Download and decompress the file + local_path = "/tmp/temp.jsonl.gz" + try: + s3.download_file(bucket_name, key, local_path) + except Exception as e: + log.error( + "Failed to download s3://{%s}/%s: %s", + bucket_name, + key, + e, + ) + continue + + # Open, decompress and parse + records_in_file = 0 + try: + records_in_file, total_records = handle_file( + local_path, + dyndb_table, + rate_limiter, + records_in_file, + total_records, + ) + + log.info( + "Successfully imported %s records from %s", + records_in_file, + key, + ) + except Exception as e: + log.error("Error processing file %s: %s", key, e) + finally: + if os.path.exists(local_path): + os.remove(local_path) + + log.info( + "Import complete. Processed %s files, imported %s total records.", + total_files, + total_records, + ) + + +if __name__ == "__main__": + main() diff --git a/ctorm/infra/README.md b/ctorm/infra/README.md new file mode 100644 index 0000000..e9c6d92 --- /dev/null +++ b/ctorm/infra/README.md @@ -0,0 +1,46 @@ +# CTORM infrastructure + +```bash +export AWS_PROFILE="cumulus-sbx-7522" +export VARFILE=dev.tfvars + +# create workspaces and select the one to use +terraform workspace new ctorm-dev + +terraform workspace new ctorm-prod + +export TF_WORKSPACE=ctorm-dev + +terraform init + +terraform plan \ + -var-file="${VARFILE}" + +terraform apply \ + -var-file="${VARFILE}" + +terraform destroy +``` + +## Increasing DynamoDB table capacity + +For loading the DynamoDB table with the .jsonl files, we'll need to increase the capacity of the table. +AWS will throttle drastically if we don't. There is some limitation on how often we can switch between +PROVISIONED and PAY_PER_REQUEST, so don't do it willy-nilly. + +```bash +# For temporary use while loading granule metadata: +terraform plan \ + -var-file="${VARFILE}" \ + -var="granule_table_billing_mode=PROVISIONED" \ + -var="granule_table_write_capacity=5000" + +# To set it back to the default: +terraform plan \ + -var-file="${VARFILE}" + + +terraform destroy + + +``` diff --git a/ctorm/infra/terraform/.terraform.lock.hcl b/ctorm/infra/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..e6e8e02 --- /dev/null +++ b/ctorm/infra/terraform/.terraform.lock.hcl @@ -0,0 +1,46 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.0" + constraints = "~> 2.4" + hashes = [ + "h1:WB6H5ksIZiyq1lQlD/PWeh+tn4FLsbSjVnRW3+4xe2Y=", + "zh:0d14713fdc259fb377d0b899ad3c650a34194bd52194c863303ef22a65a580e2", + "zh:369b56040c7a8085d04e7e8ffac1e2b321a3170e502f788819bc34b868ec016f", + "zh:4d1a3b983ed6af5a52bfe12794674ae55cbadfa6021b37106ade68b433ad216a", + "zh:5c547549e26e083573c78a966ca68ce6d7df6bb8f3948f66a575f07da46b74ea", + "zh:6de093e62a975eb19a5e3017ce38e6e3cb639c17b79648d2000e0a8348f0e997", + "zh:7267936c2cdbc448efeb594d73e6b56a53d6a7ae14fe88cdd2a4133adc3302f0", + "zh:7482f023050ed426b4b45116e1761643bc33b1fd4ce4a6fab207ae2571f35940", + "zh:76bbd93b234e5a2927d98b511d86565700f549b570871a194c35f944b96cefb7", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:c6afc4bc1f002bac9c173007dd4da05fde788cd14c2916089f958c33fedb0dfa", + "zh:d3ba40bd806a3a08e9237dece679193c99afb2085de6b45d7f5d1f673cfcd368", + "zh:e1ad7ded53ecd6f0e5b473a3b44eae2b2e885653a56050ab583d387332be02e4", + "zh:e93e78575ce82be6084cc153c24ba8f385dc8d6880888ee66e918460c870953d", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} diff --git a/ctorm/infra/terraform/dev.tfvars b/ctorm/infra/terraform/dev.tfvars new file mode 100644 index 0000000..2d27db1 --- /dev/null +++ b/ctorm/infra/terraform/dev.tfvars @@ -0,0 +1,5 @@ +aws_profile = "cumulus-sbx-7522" +name_prefix = "ctorm-dev" + +cumulus_ingest_queue_arn = "arn:aws:sqs:us-west-2:123456789012:cumulus-ingest" +cumulus_ingest_queue_url = "https://sqs.us-west-2.amazonaws.com/123456789012/cumulus-ingest" diff --git a/ctorm/infra/terraform/main.tf b/ctorm/infra/terraform/main.tf new file mode 100644 index 0000000..c0945fb --- /dev/null +++ b/ctorm/infra/terraform/main.tf @@ -0,0 +1,453 @@ +locals { + lambda_name = "${var.name_prefix}-cnm-sender" +} + + +resource "aws_sqs_queue" "granules_dlq" { + name = "${var.name_prefix}-granule-dlq" +} + +resource "aws_sqs_queue" "granules" { + name = "${var.name_prefix}-granules" + visibility_timeout_seconds = 180 + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.granules_dlq.arn + maxReceiveCount = 5 + }) +} + +resource "aws_dynamodb_table" "granules" { + name = "${var.name_prefix}-granules" + # billing_mode = "PROVISIONED" # Temporarily switch to PROVISIONED for bulk load + billing_mode = var.granule_table_billing_mode + + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null + hash_key = "pk" + range_key = "sk" + + attribute { + name = "pk" + type = "S" + } + + attribute { + name = "sk" + type = "S" + } + + attribute { + name = "gsi1pk" + type = "S" + } + + attribute { + name = "gsi1sk" + type = "S" + } + + global_secondary_index { + name = "gsi1" + hash_key = "gsi1pk" + range_key = "gsi1sk" + projection_type = "ALL" + + read_capacity = var.granule_table_billing_mode == "PROVISIONED" ? 5 : null + write_capacity = var.granule_table_billing_mode == "PROVISIONED" ? var.granule_table_write_capacity : null + } +} + + + +resource "aws_cloudwatch_log_group" "cnm-sender" { + name = "/aws/lambda/${local.lambda_name}" + retention_in_days = 14 +} + +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "cnm-sender" { + name = "${var.name_prefix}-cnm-sender-role" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json +} + +data "aws_iam_policy_document" "cnm-sender" { + statement { + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + + resources = [ + "${aws_cloudwatch_log_group.cnm-sender.arn}:*" + ] + } + + statement { + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ] + + resources = [ + aws_sqs_queue.granules.arn + ] + } + + statement { + actions = [ + "sqs:SendMessage", + "sqs:SendMessageBatch" + ] + + resources = [ + var.cumulus_ingest_queue_arn + ] + } + + statement { + actions = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:Query" + ] + + resources = [ + aws_dynamodb_table.granules.arn, + "${aws_dynamodb_table.granules.arn}/index/*" + ] + } +} + +resource "aws_iam_role_policy" "cnm-sender" { + name = "${var.name_prefix}-cnm-sender-policy" + role = aws_iam_role.cnm-sender.id + policy = data.aws_iam_policy_document.cnm-sender.json +} + +data "archive_file" "placeholder_lambda" { + type = "zip" + output_path = "${path.module}/placeholder-lambda.zip" + + source { + filename = "index.py" + + content = <=2.20.0", + "boto3>=1.43.26", + "mypy-boto3-s3>=1.43.14", +] + +[dependency-groups] +dev = [ + "black>=26.5.1", + "boto3-stubs>=1.43.27", + "pytest>=8.0.0", + "ruff>=0.15.13", +] + +[project.scripts] +ctorm = "ctorm.main:main" + +[build-system] +requires = ["uv_build>=0.10.3,<0.11.0"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-root = "" + + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = py312 +[testenv] +deps = + mypy-boto3-s3 + pytest + pytest-mock +commands = + pytest +""" + +[tool.ruff] +target-version = "py312" diff --git a/ctorm/tests/__init__.py b/ctorm/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ctorm/tests/test_cnm.py b/ctorm/tests/test_cnm.py new file mode 100644 index 0000000..96ed241 --- /dev/null +++ b/ctorm/tests/test_cnm.py @@ -0,0 +1,33 @@ +from ctorm.cnm import CtormCnmSGenerator +from ctorm.config import CtormPreparedGranule + + +def test_ctormcnmgenerator(): + test_input: CtormPreparedGranule = { + "bm": { + "sds-n-cumulus-prod-nisar-products": "B1", + "sds-n-cumulus-prod-nisar-jpl-private-data": "B2", + }, + "g": "NISAR_L0_RRST_VC08_20250821T101036_20250821T101041_P00408_J_001", + "c": "NISAR_L0A_RRST_BETA_V1", + "cv": "1", + "f": [ + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin", + "m": "e989430f4c5bfa04eeb26e56f4cd8375", + "s": 1073829760, + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.rc.yaml", + "s": 120597, + "m": "11bcaec780f0d246f3996b3f08680410", + }, + { + "f": "s3://$B1/NISAR_L0A_RRST_BETA_V1/$G/$G.bin.qa", + "s": 1134, + }, + ], + } + ctorm_cnmsgen = CtormCnmSGenerator() + output = ctorm_cnmsgen(test_input, "FOO") + assert output diff --git a/ctorm/tests/test_config.py b/ctorm/tests/test_config.py new file mode 100644 index 0000000..1e76c50 --- /dev/null +++ b/ctorm/tests/test_config.py @@ -0,0 +1,51 @@ +from unittest.mock import mock_open, patch + +from ctorm.config import CtormConfig, CtormPipeline + + +def test_ctorm_bucket_initialization(): + bucket = CtormPipeline( + bucketname="test_bucket", share=50, ummg_prefix="test_prefix/" + ) + + assert bucket.bucketname == "test_bucket" + assert bucket.share == 50 + assert bucket.ummg_prefix == "test_prefix/" + assert bucket.next_cont_token is None + + +def test_ctorm_config_from_file_success(): + mock_data = """ + [ctorm] + source_buckets = [{"bucketname": "test_bucket", "share": 50}] + granules_sqs_queue_url = "https://sqs.queue.url/" + """ + with ( + patch("builtins.open", mock_open(read_data=mock_data)), + patch("tomllib.load") as mock_toml, + ): + mock_toml.return_value = { + "ctorm": { + "source_buckets": [{"bucketname": "test_bucket", "share": 50}], + "granules_sqs_queue_url": "https://sqs.queue.url/", + } + } + config = CtormConfig.from_file("test.cfg") + + assert config.granules_sqs_queue_url == "https://sqs.queue.url/" + assert len(config.source_buckets) == 1 + assert isinstance(config.source_buckets[0], CtormPipeline) + assert config.source_buckets[0].bucketname == "test_bucket" + + +def test_ctorm_config_from_file_missing_section(): + mock_data = "{}" + with ( + patch("builtins.open", mock_open(read_data=mock_data)), + patch("tomllib.load") as mock_toml, + ): + mock_toml.return_value = {} + try: + CtormConfig.from_file("test.cfg") + except KeyError as e: + assert str(e) == "No 'ctorm' section in config file" diff --git a/ctorm/uv.lock b/ctorm/uv.lock new file mode 100644 index 0000000..ccd00a2 --- /dev/null +++ b/ctorm/uv.lock @@ -0,0 +1,362 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "aws-lambda-typing" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/2a/14e7a3dba738265db6eddfd0c9ec1619c0c14e48fcf70627fc75fca83305/aws-lambda-typing-2.20.0.tar.gz", hash = "sha256:78b0d8ebab73b3a6b0da98a7969f4e9c4bb497298ec50f3217da8a8dfba17154", size = 19413, upload-time = "2024-04-02T09:43:03.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/3d/4031f5950d65e89136d20a18b5ac985d5f087c29c902f285b6e8063619e4/aws_lambda_typing-2.20.0-py3-none-any.whl", hash = "sha256:1d44264cabfeab5ac38e67ddd0c874e677b2cbbae77a42d0519df470e6bbb49b", size = 35296, upload-time = "2024-04-02T09:43:05.881Z" }, +] + +[[package]] +name = "black" +version = "26.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pytokens" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/37/5628dd55bf2b34257fc7603f0fe97c40e3aaf24265f416a9c85c95ca1436/black-26.5.1.tar.gz", hash = "sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73", size = 679439, upload-time = "2026-05-18T16:53:36.107Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/99/7744b906703228264ef73bdd534df88ec1ef3de45c4e78f6d31b9e32d0c9/black-26.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8", size = 2012518, upload-time = "2026-05-18T17:05:20.108Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c0/c5a3b1636dfd09c42534f2b3cf33506814f6d3e066fb0879ffa16c1ae860/black-26.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217", size = 1816016, upload-time = "2026-05-18T17:05:21.84Z" }, + { url = "https://files.pythonhosted.org/packages/1f/0e/36044316b65ca471d3bb6d3703fd06fb50c6b727c3562f6a5a3153634f88/black-26.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d", size = 1884150, upload-time = "2026-05-18T17:05:23.546Z" }, + { url = "https://files.pythonhosted.org/packages/b3/33/dafc5808c2af43672912111d7c3354af1615f7e2be3bed7a878461abbe4d/black-26.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264", size = 1486825, upload-time = "2026-05-18T17:05:25.004Z" }, + { url = "https://files.pythonhosted.org/packages/82/14/b965ee6ad2a311f28bdbf692def3ee9848d2ae289dab28b27657fcee3e78/black-26.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418", size = 1288646, upload-time = "2026-05-18T17:05:26.477Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5c/c384363980e11e25ca6b93205949bb331fbf35f4e0dbec376dfa6326cec8/black-26.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3", size = 2009020, upload-time = "2026-05-18T17:05:28.132Z" }, + { url = "https://files.pythonhosted.org/packages/0b/df/9f31c5e0babbfed77d505fc5d120beb98b21b33feaeded3924ea941fe360/black-26.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0", size = 1813335, upload-time = "2026-05-18T17:05:31.266Z" }, + { url = "https://files.pythonhosted.org/packages/fb/24/8e7b9a2fa61b0afd82209efe937557d180a1fa055bd7f6161eb9defc3719/black-26.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294", size = 1881614, upload-time = "2026-05-18T17:05:32.718Z" }, + { url = "https://files.pythonhosted.org/packages/49/ad/b4e0d9365ba8ac34f6bbab62a4b1b2dd5d618fac3fa1b8db968c844201b5/black-26.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a", size = 1488925, upload-time = "2026-05-18T17:05:34.259Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4b/652b859bf5df88a751c30451b09338f7fd26a77d1271c666992f836b7711/black-26.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52", size = 1289883, upload-time = "2026-05-18T17:05:36.019Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a8da8eb208c51c7f4ce74609a45d0dcc6d8a2141e45e81ee5289d1bb0d59/black-26.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168", size = 2004800, upload-time = "2026-05-18T17:05:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/11/8a/a479296a19e383b70a725882a6cf3d786540601ff03cabbaaf1cce864c5a/black-26.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3", size = 1815576, upload-time = "2026-05-18T17:05:40.309Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/cfaf3d39f25132c156a068f6b805576c9103a84086019507c70e1911ee7d/black-26.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18", size = 1877927, upload-time = "2026-05-18T17:05:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/66/76/302e313964bcff7e28df329d39f84f5270095730d85ff0acc260610a0d82/black-26.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50", size = 1511860, upload-time = "2026-05-18T17:05:43.943Z" }, + { url = "https://files.pythonhosted.org/packages/27/4e/a3827e35e0e567f9f9ee59e2a0ab979267dca98718f25547ca8c6733afd4/black-26.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae", size = 1316632, upload-time = "2026-05-18T17:05:45.521Z" }, + { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/6a/fede22427b4e8e56bae102de50351f1a02519be86f8c8dd5347c377cfceb/boto3-1.43.26.tar.gz", hash = "sha256:33f027b7911dffe84f4896b66024c228eb856f4cf8c1ff3465be0eae1f413731", size = 113143, upload-time = "2026-06-09T19:34:20.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/39/06d683934d94972c070179952746a57adff5c1851e2396d505a9b8cd9754/boto3-1.43.26-py3-none-any.whl", hash = "sha256:7c88bfbde6abcf062d04ac410a1cf4eaf8675f025f35f3ad9accc9b0224ce6b9", size = 140537, upload-time = "2026-06-09T19:34:17.812Z" }, +] + +[[package]] +name = "boto3-stubs" +version = "1.43.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore-stubs" }, + { name = "types-s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/2e/29abbf67dc4778203f7b77ed6ad33c6fb524a6742d74af79f3d7534ebd7c/boto3_stubs-1.43.27.tar.gz", hash = "sha256:a756552f1764dbd29e255ab5c45c291246c413fbbe9047b4e085ef0f1010688d", size = 103002, upload-time = "2026-06-10T19:56:28.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/d0/0519155e5e6eb670cb03d422601e3df1b0becbc0d3c769a734c88d247909/boto3_stubs-1.43.27-py3-none-any.whl", hash = "sha256:120305c09879d927227547d0e26fb1226403c6cc983bef527ed7d1dc6157ff67", size = 70832, upload-time = "2026-06-10T19:56:17.347Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.26" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/ba/aa37cd4e72aa8b70fdb93f47129ef3b79887edcfb343bd41df2783a72f12/botocore-1.43.26.tar.gz", hash = "sha256:fd9280ac868194afcee59c7def62e0031fb4b599f7ef85360d30c7e42357c1dc", size = 15497173, upload-time = "2026-06-09T19:34:07.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/e6/5a5ec1033613e7812e5b19ec8c2a1889834fde336d8812d53019eac6e04a/botocore-1.43.26-py3-none-any.whl", hash = "sha256:eeb92265bae289555182a46341c998a656ab49c0dbdb762c65b30fe354fcc9e8", size = 15183593, upload-time = "2026-06-09T19:34:03.012Z" }, +] + +[[package]] +name = "botocore-stubs" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-awscrt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/81/79693e833291c00dc89ee610e5e915381b6f08233912e28df50106840780/botocore_stubs-1.43.14.tar.gz", hash = "sha256:9e3bc1fdd51da7473f0df726c82747a1b0ae913449d629659765c247fecc2039", size = 42738, upload-time = "2026-05-25T06:06:37.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/ca/f017727b11895908c5dedc829cf2ec35e0c4b2a26ba875db325fef2cefdf/botocore_stubs-1.43.14-py3-none-any.whl", hash = "sha256:fb98f1475c92fd718644e786b5c543a20f1b1f610e89e0a7191c3f1f429c75aa", size = 67093, upload-time = "2026-05-25T06:06:34.532Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "ctorm" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "aws-lambda-typing" }, + { name = "boto3" }, + { name = "mypy-boto3-s3" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "boto3-stubs" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "aws-lambda-typing", specifier = ">=2.20.0" }, + { name = "boto3", specifier = ">=1.43.26" }, + { name = "mypy-boto3-s3", specifier = ">=1.43.14" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=26.5.1" }, + { name = "boto3-stubs", specifier = ">=1.43.27" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "ruff", specifier = ">=0.15.13" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "mypy-boto3-s3" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/2c/fc409f9ff5904a02cf4c2c1518c34d20cb56f22b2368b35fd0adda2926f3/mypy_boto3_s3-1.43.14.tar.gz", hash = "sha256:73d54c1d0999c73c403dc9a9a3da4a9722715aba116595af08c0d4675f8bc670", size = 77078, upload-time = "2026-05-22T20:48:28.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/61/e8e74a3f4c729719efc8b1d00179bc16c302202eac32a1b56a842a997ed2/mypy_boto3_s3-1.43.14-py3-none-any.whl", hash = "sha256:ce77096d6c5f90020c45e34c83d2268ca2bb17726149ce5033751870f7fb4e97", size = 84277, upload-time = "2026-05-22T20:48:25.032Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytokens" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, + { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, + { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, + { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, + { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, + { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, + { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, + { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, + { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/5f7ec371001337d8fa61701c186ff8b613ecac1651848c5950f4c4d5f2e9/ruff-0.15.16.tar.gz", hash = "sha256:d05e78d38c78caf020b03789e25106c93017db5a0cb6e2819885018c61343b78", size = 4714267, upload-time = "2026-06-04T16:33:09.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/42/53ef1c3953f157956db9bf7861e3bc50b9b887ce93300aa48cdba8336fe6/ruff-0.15.16-py3-none-linux_armv6l.whl", hash = "sha256:6ac3c0b3969cc6cf6b158c4e2f8f682acb58e7d700d8a44b65ecdc72d66ab0b2", size = 10709025, upload-time = "2026-06-04T16:32:51.935Z" }, + { url = "https://files.pythonhosted.org/packages/93/9a/a79159346f19134a956607754e57d8d128f7a4c00f4ad2f7514d224c172c/ruff-0.15.16-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:197c207ed75ffba54a0dec23db4aa939a27a3053073e085e0042433cbdc58e4a", size = 11063550, upload-time = "2026-06-04T16:32:42.24Z" }, + { url = "https://files.pythonhosted.org/packages/bc/72/3ce2ac000a5299ec238e01f51397b3b653c93b077d9b1bfe8715bb895f20/ruff-0.15.16-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3a39fec45ab316cc23e7558f23fea4a70403ddb5648ea9a4a3854a16973d0071", size = 10421345, upload-time = "2026-06-04T16:32:37.251Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c2/cc7fad3ec9169373f5b6a18f1917b91080feec40c3f9658334a1d28e2f03/ruff-0.15.16-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba93191d79003116b95128c9d306e045200fdbd0bccb782b110f3cd1d4abc5cf", size = 10757217, upload-time = "2026-06-04T16:32:54.722Z" }, + { url = "https://files.pythonhosted.org/packages/69/d2/3474009eaa0a65b31fa7152a2fad5e2f050c640ceb1e6b02ee6922e94c82/ruff-0.15.16-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6ee4b90520630120ef032aa5cc10db483852dff950e78b1d717e2993a61ac8d", size = 10507035, upload-time = "2026-06-04T16:33:05.343Z" }, + { url = "https://files.pythonhosted.org/packages/ca/81/b7ae6ccbd11f0c8dc3d5d67fc4be9b57ff57ca86ba56152021378e1277f2/ruff-0.15.16-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e4215bc938bc3c8215c1472c1aa437e310fee20cd427335fec9d7e609563628", size = 11255291, upload-time = "2026-06-04T16:32:49.49Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e1/46e526f1a7cc90857ce6ddf25fbb77eb6568651ac38d71b033af07076dd5/ruff-0.15.16-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c8d26be963b090f10e29abc8b3e74a2a321f6fa34e02424e30b5af89350ecbb", size = 12124922, upload-time = "2026-06-04T16:33:07.821Z" }, + { url = "https://files.pythonhosted.org/packages/1a/da/5c791b088b596b24d0deb967fa28ae02ad751a140c0b9ea81c5ab915d6c0/ruff-0.15.16-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f198cf4123602a2280ed46c307bcbafe41758d6fee5b456b6b6058ca1514b3b4", size = 11332186, upload-time = "2026-06-04T16:33:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/72/11/5da87abe20047c8962361473923ebb2f62b595250126aadfad8c20649c1e/ruff-0.15.16-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb27515fa6240fb586ae82b901a59e67d24acff86f2190b433dc542fe0435aeb", size = 11373541, upload-time = "2026-06-04T16:32:47.007Z" }, + { url = "https://files.pythonhosted.org/packages/fe/2a/8554754c23a854ae3fd6b507e36ad61ddb121e298c6d5d617dec94ed0f14/ruff-0.15.16-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a267c46ba1593fc26b8eecbea050b39d40c0b6bb7781ee11c90a02cd10032951", size = 11353014, upload-time = "2026-06-04T16:32:34.795Z" }, + { url = "https://files.pythonhosted.org/packages/62/25/62ea41529ec89f742ea3fed9cb1059c72877ec7cf9b9e99ac9cf3294d1d9/ruff-0.15.16-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:528c68f39a91498a8d50e91ff5985df3d105782bab49cc378e73ac26bff083e8", size = 10737467, upload-time = "2026-06-04T16:32:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/90/17/334d3ad9de4d40f9dd58fdd09e35ce64553bb501e2f19a839e2fb6be14fc/ruff-0.15.16-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7ed55c58950df60589a9a7a5d2f8fa5f54ebd287163be805adfe6ee95a9de123", size = 10521910, upload-time = "2026-06-04T16:32:32.54Z" }, + { url = "https://files.pythonhosted.org/packages/4d/bd/3ac7c6ae77a885c1004b3dda2446ea401768d24f851c14b4ad4b24f6639c/ruff-0.15.16-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d482feaf51512b50f9790ceb417a56a61dd1e9d9bf967662b9ed27c01b34f53a", size = 10979190, upload-time = "2026-06-04T16:32:57.492Z" }, + { url = "https://files.pythonhosted.org/packages/33/d7/609546e6a413c3f216fbf2a50c928f97c80939154f6a0503114094a86191/ruff-0.15.16-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e15bc8c94513dae2a40cc9ef07c94fdd4ecc9e29dabebeebe170f952322c9e3", size = 11477014, upload-time = "2026-06-04T16:32:44.687Z" }, + { url = "https://files.pythonhosted.org/packages/74/0d/f2cd247ad32633a5c36e97141a2c21b11c6279f7957bc2ff360b1e08fddd/ruff-0.15.16-py3-none-win32.whl", hash = "sha256:580378f7bd4aa25f72e74aa54948a9622f142b1e509521dd10902e886681cc1e", size = 10735541, upload-time = "2026-06-04T16:32:30.145Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/02e845ef151b1dee585e55c4739f8e1734ae1d9f1221dff65761c162208b/ruff-0.15.16-py3-none-win_amd64.whl", hash = "sha256:408256017284eddf98fff77b29aa4fb30f586042d535b2d9befc6512f400aaec", size = 11843403, upload-time = "2026-06-04T16:32:39.76Z" }, + { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1f/12417f7f493fc45e1f9fd5d4a9b6c125cf8d2cf3f8ddbdfab3e76406e9d6/s3transfer-0.18.0.tar.gz", hash = "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd", size = 160560, upload-time = "2026-05-28T19:39:09.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/58/a58fc997655386daa2e25784e30c288aa3e3819e401f77029ee4899fb55a/s3transfer-0.18.0-py3-none-any.whl", hash = "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6", size = 88572, upload-time = "2026-05-28T19:39:07.999Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "types-awscrt" +version = "0.34.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/59/44409a8fc06b444ab1a6f71dcb29d49a6e17e02424345eb51b051bebb345/types_awscrt-0.34.1.tar.gz", hash = "sha256:559aa04250f6a419a617dfb788f3e10903aaf74700ef23e521b64a411b83b803", size = 19062, upload-time = "2026-06-05T04:40:10.689Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/b1/214b12162b452ed6acd230065e6c587cde6b96871e3ce6d653f40888f8df/types_awscrt-0.34.1-py3-none-any.whl", hash = "sha256:20c752b6031544d8f694803c35174aee129f1be5ddf886ae46d22f7ffd9b7d75", size = 45688, upload-time = "2026-06-05T04:40:09.198Z" }, +] + +[[package]] +name = "types-s3transfer" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/64/42689150509eb3e6e82b33ee3d89045de1592488842ddf23c56957786d05/types_s3transfer-0.16.0.tar.gz", hash = "sha256:b4636472024c5e2b62278c5b759661efeb52a81851cde5f092f24100b1ecb443", size = 13557, upload-time = "2025-12-08T08:13:09.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/27/e88220fe6274eccd3bdf95d9382918716d312f6f6cef6a46332d1ee2feff/types_s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:1c0cd111ecf6e21437cb410f5cddb631bfb2263b77ad973e79b9c6d0cb24e0ef", size = 19247, upload-time = "2025-12-08T08:13:08.426Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +]