From c20e4810e53c23a3bd822bc33ddfa6fd8904203d Mon Sep 17 00:00:00 2001 From: jlu Date: Wed, 27 Aug 2025 16:44:26 -0700 Subject: [PATCH] Work around tf files to make kubernet deploy working --- examples/eks_argo/argo_events.tf | 2 +- examples/eks_argo/eks.tf | 6 ++--- examples/eks_argo/metaflow.tf | 24 ++++++++++++-------- examples/eks_argo/versions.tf | 14 ++++++++++-- examples/eks_argo/vpc.tf | 39 ++++++++++++++++++++++---------- 5 files changed, 57 insertions(+), 28 deletions(-) diff --git a/examples/eks_argo/argo_events.tf b/examples/eks_argo/argo_events.tf index 621f3bb..da70e4d 100644 --- a/examples/eks_argo/argo_events.tf +++ b/examples/eks_argo/argo_events.tf @@ -1,5 +1,5 @@ module "argo_events" { depends_on = [helm_release.argo] - source = "git::git@github.com:outerbounds/metaflow-tools//common/terraform/argo_events?ref=v2.0.0" + source = "git::https://github.com/outerbounds/metaflow-tools//common/terraform/argo_events?ref=v2.0.0" jobs_namespace = "default" } diff --git a/examples/eks_argo/eks.tf b/examples/eks_argo/eks.tf index 7cf4d13..635394e 100644 --- a/examples/eks_argo/eks.tf +++ b/examples/eks_argo/eks.tf @@ -4,12 +4,12 @@ module "eks" { version = "17.23.0" cluster_name = local.cluster_name - cluster_version = "1.24" - subnets = module.vpc.private_subnets + cluster_version = "1.28" + subnets = local.private_subnet_ids enable_irsa = true tags = local.tags - vpc_id = module.vpc.vpc_id + vpc_id = data.aws_vpc.existing.id node_groups_defaults = { ami_type = "AL2_x86_64" diff --git a/examples/eks_argo/metaflow.tf b/examples/eks_argo/metaflow.tf index 58805bf..5ef8345 100644 --- a/examples/eks_argo/metaflow.tf +++ b/examples/eks_argo/metaflow.tf @@ -20,7 +20,7 @@ data "aws_availability_zones" "available" { module "metaflow-datastore" { source = "outerbounds/metaflow/aws//modules/datastore" - version = "0.10.0" + version = "0.12.1" force_destroy_s3_bucket = true @@ -28,21 +28,25 @@ module "metaflow-datastore" { resource_suffix = local.resource_suffix metadata_service_security_group_id = module.metaflow-metadata-service.metadata_service_security_group_id - metaflow_vpc_id = module.vpc.vpc_id - subnet1_id = module.vpc.private_subnets[0] - subnet2_id = module.vpc.private_subnets[1] + metaflow_vpc_id = data.aws_vpc.existing.id + subnet1_id = local.private_subnet_ids[0] + subnet2_id = local.private_subnet_ids[1] + + # Override RDS configuration to use supported instance class + db_instance_type = "db.t3.small" + db_engine_version = "13.16" standard_tags = local.tags } module "metaflow-common" { source = "outerbounds/metaflow/aws//modules/common" - version = "0.10.0" + version = "0.12.1" } module "metaflow-metadata-service" { source = "outerbounds/metaflow/aws//modules/metadata-service" - version = "0.10.0" + version = "0.12.1" resource_prefix = local.resource_prefix resource_suffix = local.resource_suffix @@ -54,13 +58,13 @@ module "metaflow-metadata-service" { database_username = module.metaflow-datastore.database_username datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn fargate_execution_role_arn = aws_iam_role.ecs_execution_role.arn - metaflow_vpc_id = module.vpc.vpc_id + metaflow_vpc_id = data.aws_vpc.existing.id metadata_service_container_image = module.metaflow-common.default_metadata_service_container_image rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn - subnet1_id = module.vpc.private_subnets[0] - subnet2_id = module.vpc.private_subnets[1] - vpc_cidr_blocks = [module.vpc.vpc_cidr_block] + subnet1_id = local.private_subnet_ids[0] + subnet2_id = local.private_subnet_ids[1] + vpc_cidr_blocks = [data.aws_vpc.existing.cidr_block] with_public_ip = var.with_public_ip standard_tags = local.tags diff --git a/examples/eks_argo/versions.tf b/examples/eks_argo/versions.tf index d1c6f88..14a1e45 100644 --- a/examples/eks_argo/versions.tf +++ b/examples/eks_argo/versions.tf @@ -2,7 +2,17 @@ terraform { required_version = ">= 0.13.1" required_providers { - aws = ">= 3.54.0" - random = ">= 2.1" + aws = { + source = "hashicorp/aws" + version = "~> 5.0" # Or a more recent version + } + helm = { + source = "hashicorp/helm" + version = "~> 2.0" // Use a version that supports the correct syntax + } } } + +provider "aws" { + region = "us-west-2" +} diff --git a/examples/eks_argo/vpc.tf b/examples/eks_argo/vpc.tf index 6d937cc..34ccba9 100644 --- a/examples/eks_argo/vpc.tf +++ b/examples/eks_argo/vpc.tf @@ -1,17 +1,32 @@ -# VPC infra using https://github.com/terraform-aws-modules/terraform-aws-vpc -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "5.1.2" +# Use existing VPC instead of creating new one +data "aws_vpc" "existing" { + id = "vpc-01195e9922f40d28d" +} - name = "${local.resource_prefix}-${local.resource_suffix}" - cidr = "10.10.0.0/16" +data "aws_subnets" "all" { + filter { + name = "vpc-id" + values = [data.aws_vpc.existing.id] + } +} - azs = data.aws_availability_zones.available.names - private_subnets = ["10.10.8.0/21", "10.10.16.0/21", "10.10.24.0/21"] - public_subnets = ["10.10.128.0/21", "10.10.136.0/21", "10.10.144.0/21"] +data "aws_subnet" "all_subnets" { + for_each = toset(data.aws_subnets.all.ids) + id = each.value +} - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true +# For now, we'll use all subnets as private since none have public IP mapping +# This may need adjustment based on your specific requirements +locals { + # Using first two subnets for private (spread across AZs) + private_subnet_ids = [ + "subnet-06d24ca1cdc68006b", # us-west-2a + "subnet-0b9f28860c7718324" # us-west-2b + ] + # Using last two subnets as "public" (though they're actually private) + public_subnet_ids = [ + "subnet-0037943db9c00ae6a", # us-west-2a + "subnet-01922add8c95526d7" # us-west-2b + ] }