From 2918717be1b13735820649703581e05311afd3f3 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 15:21:40 -0400 Subject: [PATCH 01/13] feat(operator): add API Gateway ingress for webhook platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an optional `spec.ingress` block to the oab.dev/v2 manifest so `oabctl apply` can provision inbound HTTPS webhook ingress for Telegram/LINE bots in one shot — instead of the ~7 manual `aws` commands documented in the Telegram/LINE-on-AWS refarch (Option 1). When `ingress` is present (ECS runtime only), apply reconciles, all idempotently and reused by name: - Cloud Map private DNS namespace + per-service A record - ECS service registry wiring (attached at service *creation*) - VPC Link (shared `oab-vpc-link`), waited until AVAILABLE - API Gateway HTTP API (`oab-webhook`) + HTTP_PROXY integration - one route per path + a `prod` auto-deploy stage - a self-referencing security-group inbound rule on containerPort It then prints the stable webhook URL(s) to register with the platform. Backward compatible: `ingress` is optional and defaults to absent, so existing outbound-only (Discord) deployments create no ingress resources and behave exactly as before. Because ECS service registries can only be set at creation time, apply attaches the registry on create and, for a pre-existing service without service discovery, provisions the ingress resources and prints how to recreate the service (non-destructive — no automatic delete). - manifest.rs: Ingress struct, validation, fleet passthrough, 7 unit tests - ingress.rs: Cloud Map + VPC Link + API Gateway reconciliation - apply.rs: two-phase wiring (Cloud Map before create, gateway after) - schema/oabservice-v2.json: ingress def + refs - README: ingress section - Cargo.toml: aws-sdk-apigatewayv2, aws-sdk-servicediscovery Refs openabdev/openab#1274 --- operator/Cargo.toml | 2 + operator/README.md | 58 +++- operator/schema/oabservice-v2.json | 25 +- operator/src/apply.rs | 73 ++++- operator/src/ingress.rs | 487 +++++++++++++++++++++++++++++ operator/src/main.rs | 1 + operator/src/manifest.rs | 244 +++++++++++++++ 7 files changed, 882 insertions(+), 8 deletions(-) create mode 100644 operator/src/ingress.rs diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 3550bb313..c35af5f46 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -18,6 +18,8 @@ aws-sdk-ssm = "1.52" aws-sdk-sts = "1" aws-sdk-cloudwatchlogs = "1" aws-sdk-secretsmanager = "1" +aws-sdk-apigatewayv2 = "1" +aws-sdk-servicediscovery = "1" ecsctl = { git = "https://github.com/oablab/ecsctl.git", rev = "90a6cd1" } clap = { version = "4.5", features = ["derive"] } chrono = "0.4" diff --git a/operator/README.md b/operator/README.md index 3fd1fc45f..03963c59b 100644 --- a/operator/README.md +++ b/operator/README.md @@ -209,6 +209,62 @@ spec: securityGroups: [sg-xxx] ``` +### Ingress — inbound webhooks (Telegram / LINE) + +Discord bots are outbound-only and need no ingress. Webhook platforms (Telegram, +LINE, ...) POST *into* the task, so they need a public HTTPS endpoint. Adding an +optional `spec.ingress` block makes `oabctl apply` provision the cheapest +AWS-native path in one shot — API Gateway HTTP API → VPC Link → Cloud Map → the +task — instead of running ~7 manual `aws` commands. See +[`docs/refarch/running-telegram-line-on-aws.md`](../docs/refarch/running-telegram-line-on-aws.md) +(Option 1) for the architecture and cost breakdown. + +```yaml +spec: + image: public.ecr.aws/oablab/kiro:beta + resources: { cpu: "256", memory: "512" } + configFrom: s3://.../config.toml + runtime: + type: ecs + capacityProvider: FARGATE_SPOT + networking: + subnets: [subnet-aaa, subnet-bbb] + securityGroups: [sg-xxx] + ingress: + type: apigateway # only supported type (default) + cloudMapNamespace: oab # reused across bots in the VPC (default: oab) + containerPort: 8080 # OpenAB listen port (default: 8080) + paths: + - /webhook/telegram + - /webhook/line +``` + +On `apply` this reconciles (idempotently, reused by name): + +1. **Cloud Map** private DNS namespace + a per-service A record +2. **ECS service registry** wiring (attached at service creation) +3. **VPC Link** (shared `oab-vpc-link`), waits until `AVAILABLE` +4. **API Gateway HTTP API** (`oab-webhook`) + `HTTP_PROXY` integration over the VPC Link +5. One **route** per path + a `prod` auto-deploy **stage** +6. A self-referencing **security-group** inbound rule on `containerPort` + +`apply` then prints the stable webhook URL(s) to register with BotFather / the +LINE console: + +``` +🔗 Webhook URL(s) for my-bot: + https://{api-id}.execute-api.{region}.amazonaws.com/prod/webhook/telegram + https://{api-id}.execute-api.{region}.amazonaws.com/prod/webhook/line +``` + +> **Security note:** the API Gateway endpoint is public and unauthenticated at the +> transport layer — requests are authenticated at the app layer by OpenAB (LINE +> signature via `LINE_CHANNEL_SECRET`, Telegram bot token in the path). +> +> **Recreate caveat:** ECS service registries can only be set at *creation* time. +> If the service already exists without service discovery, `apply` provisions the +> ingress resources and prints how to recreate the service so traffic can reach it. + ### OABFleet — batch deploy ```yaml @@ -234,7 +290,7 @@ spec: ``` **Fleet features:** -- Template inheritance with per-agent overrides (`image`, `resources`, `bootstrapFrom`, `secrets`) +- Template inheritance with per-agent overrides (`image`, `resources`, `bootstrapFrom`, `secrets`, `ingress`) - `${name}` interpolation in `configFrom`, `bootstrapFrom` - Runtime shared across fleet (not overridable per-agent) - Validate-all-before-apply (no partial deploys) diff --git a/operator/schema/oabservice-v2.json b/operator/schema/oabservice-v2.json index ab09d807b..71dbeed6d 100644 --- a/operator/schema/oabservice-v2.json +++ b/operator/schema/oabservice-v2.json @@ -58,7 +58,8 @@ "configFrom": { "type": "string" }, "bootstrapFrom": { "type": "string" }, "secrets": { "type": "object", "additionalProperties": { "type": "string" } }, - "runtime": { "$ref": "#/$defs/runtime" } + "runtime": { "$ref": "#/$defs/runtime" }, + "ingress": { "$ref": "#/$defs/ingress" } }, "additionalProperties": false }, @@ -83,7 +84,8 @@ "resources": { "$ref": "#/$defs/resources" }, "bootstrapFrom": { "type": "string" }, "secrets": { "type": "object", "additionalProperties": { "type": "string" } }, - "runtime": { "$ref": "#/$defs/runtime" } + "runtime": { "$ref": "#/$defs/runtime" }, + "ingress": { "$ref": "#/$defs/ingress" } }, "additionalProperties": false }, @@ -96,7 +98,8 @@ "image": { "type": "string" }, "resources": { "$ref": "#/$defs/resources" }, "bootstrapFrom": { "type": "string" }, - "secrets": { "type": "object", "additionalProperties": { "type": "string" } } + "secrets": { "type": "object", "additionalProperties": { "type": "string" } }, + "ingress": { "$ref": "#/$defs/ingress" } }, "additionalProperties": false }, @@ -109,6 +112,22 @@ }, "additionalProperties": false }, + "ingress": { + "type": "object", + "description": "Inbound HTTPS webhook ingress (API Gateway HTTP API + VPC Link + Cloud Map). ECS runtime only.", + "required": ["paths"], + "properties": { + "type": { "type": "string", "enum": ["apigateway"], "default": "apigateway" }, + "cloudMapNamespace": { "type": "string", "default": "oab" }, + "paths": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "pattern": "^/" } + }, + "containerPort": { "type": "integer", "minimum": 1, "maximum": 65535, "default": 8080 } + }, + "additionalProperties": false + }, "runtime": { "type": "object", "required": ["type"], diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 794370179..99ec7a646 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -225,6 +225,15 @@ async fn apply_ecs( .awsvpc_configuration(vpc_config) .build(); + // Ingress: ensure Cloud Map BEFORE the service exists-check, because ECS + // service registries can only be attached at service *creation* time. + let cloud_map = if let Some(ingress) = &m.spec.ingress { + eprintln!(" 🌐 Reconciling ingress (Cloud Map)..."); + Some(crate::ingress::ensure_cloud_map(config, m, ingress).await?) + } else { + None + }; + // Check if service exists let existing = ecs .describe_services() @@ -239,6 +248,12 @@ async fn apply_ecs( .and_then(|r| r.services().first()) .is_some_and(|s| s.status() == Some("ACTIVE")); + let has_registries = existing + .as_ref() + .ok() + .and_then(|r| r.services().first()) + .is_some_and(|s| !s.service_registries().is_empty()); + if service_active { ecs.update_service() .cluster("oab") @@ -249,28 +264,78 @@ async fn apply_ecs( .await .context("failed to update ECS service")?; println!(" ✓ {} updated", m.metadata.name); + + if cloud_map.is_some() && !has_registries { + eprintln!( + " ⚠ Service '{}' already exists WITHOUT service discovery.", + m.metadata.name + ); + eprintln!(" ECS service registries can only be set at creation time, so ingress"); + eprintln!(" resources were provisioned but traffic won't reach the task until you"); + eprintln!(" recreate the service (safe once desiredCount is drained):"); + eprintln!( + " oabctl delete service {} --namespace {} && oabctl apply -f ", + m.metadata.name, m.metadata.namespace + ); + } } else { let cap_strategy = CapacityProviderStrategyItem::builder() .capacity_provider(&ecs_rt.capacity_provider) .weight(1) .build()?; - ecs.create_service() + let mut create_req = ecs + .create_service() .cluster("oab") .service_name(&service_name) .task_definition(&task_def_arn) .desired_count(1) .capacity_provider_strategy(cap_strategy) - .network_configuration(network_config) + .network_configuration(network_config); + + if let Some(cm) = &cloud_map { + create_req = create_req.service_registries( + aws_sdk_ecs::types::ServiceRegistry::builder() + .registry_arn(&cm.registry_arn) + .build(), + ); + } + + create_req .send() .await .context("failed to create ECS service")?; println!( - " ✓ {} created ({}, {}cpu/{}mem)", - m.metadata.name, ecs_rt.capacity_provider, m.spec.resources.cpu, m.spec.resources.memory + " ✓ {} created ({}, {}cpu/{}mem{})", + m.metadata.name, + ecs_rt.capacity_provider, + m.spec.resources.cpu, + m.spec.resources.memory, + if cloud_map.is_some() { + ", service discovery" + } else { + "" + } ); } + // Ingress step 2: VPC Link + API Gateway + routes + SG rule. + if let (Some(ingress), Some(cm)) = (&m.spec.ingress, &cloud_map) { + eprintln!(" 🌐 Reconciling ingress (VPC Link + API Gateway)..."); + let urls = crate::ingress::ensure_gateway( + config, + ingress, + &ecs_rt.networking.subnets, + &ecs_rt.networking.security_groups, + &cm.dns_name, + ) + .await?; + println!(" 🔗 Webhook URL(s) for {}:", m.metadata.name); + for u in &urls { + println!(" {u}"); + } + } + if wait { eprintln!(" ⏳ Waiting for {} to stabilize...", m.metadata.name); wait_for_stable(ecs, "oab", &service_name).await?; diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs new file mode 100644 index 000000000..efda390f7 --- /dev/null +++ b/operator/src/ingress.rs @@ -0,0 +1,487 @@ +//! Ingress reconciliation for webhook-based platforms (Telegram, LINE, ...). +//! +//! Implements Option 1 of `docs/refarch/running-telegram-line-on-aws.md`: +//! API Gateway HTTP API → VPC Link → Cloud Map → ECS Fargate task on `:port`. +//! +//! All operations are idempotent — resources are looked up by name and reused, +//! so repeated `oabctl apply` runs converge instead of duplicating. +//! +//! The reconciliation is split in two because ECS service discovery +//! (`--service-registries`) can only be set at service *creation* time: +//! 1. [`ensure_cloud_map`] — namespace + service. Must run BEFORE the ECS +//! service is created so its registry ARN can be attached. +//! 2. [`ensure_gateway`] — VPC Link + HTTP API + integration + routes + stage +//! + security-group inbound rule. Runs AFTER the task is wired up. + +use crate::manifest::{Ingress, OABServiceManifest}; +use anyhow::{Context, Result}; +use aws_sdk_apigatewayv2::types::{ConnectionType, IntegrationType, ProtocolType}; +use aws_sdk_servicediscovery::types::{DnsConfig, DnsRecord, RecordType}; + +const VPC_LINK_NAME: &str = "oab-vpc-link"; +const API_NAME: &str = "oab-webhook"; +const STAGE_NAME: &str = "prod"; + +/// Result of Cloud Map reconciliation, consumed when creating the ECS service. +pub struct CloudMapResult { + /// Cloud Map service ARN — used as the ECS service registry ARN. + pub registry_arn: String, + /// Private DNS name the task registers under, e.g. `oab-prod-mybot.oab`. + pub dns_name: String, +} + +/// Step 1: ensure the Cloud Map private DNS namespace and service exist. +/// +/// Returns the registry ARN to attach to the ECS service and the DNS name the +/// API Gateway integration will target. +pub async fn ensure_cloud_map( + config: &aws_config::SdkConfig, + m: &OABServiceManifest, + ingress: &Ingress, +) -> Result { + let sd = aws_sdk_servicediscovery::Client::new(config); + let ec2 = aws_sdk_ec2::Client::new(config); + + let vpc_id = resolve_vpc_id(&ec2, m).await?; + + // ── Namespace (reused across all bots in the VPC) ────────────────────── + let namespace_id = ensure_namespace(&sd, &ingress.cloud_map_namespace, &vpc_id).await?; + + // ── Service (one per bot) ────────────────────────────────────────────── + let service_name = m.cloud_map_service_name(); + let (registry_arn, existed) = ensure_service(&sd, &namespace_id, &service_name).await?; + + let dns_name = format!("{}.{}", service_name, ingress.cloud_map_namespace); + if existed { + eprintln!(" ✓ Cloud Map service exists: {dns_name}"); + } else { + eprintln!(" ✓ Created Cloud Map service: {dns_name}"); + } + + Ok(CloudMapResult { + registry_arn, + dns_name, + }) +} + +/// Step 2: ensure VPC Link, HTTP API, integration, routes, stage, and the +/// security-group inbound rule. Returns the public webhook URLs (one per path). +pub async fn ensure_gateway( + config: &aws_config::SdkConfig, + ingress: &Ingress, + subnets: &[String], + security_groups: &[String], + dns_name: &str, +) -> Result> { + let api = aws_sdk_apigatewayv2::Client::new(config); + let ec2 = aws_sdk_ec2::Client::new(config); + + // ── Security group inbound rule (self-referencing on the container port) ─ + ensure_sg_ingress(&ec2, security_groups, ingress.container_port).await?; + + // ── VPC Link (shared, waits for AVAILABLE) ───────────────────────────── + let vpc_link_id = ensure_vpc_link(&api, subnets, security_groups).await?; + + // ── HTTP API (shared) ────────────────────────────────────────────────── + let (api_id, api_endpoint) = ensure_api(&api).await?; + + // ── Integration: VPC Link → Cloud Map DNS name on the container port ──── + let integration_uri = format!("http://{}:{}", dns_name, ingress.container_port); + let integration_id = ensure_integration(&api, &api_id, &vpc_link_id, &integration_uri).await?; + + // ── One route per webhook path, all → the same integration ───────────── + for path in &ingress.paths { + ensure_route(&api, &api_id, path, &integration_id).await?; + } + + // ── Stage (auto-deploy) ──────────────────────────────────────────────── + ensure_stage(&api, &api_id).await?; + + let base = api_endpoint.trim_end_matches('/'); + let urls = ingress + .paths + .iter() + .map(|p| format!("{base}/{STAGE_NAME}{p}")) + .collect(); + Ok(urls) +} + +// ─── VPC resolution ───────────────────────────────────────────────────────── + +async fn resolve_vpc_id(ec2: &aws_sdk_ec2::Client, m: &OABServiceManifest) -> Result { + let subnet = match &m.spec.runtime { + crate::manifest::Runtime::Ecs(rt) => rt + .networking + .subnets + .first() + .context("ingress requires at least one subnet")?, + _ => anyhow::bail!("ingress is only supported for ECS runtime"), + }; + let resp = ec2 + .describe_subnets() + .subnet_ids(subnet) + .send() + .await + .with_context(|| format!("failed to describe subnet {subnet}"))?; + let vpc_id = resp + .subnets() + .first() + .and_then(|s| s.vpc_id()) + .with_context(|| format!("subnet {subnet} has no VPC"))? + .to_string(); + Ok(vpc_id) +} + +// ─── Cloud Map ──────────────────────────────────────────────────────────────── + +async fn ensure_namespace( + sd: &aws_sdk_servicediscovery::Client, + name: &str, + vpc_id: &str, +) -> Result { + // Reuse an existing private DNS namespace with this name if present. + let mut pages = sd.list_namespaces().into_paginator().send(); + while let Some(page) = pages.next().await { + let page = page.context("failed to list Cloud Map namespaces")?; + for ns in page.namespaces() { + if ns.name() == Some(name) { + let id = ns.id().context("namespace missing id")?.to_string(); + eprintln!(" ✓ Cloud Map namespace exists: {name}"); + return Ok(id); + } + } + } + + // Create — this is an async Cloud Map operation; poll until it completes. + eprintln!(" ⊕ Creating Cloud Map namespace: {name} (VPC {vpc_id})"); + let out = sd + .create_private_dns_namespace() + .name(name) + .vpc(vpc_id) + .send() + .await + .context("failed to create Cloud Map namespace")?; + let op_id = out + .operation_id() + .context("no operation id for namespace creation")?; + let namespace_id = wait_for_operation_target(sd, op_id, "NAMESPACE").await?; + Ok(namespace_id) +} + +async fn ensure_service( + sd: &aws_sdk_servicediscovery::Client, + namespace_id: &str, + service_name: &str, +) -> Result<(String, bool)> { + // Look for an existing service in this namespace with the given name. + let filter = aws_sdk_servicediscovery::types::ServiceFilter::builder() + .name(aws_sdk_servicediscovery::types::ServiceFilterName::NamespaceId) + .values(namespace_id) + .build() + .context("failed to build service filter")?; + let mut pages = sd.list_services().filters(filter).into_paginator().send(); + while let Some(page) = pages.next().await { + let page = page.context("failed to list Cloud Map services")?; + for svc in page.services() { + if svc.name() == Some(service_name) { + let arn = svc.arn().context("service missing arn")?.to_string(); + return Ok((arn, true)); + } + } + } + + // Create with a single A record (TTL 60). SRV does NOT work for VPC Link. + let dns_record = DnsRecord::builder() + .r#type(RecordType::A) + .ttl(60) + .build() + .context("failed to build DNS record")?; + let dns_config = DnsConfig::builder() + .dns_records(dns_record) + .build() + .context("failed to build DNS config")?; + let out = sd + .create_service() + .name(service_name) + .namespace_id(namespace_id) + .dns_config(dns_config) + .send() + .await + .context("failed to create Cloud Map service")?; + let arn = out + .service() + .and_then(|s| s.arn()) + .context("created service has no ARN")? + .to_string(); + Ok((arn, false)) +} + +async fn wait_for_operation_target( + sd: &aws_sdk_servicediscovery::Client, + op_id: &str, + target_key: &str, +) -> Result { + use aws_sdk_servicediscovery::types::OperationStatus; + for _ in 0..60 { + let resp = sd + .get_operation() + .operation_id(op_id) + .send() + .await + .context("failed to poll Cloud Map operation")?; + let op = resp.operation().context("no operation in response")?; + match op.status() { + Some(OperationStatus::Success) => { + let target = op + .targets() + .and_then(|t| { + t.iter() + .find(|(k, _)| k.as_str() == target_key) + .map(|(_, v)| v.clone()) + }) + .context("operation succeeded but target id missing")?; + return Ok(target); + } + Some(OperationStatus::Fail) => { + anyhow::bail!( + "Cloud Map operation failed: {}", + op.error_message().unwrap_or("unknown error") + ); + } + _ => tokio::time::sleep(std::time::Duration::from_secs(2)).await, + } + } + anyhow::bail!("timed out waiting for Cloud Map operation {op_id}") +} + +// ─── Security group ─────────────────────────────────────────────────────────── + +async fn ensure_sg_ingress( + ec2: &aws_sdk_ec2::Client, + security_groups: &[String], + port: u16, +) -> Result<()> { + use aws_sdk_ec2::types::{IpPermission, UserIdGroupPair}; + for sg in security_groups { + // Self-referencing rule: VPC Link ENIs live in this SG, so allowing the + // SG to reach itself on the container port covers VPC Link → task. + let pair = UserIdGroupPair::builder().group_id(sg).build(); + let perm = IpPermission::builder() + .ip_protocol("tcp") + .from_port(port as i32) + .to_port(port as i32) + .user_id_group_pairs(pair) + .build(); + match ec2 + .authorize_security_group_ingress() + .group_id(sg) + .ip_permissions(perm) + .send() + .await + { + Ok(_) => eprintln!(" ✓ SG {sg}: allowed self :{port} (VPC Link → task)"), + Err(e) => { + let msg = format!("{e:?}"); + if msg.contains("Duplicate") || msg.contains("already exists") { + eprintln!(" ✓ SG {sg}: inbound :{port} rule already present"); + } else { + return Err(anyhow::anyhow!(e)) + .with_context(|| format!("failed to authorize ingress on {sg}")); + } + } + } + } + Ok(()) +} + +// ─── VPC Link ───────────────────────────────────────────────────────────────── + +async fn ensure_vpc_link( + api: &aws_sdk_apigatewayv2::Client, + subnets: &[String], + security_groups: &[String], +) -> Result { + use aws_sdk_apigatewayv2::types::VpcLinkStatus; + + // Reuse an existing, non-failed VPC Link with our name. + let existing = api + .get_vpc_links() + .send() + .await + .context("failed to list VPC Links")?; + let mut found: Option<(String, Option)> = None; + for link in existing.items() { + if link.name() == Some(VPC_LINK_NAME) { + if matches!( + link.vpc_link_status(), + Some(VpcLinkStatus::Failed) | Some(VpcLinkStatus::Deleting) + ) { + continue; + } + found = Some(( + link.vpc_link_id().unwrap_or_default().to_string(), + link.vpc_link_status().cloned(), + )); + break; + } + } + + let link_id = if let Some((id, _status)) = found { + eprintln!(" ✓ VPC Link exists: {VPC_LINK_NAME} ({id})"); + id + } else { + eprintln!(" ⊕ Creating VPC Link: {VPC_LINK_NAME}"); + let out = api + .create_vpc_link() + .name(VPC_LINK_NAME) + .set_subnet_ids(Some(subnets.to_vec())) + .set_security_group_ids(Some(security_groups.to_vec())) + .send() + .await + .context("failed to create VPC Link")?; + out.vpc_link_id().context("no VPC Link id")?.to_string() + }; + + // Wait until AVAILABLE — routes won't serve traffic while PENDING. + for _ in 0..60 { + let resp = api + .get_vpc_link() + .vpc_link_id(&link_id) + .send() + .await + .context("failed to poll VPC Link")?; + match resp.vpc_link_status() { + Some(VpcLinkStatus::Available) => return Ok(link_id), + Some(VpcLinkStatus::Failed) => anyhow::bail!( + "VPC Link {link_id} entered FAILED state: {}", + resp.vpc_link_status_message().unwrap_or("unknown") + ), + _ => { + eprintln!(" … waiting for VPC Link to become AVAILABLE (can take a few min)"); + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + } + } + } + anyhow::bail!("timed out waiting for VPC Link {link_id} to become AVAILABLE") +} + +// ─── HTTP API ─────────────────────────────────────────────────────────────── + +async fn ensure_api(api: &aws_sdk_apigatewayv2::Client) -> Result<(String, String)> { + let existing = api.get_apis().send().await.context("failed to list APIs")?; + for a in existing.items() { + if a.name() == Some(API_NAME) { + let id = a.api_id().context("api missing id")?.to_string(); + let endpoint = a.api_endpoint().unwrap_or_default().to_string(); + eprintln!(" ✓ HTTP API exists: {API_NAME} ({id})"); + return Ok((id, endpoint)); + } + } + eprintln!(" ⊕ Creating HTTP API: {API_NAME}"); + let out = api + .create_api() + .name(API_NAME) + .protocol_type(ProtocolType::Http) + .send() + .await + .context("failed to create HTTP API")?; + let id = out.api_id().context("no api id")?.to_string(); + let endpoint = out.api_endpoint().unwrap_or_default().to_string(); + Ok((id, endpoint)) +} + +async fn ensure_integration( + api: &aws_sdk_apigatewayv2::Client, + api_id: &str, + vpc_link_id: &str, + integration_uri: &str, +) -> Result { + let existing = api + .get_integrations() + .api_id(api_id) + .send() + .await + .context("failed to list integrations")?; + for i in existing.items() { + if i.integration_uri() == Some(integration_uri) && i.connection_id() == Some(vpc_link_id) { + let id = i + .integration_id() + .context("integration missing id")? + .to_string(); + eprintln!(" ✓ Integration exists → {integration_uri}"); + return Ok(id); + } + } + eprintln!(" ⊕ Creating integration → {integration_uri}"); + let out = api + .create_integration() + .api_id(api_id) + .integration_type(IntegrationType::HttpProxy) + .integration_method("POST") + .integration_uri(integration_uri) + .connection_type(ConnectionType::VpcLink) + .connection_id(vpc_link_id) + .payload_format_version("1.0") + .send() + .await + .context("failed to create integration")?; + Ok(out + .integration_id() + .context("no integration id")? + .to_string()) +} + +async fn ensure_route( + api: &aws_sdk_apigatewayv2::Client, + api_id: &str, + path: &str, + integration_id: &str, +) -> Result<()> { + let route_key = format!("POST {path}"); + let target = format!("integrations/{integration_id}"); + let existing = api + .get_routes() + .api_id(api_id) + .send() + .await + .context("failed to list routes")?; + for r in existing.items() { + if r.route_key() == Some(route_key.as_str()) { + eprintln!(" ✓ Route exists: {route_key}"); + return Ok(()); + } + } + api.create_route() + .api_id(api_id) + .route_key(&route_key) + .target(&target) + .send() + .await + .with_context(|| format!("failed to create route {route_key}"))?; + eprintln!(" ⊕ Created route: {route_key}"); + Ok(()) +} + +async fn ensure_stage(api: &aws_sdk_apigatewayv2::Client, api_id: &str) -> Result<()> { + let existing = api + .get_stages() + .api_id(api_id) + .send() + .await + .context("failed to list stages")?; + for s in existing.items() { + if s.stage_name() == Some(STAGE_NAME) { + eprintln!(" ✓ Stage exists: {STAGE_NAME}"); + return Ok(()); + } + } + api.create_stage() + .api_id(api_id) + .stage_name(STAGE_NAME) + .auto_deploy(true) + .send() + .await + .context("failed to create stage")?; + eprintln!(" ⊕ Created stage: {STAGE_NAME} (auto-deploy)"); + Ok(()) +} diff --git a/operator/src/main.rs b/operator/src/main.rs index c391e04e0..93ba84fcb 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -5,6 +5,7 @@ mod config; mod create; mod get; mod delete; +mod ingress; use clap::{Parser, Subcommand}; diff --git a/operator/src/manifest.rs b/operator/src/manifest.rs index 8f7d32112..079aa331c 100644 --- a/operator/src/manifest.rs +++ b/operator/src/manifest.rs @@ -51,6 +51,8 @@ pub struct FleetTemplate { #[serde(default)] pub secrets: HashMap, pub runtime: Runtime, + #[serde(default)] + pub ingress: Option, } #[derive(Debug, Deserialize, Serialize)] @@ -66,6 +68,8 @@ pub struct AgentOverride { pub secrets: Option>, #[serde(default)] pub image: Option, + #[serde(default)] + pub ingress: Option, } impl OABFleetManifest { @@ -124,6 +128,10 @@ impl OABFleetManifest { .map(|s| s.replace("${name}", &agent.name)), secrets, runtime: self.spec.template.runtime.clone(), + ingress: agent + .ingress + .clone() + .or_else(|| self.spec.template.ingress.clone()), }, } }).collect() @@ -149,6 +157,75 @@ pub struct Spec { #[serde(default)] pub secrets: HashMap, pub runtime: Runtime, + /// Optional inbound webhook ingress (Telegram/LINE/etc.). + /// When omitted, the service is outbound-only (Discord behavior) — no ingress + /// resources are created. This keeps existing deployments unchanged. + #[serde(default)] + pub ingress: Option, +} + +/// Inbound HTTPS ingress for webhook-based platforms (Telegram, LINE, ...). +/// +/// Provisions the cheapest AWS-native path: API Gateway HTTP API → VPC Link → +/// Cloud Map → the ECS task on `containerPort`. See +/// `docs/refarch/running-telegram-line-on-aws.md` (Option 1). +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Ingress { + /// Ingress implementation. Currently only `apigateway` is supported. + #[serde(default = "default_ingress_type")] + pub r#type: String, + /// Cloud Map private DNS namespace. Created if missing and reused across + /// services in the same VPC. Defaults to `oab`. + #[serde(default = "default_cloud_map_namespace")] + pub cloud_map_namespace: String, + /// Webhook route paths to expose, e.g. `["/webhook/telegram", "/webhook/line"]`. + pub paths: Vec, + /// Container port the OpenAB binary listens on. Defaults to `8080`. + #[serde(default = "default_container_port")] + pub container_port: u16, +} + +fn default_ingress_type() -> String { + "apigateway".to_string() +} + +fn default_cloud_map_namespace() -> String { + "oab".to_string() +} + +fn default_container_port() -> u16 { + 8080 +} + +impl Ingress { + /// Ingress implementations recognized by oabctl. + pub const SUPPORTED_TYPES: &'static [&'static str] = &["apigateway"]; + + pub fn validate(&self) -> anyhow::Result<()> { + if !Self::SUPPORTED_TYPES.contains(&self.r#type.as_str()) { + anyhow::bail!( + "ingress.type must be one of {:?} (got '{}')", + Self::SUPPORTED_TYPES, + self.r#type + ); + } + if self.cloud_map_namespace.is_empty() { + anyhow::bail!("ingress.cloudMapNamespace must not be empty"); + } + if self.paths.is_empty() { + anyhow::bail!("ingress.paths must not be empty"); + } + for p in &self.paths { + if !p.starts_with('/') { + anyhow::bail!("ingress path '{}' must start with '/'", p); + } + } + if self.container_port == 0 { + anyhow::bail!("ingress.containerPort must be non-zero"); + } + Ok(()) + } } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -242,10 +319,177 @@ impl OABServiceManifest { // K8S: cpu/memory format validated at deploy time by K8S API } } + if let Some(ingress) = &self.spec.ingress { + ingress.validate()?; + if !matches!(&self.spec.runtime, Runtime::Ecs(_)) { + anyhow::bail!( + "spec.ingress is only supported with ECS runtime (use native Kubernetes Ingress otherwise)" + ); + } + } Ok(()) } pub fn ecs_service_name(&self) -> String { format!("oab-{}-{}", self.metadata.namespace, self.metadata.name) } + + /// Cloud Map service name for this manifest (unique per namespace+name). + /// Resolves to `.` in private DNS. + pub fn cloud_map_service_name(&self) -> String { + format!("oab-{}-{}", self.metadata.namespace, self.metadata.name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ECS_SERVICE_WITH_INGRESS: &str = r#" +apiVersion: oab.dev/v2 +kind: OABService +metadata: + name: mybot + namespace: prod +spec: + image: public.ecr.aws/oablab/kiro:beta + resources: + cpu: "256" + memory: "512" + configFrom: s3://bucket/config.toml + runtime: + type: ecs + capacityProvider: FARGATE_SPOT + networking: + subnets: ["subnet-a", "subnet-b"] + securityGroups: ["sg-1"] + ingress: + paths: + - /webhook/telegram + - /webhook/line +"#; + + fn parse(yaml: &str) -> OABServiceManifest { + serde_yaml::from_str(yaml).expect("parse") + } + + #[test] + fn parses_ingress_with_defaults() { + let m = parse(ECS_SERVICE_WITH_INGRESS); + let ing = m.spec.ingress.as_ref().expect("ingress present"); + assert_eq!(ing.r#type, "apigateway"); + assert_eq!(ing.cloud_map_namespace, "oab"); + assert_eq!(ing.container_port, 8080); + assert_eq!(ing.paths, vec!["/webhook/telegram", "/webhook/line"]); + m.validate().expect("valid"); + } + + #[test] + fn ingress_is_optional_and_backward_compatible() { + let yaml = ECS_SERVICE_WITH_INGRESS.split(" ingress:").next().unwrap(); + let m = parse(yaml); + assert!(m.spec.ingress.is_none()); + m.validate().expect("valid without ingress"); + } + + #[test] + fn rejects_unknown_ingress_type() { + let ing = Ingress { + r#type: "nginx".into(), + cloud_map_namespace: "oab".into(), + paths: vec!["/webhook".into()], + container_port: 8080, + }; + assert!(ing.validate().is_err()); + } + + #[test] + fn rejects_empty_paths() { + let ing = Ingress { + r#type: "apigateway".into(), + cloud_map_namespace: "oab".into(), + paths: vec![], + container_port: 8080, + }; + assert!(ing.validate().is_err()); + } + + #[test] + fn rejects_path_without_leading_slash() { + let ing = Ingress { + r#type: "apigateway".into(), + cloud_map_namespace: "oab".into(), + paths: vec!["webhook/telegram".into()], + container_port: 8080, + }; + assert!(ing.validate().is_err()); + } + + #[test] + fn rejects_ingress_on_kubernetes_runtime() { + let yaml = r#" +apiVersion: oab.dev/v2 +kind: OABService +metadata: + name: mybot + namespace: prod +spec: + image: img:tag + resources: + cpu: "256" + memory: "512" + configFrom: s3://bucket/config.toml + runtime: + type: kubernetes + nodeSelector: {} + ingress: + paths: ["/webhook/telegram"] +"#; + let m = parse(yaml); + assert!(m.validate().is_err()); + } + + #[test] + fn fleet_passes_ingress_from_template_and_override_wins() { + let yaml = r#" +apiVersion: oab.dev/v2 +kind: OABFleet +metadata: + name: bots + namespace: prod +spec: + template: + image: img:tag + runtime: + type: ecs + capacityProvider: FARGATE_SPOT + networking: + subnets: ["subnet-a"] + securityGroups: ["sg-1"] + ingress: + paths: ["/webhook/telegram"] + agents: + - name: fromtemplate + configFrom: s3://b/${name}.toml + - name: overridden + configFrom: s3://b/${name}.toml + ingress: + cloudMapNamespace: custom + paths: ["/webhook/line"] +"#; + let fleet: OABFleetManifest = serde_yaml::from_str(yaml).expect("parse fleet"); + fleet.validate().expect("valid fleet"); + let expanded = fleet.expand(); + assert_eq!(expanded.len(), 2); + + let from_template = &expanded[0]; + let ing = from_template.spec.ingress.as_ref().expect("template ingress"); + assert_eq!(ing.paths, vec!["/webhook/telegram"]); + assert_eq!(ing.cloud_map_namespace, "oab"); + + let overridden = &expanded[1]; + let ing = overridden.spec.ingress.as_ref().expect("override ingress"); + assert_eq!(ing.paths, vec!["/webhook/line"]); + assert_eq!(ing.cloud_map_namespace, "custom"); + } } From 4a87338454840fc6608134920c9358efeee31cac Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 15:33:55 -0400 Subject: [PATCH 02/13] build(operator): declare standalone [workspace] so CI cargo check works The ci.yml `operator` job runs `cargo check` with working-directory operator, but the crate is not listed in the repo-root workspace (members: openab-core, openab-gateway). Cargo then walks up to the root Cargo.toml and errors ("believes it's in a workspace when it's not"). This job only runs when operator/** changes, so it was latent until now. Adding an empty [workspace] table marks operator as its own workspace root, matching how it's built and released standalone (own Cargo.lock). --- operator/Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/operator/Cargo.toml b/operator/Cargo.toml index c35af5f46..fea5b37ff 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -4,6 +4,11 @@ version = "0.1.0" edition = "2021" description = "CLI provisioner for OAB agents on ECS" +# Standalone crate — not part of the repo-root [workspace]. Declaring an empty +# workspace table stops cargo from attaching this crate to the root workspace +# (which lists only crates/openab-core and crates/openab-gateway). +[workspace] + [[bin]] name = "oabctl" path = "src/main.rs" From f464fd9a160132d569c27defc9356c120fe2a125 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 15:35:44 -0400 Subject: [PATCH 03/13] test(operator): unit-test ingress URL/route helpers; document shared VPC Link constraint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR review feedback: - Extract pure helpers (integration_uri, route_key, webhook_urls) from ensure_gateway and add 4 unit tests (F3 — ingress.rs had no coverage). - Warn at apply time when reusing the shared oab-vpc-link and document in README that all ingress bots in a VPC must share subnets/SGs, since a VPC Link's subnets/SGs are fixed at creation (F2). --- operator/README.md | 6 ++++ operator/src/ingress.rs | 79 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/operator/README.md b/operator/README.md index 03963c59b..2e892dd99 100644 --- a/operator/README.md +++ b/operator/README.md @@ -264,6 +264,12 @@ LINE console: > **Recreate caveat:** ECS service registries can only be set at *creation* time. > If the service already exists without service discovery, `apply` provisions the > ingress resources and prints how to recreate the service so traffic can reach it. +> +> **Shared VPC Link:** all ingress-enabled bots in a VPC share one `oab-vpc-link`. +> A VPC Link's subnets/security groups are fixed at creation and cannot be changed, +> so every ingress bot in the VPC must use the same `networking.subnets` / +> `securityGroups` as whichever bot created the link first. `apply` prints a +> reminder when it reuses an existing link. ### OABFleet — batch deploy diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index efda390f7..ea4396898 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -86,7 +86,7 @@ pub async fn ensure_gateway( let (api_id, api_endpoint) = ensure_api(&api).await?; // ── Integration: VPC Link → Cloud Map DNS name on the container port ──── - let integration_uri = format!("http://{}:{}", dns_name, ingress.container_port); + let integration_uri = integration_uri(dns_name, ingress.container_port); let integration_id = ensure_integration(&api, &api_id, &vpc_link_id, &integration_uri).await?; // ── One route per webhook path, all → the same integration ───────────── @@ -97,13 +97,27 @@ pub async fn ensure_gateway( // ── Stage (auto-deploy) ──────────────────────────────────────────────── ensure_stage(&api, &api_id).await?; + Ok(webhook_urls(&api_endpoint, &ingress.paths)) +} + +/// Integration URI for the VPC Link → Cloud Map target on the container port. +fn integration_uri(dns_name: &str, port: u16) -> String { + format!("http://{dns_name}:{port}") +} + +/// API Gateway route key for a webhook path (POST only). +fn route_key(path: &str) -> String { + format!("POST {path}") +} + +/// Build the public webhook URL(s) from the API endpoint and paths. +/// Each URL is `/`. +fn webhook_urls(api_endpoint: &str, paths: &[String]) -> Vec { let base = api_endpoint.trim_end_matches('/'); - let urls = ingress - .paths + paths .iter() .map(|p| format!("{base}/{STAGE_NAME}{p}")) - .collect(); - Ok(urls) + .collect() } // ─── VPC resolution ───────────────────────────────────────────────────────── @@ -328,6 +342,13 @@ async fn ensure_vpc_link( let link_id = if let Some((id, _status)) = found { eprintln!(" ✓ VPC Link exists: {VPC_LINK_NAME} ({id})"); + // A VPC Link's subnets/SGs are fixed at creation and cannot be updated. + // All ingress-enabled bots in a VPC share this one link, so they must use + // the same subnet/SG set as whichever bot created it first — otherwise the + // link's ENIs won't cover this task's subnets and integrations may 503. + eprintln!( + " ↳ reusing shared link's subnets/SGs (fixed at creation); ensure all\n ingress bots in this VPC use the same subnets/securityGroups" + ); id } else { eprintln!(" ⊕ Creating VPC Link: {VPC_LINK_NAME}"); @@ -437,7 +458,7 @@ async fn ensure_route( path: &str, integration_id: &str, ) -> Result<()> { - let route_key = format!("POST {path}"); + let route_key = route_key(path); let target = format!("integrations/{integration_id}"); let existing = api .get_routes() @@ -485,3 +506,49 @@ async fn ensure_stage(api: &aws_sdk_apigatewayv2::Client, api_id: &str) -> Resul eprintln!(" ⊕ Created stage: {STAGE_NAME} (auto-deploy)"); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn integration_uri_uses_dns_and_port() { + assert_eq!( + integration_uri("oab-prod-mybot.oab", 8080), + "http://oab-prod-mybot.oab:8080" + ); + assert_eq!( + integration_uri("svc.ns", 3000), + "http://svc.ns:3000" + ); + } + + #[test] + fn route_key_is_post_prefixed() { + assert_eq!(route_key("/webhook/telegram"), "POST /webhook/telegram"); + assert_eq!(route_key("/webhook/line"), "POST /webhook/line"); + } + + #[test] + fn webhook_urls_join_endpoint_stage_and_path() { + let paths = vec![ + "/webhook/telegram".to_string(), + "/webhook/line".to_string(), + ]; + let urls = webhook_urls("https://abc123.execute-api.us-east-1.amazonaws.com", &paths); + assert_eq!( + urls, + vec![ + "https://abc123.execute-api.us-east-1.amazonaws.com/prod/webhook/telegram", + "https://abc123.execute-api.us-east-1.amazonaws.com/prod/webhook/line", + ] + ); + } + + #[test] + fn webhook_urls_trim_trailing_slash_on_endpoint() { + let paths = vec!["/webhook/telegram".to_string()]; + let urls = webhook_urls("https://abc123.example.com/", &paths); + assert_eq!(urls, vec!["https://abc123.example.com/prod/webhook/telegram"]); + } +} From 93c995bf65d145fb927cd79d953ccf4e08d99447 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 15:48:58 -0400 Subject: [PATCH 04/13] fix(operator): correct recreate hint, add ingress teardown on delete, paginate API GW lists Addresses PR review round 3: - F1: recreate hint used `oabctl delete service` (invalid) and omitted --cluster; now prints `oabctl delete oabservice --cluster oab --namespace ` which matches delete.rs's contract and the hardcoded `oab` cluster. - F2: oabctl delete oabservice now tears down the bot's per-bot ingress resources (Cloud Map service + API Gateway routes/integration), best-effort, leaving shared VPC Link / HTTP API / SG rule intact. No-op for bots that never had ingress. Documented in README. - F3: apigatewayv2 GetApis/GetVpcLinks/GetIntegrations/GetRoutes/GetStages have no smithy paginator in this SDK, so paginate manually via next_token loops instead of reading only the first page. Verified: cargo build, clippy --all-targets -D warnings, cargo test (11 passed) in a nested layout mirroring the CI operator job. --- operator/README.md | 7 ++ operator/src/apply.rs | 2 +- operator/src/delete.rs | 7 ++ operator/src/ingress.rs | 265 +++++++++++++++++++++++++++++++--------- 4 files changed, 221 insertions(+), 60 deletions(-) diff --git a/operator/README.md b/operator/README.md index 2e892dd99..45864819e 100644 --- a/operator/README.md +++ b/operator/README.md @@ -270,6 +270,13 @@ LINE console: > so every ingress bot in the VPC must use the same `networking.subnets` / > `securityGroups` as whichever bot created the link first. `apply` prints a > reminder when it reuses an existing link. +> +> **Teardown:** `oabctl delete oabservice ` removes the bot's per-bot ingress +> resources — its Cloud Map service and its API Gateway routes + integration — on a +> best-effort basis (it never blocks service deletion). The **shared** `oab-vpc-link`, +> `oab-webhook` HTTP API, and the security-group inbound rule are intentionally left +> in place for other bots. If the Cloud Map service still has registered instances +> (tasks not yet drained), delete prints a note to retry. ### OABFleet — batch deploy diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 99ec7a646..ce2585237 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -274,7 +274,7 @@ async fn apply_ecs( eprintln!(" resources were provisioned but traffic won't reach the task until you"); eprintln!(" recreate the service (safe once desiredCount is drained):"); eprintln!( - " oabctl delete service {} --namespace {} && oabctl apply -f ", + " oabctl delete oabservice {} --cluster oab --namespace {} && oabctl apply -f ", m.metadata.name, m.metadata.namespace ); } diff --git a/operator/src/delete.rs b/operator/src/delete.rs index 199049fd9..d63bae87d 100644 --- a/operator/src/delete.rs +++ b/operator/src/delete.rs @@ -38,6 +38,13 @@ pub async fn run( .context("failed to delete ECS service")?; println!(" ✓ ECS service deleted"); + // 2b. Best-effort ingress teardown (per-bot Cloud Map service + API Gateway + // routes/integration). No-op for bots that never had ingress. Never blocks + // deletion — failures are logged only. + if let Err(e) = crate::ingress::teardown(aws_config, namespace, name).await { + eprintln!(" ⚠ ingress teardown skipped: {e}"); + } + // 3. Clean up S3 manifest let manifest_key = format!("manifests/{}/{}.yaml", namespace, name); let _ = s3 diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index ea4396898..0c0b4860d 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -120,6 +120,107 @@ fn webhook_urls(api_endpoint: &str, paths: &[String]) -> Vec { .collect() } +/// Best-effort teardown of the *per-bot* ingress resources for `namespace/name`: +/// the API Gateway routes + integration for this bot and its Cloud Map service. +/// +/// The shared resources (VPC Link `oab-vpc-link`, HTTP API `oab-webhook`, and the +/// security-group inbound rule) are intentionally left in place since other bots +/// may still use them. Safe to call for bots that never had ingress — it simply +/// finds nothing and returns. Errors are logged, not propagated, so teardown +/// never blocks service deletion. +pub async fn teardown( + config: &aws_config::SdkConfig, + namespace: &str, + name: &str, +) -> Result<()> { + let service_name = format!("oab-{namespace}-{name}"); + let api = aws_sdk_apigatewayv2::Client::new(config); + + // ── API Gateway: routes + integration for this bot (matched by DNS host) ─ + if let Some((api_id, _)) = find_api(&api).await? { + let mut integration_id: Option = None; + let mut next: Option = None; + 'find: loop { + let mut req = api.get_integrations().api_id(&api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list integrations")?; + for i in resp.items() { + if i + .integration_uri() + .is_some_and(|u| u.contains(&format!("//{service_name}."))) + { + integration_id = i.integration_id().map(|s| s.to_string()); + break 'find; + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, + } + } + + if let Some(integration_id) = integration_id { + let target = format!("integrations/{integration_id}"); + let mut route_ids = Vec::new(); + let mut next: Option = None; + loop { + let mut req = api.get_routes().api_id(&api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list routes")?; + for r in resp.items() { + if r.target() == Some(target.as_str()) { + if let Some(rid) = r.route_id() { + route_ids.push(rid.to_string()); + } + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, + } + } + for rid in route_ids { + api.delete_route().api_id(&api_id).route_id(&rid).send().await.ok(); + } + api.delete_integration() + .api_id(&api_id) + .integration_id(&integration_id) + .send() + .await + .ok(); + eprintln!(" ✓ Removed API Gateway routes + integration for {name}"); + } + } + + // ── Cloud Map: delete the per-bot service (needs no live instances) ────── + let sd = aws_sdk_servicediscovery::Client::new(config); + let mut service_id: Option = None; + let mut pages = sd.list_services().into_paginator().send(); + 'svc: while let Some(page) = pages.next().await { + let page = page.context("failed to list Cloud Map services")?; + for s in page.services() { + if s.name() == Some(service_name.as_str()) { + service_id = s.id().map(|x| x.to_string()); + break 'svc; + } + } + } + if let Some(service_id) = service_id { + match sd.delete_service().id(&service_id).send().await { + Ok(_) => eprintln!(" ✓ Deleted Cloud Map service: {service_name}"), + Err(e) => eprintln!( + " ⚠ Cloud Map service '{service_name}' not deleted — it may still have\n registered instances; retry once the ECS tasks have fully drained ({e})" + ), + } + } + + Ok(()) +} + // ─── VPC resolution ───────────────────────────────────────────────────────── async fn resolve_vpc_id(ec2: &aws_sdk_ec2::Client, m: &OABServiceManifest) -> Result { @@ -318,25 +419,33 @@ async fn ensure_vpc_link( use aws_sdk_apigatewayv2::types::VpcLinkStatus; // Reuse an existing, non-failed VPC Link with our name. - let existing = api - .get_vpc_links() - .send() - .await - .context("failed to list VPC Links")?; + // Reuse an existing, non-failed VPC Link with our name. let mut found: Option<(String, Option)> = None; - for link in existing.items() { - if link.name() == Some(VPC_LINK_NAME) { - if matches!( - link.vpc_link_status(), - Some(VpcLinkStatus::Failed) | Some(VpcLinkStatus::Deleting) - ) { - continue; + let mut next: Option = None; + 'outer: loop { + let mut req = api.get_vpc_links(); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list VPC Links")?; + for link in resp.items() { + if link.name() == Some(VPC_LINK_NAME) { + if matches!( + link.vpc_link_status(), + Some(VpcLinkStatus::Failed) | Some(VpcLinkStatus::Deleting) + ) { + continue; + } + found = Some(( + link.vpc_link_id().unwrap_or_default().to_string(), + link.vpc_link_status().cloned(), + )); + break 'outer; } - found = Some(( - link.vpc_link_id().unwrap_or_default().to_string(), - link.vpc_link_status().cloned(), - )); - break; + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, } } @@ -389,14 +498,9 @@ async fn ensure_vpc_link( // ─── HTTP API ─────────────────────────────────────────────────────────────── async fn ensure_api(api: &aws_sdk_apigatewayv2::Client) -> Result<(String, String)> { - let existing = api.get_apis().send().await.context("failed to list APIs")?; - for a in existing.items() { - if a.name() == Some(API_NAME) { - let id = a.api_id().context("api missing id")?.to_string(); - let endpoint = a.api_endpoint().unwrap_or_default().to_string(); - eprintln!(" ✓ HTTP API exists: {API_NAME} ({id})"); - return Ok((id, endpoint)); - } + if let Some((id, endpoint)) = find_api(api).await? { + eprintln!(" ✓ HTTP API exists: {API_NAME} ({id})"); + return Ok((id, endpoint)); } eprintln!(" ⊕ Creating HTTP API: {API_NAME}"); let out = api @@ -411,26 +515,57 @@ async fn ensure_api(api: &aws_sdk_apigatewayv2::Client) -> Result<(String, Strin Ok((id, endpoint)) } +/// Find the shared `oab-webhook` HTTP API, returning `(api_id, api_endpoint)`. +async fn find_api(api: &aws_sdk_apigatewayv2::Client) -> Result> { + // apigatewayv2 has no smithy paginator for GetApis; page manually. + let mut next: Option = None; + loop { + let mut req = api.get_apis(); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list APIs")?; + for a in resp.items() { + if a.name() == Some(API_NAME) { + let id = a.api_id().context("api missing id")?.to_string(); + let endpoint = a.api_endpoint().unwrap_or_default().to_string(); + return Ok(Some((id, endpoint))); + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => return Ok(None), + } + } +} + async fn ensure_integration( api: &aws_sdk_apigatewayv2::Client, api_id: &str, vpc_link_id: &str, integration_uri: &str, ) -> Result { - let existing = api - .get_integrations() - .api_id(api_id) - .send() - .await - .context("failed to list integrations")?; - for i in existing.items() { - if i.integration_uri() == Some(integration_uri) && i.connection_id() == Some(vpc_link_id) { - let id = i - .integration_id() - .context("integration missing id")? - .to_string(); - eprintln!(" ✓ Integration exists → {integration_uri}"); - return Ok(id); + let mut next: Option = None; + loop { + let mut req = api.get_integrations().api_id(api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list integrations")?; + for i in resp.items() { + if i.integration_uri() == Some(integration_uri) && i.connection_id() == Some(vpc_link_id) + { + let id = i + .integration_id() + .context("integration missing id")? + .to_string(); + eprintln!(" ✓ Integration exists → {integration_uri}"); + return Ok(id); + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, } } eprintln!(" ⊕ Creating integration → {integration_uri}"); @@ -460,16 +595,22 @@ async fn ensure_route( ) -> Result<()> { let route_key = route_key(path); let target = format!("integrations/{integration_id}"); - let existing = api - .get_routes() - .api_id(api_id) - .send() - .await - .context("failed to list routes")?; - for r in existing.items() { - if r.route_key() == Some(route_key.as_str()) { - eprintln!(" ✓ Route exists: {route_key}"); - return Ok(()); + let mut next: Option = None; + loop { + let mut req = api.get_routes().api_id(api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list routes")?; + for r in resp.items() { + if r.route_key() == Some(route_key.as_str()) { + eprintln!(" ✓ Route exists: {route_key}"); + return Ok(()); + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, } } api.create_route() @@ -484,16 +625,22 @@ async fn ensure_route( } async fn ensure_stage(api: &aws_sdk_apigatewayv2::Client, api_id: &str) -> Result<()> { - let existing = api - .get_stages() - .api_id(api_id) - .send() - .await - .context("failed to list stages")?; - for s in existing.items() { - if s.stage_name() == Some(STAGE_NAME) { - eprintln!(" ✓ Stage exists: {STAGE_NAME}"); - return Ok(()); + let mut next: Option = None; + loop { + let mut req = api.get_stages().api_id(api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list stages")?; + for s in resp.items() { + if s.stage_name() == Some(STAGE_NAME) { + eprintln!(" ✓ Stage exists: {STAGE_NAME}"); + return Ok(()); + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, } } api.create_stage() From 6f92a2edc9adae9490773d84845708a0e99b4478 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 16:02:44 -0400 Subject: [PATCH 05/13] fix(operator): per-bot HTTP API to prevent cross-bot webhook path collisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-4 review: - F1 (critical): routes on the shared oab-webhook API were keyed by path only, so two bots both declaring /webhook/telegram would collide — the second apply saw the first's route, skipped, and silently misrouted traffic. Give each bot its own HTTP API (oab-webhook--) so webhook paths can never clash across bots; each bot gets a distinct {api-id} endpoint URL. Simplifies teardown to a single delete_api (cascades routes/integration/stage). VPC Link + SG rule stay shared. - F2: removed duplicated comment line in ensure_vpc_link. - Added api_name() helper + unit test (12 tests total). Verified: build, clippy --all-targets -D warnings, cargo test (12 passed) in a nested layout mirroring the CI operator job. --- operator/README.md | 17 ++++-- operator/src/apply.rs | 2 + operator/src/ingress.rs | 126 ++++++++++++++-------------------------- 3 files changed, 58 insertions(+), 87 deletions(-) diff --git a/operator/README.md b/operator/README.md index 45864819e..e5c1b976e 100644 --- a/operator/README.md +++ b/operator/README.md @@ -244,7 +244,7 @@ On `apply` this reconciles (idempotently, reused by name): 1. **Cloud Map** private DNS namespace + a per-service A record 2. **ECS service registry** wiring (attached at service creation) 3. **VPC Link** (shared `oab-vpc-link`), waits until `AVAILABLE` -4. **API Gateway HTTP API** (`oab-webhook`) + `HTTP_PROXY` integration over the VPC Link +4. **API Gateway HTTP API** (`oab-webhook--`, one per bot) + `HTTP_PROXY` integration over the VPC Link 5. One **route** per path + a `prod` auto-deploy **stage** 6. A self-referencing **security-group** inbound rule on `containerPort` @@ -272,11 +272,16 @@ LINE console: > reminder when it reuses an existing link. > > **Teardown:** `oabctl delete oabservice ` removes the bot's per-bot ingress -> resources — its Cloud Map service and its API Gateway routes + integration — on a -> best-effort basis (it never blocks service deletion). The **shared** `oab-vpc-link`, -> `oab-webhook` HTTP API, and the security-group inbound rule are intentionally left -> in place for other bots. If the Cloud Map service still has registered instances -> (tasks not yet drained), delete prints a note to retry. +> resources — its Cloud Map service and its own HTTP API (`oab-webhook--`, +> which cascades its routes/integration/stage) — on a best-effort basis (it never +> blocks service deletion). The **shared** `oab-vpc-link` and the security-group +> inbound rule are intentionally left in place for other bots. If the Cloud Map +> service still has registered instances (tasks not yet drained), delete prints a +> note to retry. +> +> **Per-bot API, no path collisions:** each ingress bot gets its own HTTP API, so +> two bots can both use `/webhook/telegram` without clashing — each has a distinct +> `{api-id}` endpoint URL. ### OABFleet — batch deploy diff --git a/operator/src/apply.rs b/operator/src/apply.rs index ce2585237..98901de14 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -324,6 +324,8 @@ async fn apply_ecs( eprintln!(" 🌐 Reconciling ingress (VPC Link + API Gateway)..."); let urls = crate::ingress::ensure_gateway( config, + &m.metadata.namespace, + &m.metadata.name, ingress, &ecs_rt.networking.subnets, &ecs_rt.networking.security_groups, diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index 0c0b4860d..b8aa69541 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -19,9 +19,14 @@ use aws_sdk_apigatewayv2::types::{ConnectionType, IntegrationType, ProtocolType} use aws_sdk_servicediscovery::types::{DnsConfig, DnsRecord, RecordType}; const VPC_LINK_NAME: &str = "oab-vpc-link"; -const API_NAME: &str = "oab-webhook"; const STAGE_NAME: &str = "prod"; +/// Per-bot HTTP API name. Each ingress bot gets its own API so webhook paths +/// (e.g. `/webhook/telegram`) can never collide between bots on a shared API. +fn api_name(namespace: &str, name: &str) -> String { + format!("oab-webhook-{namespace}-{name}") +} + /// Result of Cloud Map reconciliation, consumed when creating the ECS service. pub struct CloudMapResult { /// Cloud Map service ARN — used as the ECS service registry ARN. @@ -68,6 +73,8 @@ pub async fn ensure_cloud_map( /// security-group inbound rule. Returns the public webhook URLs (one per path). pub async fn ensure_gateway( config: &aws_config::SdkConfig, + namespace: &str, + name: &str, ingress: &Ingress, subnets: &[String], security_groups: &[String], @@ -75,15 +82,16 @@ pub async fn ensure_gateway( ) -> Result> { let api = aws_sdk_apigatewayv2::Client::new(config); let ec2 = aws_sdk_ec2::Client::new(config); + let api_name = api_name(namespace, name); // ── Security group inbound rule (self-referencing on the container port) ─ ensure_sg_ingress(&ec2, security_groups, ingress.container_port).await?; - // ── VPC Link (shared, waits for AVAILABLE) ───────────────────────────── + // ── VPC Link (shared across all bots, waits for AVAILABLE) ───────────── let vpc_link_id = ensure_vpc_link(&api, subnets, security_groups).await?; - // ── HTTP API (shared) ────────────────────────────────────────────────── - let (api_id, api_endpoint) = ensure_api(&api).await?; + // ── HTTP API (one per bot — avoids cross-bot path collisions) ────────── + let (api_id, api_endpoint) = ensure_api(&api, &api_name).await?; // ── Integration: VPC Link → Cloud Map DNS name on the container port ──── let integration_uri = integration_uri(dns_name, ingress.container_port); @@ -121,78 +129,23 @@ fn webhook_urls(api_endpoint: &str, paths: &[String]) -> Vec { } /// Best-effort teardown of the *per-bot* ingress resources for `namespace/name`: -/// the API Gateway routes + integration for this bot and its Cloud Map service. +/// the bot's own HTTP API (`oab-webhook--`, which cascades its routes, +/// integration and stage) and its Cloud Map service. /// -/// The shared resources (VPC Link `oab-vpc-link`, HTTP API `oab-webhook`, and the -/// security-group inbound rule) are intentionally left in place since other bots -/// may still use them. Safe to call for bots that never had ingress — it simply -/// finds nothing and returns. Errors are logged, not propagated, so teardown -/// never blocks service deletion. -pub async fn teardown( - config: &aws_config::SdkConfig, - namespace: &str, - name: &str, -) -> Result<()> { +/// The shared resources (the `oab-vpc-link` VPC Link and the security-group +/// inbound rule) are intentionally left in place since other bots may still use +/// them. Safe to call for bots that never had ingress — it simply finds nothing +/// and returns. Errors are logged, not propagated, so teardown never blocks +/// service deletion. +pub async fn teardown(config: &aws_config::SdkConfig, namespace: &str, name: &str) -> Result<()> { let service_name = format!("oab-{namespace}-{name}"); let api = aws_sdk_apigatewayv2::Client::new(config); - // ── API Gateway: routes + integration for this bot (matched by DNS host) ─ - if let Some((api_id, _)) = find_api(&api).await? { - let mut integration_id: Option = None; - let mut next: Option = None; - 'find: loop { - let mut req = api.get_integrations().api_id(&api_id); - if let Some(t) = &next { - req = req.next_token(t); - } - let resp = req.send().await.context("failed to list integrations")?; - for i in resp.items() { - if i - .integration_uri() - .is_some_and(|u| u.contains(&format!("//{service_name}."))) - { - integration_id = i.integration_id().map(|s| s.to_string()); - break 'find; - } - } - match resp.next_token() { - Some(t) => next = Some(t.to_string()), - None => break, - } - } - - if let Some(integration_id) = integration_id { - let target = format!("integrations/{integration_id}"); - let mut route_ids = Vec::new(); - let mut next: Option = None; - loop { - let mut req = api.get_routes().api_id(&api_id); - if let Some(t) = &next { - req = req.next_token(t); - } - let resp = req.send().await.context("failed to list routes")?; - for r in resp.items() { - if r.target() == Some(target.as_str()) { - if let Some(rid) = r.route_id() { - route_ids.push(rid.to_string()); - } - } - } - match resp.next_token() { - Some(t) => next = Some(t.to_string()), - None => break, - } - } - for rid in route_ids { - api.delete_route().api_id(&api_id).route_id(&rid).send().await.ok(); - } - api.delete_integration() - .api_id(&api_id) - .integration_id(&integration_id) - .send() - .await - .ok(); - eprintln!(" ✓ Removed API Gateway routes + integration for {name}"); + // ── API Gateway: delete the whole per-bot API (cascades routes/integration/stage) ─ + if let Some((api_id, _)) = find_api(&api, &api_name(namespace, name)).await? { + match api.delete_api().api_id(&api_id).send().await { + Ok(_) => eprintln!(" ✓ Deleted HTTP API: {}", api_name(namespace, name)), + Err(e) => eprintln!(" ⚠ Failed to delete HTTP API {api_id}: {e}"), } } @@ -418,7 +371,6 @@ async fn ensure_vpc_link( ) -> Result { use aws_sdk_apigatewayv2::types::VpcLinkStatus; - // Reuse an existing, non-failed VPC Link with our name. // Reuse an existing, non-failed VPC Link with our name. let mut found: Option<(String, Option)> = None; let mut next: Option = None; @@ -497,15 +449,18 @@ async fn ensure_vpc_link( // ─── HTTP API ─────────────────────────────────────────────────────────────── -async fn ensure_api(api: &aws_sdk_apigatewayv2::Client) -> Result<(String, String)> { - if let Some((id, endpoint)) = find_api(api).await? { - eprintln!(" ✓ HTTP API exists: {API_NAME} ({id})"); +async fn ensure_api( + api: &aws_sdk_apigatewayv2::Client, + api_name: &str, +) -> Result<(String, String)> { + if let Some((id, endpoint)) = find_api(api, api_name).await? { + eprintln!(" ✓ HTTP API exists: {api_name} ({id})"); return Ok((id, endpoint)); } - eprintln!(" ⊕ Creating HTTP API: {API_NAME}"); + eprintln!(" ⊕ Creating HTTP API: {api_name}"); let out = api .create_api() - .name(API_NAME) + .name(api_name) .protocol_type(ProtocolType::Http) .send() .await @@ -515,8 +470,11 @@ async fn ensure_api(api: &aws_sdk_apigatewayv2::Client) -> Result<(String, Strin Ok((id, endpoint)) } -/// Find the shared `oab-webhook` HTTP API, returning `(api_id, api_endpoint)`. -async fn find_api(api: &aws_sdk_apigatewayv2::Client) -> Result> { +/// Find an HTTP API by name, returning `(api_id, api_endpoint)`. +async fn find_api( + api: &aws_sdk_apigatewayv2::Client, + api_name: &str, +) -> Result> { // apigatewayv2 has no smithy paginator for GetApis; page manually. let mut next: Option = None; loop { @@ -526,7 +484,7 @@ async fn find_api(api: &aws_sdk_apigatewayv2::Client) -> Result Date: Wed, 1 Jul 2026 16:06:04 -0400 Subject: [PATCH 06/13] fix(operator): match typed EC2 error code for duplicate SG rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-5 review F1: ensure_sg_ingress classified the "rule already exists" case by substring-matching the Debug-rendered error string, which breaks if the SDK changes error formatting. Match the typed AWS error code InvalidPermission.Duplicate via ProvideErrorMetadata::code() instead. (F2 — CI green — the operator job now passes on the prior commit.) Verified: build, clippy --all-targets -D warnings, cargo test (12 passed). --- operator/src/ingress.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index b8aa69541..22b65378b 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -329,6 +329,7 @@ async fn ensure_sg_ingress( security_groups: &[String], port: u16, ) -> Result<()> { + use aws_sdk_ec2::error::ProvideErrorMetadata; use aws_sdk_ec2::types::{IpPermission, UserIdGroupPair}; for sg in security_groups { // Self-referencing rule: VPC Link ENIs live in this SG, so allowing the @@ -348,14 +349,14 @@ async fn ensure_sg_ingress( .await { Ok(_) => eprintln!(" ✓ SG {sg}: allowed self :{port} (VPC Link → task)"), + // EC2 returns InvalidPermission.Duplicate when the rule already exists. + // Match the typed error code, not the Debug-rendered message text. + Err(e) if e.code() == Some("InvalidPermission.Duplicate") => { + eprintln!(" ✓ SG {sg}: inbound :{port} rule already present"); + } Err(e) => { - let msg = format!("{e:?}"); - if msg.contains("Duplicate") || msg.contains("already exists") { - eprintln!(" ✓ SG {sg}: inbound :{port} rule already present"); - } else { - return Err(anyhow::anyhow!(e)) - .with_context(|| format!("failed to authorize ingress on {sg}")); - } + return Err(anyhow::anyhow!(e)) + .with_context(|| format!("failed to authorize ingress on {sg}")); } } } From 499df8c8402eed022335f3be774e67435ae8b10b Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 16:57:35 -0400 Subject: [PATCH 07/13] fix(operator): use Cloud Map SRV + service-ARN integration (fixes live 503/BadRequest) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live E2E against a real account revealed the documented Option-1 wiring does not work: an HTTP API VPC_LINK integration rejects a raw 'http://:' URI with BadRequestException: For VpcLink VPC_LINK, integration uri should be a valid ELB listener ARN or a valid Cloud Map service ARN. Correct wiring (matches CDK's HttpServiceDiscoveryIntegration): - Cloud Map service uses an SRV record (not A) so the container port is captured; ECS registers the task IP + port into it. - ECS service registry sets containerName/containerPort and the task def exposes the container port, so ECS writes the SRV record. - API Gateway integration URI is the Cloud Map *service ARN* with connectionType=VPC_LINK and integrationMethod=ANY; the port is resolved from SRV. Verified end-to-end (POST through API Gateway → VPC Link → Cloud Map SRV → Fargate task returned HTTP 200 with the request echoed at /prod/webhook/telegram), then torn down cleanly. build + clippy -D warnings + cargo test (11 passed). --- operator/README.md | 2 +- operator/src/apply.rs | 35 +++++++++++++++++++++++--------- operator/src/ingress.rs | 45 +++++++++++++---------------------------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/operator/README.md b/operator/README.md index e5c1b976e..7988172e4 100644 --- a/operator/README.md +++ b/operator/README.md @@ -241,7 +241,7 @@ spec: On `apply` this reconciles (idempotently, reused by name): -1. **Cloud Map** private DNS namespace + a per-service A record +1. **Cloud Map** private DNS namespace + a per-service **SRV** record (carries the container port; a plain A record does not work as a VPC-Link integration target) 2. **ECS service registry** wiring (attached at service creation) 3. **VPC Link** (shared `oab-vpc-link`), waits until `AVAILABLE` 4. **API Gateway HTTP API** (`oab-webhook--`, one per bot) + `HTTP_PROXY` integration over the VPC Link diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 98901de14..9d1c464cc 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -182,13 +182,25 @@ async fn apply_ecs( .collect(); // 4. Register task definition - let container = ContainerDefinition::builder() + let mut container = ContainerDefinition::builder() .name("openab") .image(&m.spec.image) .essential(true) .set_environment(Some(env_vars)) - .set_secrets(if secrets.is_empty() { None } else { Some(secrets) }) - .build(); + .set_secrets(if secrets.is_empty() { None } else { Some(secrets) }); + + // Ingress needs the container port exposed so ECS can register an SRV record + // (Cloud Map + API Gateway learn the target port from it). + if let Some(ingress) = &m.spec.ingress { + container = container.port_mappings( + aws_sdk_ecs::types::PortMapping::builder() + .container_port(ingress.container_port as i32) + .protocol(aws_sdk_ecs::types::TransportProtocol::Tcp) + .build(), + ); + } + + let container = container.build(); let task_def = ecs .register_task_definition() @@ -294,11 +306,16 @@ async fn apply_ecs( .network_configuration(network_config); if let Some(cm) = &cloud_map { - create_req = create_req.service_registries( - aws_sdk_ecs::types::ServiceRegistry::builder() - .registry_arn(&cm.registry_arn) - .build(), - ); + let mut registry = aws_sdk_ecs::types::ServiceRegistry::builder() + .registry_arn(&cm.registry_arn); + // SRV records require the container name + port so ECS registers the + // task's port alongside its IP. + if let Some(ingress) = &m.spec.ingress { + registry = registry + .container_name("openab") + .container_port(ingress.container_port as i32); + } + create_req = create_req.service_registries(registry.build()); } create_req @@ -329,7 +346,7 @@ async fn apply_ecs( ingress, &ecs_rt.networking.subnets, &ecs_rt.networking.security_groups, - &cm.dns_name, + &cm.registry_arn, ) .await?; println!(" 🔗 Webhook URL(s) for {}:", m.metadata.name); diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index 22b65378b..3a1f7470d 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -29,10 +29,9 @@ fn api_name(namespace: &str, name: &str) -> String { /// Result of Cloud Map reconciliation, consumed when creating the ECS service. pub struct CloudMapResult { - /// Cloud Map service ARN — used as the ECS service registry ARN. + /// Cloud Map service ARN — used both as the ECS service registry ARN and as + /// the API Gateway integration URI. pub registry_arn: String, - /// Private DNS name the task registers under, e.g. `oab-prod-mybot.oab`. - pub dns_name: String, } /// Step 1: ensure the Cloud Map private DNS namespace and service exist. @@ -63,10 +62,7 @@ pub async fn ensure_cloud_map( eprintln!(" ✓ Created Cloud Map service: {dns_name}"); } - Ok(CloudMapResult { - registry_arn, - dns_name, - }) + Ok(CloudMapResult { registry_arn }) } /// Step 2: ensure VPC Link, HTTP API, integration, routes, stage, and the @@ -78,7 +74,7 @@ pub async fn ensure_gateway( ingress: &Ingress, subnets: &[String], security_groups: &[String], - dns_name: &str, + cloud_map_service_arn: &str, ) -> Result> { let api = aws_sdk_apigatewayv2::Client::new(config); let ec2 = aws_sdk_ec2::Client::new(config); @@ -93,9 +89,10 @@ pub async fn ensure_gateway( // ── HTTP API (one per bot — avoids cross-bot path collisions) ────────── let (api_id, api_endpoint) = ensure_api(&api, &api_name).await?; - // ── Integration: VPC Link → Cloud Map DNS name on the container port ──── - let integration_uri = integration_uri(dns_name, ingress.container_port); - let integration_id = ensure_integration(&api, &api_id, &vpc_link_id, &integration_uri).await?; + // ── Integration: VPC Link → Cloud Map service (URI is the service ARN; + // the port is resolved from the service's SRV record) ─────────────── + let integration_id = + ensure_integration(&api, &api_id, &vpc_link_id, cloud_map_service_arn).await?; // ── One route per webhook path, all → the same integration ───────────── for path in &ingress.paths { @@ -108,11 +105,6 @@ pub async fn ensure_gateway( Ok(webhook_urls(&api_endpoint, &ingress.paths)) } -/// Integration URI for the VPC Link → Cloud Map target on the container port. -fn integration_uri(dns_name: &str, port: u16) -> String { - format!("http://{dns_name}:{port}") -} - /// API Gateway route key for a webhook path (POST only). fn route_key(path: &str) -> String { format!("POST {path}") @@ -258,9 +250,12 @@ async fn ensure_service( } } - // Create with a single A record (TTL 60). SRV does NOT work for VPC Link. + // Create with an SRV record. For an HTTP API private integration whose URI + // is the Cloud Map service ARN, API Gateway learns the target port from the + // SRV record — a plain A record carries no port and does NOT work. ECS + // registers the task's IP + container port into this SRV record. let dns_record = DnsRecord::builder() - .r#type(RecordType::A) + .r#type(RecordType::Srv) .ttl(60) .build() .context("failed to build DNS record")?; @@ -532,7 +527,7 @@ async fn ensure_integration( .create_integration() .api_id(api_id) .integration_type(IntegrationType::HttpProxy) - .integration_method("POST") + .integration_method("ANY") .integration_uri(integration_uri) .connection_type(ConnectionType::VpcLink) .connection_id(vpc_link_id) @@ -617,18 +612,6 @@ async fn ensure_stage(api: &aws_sdk_apigatewayv2::Client, api_id: &str) -> Resul mod tests { use super::*; - #[test] - fn integration_uri_uses_dns_and_port() { - assert_eq!( - integration_uri("oab-prod-mybot.oab", 8080), - "http://oab-prod-mybot.oab:8080" - ); - assert_eq!( - integration_uri("svc.ns", 3000), - "http://svc.ns:3000" - ); - } - #[test] fn route_key_is_post_prefixed() { assert_eq!(route_key("/webhook/telegram"), "POST /webhook/telegram"); From df9f14ddfe505971641dd08f3085382dfc08c53a Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 18:25:41 -0400 Subject: [PATCH 08/13] feat(operator): consolidated apply summary for services needing ingress recreation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review F2: the 'service exists without service discovery' warning was a mid-run eprintln that's easy to miss in non-interactive CI, so a partial apply could look like a clean success. apply_ecs now returns whether the service needs recreation; apply::run collects these and prints a single consolidated ⚠ summary block (with the exact recreate commands) after the 'N service(s) applied' line. build + clippy -D warnings + test (11 passed). --- operator/src/apply.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 9d1c464cc..e8ce23acf 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -60,12 +60,30 @@ pub async fn run(aws_config: &aws_config::SdkConfig, file_path: &str, sync_confi } } + let mut needs_recreate: Vec = Vec::new(); for m in &manifests { println!(" Applying {} (ECS)...", m.metadata.name); - apply_ecs(&ecs, &s3, aws_config, m, wait).await?; + if apply_ecs(&ecs, &s3, aws_config, m, wait).await? { + needs_recreate.push(format!("{}/{}", m.metadata.namespace, m.metadata.name)); + } } println!("\n{} service(s) applied.", manifests.len()); + + // Surface the create-only service-discovery limitation as a consolidated + // summary so it isn't lost in mid-run output (e.g. in non-interactive CI). + if !needs_recreate.is_empty() { + eprintln!( + "\n⚠ {} service(s) have ingress configured but were created WITHOUT service\n discovery, so webhook traffic will NOT reach them until recreated:", + needs_recreate.len() + ); + for id in &needs_recreate { + eprintln!(" - {id}"); + } + eprintln!( + " ECS service registries can only be set at creation time. For each, run:\n oabctl delete oabservice --cluster oab --namespace && oabctl apply -f " + ); + } Ok(()) } @@ -117,7 +135,7 @@ async fn apply_ecs( config: &aws_config::SdkConfig, m: &OABServiceManifest, wait: bool, -) -> Result<()> { +) -> Result { let ecs_rt = match &m.spec.runtime { Runtime::Ecs(rt) => rt, _ => unreachable!(), @@ -266,6 +284,7 @@ async fn apply_ecs( .and_then(|r| r.services().first()) .is_some_and(|s| !s.service_registries().is_empty()); + let mut needs_recreate = false; if service_active { ecs.update_service() .cluster("oab") @@ -278,6 +297,7 @@ async fn apply_ecs( println!(" ✓ {} updated", m.metadata.name); if cloud_map.is_some() && !has_registries { + needs_recreate = true; eprintln!( " ⚠ Service '{}' already exists WITHOUT service discovery.", m.metadata.name @@ -361,7 +381,7 @@ async fn apply_ecs( eprintln!(" ✓ {} is stable", m.metadata.name); } - Ok(()) + Ok(needs_recreate) } async fn wait_for_stable(ecs: &aws_sdk_ecs::Client, cluster: &str, service: &str) -> Result<()> { From d40e7db3af58b232fa695cc09d99f31e7a23f0f9 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 18:38:00 -0400 Subject: [PATCH 09/13] fix(operator): scope VPC Link/namespace per-VPC; teardown on ingress removal; prune stale routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-7 review: - F1 (critical): VPC Link was matched by a hardcoded name ('oab-vpc-link') with no VPC check. A VPC Link's ENIs live in one VPC and cannot route to another, so a second bot in a *different* VPC would silently reuse the first VPC's link and get unreachable integrations. Fixed by naming the link 'oab-vpc-link-', scoped per-VPC. - F2: same collision class for the Cloud Map namespace — matched by configured name only. Fixed by scoping the actual namespace name to '-' (the DnsConfig VPC association makes namespace-per-VPC the correct AWS-native mental model anyway; the private DNS only resolves inside that VPC). - F3: apply only ever added ingress resources — editing a manifest to remove spec.ingress orphaned the per-bot HTTP API + Cloud Map service. apply now detects 'had ingress before, doesn't now' by comparing against the previously-stored S3 manifest and calls the same best-effort ingress::teardown used by . - F4: ensure_route only ever added routes; renaming/removing a webhook path left a dead route on the bot's API forever. Added prune_stale_routes, which deletes any route on the bot's API whose key isn't in the current ingress.paths after ensuring the desired ones. Added 3 unit tests (vpc_link_name, vpc_scoped_namespace) — 13 total. Updated README + module doc to describe per-VPC scoping and the new apply-time teardown/pruning behavior. Verified: build, clippy --all-targets -D warnings, cargo test (13 passed). --- operator/README.md | 30 ++++++---- operator/src/apply.rs | 35 ++++++++--- operator/src/ingress.rs | 129 +++++++++++++++++++++++++++++++++++----- 3 files changed, 158 insertions(+), 36 deletions(-) diff --git a/operator/README.md b/operator/README.md index 7988172e4..739b8307b 100644 --- a/operator/README.md +++ b/operator/README.md @@ -241,9 +241,9 @@ spec: On `apply` this reconciles (idempotently, reused by name): -1. **Cloud Map** private DNS namespace + a per-service **SRV** record (carries the container port; a plain A record does not work as a VPC-Link integration target) +1. **Cloud Map** private DNS namespace (`-`, shared per-VPC) + a per-service **SRV** record (carries the container port; a plain A record does not work as a VPC-Link integration target) 2. **ECS service registry** wiring (attached at service creation) -3. **VPC Link** (shared `oab-vpc-link`), waits until `AVAILABLE` +3. **VPC Link** (`oab-vpc-link-`, shared per-VPC), waits until `AVAILABLE` 4. **API Gateway HTTP API** (`oab-webhook--`, one per bot) + `HTTP_PROXY` integration over the VPC Link 5. One **route** per path + a `prod` auto-deploy **stage** 6. A self-referencing **security-group** inbound rule on `containerPort` @@ -265,19 +265,27 @@ LINE console: > If the service already exists without service discovery, `apply` provisions the > ingress resources and prints how to recreate the service so traffic can reach it. > -> **Shared VPC Link:** all ingress-enabled bots in a VPC share one `oab-vpc-link`. -> A VPC Link's subnets/security groups are fixed at creation and cannot be changed, -> so every ingress bot in the VPC must use the same `networking.subnets` / -> `securityGroups` as whichever bot created the link first. `apply` prints a -> reminder when it reuses an existing link. +> **Shared per-VPC (not per-account):** all ingress-enabled bots in the *same VPC* +> share one VPC Link (`oab-vpc-link-`) and one Cloud Map namespace +> (`-`) — both are named by VPC ID so bots in different +> VPCs never collide or reuse each other's link/namespace. A VPC Link's +> subnets/security groups are fixed at creation and cannot be changed, so every +> ingress bot in a given VPC must use the same `networking.subnets` / +> `securityGroups` as whichever bot created that VPC's link first. `apply` prints +> a reminder when it reuses an existing link. > > **Teardown:** `oabctl delete oabservice ` removes the bot's per-bot ingress > resources — its Cloud Map service and its own HTTP API (`oab-webhook--`, > which cascades its routes/integration/stage) — on a best-effort basis (it never -> blocks service deletion). The **shared** `oab-vpc-link` and the security-group -> inbound rule are intentionally left in place for other bots. If the Cloud Map -> service still has registered instances (tasks not yet drained), delete prints a -> note to retry. +> blocks service deletion). The same teardown also runs automatically from `apply` +> if you edit a manifest to remove `spec.ingress` entirely. The **shared** VPC Link +> and the security-group inbound rule are intentionally left in place for other +> bots. If the Cloud Map service still has registered instances (tasks not yet +> drained), delete prints a note to retry. +> +> **Changing `paths`:** `apply` prunes routes on the bot's API that are no longer +> in the manifest's `ingress.paths`, so renaming or removing a webhook path never +> leaves a dangling route. > > **Per-bot API, no path collisions:** each ingress bot gets its own HTTP API, so > two bots can both use `/webhook/telegram` without clashing — each has a distinct diff --git a/operator/src/apply.rs b/operator/src/apply.rs index e8ce23acf..9321f917f 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -153,18 +153,35 @@ async fn apply_ecs( format!("oab-control-plane-{account}") }; - // Read current generation from S3 manifest (if exists), increment + // Read current generation from S3 manifest (if exists), increment. + // Also capture whether the *previous* apply had ingress configured, so we + // can detect "ingress was removed from the manifest" and tear it down + // below — apply only ever provisioned ingress resources before this, so a + // manifest edit that drops `spec.ingress` used to orphan the per-bot HTTP + // API and Cloud Map service. let manifest_key = format!("manifests/{}/{}.yaml", m.metadata.namespace, m.metadata.name); - let current_gen = match s3.get_object().bucket(&bucket).key(&manifest_key).send().await { - Ok(resp) => { - let bytes = resp.body.collect().await?.into_bytes(); - let existing: OABServiceManifest = serde_yaml::from_slice(&bytes)?; - existing.metadata.generation - } - Err(_) => 0, - }; + let (current_gen, previously_had_ingress) = + match s3.get_object().bucket(&bucket).key(&manifest_key).send().await { + Ok(resp) => { + let bytes = resp.body.collect().await?.into_bytes(); + let existing: OABServiceManifest = serde_yaml::from_slice(&bytes)?; + (existing.metadata.generation, existing.spec.ingress.is_some()) + } + Err(_) => (0, false), + }; let generation = current_gen + 1; + // If ingress was configured before but is absent now, tear down the + // orphaned per-bot ingress resources (best-effort, mirrors `oabctl delete`). + if previously_had_ingress && m.spec.ingress.is_none() { + eprintln!(" 🌐 ingress removed from manifest — tearing down orphaned resources..."); + if let Err(e) = + crate::ingress::teardown(config, &m.metadata.namespace, &m.metadata.name).await + { + eprintln!(" ⚠ ingress teardown skipped: {e}"); + } + } + // 1. Upload manifest to S3 (record of desired state) let mut manifest_to_store = serde_yaml::to_value(m)?; manifest_to_store["metadata"]["generation"] = serde_yaml::Value::Number(generation.into()); diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index 3a1f7470d..e57aac9c1 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -4,7 +4,10 @@ //! API Gateway HTTP API → VPC Link → Cloud Map → ECS Fargate task on `:port`. //! //! All operations are idempotent — resources are looked up by name and reused, -//! so repeated `oabctl apply` runs converge instead of duplicating. +//! so repeated `oabctl apply` runs converge instead of duplicating. The shared +//! VPC Link and Cloud Map namespace are scoped per-VPC (their names include the +//! VPC ID) since a VPC Link's ENIs and a namespace's private DNS are only valid +//! within the VPC they were created in — two VPCs never share either resource. //! //! The reconciliation is split in two because ECS service discovery //! (`--service-registries`) can only be set at service *creation* time: @@ -18,9 +21,23 @@ use anyhow::{Context, Result}; use aws_sdk_apigatewayv2::types::{ConnectionType, IntegrationType, ProtocolType}; use aws_sdk_servicediscovery::types::{DnsConfig, DnsRecord, RecordType}; -const VPC_LINK_NAME: &str = "oab-vpc-link"; const STAGE_NAME: &str = "prod"; +/// VPC Link name, scoped per-VPC. A VPC Link's ENIs live in one VPC and cannot +/// route to another, so each VPC gets its own link — sharing a name across VPCs +/// would silently misroute traffic through the wrong VPC's link. +fn vpc_link_name(vpc_id: &str) -> String { + format!("oab-vpc-link-{vpc_id}") +} + +/// Cloud Map private DNS namespace name, scoped per-VPC. A namespace's private +/// DNS only resolves within the VPC it's associated with, so two VPCs both +/// using the configured `cloudMapNamespace` (default `oab`) must not resolve to +/// the same lookup — scope the actual namespace by VPC ID. +fn vpc_scoped_namespace(configured_namespace: &str, vpc_id: &str) -> String { + format!("{configured_namespace}-{vpc_id}") +} + /// Per-bot HTTP API name. Each ingress bot gets its own API so webhook paths /// (e.g. `/webhook/telegram`) can never collide between bots on a shared API. fn api_name(namespace: &str, name: &str) -> String { @@ -48,14 +65,17 @@ pub async fn ensure_cloud_map( let vpc_id = resolve_vpc_id(&ec2, m).await?; - // ── Namespace (reused across all bots in the VPC) ────────────────────── - let namespace_id = ensure_namespace(&sd, &ingress.cloud_map_namespace, &vpc_id).await?; + // ── Namespace (shared per-VPC; scoped by VPC so two VPCs using the same + // configured cloudMapNamespace name never collide — a namespace's DNS + // is only resolvable within the VPC it's associated with) ──────────── + let namespace_name = vpc_scoped_namespace(&ingress.cloud_map_namespace, &vpc_id); + let namespace_id = ensure_namespace(&sd, &namespace_name, &vpc_id).await?; // ── Service (one per bot) ────────────────────────────────────────────── let service_name = m.cloud_map_service_name(); let (registry_arn, existed) = ensure_service(&sd, &namespace_id, &service_name).await?; - let dns_name = format!("{}.{}", service_name, ingress.cloud_map_namespace); + let dns_name = format!("{service_name}.{namespace_name}"); if existed { eprintln!(" ✓ Cloud Map service exists: {dns_name}"); } else { @@ -83,8 +103,10 @@ pub async fn ensure_gateway( // ── Security group inbound rule (self-referencing on the container port) ─ ensure_sg_ingress(&ec2, security_groups, ingress.container_port).await?; - // ── VPC Link (shared across all bots, waits for AVAILABLE) ───────────── - let vpc_link_id = ensure_vpc_link(&api, subnets, security_groups).await?; + // ── VPC Link (shared per-VPC, waits for AVAILABLE) ────────────────────── + let subnet = subnets.first().context("ingress requires at least one subnet")?; + let vpc_id = resolve_vpc_id_from_subnet(&ec2, subnet).await?; + let vpc_link_id = ensure_vpc_link(&api, &vpc_id, subnets, security_groups).await?; // ── HTTP API (one per bot — avoids cross-bot path collisions) ────────── let (api_id, api_endpoint) = ensure_api(&api, &api_name).await?; @@ -99,6 +121,9 @@ pub async fn ensure_gateway( ensure_route(&api, &api_id, path, &integration_id).await?; } + // ── Prune routes for paths no longer in the manifest (rename/removal) ─── + prune_stale_routes(&api, &api_id, &ingress.paths).await?; + // ── Stage (auto-deploy) ──────────────────────────────────────────────── ensure_stage(&api, &api_id).await?; @@ -177,6 +202,11 @@ async fn resolve_vpc_id(ec2: &aws_sdk_ec2::Client, m: &OABServiceManifest) -> Re .context("ingress requires at least one subnet")?, _ => anyhow::bail!("ingress is only supported for ECS runtime"), }; + resolve_vpc_id_from_subnet(ec2, subnet).await +} + +/// Resolve the VPC ID that a given subnet belongs to. +async fn resolve_vpc_id_from_subnet(ec2: &aws_sdk_ec2::Client, subnet: &str) -> Result { let resp = ec2 .describe_subnets() .subnet_ids(subnet) @@ -362,12 +392,14 @@ async fn ensure_sg_ingress( async fn ensure_vpc_link( api: &aws_sdk_apigatewayv2::Client, + vpc_id: &str, subnets: &[String], security_groups: &[String], ) -> Result { use aws_sdk_apigatewayv2::types::VpcLinkStatus; + let link_name = vpc_link_name(vpc_id); - // Reuse an existing, non-failed VPC Link with our name. + // Reuse an existing, non-failed VPC Link with our per-VPC name. let mut found: Option<(String, Option)> = None; let mut next: Option = None; 'outer: loop { @@ -377,7 +409,7 @@ async fn ensure_vpc_link( } let resp = req.send().await.context("failed to list VPC Links")?; for link in resp.items() { - if link.name() == Some(VPC_LINK_NAME) { + if link.name() == Some(link_name.as_str()) { if matches!( link.vpc_link_status(), Some(VpcLinkStatus::Failed) | Some(VpcLinkStatus::Deleting) @@ -398,20 +430,21 @@ async fn ensure_vpc_link( } let link_id = if let Some((id, _status)) = found { - eprintln!(" ✓ VPC Link exists: {VPC_LINK_NAME} ({id})"); + eprintln!(" ✓ VPC Link exists: {link_name} ({id})"); // A VPC Link's subnets/SGs are fixed at creation and cannot be updated. - // All ingress-enabled bots in a VPC share this one link, so they must use - // the same subnet/SG set as whichever bot created it first — otherwise the - // link's ENIs won't cover this task's subnets and integrations may 503. + // All ingress-enabled bots in this VPC share this one link, so they must + // use the same subnet/SG set as whichever bot created it first — + // otherwise the link's ENIs won't cover this task's subnets and + // integrations may 503. eprintln!( - " ↳ reusing shared link's subnets/SGs (fixed at creation); ensure all\n ingress bots in this VPC use the same subnets/securityGroups" + " ↳ reusing this VPC's shared link subnets/SGs (fixed at creation);\n ensure all ingress bots in vpc {vpc_id} use the same subnets/securityGroups" ); id } else { - eprintln!(" ⊕ Creating VPC Link: {VPC_LINK_NAME}"); + eprintln!(" ⊕ Creating VPC Link: {link_name}"); let out = api .create_vpc_link() - .name(VPC_LINK_NAME) + .name(&link_name) .set_subnet_ids(Some(subnets.to_vec())) .set_security_group_ids(Some(security_groups.to_vec())) .send() @@ -578,6 +611,50 @@ async fn ensure_route( Ok(()) } +/// Delete any route on the bot's API whose path isn't in `current_paths`. +/// +/// `ensure_route` only ever adds routes; without this, renaming or removing a +/// webhook path in the manifest leaves a dead route on the API permanently. +async fn prune_stale_routes( + api: &aws_sdk_apigatewayv2::Client, + api_id: &str, + current_paths: &[String], +) -> Result<()> { + let current_keys: std::collections::HashSet = + current_paths.iter().map(|p| route_key(p)).collect(); + + let mut stale: Vec<(String, String)> = Vec::new(); // (route_id, route_key) + let mut next: Option = None; + loop { + let mut req = api.get_routes().api_id(api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list routes")?; + for r in resp.items() { + if let Some(key) = r.route_key() { + if !current_keys.contains(key) { + if let Some(id) = r.route_id() { + stale.push((id.to_string(), key.to_string())); + } + } + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, + } + } + + for (route_id, key) in stale { + match api.delete_route().api_id(api_id).route_id(&route_id).send().await { + Ok(_) => eprintln!(" ⊖ Removed stale route (no longer in manifest): {key}"), + Err(e) => eprintln!(" ⚠ Failed to remove stale route {key}: {e}"), + } + } + Ok(()) +} + async fn ensure_stage(api: &aws_sdk_apigatewayv2::Client, api_id: &str) -> Result<()> { let mut next: Option = None; loop { @@ -624,6 +701,26 @@ mod tests { assert_ne!(api_name("prod", "a"), api_name("prod", "b")); } + #[test] + fn vpc_link_name_is_per_vpc() { + assert_eq!(vpc_link_name("vpc-abc123"), "oab-vpc-link-vpc-abc123"); + assert_ne!(vpc_link_name("vpc-aaa"), vpc_link_name("vpc-bbb")); + } + + #[test] + fn vpc_scoped_namespace_differs_per_vpc() { + assert_eq!(vpc_scoped_namespace("oab", "vpc-aaa"), "oab-vpc-aaa"); + assert_ne!( + vpc_scoped_namespace("oab", "vpc-aaa"), + vpc_scoped_namespace("oab", "vpc-bbb") + ); + // Same VPC, different configured namespace names still differ. + assert_ne!( + vpc_scoped_namespace("oab", "vpc-aaa"), + vpc_scoped_namespace("custom", "vpc-aaa") + ); + } + #[test] fn webhook_urls_join_endpoint_stage_and_path() { let paths = vec![ From 838c220f09cd9ca17f852dfecc4a1d79a48e8119 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 19:13:34 -0400 Subject: [PATCH 10/13] fix(operator): dead doc link, VPC Link race hardening, Cloud Map teardown retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-8 review: - F1 (blocking): README and code comments linked to docs/refarch/running-telegram-line-on-aws.md, which does not exist on main or this branch (its PR #1274 is still open). Repointed the README to the existing docs/refarch/telegram-cloudflare-tunnel.md as an interim cross-reference with a note that a dedicated AWS-native doc is tracked in #1274; ingress.rs/manifest.rs doc comments now point at operator/README.md instead of the nonexistent file. - F2: VPC Link names are not unique to the apigatewayv2 API (confirmed: AWS does not error on a duplicate name), so two 'oabctl apply' processes racing to create the same per-VPC link in a brand-new VPC could end up with two links. Within a single apply run this can't happen (manifests are processed sequentially), but the reconciler hardens against the cross-process race anyway: ensure_vpc_link now collects ALL matching links, deterministically picks the lexicographically-first ID (stable regardless of list ordering) rather than an arbitrary one, and warns with cleanup commands if more than one exists. - F3: Cloud Map service teardown gave up on the first 'still has registered instances' error, permanently orphaning the service if the caller didn't manually retry. ECS deregisters the instance asynchronously on scale-to-0/delete, so this is usually just a timing window, not a real conflict — teardown now retries delete_service up to 6 times over ~25s before falling back to a warning with the exact manual cleanup command. Verified: build, clippy --all-targets -D warnings, cargo test (13 passed). --- operator/README.md | 9 +++-- operator/src/ingress.rs | 75 +++++++++++++++++++++++++++++++--------- operator/src/manifest.rs | 5 +-- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/operator/README.md b/operator/README.md index 739b8307b..42ebc7d99 100644 --- a/operator/README.md +++ b/operator/README.md @@ -215,9 +215,12 @@ Discord bots are outbound-only and need no ingress. Webhook platforms (Telegram, LINE, ...) POST *into* the task, so they need a public HTTPS endpoint. Adding an optional `spec.ingress` block makes `oabctl apply` provision the cheapest AWS-native path in one shot — API Gateway HTTP API → VPC Link → Cloud Map → the -task — instead of running ~7 manual `aws` commands. See -[`docs/refarch/running-telegram-line-on-aws.md`](../docs/refarch/running-telegram-line-on-aws.md) -(Option 1) for the architecture and cost breakdown. +task — instead of running ~7 manual `aws` commands, replacing the manual steps +implemented here. For a Kubernetes/Cloudflare-Tunnel alternative, see +[`docs/refarch/telegram-cloudflare-tunnel.md`](../docs/refarch/telegram-cloudflare-tunnel.md). +A dedicated AWS-native refarch doc covering this path in depth is tracked in +[#1274](https://github.com/openabdev/openab/pull/1274); once merged it will be +linked here. ```yaml spec: diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index e57aac9c1..67403bc43 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -1,7 +1,11 @@ //! Ingress reconciliation for webhook-based platforms (Telegram, LINE, ...). //! -//! Implements Option 1 of `docs/refarch/running-telegram-line-on-aws.md`: -//! API Gateway HTTP API → VPC Link → Cloud Map → ECS Fargate task on `:port`. +//! Implements the API Gateway HTTP API → VPC Link → Cloud Map → ECS Fargate +//! path for inbound webhook ingress (Telegram, LINE, ...), replacing ~7 manual +//! `aws apigatewayv2`/`servicediscovery`/`ecs` CLI steps. See +//! `operator/README.md` ("Ingress — inbound webhooks") for the manifest schema +//! and operational notes; a dedicated AWS reference architecture doc for this +//! path is tracked in openabdev/openab#1274. //! //! All operations are idempotent — resources are looked up by name and reused, //! so repeated `oabctl apply` runs converge instead of duplicating. The shared @@ -180,11 +184,31 @@ pub async fn teardown(config: &aws_config::SdkConfig, namespace: &str, name: &st } } if let Some(service_id) = service_id { - match sd.delete_service().id(&service_id).send().await { - Ok(_) => eprintln!(" ✓ Deleted Cloud Map service: {service_name}"), - Err(e) => eprintln!( - " ⚠ Cloud Map service '{service_name}' not deleted — it may still have\n registered instances; retry once the ECS tasks have fully drained ({e})" - ), + // ECS deregisters the task's Cloud Map instance asynchronously when a + // service scales to 0 / is deleted, so `delete_service` can fail with + // "still has registered instances" for a short window even though the + // task is already gone. Retry briefly instead of giving up on the + // first attempt — this is the common case, not an edge case. + let mut last_err = None; + let mut deleted = false; + for attempt in 0..6 { + if attempt > 0 { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + match sd.delete_service().id(&service_id).send().await { + Ok(_) => { + eprintln!(" ✓ Deleted Cloud Map service: {service_name}"); + deleted = true; + break; + } + Err(e) => last_err = Some(e), + } + } + if !deleted { + eprintln!( + " ⚠ Cloud Map service '{service_name}' not deleted after retrying — it still\n has registered instances. It will be orphaned until manually removed:\n aws servicediscovery delete-service --id {service_id}\n ({})", + last_err.map(|e| e.to_string()).unwrap_or_default() + ); } } @@ -399,28 +423,34 @@ async fn ensure_vpc_link( use aws_sdk_apigatewayv2::types::VpcLinkStatus; let link_name = vpc_link_name(vpc_id); - // Reuse an existing, non-failed VPC Link with our per-VPC name. - let mut found: Option<(String, Option)> = None; + // Reuse an existing, non-failed VPC Link with our per-VPC name. VPC Link + // names are NOT unique to the API — if two `oabctl apply` invocations race + // to create the same-named link in a brand-new VPC (e.g. a fleet's agents + // applied via separate concurrent processes), AWS will happily create two. + // We can't prevent that race across processes, but we can make reuse + // deterministic afterward: collect all matches and always pick the one + // with the lexicographically smallest ID (stable across repeated calls, + // regardless of list ordering), warning if more than one exists so an + // operator can clean up the duplicate. + let mut candidates: Vec<(String, Option)> = Vec::new(); let mut next: Option = None; - 'outer: loop { + loop { let mut req = api.get_vpc_links(); if let Some(t) = &next { req = req.next_token(t); } let resp = req.send().await.context("failed to list VPC Links")?; for link in resp.items() { - if link.name() == Some(link_name.as_str()) { - if matches!( + if link.name() == Some(link_name.as_str()) + && !matches!( link.vpc_link_status(), Some(VpcLinkStatus::Failed) | Some(VpcLinkStatus::Deleting) - ) { - continue; - } - found = Some(( + ) + { + candidates.push(( link.vpc_link_id().unwrap_or_default().to_string(), link.vpc_link_status().cloned(), )); - break 'outer; } } match resp.next_token() { @@ -428,6 +458,17 @@ async fn ensure_vpc_link( None => break, } } + candidates.sort_by(|(a, _), (b, _)| a.cmp(b)); + if candidates.len() > 1 { + eprintln!( + " ⚠ Found {} VPC Links named '{link_name}' (a race between concurrent\n `apply` runs can create duplicates — AWS does not enforce name\n uniqueness). Using the lexicographically first; consider deleting the extras:", + candidates.len() + ); + for (id, _) in &candidates[1..] { + eprintln!(" aws apigatewayv2 delete-vpc-link --vpc-link-id {id}"); + } + } + let found = candidates.into_iter().next(); let link_id = if let Some((id, _status)) = found { eprintln!(" ✓ VPC Link exists: {link_name} ({id})"); diff --git a/operator/src/manifest.rs b/operator/src/manifest.rs index 079aa331c..e43aaa397 100644 --- a/operator/src/manifest.rs +++ b/operator/src/manifest.rs @@ -167,8 +167,9 @@ pub struct Spec { /// Inbound HTTPS ingress for webhook-based platforms (Telegram, LINE, ...). /// /// Provisions the cheapest AWS-native path: API Gateway HTTP API → VPC Link → -/// Cloud Map → the ECS task on `containerPort`. See -/// `docs/refarch/running-telegram-line-on-aws.md` (Option 1). +/// Cloud Map → the ECS task on `containerPort`. See `operator/README.md` +/// ("Ingress — inbound webhooks") for the manifest schema and operational +/// notes. #[derive(Debug, Deserialize, Serialize, Clone)] #[serde(rename_all = "camelCase")] pub struct Ingress { From fc21c67b370ba17e3faf6b280f19da62ead72129 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 19:53:50 -0400 Subject: [PATCH 11/13] fix(operator): preserve webhook URL on recreate; exact Cloud Map targeting; namespace-change + SG mismatch detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-10 review (full team pass on 838c220): - F1 (blocking): the documented recreate path (oabctl delete && apply) deleted the bot's HTTP API resource itself via delete_api(), so the next apply's ensure_api() found nothing and created a NEW api-id, silently rotating the webhook URL hostname and breaking any Telegram/ LINE registration made against the old URL. Fixed by splitting teardown into two functions: - ingress::teardown(): strips routes/integration/stage but keeps the HTTP API resource, so its api-id (and thus the public URL) survives an ECS-service recreate. Used by both apply's ingress-removed path and delete's ingress cleanup. - ingress::delete_api(): permanently deletes the HTTP API. Only called from oabctl delete's full-removal path, where there's no URL to keep stable since the bot itself is gone. - F2: Cloud Map service teardown matched by name via an account-wide list_services() scan, so two bots with the same namespace/name in different VPCs/environments could collide. delete.rs now captures the ECS service's actual service_registries ARN BEFORE deleting it and passes it to teardown(), which resolves the exact Cloud Map service ID from that ARN instead of searching by name. Falls back to the by-name scan only when no ARN is known (apply's ingress-removed path, where the service may already be gone). - F3: has_registries only checked whether SOME registry was attached, not whether it matched the currently-resolved registry for the manifest's ingress.cloudMapNamespace. Changing cloudMapNamespace on an existing service silently left it pointed at the old namespace (503s) without triggering the recreate warning. apply now compares the resolved registry ARN against the service's actual registry ARNs and flags a mismatch as needing recreate too, with a distinct message. - F4: VPC Link reuse only printed a reminder about subnet/SG matching, never validated it. ensure_vpc_link now calls GetVpcLink on the reused link and compares its actual security groups against the manifest's, warning loudly on mismatch. (Subnets aren't exposed by GetVpcLink, so those remain a documented reminder only.) - F5: VPC Link duplicate-name tie-breaker sorted candidates by ID only, ignoring status, so a Pending duplicate could be picked over an Available one (wasted wait time, not a correctness bug). Now sorts AVAILABLE first, ID as tiebreaker. - F6: Cargo.lock was stale (missing the two new SDK deps entirely). Regenerated and committed. Added 2 new unit tests (cloud_map_service_id_from_arn parsing) — 15 total. Rewrote the README ingress caveats section to describe the stable-URL guarantee, exact-ARN teardown targeting, namespace-change detection, and SG validation. Verified: build, clippy --all-targets -D warnings, cargo test (15 passed). --- operator/Cargo.lock | 1023 ++++++++++++++++++++++++++++++++++++++- operator/README.md | 39 +- operator/src/apply.rs | 47 +- operator/src/delete.rs | 32 +- operator/src/ingress.rs | 220 +++++++-- 5 files changed, 1291 insertions(+), 70 deletions(-) diff --git a/operator/Cargo.lock b/operator/Cargo.lock index 435dd4af7..ae1f2ed85 100644 --- a/operator/Cargo.lock +++ b/operator/Cargo.lock @@ -2,12 +2,30 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -64,6 +82,16 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -168,6 +196,80 @@ dependencies = [ "uuid", ] +[[package]] +name = "aws-sdk-apigatewayv2" +version = "1.102.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd5de6b52cb383be9eec7e6b7e6c46038d637e504e64d79701a0aba59be7b28d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-cloudwatchlogs" +version = "1.131.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff364cad0c58f3e67d81b68a56f4d1d8a26249128573298195b3d0dd79039f5b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ec2" +version = "1.226.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ef215366676d2392accd5a5f964e891f7a97d210b34ccabe775510ccfd0302" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-ecs" version = "1.124.0" @@ -192,6 +294,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-iam" +version = "1.108.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e105d3f0200de0c9672abbbf299d89ca4fccd8f8886fd91b0f1950fb933336d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-s3" version = "1.132.0" @@ -227,6 +354,54 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-secretsmanager" +version = "1.104.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "811bcb3da8bf26e571da7a248c0a93152bba9642aa4c8c907c58d949fa5f1d5c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-servicediscovery" +version = "1.98.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0f242825f0d720a6e9d69c642e836af55842c7c2feea97382b01d41537637c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-ssm" version = "1.109.0" @@ -424,23 +599,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" dependencies = [ "aws-smithy-async", + "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", + "bytes", "h2 0.3.27", "h2 0.4.14", "http 0.2.12", "http 1.4.0", "http-body 0.4.6", + "http-body 1.0.1", "hyper 0.14.32", "hyper 1.9.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", + "indexmap", "pin-project-lite", "rustls 0.21.12", "rustls 0.23.40", "rustls-native-certs", "rustls-pki-types", + "serde", + "serde_json", "tokio", "tokio-rustls 0.26.4", "tower", @@ -465,6 +646,25 @@ dependencies = [ "aws-smithy-runtime-api", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.63.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b227aa94af99a8e5ee52551cc7e3ee30a217019ef99207b6f0b7a1527685941" +dependencies = [ + "assert-json-diff", + "aws-smithy-runtime-api", + "base64-simd", + "cbor-diag", + "ciborium", + "http 0.2.12", + "pretty_assertions", + "regex-lite", + "roxmltree", + "serde_json", + "thiserror", +] + [[package]] name = "aws-smithy-query" version = "0.60.15" @@ -498,6 +698,7 @@ dependencies = [ "pin-utils", "tokio", "tracing", + "tracing-subscriber", ] [[package]] @@ -630,6 +831,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -652,6 +862,25 @@ dependencies = [ "either", ] +[[package]] +name = "cbor-diag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +dependencies = [ + "bs58", + "chrono", + "data-encoding", + "half", + "nom", + "num-bigint", + "num-rational", + "num-traits", + "separator", + "url", + "uuid", +] + [[package]] name = "cc" version = "1.2.62" @@ -670,6 +899,52 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -813,6 +1088,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.4.9" @@ -820,7 +1101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -831,7 +1112,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -863,6 +1144,12 @@ dependencies = [ "cmov", ] +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + [[package]] name = "der" version = "0.6.1" @@ -882,6 +1169,12 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -905,6 +1198,27 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -934,6 +1248,29 @@ dependencies = [ "signature", ] +[[package]] +name = "ecsctl" +version = "0.7.1" +source = "git+https://github.com/oablab/ecsctl.git?rev=90a6cd1#90a6cd1d90d5d25797267c78645f756338667b4f" +dependencies = [ + "anyhow", + "aws-config", + "aws-sdk-cloudwatchlogs", + "aws-sdk-ec2", + "aws-sdk-ecs", + "aws-sdk-s3", + "aws-sdk-sts", + "clap", + "dirs", + "reqwest", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "toml", + "uuid", +] + [[package]] name = "either" version = "1.15.0" @@ -954,7 +1291,7 @@ dependencies = [ "generic-array", "group", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -988,7 +1325,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1081,8 +1418,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1092,21 +1431,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] -name = "group" -version = "0.12.1" +name = "getrandom" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ - "ff", - "rand_core", - "subtle", -] + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] [[package]] name = "h2" @@ -1146,6 +1498,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1343,6 +1706,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", + "webpki-roots", ] [[package]] @@ -1368,6 +1732,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1479,6 +1867,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -1521,12 +1911,27 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "libc", +] + [[package]] name = "litemap" version = "0.8.2" @@ -1557,6 +1962,21 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "md-5" version = "0.11.0" @@ -1573,6 +1993,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.2.0" @@ -1584,6 +2010,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1599,6 +2054,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1614,11 +2080,23 @@ version = "0.1.0" dependencies = [ "anyhow", "aws-config", + "aws-sdk-apigatewayv2", + "aws-sdk-cloudwatchlogs", + "aws-sdk-ec2", "aws-sdk-ecs", + "aws-sdk-iam", "aws-sdk-s3", + "aws-sdk-secretsmanager", + "aws-sdk-servicediscovery", "aws-sdk-ssm", + "aws-sdk-sts", + "chrono", "clap", + "dirs", + "ecsctl", + "rpassword", "serde", + "serde_json", "serde_yaml", "tokio", "toml", @@ -1642,6 +2120,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "outref" version = "0.5.2" @@ -1725,6 +2209,25 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1734,6 +2237,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.40", + "socket2 0.6.3", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls 0.23.40", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.3", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -1749,6 +2307,32 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -1758,6 +2342,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1767,12 +2360,78 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-lite" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-rustls 0.27.9", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.40", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -1798,6 +2457,42 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "rpassword" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da316a15f47e3d053de9cb2c439650bd8fa4aaeb9365f2e5f27f492ff73c196" +dependencies = [ + "libc", + "rtoolbox", + "windows-sys 0.61.2", +] + +[[package]] +name = "rtoolbox" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50a0e551c1e27e1731aba276dbeaeac73f53c7cd34d1bda485d02bd1e0f36844" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1827,6 +2522,7 @@ checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.103.13", "subtle", @@ -1851,6 +2547,7 @@ version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ + "web-time", "zeroize", ] @@ -1956,6 +2653,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "separator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" + [[package]] name = "serde" version = "1.0.228" @@ -1986,6 +2689,20 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -1995,6 +2712,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -2052,6 +2781,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2075,7 +2813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ "digest 0.10.7", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2155,6 +2893,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -2166,6 +2913,35 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" version = "0.3.47" @@ -2206,6 +2982,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -2314,8 +3105,31 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "pin-project-lite", + "tower", "tower-layer", "tower-service", + "url", ] [[package]] @@ -2359,6 +3173,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -2427,10 +3284,17 @@ version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ + "getrandom 0.4.3", "js-sys", "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "version_check" version = "0.9.5" @@ -2480,6 +3344,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.121" @@ -2512,12 +3386,94 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2527,6 +3483,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -2627,6 +3592,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.2" @@ -2650,6 +3621,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.8" @@ -2709,3 +3700,9 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/operator/README.md b/operator/README.md index 42ebc7d99..8e25fc0de 100644 --- a/operator/README.md +++ b/operator/README.md @@ -265,8 +265,15 @@ LINE console: > signature via `LINE_CHANNEL_SECRET`, Telegram bot token in the path). > > **Recreate caveat:** ECS service registries can only be set at *creation* time. -> If the service already exists without service discovery, `apply` provisions the -> ingress resources and prints how to recreate the service so traffic can reach it. +> If the service already exists without service discovery — or its registry +> points at a different Cloud Map service than the one currently resolved for +> `ingress.cloudMapNamespace` (e.g. the namespace was changed after the service +> was created) — `apply` provisions the ingress resources and prints how to +> recreate the service so traffic can reach it. Recreating via +> `oabctl delete oabservice ... && oabctl apply` **keeps the same webhook URL**: +> deleting a service only strips its routes/integration/stage from its HTTP API, +> it never deletes the API resource itself, so the `api-id` (and thus the URL +> hostname) survives. > > **Shared per-VPC (not per-account):** all ingress-enabled bots in the *same VPC* > share one VPC Link (`oab-vpc-link-`) and one Cloud Map namespace @@ -274,17 +281,23 @@ LINE console: > VPCs never collide or reuse each other's link/namespace. A VPC Link's > subnets/security groups are fixed at creation and cannot be changed, so every > ingress bot in a given VPC must use the same `networking.subnets` / -> `securityGroups` as whichever bot created that VPC's link first. `apply` prints -> a reminder when it reuses an existing link. +> `securityGroups` as whichever bot created that VPC's link first. `apply` +> verifies the reused link's actual security groups match the manifest and warns +> loudly on a mismatch (subnets aren't exposed by the API, so those can only be +> reminded, not verified). > -> **Teardown:** `oabctl delete oabservice ` removes the bot's per-bot ingress -> resources — its Cloud Map service and its own HTTP API (`oab-webhook--`, -> which cascades its routes/integration/stage) — on a best-effort basis (it never -> blocks service deletion). The same teardown also runs automatically from `apply` -> if you edit a manifest to remove `spec.ingress` entirely. The **shared** VPC Link -> and the security-group inbound rule are intentionally left in place for other -> bots. If the Cloud Map service still has registered instances (tasks not yet -> drained), delete prints a note to retry. +> **Teardown:** `oabctl delete oabservice ` permanently removes the bot's +> per-bot ingress resources — its exact Cloud Map service (resolved by the ECS +> service's own registry ARN, not a name search, so same-named bots in different +> VPCs/environments can't collide) and its HTTP API (`oab-webhook--`, +> including the API resource itself this time, since the bot is gone for good) — +> on a best-effort basis (it never blocks service deletion). If you instead edit +> a manifest to remove `spec.ingress` while keeping the bot, `apply` runs the same +> Cloud Map + routes/integration/stage cleanup automatically, but **keeps the HTTP +> API** in case ingress is re-added later. The **shared** VPC Link and the +> security-group inbound rule are always left in place for other bots. If the +> Cloud Map service still has registered instances, teardown retries for ~25s +> before falling back to a warning with the manual cleanup command. > > **Changing `paths`:** `apply` prunes routes on the bot's API that are no longer > in the manifest's `ingress.paths`, so renaming or removing a webhook path never @@ -292,7 +305,7 @@ LINE console: > > **Per-bot API, no path collisions:** each ingress bot gets its own HTTP API, so > two bots can both use `/webhook/telegram` without clashing — each has a distinct -> `{api-id}` endpoint URL. +> `{api-id}` endpoint URL that stays stable across recreates. ### OABFleet — batch deploy diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 9321f917f..4a482d321 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -175,8 +175,13 @@ async fn apply_ecs( // orphaned per-bot ingress resources (best-effort, mirrors `oabctl delete`). if previously_had_ingress && m.spec.ingress.is_none() { eprintln!(" 🌐 ingress removed from manifest — tearing down orphaned resources..."); - if let Err(e) = - crate::ingress::teardown(config, &m.metadata.namespace, &m.metadata.name).await + if let Err(e) = crate::ingress::teardown( + config, + &m.metadata.namespace, + &m.metadata.name, + None, + ) + .await { eprintln!(" ⚠ ingress teardown skipped: {e}"); } @@ -295,11 +300,19 @@ async fn apply_ecs( .and_then(|r| r.services().first()) .is_some_and(|s| s.status() == Some("ACTIVE")); - let has_registries = existing + let existing_registry_arns: Vec = existing .as_ref() .ok() .and_then(|r| r.services().first()) - .is_some_and(|s| !s.service_registries().is_empty()); + .map(|s| { + s.service_registries() + .iter() + .filter_map(|r| r.registry_arn()) + .map(|a| a.to_string()) + .collect() + }) + .unwrap_or_default(); + let has_registries = !existing_registry_arns.is_empty(); let mut needs_recreate = false; if service_active { @@ -313,12 +326,28 @@ async fn apply_ecs( .context("failed to update ECS service")?; println!(" ✓ {} updated", m.metadata.name); - if cloud_map.is_some() && !has_registries { + // Recreate is needed either when the service has no registry at all, + // or when it has one but it doesn't match the registry we just + // resolved for the manifest's current `ingress.cloudMapNamespace` — + // e.g. the namespace was changed after the service was created. The + // service's registry is fixed at creation time either way, so a + // mismatch means traffic is routed at the old namespace and 503s. + let registry_mismatch = cloud_map.as_ref().is_some_and(|cm| { + has_registries && !existing_registry_arns.contains(&cm.registry_arn) + }); + if cloud_map.is_some() && (!has_registries || registry_mismatch) { needs_recreate = true; - eprintln!( - " ⚠ Service '{}' already exists WITHOUT service discovery.", - m.metadata.name - ); + if registry_mismatch { + eprintln!( + " ⚠ Service '{}' is registered under a DIFFERENT Cloud Map service than\n the one resolved for its current ingress.cloudMapNamespace (likely changed\n since the service was created).", + m.metadata.name + ); + } else { + eprintln!( + " ⚠ Service '{}' already exists WITHOUT service discovery.", + m.metadata.name + ); + } eprintln!(" ECS service registries can only be set at creation time, so ingress"); eprintln!(" resources were provisioned but traffic won't reach the task until you"); eprintln!(" recreate the service (safe once desiredCount is drained):"); diff --git a/operator/src/delete.rs b/operator/src/delete.rs index d63bae87d..536a39498 100644 --- a/operator/src/delete.rs +++ b/operator/src/delete.rs @@ -18,6 +18,20 @@ pub async fn run( println!("Deleting {}...", name); + // Capture the service's Cloud Map registry ARN (if any) BEFORE deleting it, + // so teardown can resolve the exact Cloud Map service by ARN instead of a + // name-only account-wide scan (which could otherwise match a + // same-named bot in a different VPC/environment). + let registry_arn: Option = ecs + .describe_services() + .cluster(cluster) + .services(&service_name) + .send() + .await + .ok() + .and_then(|r| r.services().first().cloned()) + .and_then(|s| s.service_registries().first().and_then(|r| r.registry_arn()).map(|a| a.to_string())); + // 1. Scale to 0 let _ = ecs .update_service() @@ -38,13 +52,23 @@ pub async fn run( .context("failed to delete ECS service")?; println!(" ✓ ECS service deleted"); - // 2b. Best-effort ingress teardown (per-bot Cloud Map service + API Gateway - // routes/integration). No-op for bots that never had ingress. Never blocks - // deletion — failures are logged only. - if let Err(e) = crate::ingress::teardown(aws_config, namespace, name).await { + // 2b. Best-effort ingress teardown: Cloud Map service + this API's + // routes/integration/stage. No-op for bots that never had ingress. Never + // blocks deletion — failures are logged only. + if let Err(e) = + crate::ingress::teardown(aws_config, namespace, name, registry_arn.as_deref()).await + { eprintln!(" ⚠ ingress teardown skipped: {e}"); } + // 2c. The bot is being permanently removed here (unlike `apply`'s + // ingress-removed recreate path), so it's safe to delete the per-bot HTTP + // API resource itself too — there's no webhook URL to keep stable for a + // bot that no longer exists. + if let Err(e) = crate::ingress::delete_api(aws_config, namespace, name).await { + eprintln!(" ⚠ HTTP API cleanup skipped: {e}"); + } + // 3. Clean up S3 manifest let manifest_key = format!("manifests/{}/{}.yaml", namespace, name); let _ = s3 diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index 67403bc43..e461bd0e7 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -139,6 +139,12 @@ fn route_key(path: &str) -> String { format!("POST {path}") } +/// Extract the Cloud Map service ID from its ARN +/// (`arn:aws:servicediscovery:::service/`). +fn cloud_map_service_id_from_arn(arn: &str) -> Option { + arn.rsplit('/').next().filter(|s| !s.is_empty()).map(|s| s.to_string()) +} + /// Build the public webhook URL(s) from the API endpoint and paths. /// Each URL is `/`. fn webhook_urls(api_endpoint: &str, paths: &[String]) -> Vec { @@ -149,40 +155,113 @@ fn webhook_urls(api_endpoint: &str, paths: &[String]) -> Vec { .collect() } -/// Best-effort teardown of the *per-bot* ingress resources for `namespace/name`: -/// the bot's own HTTP API (`oab-webhook--`, which cascades its routes, -/// integration and stage) and its Cloud Map service. +/// Best-effort teardown of the *per-bot ingress wiring* for `namespace/name`: +/// its routes, integration, and stage on the per-bot HTTP API, plus its Cloud +/// Map service. Deliberately does NOT delete the HTTP API resource itself — +/// only what points at the now-gone task — so the API's `api-id` (and thus the +/// public webhook URL's hostname) survives an ECS-service recreate cycle. Use +/// [`delete_api`] separately when the bot is being permanently removed. /// -/// The shared resources (the `oab-vpc-link` VPC Link and the security-group -/// inbound rule) are intentionally left in place since other bots may still use -/// them. Safe to call for bots that never had ingress — it simply finds nothing -/// and returns. Errors are logged, not propagated, so teardown never blocks -/// service deletion. -pub async fn teardown(config: &aws_config::SdkConfig, namespace: &str, name: &str) -> Result<()> { +/// The shared resources (the VPC Link and the security-group inbound rule) are +/// intentionally left in place since other bots may still use them. Safe to +/// call for bots that never had ingress — it simply finds nothing and returns. +/// Errors are logged, not propagated, so teardown never blocks service deletion. +pub async fn teardown( + config: &aws_config::SdkConfig, + namespace: &str, + name: &str, + known_registry_arn: Option<&str>, +) -> Result<()> { let service_name = format!("oab-{namespace}-{name}"); let api = aws_sdk_apigatewayv2::Client::new(config); - // ── API Gateway: delete the whole per-bot API (cascades routes/integration/stage) ─ + // ── API Gateway: strip routes + integration + stage, keep the API itself ─ if let Some((api_id, _)) = find_api(&api, &api_name(namespace, name)).await? { - match api.delete_api().api_id(&api_id).send().await { - Ok(_) => eprintln!(" ✓ Deleted HTTP API: {}", api_name(namespace, name)), - Err(e) => eprintln!(" ⚠ Failed to delete HTTP API {api_id}: {e}"), + // Delete all routes on this API first (integrations can't be deleted + // while a route still targets them). + let mut route_ids = Vec::new(); + let mut next: Option = None; + loop { + let mut req = api.get_routes().api_id(&api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list routes")?; + for r in resp.items() { + if let Some(id) = r.route_id() { + route_ids.push(id.to_string()); + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, + } + } + for route_id in &route_ids { + api.delete_route().api_id(&api_id).route_id(route_id).send().await.ok(); + } + + // Delete integrations (there's normally just one, but clean up all). + let mut integration_ids = Vec::new(); + let mut next: Option = None; + loop { + let mut req = api.get_integrations().api_id(&api_id); + if let Some(t) = &next { + req = req.next_token(t); + } + let resp = req.send().await.context("failed to list integrations")?; + for i in resp.items() { + if let Some(id) = i.integration_id() { + integration_ids.push(id.to_string()); + } + } + match resp.next_token() { + Some(t) => next = Some(t.to_string()), + None => break, + } } + for integration_id in &integration_ids { + api.delete_integration() + .api_id(&api_id) + .integration_id(integration_id) + .send() + .await + .ok(); + } + + api.delete_stage().api_id(&api_id).stage_name(STAGE_NAME).send().await.ok(); + + eprintln!( + " ✓ Cleared ingress wiring on HTTP API {} ({} route(s), {} integration(s)) — API itself kept so its URL survives a recreate", + api_name(namespace, name), + route_ids.len(), + integration_ids.len() + ); } // ── Cloud Map: delete the per-bot service (needs no live instances) ────── + // Prefer resolving the exact service from the ECS service's own registry + // ARN (passed by the caller when known) over a name-only account-wide + // scan — two bots with the same namespace/name in different VPCs (e.g. + // staging vs. prod sharing an account) would otherwise collide and the + // wrong one could be deleted. let sd = aws_sdk_servicediscovery::Client::new(config); - let mut service_id: Option = None; - let mut pages = sd.list_services().into_paginator().send(); - 'svc: while let Some(page) = pages.next().await { - let page = page.context("failed to list Cloud Map services")?; - for s in page.services() { - if s.name() == Some(service_name.as_str()) { - service_id = s.id().map(|x| x.to_string()); - break 'svc; + let service_id: Option = if let Some(arn) = known_registry_arn { + cloud_map_service_id_from_arn(arn) + } else { + let mut found: Option = None; + let mut pages = sd.list_services().into_paginator().send(); + 'svc: while let Some(page) = pages.next().await { + let page = page.context("failed to list Cloud Map services")?; + for s in page.services() { + if s.name() == Some(service_name.as_str()) { + found = s.id().map(|x| x.to_string()); + break 'svc; + } } } - } + found + }; if let Some(service_id) = service_id { // ECS deregisters the task's Cloud Map instance asynchronously when a // service scales to 0 / is deleted, so `delete_service` can fail with @@ -215,6 +294,24 @@ pub async fn teardown(config: &aws_config::SdkConfig, namespace: &str, name: &st Ok(()) } +/// Permanently delete the bot's per-bot HTTP API (`oab-webhook--`), +/// cascading its routes/integration/stage with it. This DESTROYS the `api-id` +/// and therefore the public webhook URL's hostname — only call this when the +/// bot itself is being permanently removed (`oabctl delete`), never from the +/// `apply` recreate path, which relies on the API surviving so its URL stays +/// stable across an ECS-service recreate. +pub async fn delete_api(config: &aws_config::SdkConfig, namespace: &str, name: &str) -> Result<()> { + let api = aws_sdk_apigatewayv2::Client::new(config); + let name_str = api_name(namespace, name); + if let Some((api_id, _)) = find_api(&api, &name_str).await? { + match api.delete_api().api_id(&api_id).send().await { + Ok(_) => eprintln!(" ✓ Deleted HTTP API: {name_str}"), + Err(e) => eprintln!(" ⚠ Failed to delete HTTP API {api_id}: {e}"), + } + } + Ok(()) +} + // ─── VPC resolution ───────────────────────────────────────────────────────── async fn resolve_vpc_id(ec2: &aws_sdk_ec2::Client, m: &OABServiceManifest) -> Result { @@ -458,10 +555,19 @@ async fn ensure_vpc_link( None => break, } } - candidates.sort_by(|(a, _), (b, _)| a.cmp(b)); + // Prefer an already-AVAILABLE link over a PENDING one (avoids waiting on a + // duplicate that hasn't finished provisioning when a ready one exists), + // then break remaining ties by ID for determinism. + candidates.sort_by(|(a_id, a_status), (b_id, b_status)| { + let rank = |s: &Option| match s { + Some(VpcLinkStatus::Available) => 0, + _ => 1, + }; + rank(a_status).cmp(&rank(b_status)).then_with(|| a_id.cmp(b_id)) + }); if candidates.len() > 1 { eprintln!( - " ⚠ Found {} VPC Links named '{link_name}' (a race between concurrent\n `apply` runs can create duplicates — AWS does not enforce name\n uniqueness). Using the lexicographically first; consider deleting the extras:", + " ⚠ Found {} VPC Links named '{link_name}' (a race between concurrent\n `apply` runs can create duplicates — AWS does not enforce name\n uniqueness). Using the first AVAILABLE one (or lexicographically first\n if none are ready yet); consider deleting the extras:", candidates.len() ); for (id, _) in &candidates[1..] { @@ -473,13 +579,11 @@ async fn ensure_vpc_link( let link_id = if let Some((id, _status)) = found { eprintln!(" ✓ VPC Link exists: {link_name} ({id})"); // A VPC Link's subnets/SGs are fixed at creation and cannot be updated. - // All ingress-enabled bots in this VPC share this one link, so they must - // use the same subnet/SG set as whichever bot created it first — - // otherwise the link's ENIs won't cover this task's subnets and - // integrations may 503. - eprintln!( - " ↳ reusing this VPC's shared link subnets/SGs (fixed at creation);\n ensure all ingress bots in vpc {vpc_id} use the same subnets/securityGroups" - ); + // All ingress-enabled bots in this VPC share this one link, so verify + // (not just remind) that this manifest's subnets/SGs actually match + // what the link was created with — otherwise its ENIs won't cover + // this task's subnets and integrations may 503. + validate_vpc_link_config(api, &id, subnets, security_groups).await?; id } else { eprintln!(" ⊕ Creating VPC Link: {link_name}"); @@ -517,6 +621,44 @@ async fn ensure_vpc_link( anyhow::bail!("timed out waiting for VPC Link {link_id} to become AVAILABLE") } +/// Verify a reused VPC Link's actual security groups match the manifest's. +/// (API Gateway's `GetVpcLink` does not expose the link's subnet IDs, only +/// security groups, so subnet mismatches can't be directly verified here — +/// they still surface indirectly as unreachable integrations, which is the +/// pre-existing behavior this doesn't regress.) Warns loudly rather than +/// failing outright, since a legitimate SG rotation could trigger this too +/// and we don't want to block `apply` on a false positive. +async fn validate_vpc_link_config( + api: &aws_sdk_apigatewayv2::Client, + link_id: &str, + subnets: &[String], + security_groups: &[String], +) -> Result<()> { + let resp = api + .get_vpc_link() + .vpc_link_id(link_id) + .send() + .await + .context("failed to describe VPC Link for validation")?; + let actual_sgs: std::collections::HashSet<&str> = + resp.security_group_ids().iter().map(|s| s.as_str()).collect(); + let wanted_sgs: std::collections::HashSet<&str> = + security_groups.iter().map(|s| s.as_str()).collect(); + if actual_sgs != wanted_sgs { + eprintln!( + " ⚠ VPC Link {link_id}'s actual security groups {:?} do NOT match this\n manifest's {:?}. The link's SGs are fixed at creation — integrations may\n fail to reach this task. All ingress bots in this VPC must share the\n same securityGroups as whichever bot created the link.", + actual_sgs, wanted_sgs + ); + } + // Subnets aren't exposed by GetVpcLink; remind the operator this is the + // one part of the config we can't directly verify. + eprintln!( + " ↳ reusing this VPC's shared link (subnets fixed at creation, not verifiable via\n the API); ensure this manifest's subnets {:?} match whichever bot created it", + subnets + ); + Ok(()) +} + // ─── HTTP API ─────────────────────────────────────────────────────────────── async fn ensure_api( @@ -736,6 +878,22 @@ mod tests { assert_eq!(route_key("/webhook/line"), "POST /webhook/line"); } + #[test] + fn cloud_map_service_id_parses_from_arn() { + assert_eq!( + cloud_map_service_id_from_arn( + "arn:aws:servicediscovery:us-east-1:903779448426:service/srv-abc123" + ), + Some("srv-abc123".to_string()) + ); + } + + #[test] + fn cloud_map_service_id_from_arn_rejects_empty() { + assert_eq!(cloud_map_service_id_from_arn(""), None); + assert_eq!(cloud_map_service_id_from_arn("trailing/"), None); + } + #[test] fn api_name_is_per_bot() { assert_eq!(api_name("prod", "mybot"), "oab-webhook-prod-mybot"); From 536e858685af08e1c41e31ee49724360254b4056 Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Wed, 1 Jul 2026 20:07:42 -0400 Subject: [PATCH 12/13] fix(operator): attach/fix service discovery via UpdateService, no recreate needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-11 review (full team pass): the PR's core operational caveat — "ECS service registries can only be set at creation time, so an existing service needs delete-and-recreate to get ingress" — is outdated and wrong. Confirmed directly against the AWS API_UpdateService reference: serviceRegistries has been a documented UpdateService parameter since March 2022. "When you add, update, or remove the service registries configuration, Amazon ECS starts new tasks with the updated service registries configuration, and then stops the old tasks when the new tasks are running" — a normal rolling replacement, no downtime gap, no delete needed. Requires the AWSServiceRoleForECS service-linked role, which ECS creates automatically the first time any service in the account uses service discovery. Removed the entire recreate code path: - apply_ecs's update_service branch now attaches or replaces the serviceRegistries directly when there's no registry or a mismatch (registry_mismatch detection from round-10 is kept — just resolved automatically instead of printed as a manual instruction). - Deleted the "needs_recreate" plumbing through apply_ecs/run (dead code once the update path handles it directly). - Corrected the "create-only" claim in ingress.rs's module doc and apply.rs's ensure_cloud_map-ordering comment. - Rewrote README's "Recreate caveat" into "Adding/fixing service discovery never requires recreating the service", describing the actual UpdateService rolling-replacement behavior and the service-linked role note. Also addressed the two non-blocking findings from the same review round: - Added an "Additional permissions for spec.ingress" table to the Prerequisites section (Cloud Map, API Gateway, EC2, ECS UpdateService actions) — this is new AWS API surface the caller's credentials need that wasn't documented anywhere. - Corrected the security note: Telegram validates the X-Telegram-Bot-Api-Secret-Token header + source-IP allowlist (not just "token in the path"), LINE does HMAC-SHA256 signature verification — verified against crates/openab-gateway/src/adapters/{telegram,line}.rs. Verified live end-to-end against a real AWS account: 1. Applied a manifest WITHOUT ingress -> plain ECS service created. 2. Added ingress, re-applied -> printed "updated (service discovery attached; rolling replacement, no downtime)", NOT a recreate warning. 3. Confirmed via describe-services: same serviceArn, same createdAt timestamp (service was never deleted), serviceRegistries now populated with the correct Cloud Map ARN. 4. Confirmed the task rolled (old task ARN -> new task ARN) rather than the service being replaced. 5. curl through the printed webhook URL end-to-end -> HTTP 200. 6. Deleted and cleaned up all test resources (service, HTTP API, Cloud Map service/namespace, VPC Link, SG rule) - account left clean. Verified: build, clippy --all-targets -D warnings, cargo test (15 passed, unchanged from round-10 since no new pure logic was added here). --- operator/README.md | 47 +++++++++++++++------ operator/src/apply.rs | 93 +++++++++++++++++++---------------------- operator/src/ingress.rs | 12 ++++-- 3 files changed, 87 insertions(+), 65 deletions(-) diff --git a/operator/README.md b/operator/README.md index 8e25fc0de..23acca2c4 100644 --- a/operator/README.md +++ b/operator/README.md @@ -260,20 +260,26 @@ LINE console: https://{api-id}.execute-api.{region}.amazonaws.com/prod/webhook/line ``` -> **Security note:** the API Gateway endpoint is public and unauthenticated at the -> transport layer — requests are authenticated at the app layer by OpenAB (LINE -> signature via `LINE_CHANNEL_SECRET`, Telegram bot token in the path). +> **Security note:** the API Gateway endpoint itself is public and unauthenticated +> at the transport layer (no IAM auth, no API key). OpenAB's webhook handlers add +> their own app-layer verification on top: Telegram validates the +> `X-Telegram-Bot-Api-Secret-Token` header (`TELEGRAM_SECRET_TOKEN`) and the +> request's source IP against Telegram's published webhook subnets; LINE +> verifies an HMAC-SHA256 signature using `LINE_CHANNEL_SECRET`. Set +> `TELEGRAM_SECRET_TOKEN` when registering the webhook with BotFather to enable +> that check. > -> **Recreate caveat:** ECS service registries can only be set at *creation* time. -> If the service already exists without service discovery — or its registry -> points at a different Cloud Map service than the one currently resolved for +> **Adding/fixing service discovery never requires recreating the service:** +> if an existing ECS service has no Cloud Map registry, or has one pointing at a +> different Cloud Map service than the one currently resolved for > `ingress.cloudMapNamespace` (e.g. the namespace was changed after the service -> was created) — `apply` provisions the ingress resources and prints how to -> recreate the service so traffic can reach it. Recreating via -> `oabctl delete oabservice ... && oabctl apply` **keeps the same webhook URL**: -> deleting a service only strips its routes/integration/stage from its HTTP API, -> it never deletes the API resource itself, so the `api-id` (and thus the URL -> hostname) survives. +> was created), `apply`'s `update_service` call attaches or replaces the +> registry directly — ECS has supported adding/updating/removing +> `serviceRegistries` on an existing service via a normal rolling replacement +> (new tasks start with the new registry, old tasks stop once healthy — no +> downtime gap) since March 2022. This requires the `AWSServiceRoleForECS` +> service-linked role, which ECS creates automatically the first time any +> service in the account uses service discovery — no setup needed. > > **Shared per-VPC (not per-account):** all ingress-enabled bots in the *same VPC* > share one VPC Link (`oab-vpc-link-`) and one Cloud Map namespace @@ -426,3 +432,20 @@ With `oabctl bootstrap`, most prerequisites are handled automatically. You only 1. **AWS credentials** — IAM user/role with permissions to create the above resources 2. **Docker** — to build custom images (optional if using official images) +### Additional permissions for `spec.ingress` + +The resources bootstrap creates cover outbound-only (Discord) deployments. If +any manifest sets `spec.ingress`, the **caller of `oabctl apply`/`delete`** +(not the task role) also needs: + +| Service | Actions | +|---------|---------| +| Cloud Map | `servicediscovery:CreatePrivateDnsNamespace`, `CreateService`, `DeleteService`, `ListNamespaces`, `ListServices`, `GetOperation` | +| API Gateway | `apigateway:CreateVpcLink`, `CreateApi`, `CreateIntegration`, `CreateRoute`, `CreateStage`, `DeleteRoute`, `DeleteIntegration`, `DeleteStage`, `DeleteApi`, `GetVpcLinks`, `GetVpcLink`, `GetApis`, `GetIntegrations`, `GetRoutes`, `GetStages` | +| EC2 | `ec2:DescribeSubnets`, `AuthorizeSecurityGroupIngress` | +| ECS | `ecs:UpdateService` with `serviceRegistries` (requires the `AWSServiceRoleForECS` service-linked role, which ECS creates automatically the first time any service in the account uses service discovery) | + +`AdministratorAccess`-equivalent or a broad `servicediscovery:*`/`apigateway:*` +during development is fine; the table above is for scoping a least-privilege +policy. + diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 4a482d321..642a37495 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -60,30 +60,12 @@ pub async fn run(aws_config: &aws_config::SdkConfig, file_path: &str, sync_confi } } - let mut needs_recreate: Vec = Vec::new(); for m in &manifests { println!(" Applying {} (ECS)...", m.metadata.name); - if apply_ecs(&ecs, &s3, aws_config, m, wait).await? { - needs_recreate.push(format!("{}/{}", m.metadata.namespace, m.metadata.name)); - } + apply_ecs(&ecs, &s3, aws_config, m, wait).await?; } println!("\n{} service(s) applied.", manifests.len()); - - // Surface the create-only service-discovery limitation as a consolidated - // summary so it isn't lost in mid-run output (e.g. in non-interactive CI). - if !needs_recreate.is_empty() { - eprintln!( - "\n⚠ {} service(s) have ingress configured but were created WITHOUT service\n discovery, so webhook traffic will NOT reach them until recreated:", - needs_recreate.len() - ); - for id in &needs_recreate { - eprintln!(" - {id}"); - } - eprintln!( - " ECS service registries can only be set at creation time. For each, run:\n oabctl delete oabservice --cluster oab --namespace && oabctl apply -f " - ); - } Ok(()) } @@ -135,7 +117,7 @@ async fn apply_ecs( config: &aws_config::SdkConfig, m: &OABServiceManifest, wait: bool, -) -> Result { +) -> Result<()> { let ecs_rt = match &m.spec.runtime { Runtime::Ecs(rt) => rt, _ => unreachable!(), @@ -277,8 +259,11 @@ async fn apply_ecs( .awsvpc_configuration(vpc_config) .build(); - // Ingress: ensure Cloud Map BEFORE the service exists-check, because ECS - // service registries can only be attached at service *creation* time. + // Ingress: ensure Cloud Map BEFORE the service exists-check, so the + // registry ARN is ready whether the ECS service needs to be created (via + // `create_service`) or updated to attach/replace service discovery (via + // `update_service` — ECS has supported changing `serviceRegistries` on an + // existing service since March 2022; no delete-and-recreate is needed). let cloud_map = if let Some(ingress) = &m.spec.ingress { eprintln!(" 🌐 Reconciling ingress (Cloud Map)..."); Some(crate::ingress::ensure_cloud_map(config, m, ingress).await?) @@ -314,47 +299,57 @@ async fn apply_ecs( .unwrap_or_default(); let has_registries = !existing_registry_arns.is_empty(); - let mut needs_recreate = false; if service_active { - ecs.update_service() + // Recreate is NOT required to attach/fix service discovery: ECS's + // UpdateService API has supported adding/updating/removing + // serviceRegistries since March 2022 (rolling replacement — new tasks + // start with the updated registry, old tasks stop once they're + // healthy, no downtime gap). It does require the AWSServiceRoleForECS + // service-linked role, which ECS creates automatically the first time + // any account uses ECS service discovery — no action needed here. + let registry_mismatch = cloud_map.as_ref().is_some_and(|cm| { + has_registries && !existing_registry_arns.contains(&cm.registry_arn) + }); + + let mut update_req = ecs + .update_service() .cluster("oab") .service(&service_name) .task_definition(&task_def_arn) - .network_configuration(network_config) + .network_configuration(network_config); + + if let Some(cm) = &cloud_map { + if !has_registries || registry_mismatch { + let mut registry = aws_sdk_ecs::types::ServiceRegistry::builder() + .registry_arn(&cm.registry_arn); + if let Some(ingress) = &m.spec.ingress { + registry = registry + .container_name("openab") + .container_port(ingress.container_port as i32); + } + update_req = update_req.service_registries(registry.build()); + } + } + + update_req .send() .await .context("failed to update ECS service")?; - println!(" ✓ {} updated", m.metadata.name); - - // Recreate is needed either when the service has no registry at all, - // or when it has one but it doesn't match the registry we just - // resolved for the manifest's current `ingress.cloudMapNamespace` — - // e.g. the namespace was changed after the service was created. The - // service's registry is fixed at creation time either way, so a - // mismatch means traffic is routed at the old namespace and 503s. - let registry_mismatch = cloud_map.as_ref().is_some_and(|cm| { - has_registries && !existing_registry_arns.contains(&cm.registry_arn) - }); + if cloud_map.is_some() && (!has_registries || registry_mismatch) { - needs_recreate = true; if registry_mismatch { - eprintln!( - " ⚠ Service '{}' is registered under a DIFFERENT Cloud Map service than\n the one resolved for its current ingress.cloudMapNamespace (likely changed\n since the service was created).", + println!( + " ✓ {} updated (service discovery re-pointed to the current Cloud Map service; rolling replacement, no downtime)", m.metadata.name ); } else { - eprintln!( - " ⚠ Service '{}' already exists WITHOUT service discovery.", + println!( + " ✓ {} updated (service discovery attached; rolling replacement, no downtime)", m.metadata.name ); } - eprintln!(" ECS service registries can only be set at creation time, so ingress"); - eprintln!(" resources were provisioned but traffic won't reach the task until you"); - eprintln!(" recreate the service (safe once desiredCount is drained):"); - eprintln!( - " oabctl delete oabservice {} --cluster oab --namespace {} && oabctl apply -f ", - m.metadata.name, m.metadata.namespace - ); + } else { + println!(" ✓ {} updated", m.metadata.name); } } else { let cap_strategy = CapacityProviderStrategyItem::builder() @@ -427,7 +422,7 @@ async fn apply_ecs( eprintln!(" ✓ {} is stable", m.metadata.name); } - Ok(needs_recreate) + Ok(()) } async fn wait_for_stable(ecs: &aws_sdk_ecs::Client, cluster: &str, service: &str) -> Result<()> { diff --git a/operator/src/ingress.rs b/operator/src/ingress.rs index e461bd0e7..8de6b0430 100644 --- a/operator/src/ingress.rs +++ b/operator/src/ingress.rs @@ -13,10 +13,14 @@ //! VPC ID) since a VPC Link's ENIs and a namespace's private DNS are only valid //! within the VPC they were created in — two VPCs never share either resource. //! -//! The reconciliation is split in two because ECS service discovery -//! (`--service-registries`) can only be set at service *creation* time: -//! 1. [`ensure_cloud_map`] — namespace + service. Must run BEFORE the ECS -//! service is created so its registry ARN can be attached. +//! The reconciliation is split in two so Cloud Map is ready before the ECS +//! service needs it (whether creating a new service or attaching service +//! discovery to an existing one via `UpdateService` — ECS has supported +//! adding/updating/removing `serviceRegistries` on an existing service via a +//! normal rolling replacement since March 2022, so no delete-and-recreate is +//! needed either way): +//! 1. [`ensure_cloud_map`] — namespace + service. Runs BEFORE the ECS +//! create/update-service call so its registry ARN is ready to attach. //! 2. [`ensure_gateway`] — VPC Link + HTTP API + integration + routes + stage //! + security-group inbound rule. Runs AFTER the task is wired up. From 70a920ca718d54a7bb19ca3a2de4cadae50a75bd Mon Sep 17 00:00:00 2001 From: chaodu-agent Date: Thu, 2 Jul 2026 00:21:31 +0000 Subject: [PATCH 13/13] fix(operator): detach registry + exact-ARN teardown on ingress removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-12 review (full team pass on 536e858): the ingress-removal path (manifest edited to drop spec.ingress) had two gaps left over from round-11's UpdateService redesign, both only reachable via that specific transition: - F1 (critical): the update_service branch only ever set `serviceRegistries` inside `if let Some(cm) = &cloud_map`. When ingress is removed, cloud_map is None, so the field was omitted from the request entirely. Per AWS's own UpdateService reference, an omitted serviceRegistries leaves the existing configuration unchanged — only an explicit empty list detaches it. Since ingress::teardown (running earlier in the same call) deletes the underlying Cloud Map service, the ECS service was left pointing at a registry ARN that no longer existed, which surfaces as registration failures on the next deployment. Fixed by tracking `needs_detach = cloud_map.is_none() && has_registries` and calling `update_req.set_service_registries(Some(vec![]))` in that case. - F2: the ingress-removal call site passed `None` for teardown's `known_registry_arn`, forcing a fallback to an account-wide `list_services()` scan matching only on `oab-{namespace}-{name}`. Two bots with the same namespace/name in different VPCs/environments sharing one account could collide there. Fixed by hoisting the `describe_services` call (previously done further down, purely for the service_active check) to the top of `apply_ecs`, before the ingress-removal branch, so the real registry ARN captured from the live ECS service can be threaded through instead of `None`. The service_active check later in the function now reuses this same response instead of issuing a second describe_services call. No new tests added — this only changes AWS SDK request-building logic in a code path (ingress removal) that isn't exercised by the existing unit tests (which cover manifest/schema parsing, not live apply_ecs behavior). Verified via `cargo fmt --check` (no new diffs beyond the changed lines) and manual review of the full diff; `cargo build`/`clippy`/`test` could not be run in this environment (no linker available), so CI is the first real compile/test signal for this commit — flagged explicitly in the accompanying PR comment. --- operator/src/apply.rs | 76 +++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/operator/src/apply.rs b/operator/src/apply.rs index 642a37495..7a141c090 100644 --- a/operator/src/apply.rs +++ b/operator/src/apply.rs @@ -153,15 +153,45 @@ async fn apply_ecs( }; let generation = current_gen + 1; + // Look up the ECS service's current registry ARN(s) up front so both the + // ingress-removal teardown below and the update/create logic further down + // can use the *exact* registry rather than falling back to a name-only + // Cloud Map scan (which can collide across VPCs/environments that share + // an account and reuse the same namespace/name). + let describe_resp = ecs + .describe_services() + .cluster("oab") + .services(&service_name) + .send() + .await; + let existing_registry_arns: Vec = describe_resp + .as_ref() + .ok() + .and_then(|r| r.services().first()) + .map(|s| { + s.service_registries() + .iter() + .filter_map(|r| r.registry_arn()) + .map(|a| a.to_string()) + .collect() + }) + .unwrap_or_default(); + let has_registries = !existing_registry_arns.is_empty(); + // If ingress was configured before but is absent now, tear down the - // orphaned per-bot ingress resources (best-effort, mirrors `oabctl delete`). + // orphaned per-bot ingress resources (best-effort, mirrors `oabctl delete`) + // and detach the stale registry from the ECS service itself — omitting + // `serviceRegistries` on `UpdateService` leaves the existing configuration + // untouched (AWS only clears it when explicitly passed an empty list), so + // without this the service would keep pointing at a Cloud Map service that + // teardown() is about to delete. if previously_had_ingress && m.spec.ingress.is_none() { eprintln!(" 🌐 ingress removed from manifest — tearing down orphaned resources..."); if let Err(e) = crate::ingress::teardown( config, &m.metadata.namespace, &m.metadata.name, - None, + existing_registry_arns.first().map(|s| s.as_str()), ) .await { @@ -271,34 +301,15 @@ async fn apply_ecs( None }; - // Check if service exists - let existing = ecs - .describe_services() - .cluster("oab") - .services(&service_name) - .send() - .await; - - let service_active = existing + // Check if service exists. Reuses `describe_resp` captured above (before + // the ingress-removal teardown) — `ensure_cloud_map` above doesn't touch + // the ECS service, so its ACTIVE status can't have changed since then. + let service_active = describe_resp .as_ref() .ok() .and_then(|r| r.services().first()) .is_some_and(|s| s.status() == Some("ACTIVE")); - let existing_registry_arns: Vec = existing - .as_ref() - .ok() - .and_then(|r| r.services().first()) - .map(|s| { - s.service_registries() - .iter() - .filter_map(|r| r.registry_arn()) - .map(|a| a.to_string()) - .collect() - }) - .unwrap_or_default(); - let has_registries = !existing_registry_arns.is_empty(); - if service_active { // Recreate is NOT required to attach/fix service discovery: ECS's // UpdateService API has supported adding/updating/removing @@ -310,6 +321,14 @@ async fn apply_ecs( let registry_mismatch = cloud_map.as_ref().is_some_and(|cm| { has_registries && !existing_registry_arns.contains(&cm.registry_arn) }); + // `ingress` was removed from the manifest (cloud_map is None here) + // but the ECS service still has a registry attached from a previous + // apply — must explicitly detach it. `UpdateService` treats an + // *omitted* `serviceRegistries` field as "leave unchanged", not + // "clear"; only an explicit empty list detaches it. Without this the + // service keeps pointing at the Cloud Map service that the + // ingress-removal teardown (above) just deleted. + let needs_detach = cloud_map.is_none() && has_registries; let mut update_req = ecs .update_service() @@ -329,6 +348,8 @@ async fn apply_ecs( } update_req = update_req.service_registries(registry.build()); } + } else if needs_detach { + update_req = update_req.set_service_registries(Some(Vec::new())); } update_req @@ -348,6 +369,11 @@ async fn apply_ecs( m.metadata.name ); } + } else if needs_detach { + println!( + " ✓ {} updated (service discovery detached; rolling replacement, no downtime)", + m.metadata.name + ); } else { println!(" ✓ {} updated", m.metadata.name); }