From 11ef86c490282f5bb3d7f9824d388086c030a2b7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:12:45 +0200 Subject: [PATCH 1/2] fix(ingest): stop rolling the EC2 fleet on every merge and right-size hosts Every prd deploy stamped the merged commit into ingest's task definition as COMMIT_SHA. The image is content-hashed and usually unchanged, but the new env value still registered a revision and forced a rollout, and with host networking each rollout launches two fresh c7gd hosts and drains the old ones. That was ~15 rollouts a day. - Ingest now reports INGEST_COMMIT_SHA, the last commit touching apps/ingest, resolved in deploy-prd-instance.yml (blobless full-history checkout). It leaves PRD_LOCKSTEP_REVISION_SERVICES; the skew rule's SQL is updated. - US prd ingest moves to c7gd.medium with 1 vCPU / 1.5 GiB tasks, as EU runs. Over 14 days the c7gd.large tasks peaked at 25% CPU and 6% memory. - Preview ingest uses the same smaller size. --- .github/workflows/deploy-prd-instance.yml | 8 ++++++++ apps/ingest/alchemy.run.ts | 15 ++++++++++++--- docs/infra.md | 5 +++++ packages/infra/src/env.test.ts | 9 +++------ packages/infra/src/env.ts | 4 ++-- packages/infra/src/profile.ts | 10 ++++++---- 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/.github/workflows/deploy-prd-instance.yml b/.github/workflows/deploy-prd-instance.yml index 94a4454a94..471c934b29 100644 --- a/.github/workflows/deploy-prd-instance.yml +++ b/.github/workflows/deploy-prd-instance.yml @@ -38,6 +38,14 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 with: ref: ${{ inputs.commit-sha }} + # History without blobs, for the ingest revision below. + fetch-depth: 0 + filter: blob:none + + # Ingest reports the last commit that touched its image context, not the deployed + # commit: a new env value would roll the EC2 fleet (fresh hosts) on every merge. + - name: Resolve the ingest revision + run: echo "INGEST_COMMIT_SHA=$(git log -1 --format=%H -- apps/ingest)" >> "$GITHUB_ENV" # Toolchain, Infisical, AWS OIDC (after Infisical so it wins), deps, ingest binary. - name: Deploy setup diff --git a/apps/ingest/alchemy.run.ts b/apps/ingest/alchemy.run.ts index 0ddc9bfab0..b32d25b3d2 100644 --- a/apps/ingest/alchemy.run.ts +++ b/apps/ingest/alchemy.run.ts @@ -275,8 +275,8 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }: roleName: instanceRole.roleName, }) - // Newest ECS-optimized AL2023 arm64 image. Every deploy launches fresh hosts - // (the old task holds the port), so AMI patching rides the deploys. + // Newest ECS-optimized AL2023 arm64 image. Every ingest rollout launches fresh hosts + // (the old task holds the port), so AMI patching rides the ingest changes. const imageId = AWS.EC2.getAmi({ owners: ["amazon"], name: ["al2023-ami-ecs-hvm-*-kernel-6.1-arm64"], @@ -332,6 +332,13 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }: tags, }) + const revisions = { + ...(yield* optionalPlain("INGEST_COMMIT_SHA")), + ...(yield* optionalPlain("COMMIT_SHA")), + ...(yield* optionalPlain("GITHUB_SHA")), + } + const ingestRevision = revisions.INGEST_COMMIT_SHA || revisions.COMMIT_SHA || revisions.GITHUB_SHA + const secret = ecsSecrets(name("ingest"), tags) const tinybirdToken = yield* secret("tinybird-token", yield* requiredPlain("TINYBIRD_TOKEN")) @@ -644,7 +651,9 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }: // 30s instead of the binary's 1s cuts Autumn track calls ~30x. ...(yield* optionalPlain("AUTUMN_FLUSH_INTERVAL_SECS", "30")), ...(yield* optionalPlain("INGEST_SHUTDOWN_DRAIN_SECS")), - ...(yield* optionalPlain("COMMIT_SHA", (yield* optionalPlain("GITHUB_SHA")).GITHUB_SHA)), + // `INGEST_COMMIT_SHA` (prd) is the image's own revision, so a deploy that leaves + // `apps/ingest` alone does not change the task definition and roll the hosts. + ...(ingestRevision ? { COMMIT_SHA: ingestRevision } : undefined), } satisfies Record>, tags, diff --git a/docs/infra.md b/docs/infra.md index 52ee082dd4..16909e9df4 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -570,6 +570,11 @@ architecture. one c7gd.medium (autoscaling 1-3) and the collector at the non-prd size; Electric keeps the prd size. Managed scaling adds a host to roll a deploy. To scale it, raise the EU values there, or point `prd-eu` at `US_PRD`. +- **Ingest stamps its own revision.** Every ingest rollout launches fresh hosts, so its + `COMMIT_SHA` is `INGEST_COMMIT_SHA` (the last commit touching `apps/ingest`, resolved in + `deploy-prd-instance.yml`), not the deployed commit. A merge that leaves `apps/ingest` alone + keeps the task definition and the fleet. That is also why ingest is not in + `PRD_LOCKSTEP_REVISION_SERVICES`. - **Graviton (ARM64).** The gateway, collector and Electric run `cpuArchitecture: "ARM64"` (`runtimePlatform`), cheaper than x86_64. `build-ingest-binary.yml` compiles natively on an arm64 runner. diff --git a/packages/infra/src/env.test.ts b/packages/infra/src/env.test.ts index f576c69d89..14e097338c 100644 --- a/packages/infra/src/env.test.ts +++ b/packages/infra/src/env.test.ts @@ -238,19 +238,16 @@ describe("the prd revision lockstep the skew alert depends on", () => { "alerting", "maple-ai", "electric-sync", - "ingest", "maple-api", "maple-web", ]) }) it("only claims lockstep for the stage-gated services prd actually deploys", () => { - // `ingest` and `electric-sync` are the two members behind a stage - // predicate. Flip either away from prd and it stops tracking the other - // three, so it has to leave the list — and the rule's SQL — in the same - // change. + // `electric-sync` is the member behind a stage predicate. Flip it away from + // prd and it stops tracking the others, so it has to leave the list (and the + // rule's SQL) in the same change. Ingest stamps its own revision. const { deploys } = resolveMapleProfile({ stage: { kind: "prd" }, region: "us" }) - expect(PRD_LOCKSTEP_REVISION_SERVICES.includes("ingest")).toBe(deploys.ingest) expect(PRD_LOCKSTEP_REVISION_SERVICES.includes("electric-sync")).toBe(deploys.electric) }) }) diff --git a/packages/infra/src/env.ts b/packages/infra/src/env.ts index f6c50b8fda..7ffa8d5bb6 100644 --- a/packages/infra/src/env.ts +++ b/packages/infra/src/env.ts @@ -164,7 +164,8 @@ export const selfObservabilityEnv = ( ) /** - * prd services deployed together that must report the same revision. The "Prod revision skew" + * prd services deployed together that must report the same revision. Not ingest: it reports + * the last commit that touched `apps/ingest` (`INGEST_COMMIT_SHA`). The "Prod revision skew" * alert rule (id `2a6e9529-5f73-4478-9fa0-432904ff15c8`) lives in the Maple database, not this * repo: if you change this list, you MUST edit that rule's SQL to match. */ @@ -172,7 +173,6 @@ export const PRD_LOCKSTEP_REVISION_SERVICES = [ "alerting", "maple-ai", "electric-sync", - "ingest", "maple-api", "maple-web", ] as const diff --git a/packages/infra/src/profile.ts b/packages/infra/src/profile.ts index ae4c4cf43b..cbf2853a47 100644 --- a/packages/infra/src/profile.ts +++ b/packages/infra/src/profile.ts @@ -106,9 +106,11 @@ const US_PRD: MapleProfile = { }, ingest: { desiredCount: 2, + // Two hosts for AZ redundancy; scaling absorbs bursts. On c7gd.large the tasks + // peaked at 25% CPU and 6% memory (Oct 2026). scaling: scaling(2, 6), - instanceType: "c7gd.large", - taskSize: { cpu: 2048, memory: 3072 }, + instanceType: "c7gd.medium", + taskSize: { cpu: 1024, memory: 1536 }, collectorTaskSize: { cpu: 512, memory: 1024 }, selfTraceSampleRatio: "0.05", }, @@ -140,8 +142,8 @@ const EU_PRD: MapleProfile = { const SMALL_INGEST: MapleProfile["ingest"] = { desiredCount: 1, scaling: undefined, - instanceType: "c7gd.large", - taskSize: { cpu: 2048, memory: 3072 }, + instanceType: "c7gd.medium", + taskSize: { cpu: 1024, memory: 1536 }, collectorTaskSize: { cpu: 256, memory: 1024 }, selfTraceSampleRatio: undefined, } From 7dae4b994747a9c72583781206b3f26a139e9497 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 23 Oct 2026 01:16:10 +0200 Subject: [PATCH 2/2] feat(ingest): run preview ingest on a t4g.small with the WAL on the root volume c7gd.medium was the floor only because the WAL needs local NVMe. A preview carries almost no traffic, so it now runs a burstable t4g.small with a 4 GiB WAL in a root-volume directory. WAL placement and cap come from the instance type (WAL_STORAGE), so a c7gd host still refuses to join the cluster when its instance store does not mount. --- apps/ingest/alchemy.run.ts | 38 ++++++++++++++++++++++++----------- docs/infra.md | 3 ++- packages/infra/src/profile.ts | 8 ++++---- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/apps/ingest/alchemy.run.ts b/apps/ingest/alchemy.run.ts index b32d25b3d2..d7367a1b00 100644 --- a/apps/ingest/alchemy.run.ts +++ b/apps/ingest/alchemy.run.ts @@ -49,11 +49,21 @@ const COLLECTOR_DOCKERFILE = resolve(COLLECTOR_CONTEXT, "Dockerfile") /** Port the gateway binds (`apps/ingest/Dockerfile` EXPOSEs the same). */ const INGEST_PORT = 3474 +const GIB = 1024 * 1024 * 1024 + /** - * WAL cap, sized to fit the smallest host's NVMe instance store (c7gd.medium, 59 GB). + * Where each host type keeps the WAL, and its cap. c7gd hosts mount their NVMe instance + * store (59 GB on a medium); a t4g has none, so the WAL shares the 30 GB root volume. * The per-lane budget is `INGEST_QUEUE_MAX_BYTES / (WAL_SHARDS * lanes)`. */ -const WAL_MAX_BYTES = 48 * 1024 * 1024 * 1024 +const WAL_STORAGE = { + "c7gd.large": { disk: "instance-store", maxBytes: 48 * GIB }, + "c7gd.medium": { disk: "instance-store", maxBytes: 48 * GIB }, + "t4g.small": { disk: "root-volume", maxBytes: 4 * GIB }, +} as const satisfies Record< + MapleStackContext["profile"]["ingest"]["instanceType"], + { disk: "instance-store" | "root-volume"; maxBytes: number } +> /** * Pinned rather than derived: the gateway defaults to `num_cpus * 2`, which @@ -64,22 +74,26 @@ const WAL_SHARDS = 4 /** Where the gateway keeps its WAL inside the container (the binary's default `INGEST_QUEUE_DIR`). */ const WAL_CONTAINER_DIR = "/var/lib/maple-ingest/wal" -/** The host's instance-store NVMe, mounted by `ec2UserData`. */ +/** The host's instance-store NVMe (mounted by `ec2UserData`) or a root-volume directory. */ const WAL_HOST_DIR = "/mnt/wal" /** - * Boot script for a gateway host (ECS-optimized AL2023). Mounts the NVMe at - * WAL_HOST_DIR and only THEN joins the cluster, so a host whose disk did not - * come up never gets a task. The S3 tier covers the disk being wiped on replace. + * Boot script for a gateway host (ECS-optimized AL2023). An instance-store host mounts the + * NVMe at WAL_HOST_DIR and only THEN joins the cluster, so a host whose disk did not come + * up never gets a task. The S3 tier covers the disk being wiped on replace. */ -const ec2UserData = (clusterName: string) => `#!/bin/bash +const ec2UserData = (clusterName: string, disk: "instance-store" | "root-volume") => `#!/bin/bash set -euxo pipefail -disk=$(ls /dev/disk/by-id/nvme-Amazon_EC2_NVMe_Instance_Storage_* | grep -v -- -part | head -n1) -mkfs.xfs -f "$disk" mkdir -p ${WAL_HOST_DIR} +${ + disk === "instance-store" + ? `disk=$(ls /dev/disk/by-id/nvme-Amazon_EC2_NVMe_Instance_Storage_* | grep -v -- -part | head -n1) +mkfs.xfs -f "$disk" echo "UUID=$(blkid -s UUID -o value "$disk") ${WAL_HOST_DIR} xfs noatime,nofail 0 2" >> /etc/fstab -mount ${WAL_HOST_DIR} +mount ${WAL_HOST_DIR}` + : "" +} cat >> /etc/ecs/ecs.config <<'CONFIG' ECS_CLUSTER=${clusterName} @@ -290,7 +304,7 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }: securityGroupIds: [instanceSecurityGroup.groupId], instanceProfileName: instanceProfile.instanceProfileName, associatePublicIpAddress: true, - userData: ec2UserData(clusterName), + userData: ec2UserData(clusterName, WAL_STORAGE[instanceType].disk), tags, }) @@ -597,7 +611,7 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }: // nothing but Cloudflare; a preview's open ALB would let any client set it. ...(domains.ingest && { MAPLE_INGEST_TRUST_PROXY_GEO: "true" }), - INGEST_QUEUE_MAX_BYTES: String(WAL_MAX_BYTES), + INGEST_QUEUE_MAX_BYTES: String(WAL_STORAGE[instanceType].maxBytes), INGEST_WAL_SHARDS: String(WAL_SHARDS), // The task role signs these requests; the S3 gateway endpoint keeps them private. INGEST_WAL_S3_BUCKET: walBucketName, diff --git a/docs/infra.md b/docs/infra.md index 16909e9df4..b7eb400f83 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -488,7 +488,8 @@ list of yields. What it composes lives beside it: ## AWS: ingest, collector and Electric The Rust OTLP gateway (`apps/ingest`) runs on an ECS EC2 fleet of Graviton c7gd instances, -one task per host in host networking, with the WAL on the local NVMe instance store. That is +one task per host in host networking, with the WAL on the local NVMe instance store. PR +previews run one t4g.small instead, with a 4 GiB WAL on the root volume (`WAL_STORAGE`). That is the only ingest path. The OTel collector and Electric (`apps/electric`) run as ECS Fargate services. The Prometheus scraper (`apps/scraper`, US prd only) is a Fargate singleton in the same VPC with no inbound traffic; its image is the single-file bundle `alchemy:build-deps` diff --git a/packages/infra/src/profile.ts b/packages/infra/src/profile.ts index cbf2853a47..43f350f2bc 100644 --- a/packages/infra/src/profile.ts +++ b/packages/infra/src/profile.ts @@ -62,8 +62,8 @@ export interface MapleProfile { readonly desiredCount: number /** `undefined` keeps a fixed count. */ readonly scaling: IngestScaling | undefined - /** Graviton with local NVMe for the WAL; the 48 GiB WAL cap must fit the smallest disk. */ - readonly instanceType: "c7gd.large" | "c7gd.medium" + /** Graviton. The c7gd types keep the WAL on local NVMe; a t4g on its root volume (`WAL_STORAGE`). */ + readonly instanceType: "c7gd.large" | "c7gd.medium" | "t4g.small" /** One task per host: claims the host's vCPU and most of its memory. Change with `instanceType`. */ readonly taskSize: TaskSize /** 1 GiB everywhere: the collector config's `memory_limiter` is sized for it. */ @@ -138,11 +138,11 @@ const EU_PRD: MapleProfile = { electric: { taskSize: { cpu: 512, memory: 1024 }, dbPoolSize: 4 }, } -/** Non-prd sizing; a preview or dev stage never bursts. */ +/** Non-prd sizing; a preview or dev stage never bursts, so a burstable host with a small WAL. */ const SMALL_INGEST: MapleProfile["ingest"] = { desiredCount: 1, scaling: undefined, - instanceType: "c7gd.medium", + instanceType: "t4g.small", taskSize: { cpu: 1024, memory: 1536 }, collectorTaskSize: { cpu: 256, memory: 1024 }, selfTraceSampleRatio: undefined,