fix(ingest): stop rolling the EC2 fleet on every merge and right-size hosts

.github/workflows/deploy-prd-instance.yml

@@ -38,6 +38,14 @@ jobs:
               uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
               with:
                   ref: ${{ inputs.commit-sha }}
+                  # History without blobs, for the ingest revision below.
+                  fetch-depth: 0
+                  filter: blob:none
+
+            # Ingest reports the last commit that touched its image context, not the deployed
+            # commit: a new env value would roll the EC2 fleet (fresh hosts) on every merge.
+            - name: Resolve the ingest revision
+              run: echo "INGEST_COMMIT_SHA=$(git log -1 --format=%H -- apps/ingest)" >> "$GITHUB_ENV"
 
             # Toolchain, Infisical, AWS OIDC (after Infisical so it wins), deps, ingest binary.
             - name: Deploy setup

apps/ingest/alchemy.run.ts

@@ -49,11 +49,21 @@ const COLLECTOR_DOCKERFILE = resolve(COLLECTOR_CONTEXT, "Dockerfile")
 /** Port the gateway binds (`apps/ingest/Dockerfile` EXPOSEs the same). */
 const INGEST_PORT = 3474
 
+const GIB = 1024 * 1024 * 1024
+
 /**
- * WAL cap, sized to fit the smallest host's NVMe instance store (c7gd.medium, 59 GB).
+ * Where each host type keeps the WAL, and its cap. c7gd hosts mount their NVMe instance
+ * store (59 GB on a medium); a t4g has none, so the WAL shares the 30 GB root volume.
  * The per-lane budget is `INGEST_QUEUE_MAX_BYTES / (WAL_SHARDS * lanes)`.
  */
-const WAL_MAX_BYTES = 48 * 1024 * 1024 * 1024
+const WAL_STORAGE = {
+	"c7gd.large": { disk: "instance-store", maxBytes: 48 * GIB },
+	"c7gd.medium": { disk: "instance-store", maxBytes: 48 * GIB },
+	"t4g.small": { disk: "root-volume", maxBytes: 4 * GIB },
+} as const satisfies Record<
+	MapleStackContext["profile"]["ingest"]["instanceType"],
+	{ disk: "instance-store" | "root-volume"; maxBytes: number }
+>
 
 /**
  * Pinned rather than derived: the gateway defaults to `num_cpus * 2`, which
@@ -64,22 +74,26 @@ const WAL_SHARDS = 4
 /** Where the gateway keeps its WAL inside the container (the binary's default `INGEST_QUEUE_DIR`). */
 const WAL_CONTAINER_DIR = "/var/lib/maple-ingest/wal"
 
-/** The host's instance-store NVMe, mounted by `ec2UserData`. */
+/** The host's instance-store NVMe (mounted by `ec2UserData`) or a root-volume directory. */
 const WAL_HOST_DIR = "/mnt/wal"
 
 /**
- * Boot script for a gateway host (ECS-optimized AL2023). Mounts the NVMe at
- * WAL_HOST_DIR and only THEN joins the cluster, so a host whose disk did not
- * come up never gets a task. The S3 tier covers the disk being wiped on replace.
+ * Boot script for a gateway host (ECS-optimized AL2023). An instance-store host mounts the
+ * NVMe at WAL_HOST_DIR and only THEN joins the cluster, so a host whose disk did not come
+ * up never gets a task. The S3 tier covers the disk being wiped on replace.
  */
-const ec2UserData = (clusterName: string) => `#!/bin/bash
+const ec2UserData = (clusterName: string, disk: "instance-store" | "root-volume") => `#!/bin/bash
 set -euxo pipefail
 
-disk=$(ls /dev/disk/by-id/nvme-Amazon_EC2_NVMe_Instance_Storage_* | grep -v -- -part | head -n1)
-mkfs.xfs -f "$disk"
 mkdir -p ${WAL_HOST_DIR}
+${
+	disk === "instance-store"
+		? `disk=$(ls /dev/disk/by-id/nvme-Amazon_EC2_NVMe_Instance_Storage_* | grep -v -- -part | head -n1)
+mkfs.xfs -f "$disk"
 echo "UUID=$(blkid -s UUID -o value "$disk") ${WAL_HOST_DIR} xfs noatime,nofail 0 2" >> /etc/fstab
-mount ${WAL_HOST_DIR}
+mount ${WAL_HOST_DIR}`
+		: ""
+}
 
 cat >> /etc/ecs/ecs.config <<'CONFIG'
 ECS_CLUSTER=${clusterName}
@@ -275,8 +289,8 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }:
 			roleName: instanceRole.roleName,
 		})
 
-		// Newest ECS-optimized AL2023 arm64 image. Every deploy launches fresh hosts
-		// (the old task holds the port), so AMI patching rides the deploys.
+		// Newest ECS-optimized AL2023 arm64 image. Every ingest rollout launches fresh hosts
+		// (the old task holds the port), so AMI patching rides the ingest changes.
 		const imageId = AWS.EC2.getAmi({
 			owners: ["amazon"],
 			name: ["al2023-ami-ecs-hvm-*-kernel-6.1-arm64"],
@@ -290,7 +304,7 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }:
 			securityGroupIds: [instanceSecurityGroup.groupId],
 			instanceProfileName: instanceProfile.instanceProfileName,
 			associatePublicIpAddress: true,
-			userData: ec2UserData(clusterName),
+			userData: ec2UserData(clusterName, WAL_STORAGE[instanceType].disk),
 			tags,
 		})
 
@@ -332,6 +346,13 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }:
 			tags,
 		})
 
+		const revisions = {
+			...(yield* optionalPlain("INGEST_COMMIT_SHA")),
+			...(yield* optionalPlain("COMMIT_SHA")),
+			...(yield* optionalPlain("GITHUB_SHA")),
+		}
+		const ingestRevision = revisions.INGEST_COMMIT_SHA || revisions.COMMIT_SHA || revisions.GITHUB_SHA
+
 		const secret = ecsSecrets(name("ingest"), tags)
 
 		const tinybirdToken = yield* secret("tinybird-token", yield* requiredPlain("TINYBIRD_TOKEN"))
@@ -590,7 +611,7 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }:
 				// nothing but Cloudflare; a preview's open ALB would let any client set it.
 				...(domains.ingest && { MAPLE_INGEST_TRUST_PROXY_GEO: "true" }),
 
-				INGEST_QUEUE_MAX_BYTES: String(WAL_MAX_BYTES),
+				INGEST_QUEUE_MAX_BYTES: String(WAL_STORAGE[instanceType].maxBytes),
 				INGEST_WAL_SHARDS: String(WAL_SHARDS),
 				// The task role signs these requests; the S3 gateway endpoint keeps them private.
 				INGEST_WAL_S3_BUCKET: walBucketName,
@@ -644,7 +665,9 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbLogin }:
 				// 30s instead of the binary's 1s cuts Autumn track calls ~30x.
 				...(yield* optionalPlain("AUTUMN_FLUSH_INTERVAL_SECS", "30")),
 				...(yield* optionalPlain("INGEST_SHUTDOWN_DRAIN_SECS")),
-				...(yield* optionalPlain("COMMIT_SHA", (yield* optionalPlain("GITHUB_SHA")).GITHUB_SHA)),
+				// `INGEST_COMMIT_SHA` (prd) is the image's own revision, so a deploy that leaves
+				// `apps/ingest` alone does not change the task definition and roll the hosts.
+				...(ingestRevision ? { COMMIT_SHA: ingestRevision } : undefined),
 			} satisfies Record<string, string | Output.Output<string>>,
 
 			tags,

docs/infra.md

@@ -488,7 +488,8 @@ list of yields. What it composes lives beside it:
 ## AWS: ingest, collector and Electric
 
 The Rust OTLP gateway (`apps/ingest`) runs on an ECS EC2 fleet of Graviton c7gd instances,
-one task per host in host networking, with the WAL on the local NVMe instance store. That is
+one task per host in host networking, with the WAL on the local NVMe instance store. PR
+previews run one t4g.small instead, with a 4 GiB WAL on the root volume (`WAL_STORAGE`). That is
 the only ingest path. The OTel collector and Electric (`apps/electric`) run as ECS Fargate
 services. The Prometheus scraper (`apps/scraper`, US prd only) is a Fargate singleton in the
 same VPC with no inbound traffic; its image is the single-file bundle `alchemy:build-deps`
@@ -570,6 +571,11 @@ architecture.
   one c7gd.medium (autoscaling 1-3) and the collector at the non-prd size; Electric keeps
   the prd size. Managed scaling adds a host to roll a deploy. To scale it, raise the EU
   values there, or point `prd-eu` at `US_PRD`.
+- **Ingest stamps its own revision.** Every ingest rollout launches fresh hosts, so its
+  `COMMIT_SHA` is `INGEST_COMMIT_SHA` (the last commit touching `apps/ingest`, resolved in
+  `deploy-prd-instance.yml`), not the deployed commit. A merge that leaves `apps/ingest` alone
+  keeps the task definition and the fleet. That is also why ingest is not in
+  `PRD_LOCKSTEP_REVISION_SERVICES`.
 - **Graviton (ARM64).** The gateway, collector and Electric run `cpuArchitecture: "ARM64"`
   (`runtimePlatform`), cheaper than x86_64. `build-ingest-binary.yml` compiles natively on
   an arm64 runner.

packages/infra/src/env.test.ts

@@ -238,19 +238,16 @@ describe("the prd revision lockstep the skew alert depends on", () => {
 			"alerting",
 			"maple-ai",
 			"electric-sync",
-			"ingest",
 			"maple-api",
 			"maple-web",
 		])
 	})
 
 	it("only claims lockstep for the stage-gated services prd actually deploys", () => {
-		// `ingest` and `electric-sync` are the two members behind a stage
-		// predicate. Flip either away from prd and it stops tracking the other
-		// three, so it has to leave the list — and the rule's SQL — in the same
-		// change.
+		// `electric-sync` is the member behind a stage predicate. Flip it away from
+		// prd and it stops tracking the others, so it has to leave the list (and the
+		// rule's SQL) in the same change. Ingest stamps its own revision.
 		const { deploys } = resolveMapleProfile({ stage: { kind: "prd" }, region: "us" })
-		expect(PRD_LOCKSTEP_REVISION_SERVICES.includes("ingest")).toBe(deploys.ingest)
 		expect(PRD_LOCKSTEP_REVISION_SERVICES.includes("electric-sync")).toBe(deploys.electric)
 	})
 })

packages/infra/src/env.ts

@@ -164,15 +164,15 @@ export const selfObservabilityEnv = (
 	)
 
 /**
- * prd services deployed together that must report the same revision. The "Prod revision skew"
+ * prd services deployed together that must report the same revision. Not ingest: it reports
+ * the last commit that touched `apps/ingest` (`INGEST_COMMIT_SHA`). The "Prod revision skew"
  * alert rule (id `2a6e9529-5f73-4478-9fa0-432904ff15c8`) lives in the Maple database, not this
  * repo: if you change this list, you MUST edit that rule's SQL to match.
  */
 export const PRD_LOCKSTEP_REVISION_SERVICES = [
 	"alerting",
 	"maple-ai",
 	"electric-sync",
-	"ingest",
 	"maple-api",
 	"maple-web",
 ] as const

packages/infra/src/profile.ts

@@ -62,8 +62,8 @@ export interface MapleProfile {
 		readonly desiredCount: number
 		/** `undefined` keeps a fixed count. */
 		readonly scaling: IngestScaling | undefined
-		/** Graviton with local NVMe for the WAL; the 48 GiB WAL cap must fit the smallest disk. */
-		readonly instanceType: "c7gd.large" | "c7gd.medium"
+		/** Graviton. The c7gd types keep the WAL on local NVMe; a t4g on its root volume (`WAL_STORAGE`). */
+		readonly instanceType: "c7gd.large" | "c7gd.medium" | "t4g.small"
 		/** One task per host: claims the host's vCPU and most of its memory. Change with `instanceType`. */
 		readonly taskSize: TaskSize
 		/** 1 GiB everywhere: the collector config's `memory_limiter` is sized for it. */
@@ -106,9 +106,11 @@ const US_PRD: MapleProfile = {
 	},
 	ingest: {
 		desiredCount: 2,
+		// Two hosts for AZ redundancy; scaling absorbs bursts. On c7gd.large the tasks
+		// peaked at 25% CPU and 6% memory (Oct 2026).
 		scaling: scaling(2, 6),
-		instanceType: "c7gd.large",
-		taskSize: { cpu: 2048, memory: 3072 },
+		instanceType: "c7gd.medium",
+		taskSize: { cpu: 1024, memory: 1536 },
 		collectorTaskSize: { cpu: 512, memory: 1024 },
 		selfTraceSampleRatio: "0.05",
 	},
@@ -136,12 +138,12 @@ const EU_PRD: MapleProfile = {
 	electric: { taskSize: { cpu: 512, memory: 1024 }, dbPoolSize: 4 },
 }
 
-/** Non-prd sizing; a preview or dev stage never bursts. */
+/** Non-prd sizing; a preview or dev stage never bursts, so a burstable host with a small WAL. */
 const SMALL_INGEST: MapleProfile["ingest"] = {
 	desiredCount: 1,
 	scaling: undefined,
-	instanceType: "c7gd.large",
-	taskSize: { cpu: 2048, memory: 3072 },
+	instanceType: "t4g.small",
+	taskSize: { cpu: 1024, memory: 1536 },
 	collectorTaskSize: { cpu: 256, memory: 1024 },
 	selfTraceSampleRatio: undefined,
 }