fix: open every VPC port in the run's firewall and add a capacity warm-up

README.md

@@ -127,8 +127,8 @@ the end. In `infra.region` of `perfkit.yaml`, it creates, in this order:
    collide;
 3. two `ubuntu-24-04-x64` droplets in it, `perfkit-<run id>-uut` and
    `perfkit-<run id>-loadgen`, tagged `perfkit` and `perfkit-run-<run id>`;
-4. a firewall, `perfkit-<run id>`: SIP (5060-5063 tcp/udp) and OTLP (4318 tcp) from the VPC only,
-   SSH (22) from this machine's public IP (from `https://api.ipify.org`) and from the VPC, where the agent on the load generator reaches the server under test.
+4. a firewall, `perfkit-<run id>`: from the VPC: all ports (tcp and udp) and icmp, so the load
+   generator reaches the server's API and the server reaches the callees; SSH (22) from this machine's public IP (from `https://api.ipify.org`) and from the VPC, where the agent on the load generator reaches the server under test.
 
 Both droplets get Docker Engine with the compose plugin, 25 MiB socket buffers
 (`net.core.rmem_max`, `wmem_max`, `rmem_default`, `wmem_default`) and 1048576 open files, through
@@ -161,7 +161,7 @@ plan for Routr 2.31.6 (dry run, nothing created)
   region     nyc3, new VPC (range assigned by DigitalOcean)
   uut        c-8   ubuntu-24-04-x64
   loadgen    c-16  ubuntu-24-04-x64
-  firewall   SIP (5060-5063 tcp/udp) and OTLP (4318 tcp) from the VPC only; SSH (22) from 203.0.113.7 and the VPC only
+  firewall   from the VPC: all ports (tcp/udp) and icmp; SSH (22) from 203.0.113.7 and the VPC only
   tags       perfkit, perfkit-run-20261103T141210Z-bd3254a
   images     fonoster/routr-edgeport:2.31.6@sha256:8bca…1866
              fonoster/routr-echo:2.31.6@sha256:14a4…a43f
@@ -364,6 +364,10 @@ SIPp adds load in steps, such as 50 more calls/s every minute. For each step:
   in the next one.
 - Its attempted, failed and retransmitted calls are SIPp's 10-second counters (`-trace_stat`)
   that fall in the step. Failed % is failed / attempted.
+- **Warm-up.** With `warmup` set, the first step still runs its full `every`, but it is measured
+  only from `warmup` after it starts: calls sent, and 10-second counters whose period mid-point
+  falls, before that don't count. A cold JIT (EdgePort's JVM and GraalJS) makes the first seconds
+  slow; the later steps are measured whole.
 
 perfkit checks the steps in order:
 

docs/perfkit-yaml.md

@@ -65,7 +65,7 @@ the other kind rejects them.
 | `narrative.why` | string | yes | | Why it matters. |
 | `narrative.how` | string | yes | | How the load is driven and what the verdict checks. |
 | `duration` | duration | *soak*: yes | | How long the load runs. |
-| `warmup` | duration | *soak*: yes | | Excluded from every verdict. |
+| `warmup` | duration | *soak*: yes; *capacity*: no (`0s`) | | *soak*: excluded from every verdict. *capacity*: the first step still runs its full `every`, but its measurement starts `warmup` after the step starts, so a cold server's first seconds don't count. Shorter than `load.rate.every`. |
 | `uut.compose` | path | yes | | The Compose file the unit under test runs. |
 | `uut.setup` | path | no | none | A script that runs once the unit under test is up, for example to seed data. |
 | `uut.processes` | list | yes | | At least one. Sampled every 10 s. |

docs/project-setup.md

@@ -43,8 +43,8 @@ anything (exit 2):
 perfkit: sipp-uas/compose.yaml: service uas must use network_mode: host (Docker's UDP port publishing distorts the results); nothing was created
 ```
 
-The firewall lets SIP in on 5060 to 5063 (UDP and TCP) and OTLP on 4318 from the VPC, so the
-server listens on a port in that range.
+The firewall lets every TCP and UDP port (and ICMP) in from the VPC, so the server can listen on
+any port, and SSH from the runner and the VPC. Nothing else reaches the droplets.
 
 **Healthchecks.** The UUT is up when every service is `running`, and `healthy` when it has a
 `healthcheck`, within 5 minutes; otherwise the run ends with exit 3. Give a service a healthcheck

package.json

@@ -1,6 +1,6 @@
 {
   "name": "perfkit",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Benchmark harness for Fonoster projects: DigitalOcean runs, one report.json schema, one branded report page",
   "license": "MIT",
   "author": "Pedro Sanders <psanders@fonoster.com>",

schemas/perfkit-v1.schema.json

@@ -152,6 +152,11 @@
                 ],
                 "additionalProperties": false
               },
+              "warmup": {
+                "default": "0s",
+                "type": "string",
+                "pattern": "^\\d+[smh]$"
+              },
               "uut": {
                 "type": "object",
                 "properties": {

src/analyze/capacityStepWindow.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
+ */
+import type { TimeWindow } from "./types.js";
+
+/**
+ * The window a capacity step is measured over: step `index` (from 0) runs
+ * from `startedMs + index × everyMs` for `everyMs`. The first step's window
+ * starts `warmupMs` later, so a cold server's first seconds don't count; it
+ * still runs its full `everyMs`. `endedMs`, when given, cuts the window short
+ * (a ramp that ended early), and the window never starts after it ends.
+ */
+export function capacityStepWindow(input: {
+  startedMs: number;
+  index: number;
+  everyMs: number;
+  warmupMs: number;
+  endedMs?: number;
+}): TimeWindow {
+  const { startedMs, index, everyMs, warmupMs } = input;
+  const stepStartMs = startedMs + index * everyMs;
+  const stepEndMs = stepStartMs + everyMs;
+  const toMs =
+    input.endedMs === undefined
+      ? stepEndMs
+      : Math.min(stepEndMs, input.endedMs);
+  const fromMs = Math.min(stepStartMs + (index === 0 ? warmupMs : 0), toMs);
+  return { fromMs, toMs };
+}

src/analyze/index.ts

@@ -15,6 +15,7 @@ export {
 } from "./analyzeSoak.schema.js";
 export { computeHops } from "./computeHops.js";
 export { countRestarts } from "./countRestarts.js";
+export { capacityStepWindow } from "./capacityStepWindow.js";
 export { cpuPctOver } from "./cpuPctOver.js";
 export { createAnalyzeCapacity } from "./createAnalyzeCapacity.js";
 export {

src/cli/formatPlan.ts

@@ -1,7 +1,6 @@
 /**
  * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
  */
-import { OTLP_PORT, SIP_PORTS } from "../infra/constants.js";
 import type { Plan } from "../infra/index.js";
 import { parseDuration } from "../infra/parseDuration.js";
 import type { BenchmarkConfig } from "../schemas/index.js";
@@ -60,7 +59,7 @@ export function formatPlan(
       `${plan.loadgen.size.padEnd(sizeWidth)}${plan.loadgen.image}`
     ]),
     ...field("firewall", [
-      `SIP (${SIP_PORTS} tcp/udp) and OTLP (${OTLP_PORT} tcp) from the VPC only; SSH (22) from ${plan.sshSource.replace(/\/32$/, "")} and the VPC only`
+      `from the VPC: all ports (tcp/udp) and icmp; SSH (22) from ${plan.sshSource.replace(/\/32$/, "")} and the VPC only`
     ]),
     ...field("tags", [plan.tags.join(", ")]),
     ...field(

src/infra/buildFirewall.ts

@@ -1,16 +1,17 @@
 /**
  * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
  */
-import { OTLP_PORT, SIP_PORTS } from "./constants.js";
 import type { DoFirewallInput } from "./types.js";
 
 const ANYWHERE = { addresses: ["0.0.0.0/0", "::/0"] };
 
 /**
- * A run's firewall: SIP (tcp and udp) and OTLP (tcp) from the VPC's range
- * only, SSH (22) from `sshSources` and from the VPC (the agent on the load
- * generator drives the UUT over SSH at its private IP), and every outbound
- * connection. It applies to the run's droplets.
+ * A run's firewall: every tcp and udp port and icmp from the VPC's range
+ * (the seed script reaches the UUT's API, and the UUT opens new flows to the
+ * callees on the load generator), SSH (22) from `sshSources` and from the
+ * VPC (the agent on the load generator drives the UUT over SSH at its
+ * private IP), and every outbound connection. It applies to the run's
+ * droplets.
  */
 export function buildFirewall(input: {
   name: string;
@@ -22,9 +23,9 @@ export function buildFirewall(input: {
   return {
     name: input.name,
     inbound_rules: [
-      { protocol: "udp", ports: SIP_PORTS, sources: vpc },
-      { protocol: "tcp", ports: SIP_PORTS, sources: vpc },
-      { protocol: "tcp", ports: OTLP_PORT, sources: vpc },
+      { protocol: "tcp", ports: "all", sources: vpc },
+      { protocol: "udp", ports: "all", sources: vpc },
+      { protocol: "icmp", sources: vpc },
       {
         protocol: "tcp",
         ports: "22",

src/infra/constants.ts

@@ -12,11 +12,6 @@ export const PERFKIT_TAG = "perfkit";
 /** Name of the SSH key that PERFKIT_SSH_KEY registers; never deleted. */
 export const SHARED_KEY_NAME = "perfkit-shared";
 
-/** SIP from the VPC. */
-export const SIP_PORTS = "5060-5063";
-/** OTLP/HTTP from the VPC. */
-export const OTLP_PORT = "4318";
-
 /** Droplets active and cloud-init finished within this, or exit 3. */
 export const READY_TIMEOUT_MS = 10 * 60 * 1000;
 /** How long teardown waits for a VPC to have no members. */

src/orchestrate/createRunBenchmark.ts

@@ -2,6 +2,7 @@
  * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
  */
 import { basename } from "node:path";
+import { capacityStepWindow } from "../analyze/capacityStepWindow.js";
 import { durationToSeconds } from "../analyze/durationToSeconds.js";
 import type { Now, Sleep } from "../infra/index.js";
 import type { BenchmarkConfig } from "../schemas/index.js";
@@ -543,18 +544,18 @@ export function createRunBenchmark(deps: AgentDeps, state: RunState) {
             ...base,
             kind: "capacity",
             steps: Array.from({ length: Math.max(steps, 1) }, (_, index) => {
-              const everyMs =
-                durationToSeconds(benchmark.load.rate.every) * 1000;
+              const window = capacityStepWindow({
+                startedMs,
+                index,
+                everyMs: durationToSeconds(benchmark.load.rate.every) * 1000,
+                warmupMs: durationToSeconds(benchmark.warmup) * 1000,
+                endedMs: Math.max(endedMs, startedMs + 1)
+              });
               return {
                 offeredCps:
                   benchmark.load.rate.start + index * benchmark.load.rate.step,
-                from: iso(startedMs + index * everyMs),
-                to: iso(
-                  Math.min(
-                    startedMs + (index + 1) * everyMs,
-                    Math.max(endedMs, startedMs + 1)
-                  )
-                )
+                from: iso(window.fromMs),
+                to: iso(window.toMs)
               };
             })
           }

src/orchestrate/evaluateCapacityStep.ts

@@ -3,6 +3,7 @@
  */
 import { join } from "node:path";
 import {
+  capacityStepWindow,
   measureCapacityStep,
   parseSamplerJsonl,
   parseUdpJsonl,
@@ -52,10 +53,12 @@ export async function evaluateCapacityStep(input: {
     names.includes(file) ? deps.readFile(join(rawDir, file)) : "";
   const measured = measureCapacityStep({
     offeredCps,
-    window: {
-      fromMs: input.startMs + index * everyMs,
-      toMs: input.startMs + (index + 1) * everyMs
-    },
+    window: capacityStepWindow({
+      startedMs: input.startMs,
+      index,
+      everyMs,
+      warmupMs: durationMs(benchmark.warmup)
+    }),
     uac,
     samples: parseSamplerJsonl(
       await read("sampler-loadgen.jsonl"),

src/schemas/checkWarmupShorterThanStep.ts

@@ -0,0 +1,33 @@
+/**
+ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License.
+ */
+import type { z } from "zod/v4";
+
+const UNIT_S = { s: 1, m: 60, h: 3600 } as const;
+
+/**
+ * Adds an issue at `warmup` when a capacity benchmark's warm-up is not
+ * shorter than one step (`load.rate.every`): the first step would have
+ * nothing left to measure. A `0s` warm-up is always fine. For use inside the
+ * capacity benchmark's `superRefine`.
+ */
+export function checkWarmupShorterThanStep(
+  benchmark: { warmup: string; load: { rate: { every: string } } },
+  ctx: z.core.$RefinementCtx
+): void {
+  const warmupS = seconds(benchmark.warmup);
+  const everyS = seconds(benchmark.load.rate.every);
+  if (warmupS > 0 && warmupS >= everyS) {
+    ctx.addIssue({
+      code: "custom",
+      path: ["warmup"],
+      message: `expected a warm-up shorter than load.rate.every (${benchmark.load.rate.every})`,
+      input: benchmark.warmup
+    });
+  }
+}
+
+function seconds(duration: string): number {
+  const match = /^(\d+)([smh])$/.exec(duration);
+  return match ? Number(match[1]) * UNIT_S[match[2] as keyof typeof UNIT_S] : 0;
+}

src/schemas/perfkitConfig.schema.ts

@@ -9,6 +9,7 @@
  */
 import { z } from "zod/v4";
 import { checkUniqueIds } from "./checkUniqueIds.js";
+import { checkWarmupShorterThanStep } from "./checkWarmupShorterThanStep.js";
 import {
   benchmarkIdSchema,
   count,
@@ -175,16 +176,20 @@ const narrativeSchema = z.strictObject({
   how: nonEmptyString
 });
 
-export const capacityBenchmarkConfigSchema = z.strictObject({
-  id: benchmarkIdSchema,
-  kind: z.literal("capacity"),
-  title: nonEmptyString,
-  narrative: narrativeSchema,
-  uut: configUutSchema,
-  load: capacityLoadSchema,
-  telemetry: telemetrySchema,
-  thresholds: capacityThresholdsSchema
-});
+export const capacityBenchmarkConfigSchema = z
+  .strictObject({
+    id: benchmarkIdSchema,
+    kind: z.literal("capacity"),
+    title: nonEmptyString,
+    narrative: narrativeSchema,
+    // The first step's measurement starts this long after the step does.
+    warmup: durationSchema.default("0s"),
+    uut: configUutSchema,
+    load: capacityLoadSchema,
+    telemetry: telemetrySchema,
+    thresholds: capacityThresholdsSchema
+  })
+  .superRefine(checkWarmupShorterThanStep);
 
 export const soakBenchmarkConfigSchema = z.strictObject({
   id: benchmarkIdSchema,