From 4b77806cefc3b123e1d3143df477922c2462d3e4 Mon Sep 17 00:00:00 2001 From: Pedro Sanders Date: Wed, 4 Nov 2026 09:51:15 -0400 Subject: [PATCH] feat: accept { name, tagSuffix } image entries in perfkit.yaml (0.1.6) An images entry may now be an object { name, tagSuffix }; its tag is --target-version followed by tagSuffix (like 2.32.0-go-experimental). Plain string entries behave as before. --- docs/perfkit-yaml.md | 26 +++++++-- docs/project-setup.md | 2 +- package-lock.json | 4 +- package.json | 2 +- schemas/perfkit-v1.schema.json | 26 ++++++++- src/infra/createPlanRun.ts | 4 +- src/infra/createResolveImageDigests.ts | 13 +++-- src/infra/infra.schema.ts | 3 +- src/schemas/index.ts | 2 + src/schemas/perfkitConfig.schema.ts | 19 ++++++- .../createResolveImageDigests.unit.test.ts | 53 ++++++++++++++++++ test/infra/run.unit.test.ts | 40 ++++++++++++++ test/orchestrate/buildComposeEnv.unit.test.ts | 54 +++++++++++++++++++ test/schemas/perfkitConfig.unit.test.ts | 28 ++++++++++ test/schemas/report.unit.test.ts | 6 +++ 15 files changed, 265 insertions(+), 17 deletions(-) create mode 100644 test/orchestrate/buildComposeEnv.unit.test.ts diff --git a/docs/perfkit-yaml.md b/docs/perfkit-yaml.md index a426364..cb5090f 100644 --- a/docs/perfkit-yaml.md +++ b/docs/perfkit-yaml.md @@ -46,9 +46,29 @@ Rules for the whole file: ## images -A map from a short name to an image name without a tag or digest, like -`edgeport: fonoster/routr-edgeport`. The tag comes from `--target-version`, and perfkit pins each -image by digest when the run starts. At least one image is required. +A map from a short name (the key) to an image. At least one image is required. Each entry is +either: + +- An image name without a tag or digest, like `edgeport: fonoster/routr-edgeport`. The tag is + `--target-version`. +- An object `{ name, tagSuffix }`. `name` is an image name without a tag or digest; the tag is + `--target-version` followed by `tagSuffix`. `tagSuffix` matches `[A-Za-z0-9._-]+`. With + `--target-version 2.32.0`, this entry runs `fonoster/routr-edgeport:2.32.0-go-experimental`: + + ```yaml + images: + edgeport: fonoster/routr-edgeport + edgeportGo: + name: fonoster/routr-edgeport + tagSuffix: -go-experimental + ``` + +perfkit pins each image by digest when the run starts, so `--dry-run` shows the digests the run +will use. Each image reaches the Compose file as `PERFKIT_IMAGE_`: the key in uppercase, +with every character other than `A-Z` and `0-9` turned into `_`. Uppercasing removes camelCase, +so `edgeportGo` becomes `PERFKIT_IMAGE_EDGEPORTGO` (not `EDGEPORT_GO`) and `edgeport-go` becomes +`PERFKIT_IMAGE_EDGEPORT_GO`. In `report.json`, `run.images` maps each key to the pinned image, +like `fonoster/routr-edgeport:2.32.0-go-experimental@sha256:…`. ## benchmarks diff --git a/docs/project-setup.md b/docs/project-setup.md index 7887015..63aa9d2 100644 --- a/docs/project-setup.md +++ b/docs/project-setup.md @@ -25,7 +25,7 @@ with these variables, which the compose file uses as `${PERFKIT_...}`: | Variable | Value | | :--- | :--- | -| `PERFKIT_IMAGE_` | One per entry of `images:`, pinned by digest, like `PERFKIT_IMAGE_EDGEPORT=fonoster/routr-edgeport:2.31.5@sha256:4f1c…`. `` is the key in uppercase, any other character turned into `_`. | +| `PERFKIT_IMAGE_` | One per entry of `images:`, pinned by digest, like `PERFKIT_IMAGE_EDGEPORT=fonoster/routr-edgeport:2.31.5@sha256:4f1c…`. `` is the key in uppercase, any other character turned into `_`: `edgeportGo` becomes `PERFKIT_IMAGE_EDGEPORTGO`. | | `PERFKIT_UUT_IP` | The UUT's private (VPC) IP. | | `PERFKIT_LOADGEN_IP` | The load generator's private IP. | | `PERFKIT_VPC_CIDR` | The VPC range, like `10.20.0.0/24`. | diff --git a/package-lock.json b/package-lock.json index 7ebc3e3..6fdee7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "perfkit", - "version": "0.1.5", + "version": "0.1.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "perfkit", - "version": "0.1.5", + "version": "0.1.6", "license": "MIT", "dependencies": { "commander": "^14.0.3", diff --git a/package.json b/package.json index d5a6c0f..aab21aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "perfkit", - "version": "0.1.5", + "version": "0.1.6", "description": "Benchmark harness for Fonoster projects: DigitalOcean runs, one report.json schema, one branded report page", "license": "MIT", "author": "Pedro Sanders ", diff --git a/schemas/perfkit-v1.schema.json b/schemas/perfkit-v1.schema.json index e141f80..bb38789 100644 --- a/schemas/perfkit-v1.schema.json +++ b/schemas/perfkit-v1.schema.json @@ -105,8 +105,30 @@ "type": "string" }, "additionalProperties": { - "type": "string", - "minLength": 1 + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "tagSuffix": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]+$" + } + }, + "required": [ + "name", + "tagSuffix" + ], + "additionalProperties": false + } + ] } }, "benchmarks": { diff --git a/src/infra/createPlanRun.ts b/src/infra/createPlanRun.ts index 8194625..2d3254c 100644 --- a/src/infra/createPlanRun.ts +++ b/src/infra/createPlanRun.ts @@ -3,7 +3,7 @@ */ import type { z } from "zod/v4"; import { withErrorHandlingAndValidation } from "../errors/index.js"; -import { formatRunId } from "../schemas/index.js"; +import { formatRunId, type PerfkitConfig } from "../schemas/index.js"; import { DROPLET_IMAGE, PERFKIT_TAG, @@ -23,7 +23,7 @@ import type { Plan, ResolvedImage } from "./types.js"; */ export function createPlanRun(deps: { resolveImageDigests: (input: { - images: Record; + images: PerfkitConfig["images"]; tag: string; }) => Promise; getPublicIp: () => Promise; diff --git a/src/infra/createResolveImageDigests.ts b/src/infra/createResolveImageDigests.ts index 20d2ef6..fef32f8 100644 --- a/src/infra/createResolveImageDigests.ts +++ b/src/infra/createResolveImageDigests.ts @@ -11,7 +11,8 @@ import type { RegistryClient, ResolvedImage } from "./types.js"; /** * Creates the function that pins every image of perfkit.yaml at one tag to - * its digest, so a run uses exactly the images `--dry-run` showed. + * its digest, so a run uses exactly the images `--dry-run` showed. An entry + * with a `tagSuffix` is pinned at the tag followed by its suffix. * * @param deps - The registry client (injected) * @returns A validated function; it throws ImageNotFoundError for a missing @@ -22,11 +23,15 @@ export function createResolveImageDigests(deps: { registry: RegistryClient }) { images, tag }: ResolveImageDigestsInput): Promise => { - const entries = Object.entries(images); + const entries = Object.entries(images).map(([key, entry]) => + typeof entry === "string" + ? { key, name: entry, tag } + : { key, name: entry.name, tag: `${tag}${entry.tagSuffix}` } + ); const digests = await Promise.all( - entries.map(([, name]) => deps.registry.getDigest(name, tag)) + entries.map(({ name, tag }) => deps.registry.getDigest(name, tag)) ); - return entries.map(([key, name], index) => { + return entries.map(({ key, name, tag }, index) => { const digest = digests[index]; if (!digest) throw new ImageNotFoundError(`${name}:${tag}`); return { key, name, tag, digest, ref: `${name}:${tag}@${digest}` }; diff --git a/src/infra/infra.schema.ts b/src/infra/infra.schema.ts index 8d663a4..5fafe08 100644 --- a/src/infra/infra.schema.ts +++ b/src/infra/infra.schema.ts @@ -9,6 +9,7 @@ import { z } from "zod/v4"; import { benchmarkConfigSchema, durationSchema, + imageEntrySchema, perfkitConfigSchema } from "../schemas/index.js"; import { @@ -48,7 +49,7 @@ const ipv4Cidr32Schema = z export const resolveImageDigestsInputSchema = z.strictObject({ images: z - .record(z.string(), nonEmptyString) + .record(z.string(), imageEntrySchema) .refine((images) => Object.keys(images).length > 0, { error: "expected at least one image" }), diff --git a/src/schemas/index.ts b/src/schemas/index.ts index 3566998..9df20ed 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -9,11 +9,13 @@ export { formatRunId } from "./formatRunId.js"; export { benchmarkConfigSchema, capacityBenchmarkConfigSchema, + imageEntrySchema, perfkitConfigSchema, relativePathSchema, soakBenchmarkConfigSchema, type BenchmarkConfig, type CapacityBenchmarkConfig, + type ImageEntry, type PerfkitConfig, type PerfkitConfigInput, type SoakBenchmarkConfig diff --git a/src/schemas/perfkitConfig.schema.ts b/src/schemas/perfkitConfig.schema.ts index 5588d60..2f0f5d6 100644 --- a/src/schemas/perfkitConfig.schema.ts +++ b/src/schemas/perfkitConfig.schema.ts @@ -63,8 +63,25 @@ const imageNameSchema = nonEmptyString.refine( } ); +/** + * One `images` entry: an image name, tagged with `--target-version`, or + * `{ name, tagSuffix }`, tagged with `--target-version` followed by + * `tagSuffix` (like `2.32.0-go-experimental`). + */ +export const imageEntrySchema = z.union([ + imageNameSchema, + z.strictObject({ + name: imageNameSchema, + tagSuffix: z.string().regex(/^[A-Za-z0-9._-]+$/, { + error: "expected a tag suffix of letters, digits, '.', '_' or '-'" + }) + }) +]); + +export type ImageEntry = z.output; + const configImagesSchema = z - .record(z.string(), imageNameSchema) + .record(z.string(), imageEntrySchema) .refine((images) => Object.keys(images).length > 0, { error: "expected at least one image" }); diff --git a/test/infra/createResolveImageDigests.unit.test.ts b/test/infra/createResolveImageDigests.unit.test.ts index 43d4d43..6c09144 100644 --- a/test/infra/createResolveImageDigests.unit.test.ts +++ b/test/infra/createResolveImageDigests.unit.test.ts @@ -42,6 +42,43 @@ describe("createResolveImageDigests", () => { ]); }); + it("should pin an entry with a tagSuffix at the tag followed by the suffix", async () => { + // Arrange + const registry = fakeRegistry({ + "fonoster/routr-edgeport:2.32.0": DIGEST.edgeport, + "fonoster/routr-edgeport:2.32.0-go-experimental": DIGEST.one + }); + const resolve = createResolveImageDigests({ registry }); + + // Act + const images = await resolve({ + images: { + edgeport: "fonoster/routr-edgeport", + edgeportGo: { + name: "fonoster/routr-edgeport", + tagSuffix: "-go-experimental" + } + }, + tag: "2.32.0" + }); + + // Assert + expect(images[1]).to.deep.equal({ + key: "edgeportGo", + name: "fonoster/routr-edgeport", + tag: "2.32.0-go-experimental", + digest: DIGEST.one, + ref: `fonoster/routr-edgeport:2.32.0-go-experimental@${DIGEST.one}` + }); + expect(images[0]?.ref).to.equal( + `fonoster/routr-edgeport:2.32.0@${DIGEST.edgeport}` + ); + expect(registry.asked).to.deep.equal([ + "fonoster/routr-edgeport:2.32.0", + "fonoster/routr-edgeport:2.32.0-go-experimental" + ]); + }); + it("should name the missing image", async () => { // Arrange const resolve = createResolveImageDigests({ @@ -78,5 +115,21 @@ describe("createResolveImageDigests", () => { expect(err).to.be.instanceOf(ValidationError); expect(registry.asked).to.deep.equal([]); }); + + it("should reject a tagSuffix with characters a tag can't hold", async () => { + // Arrange + const registry = fakeRegistry(); + const resolve = createResolveImageDigests({ registry }); + + // Act + const err = await resolve({ + images: { go: { name: "fonoster/routr-edgeport", tagSuffix: "/go" } }, + tag: "2.32.0" + }).catch((e: unknown) => e); + + // Assert + expect(err).to.be.instanceOf(ValidationError); + expect(registry.asked).to.deep.equal([]); + }); }); }); diff --git a/test/infra/run.unit.test.ts b/test/infra/run.unit.test.ts index 807c0ad..9398000 100644 --- a/test/infra/run.unit.test.ts +++ b/test/infra/run.unit.test.ts @@ -2,10 +2,15 @@ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. */ import { expect } from "chai"; +import { cp, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; import sinon from "sinon"; +import { parse, stringify } from "yaml"; import { fingerprintOf } from "../../src/infra/index.js"; import { ROUTR_CONFIG } from "../fixtures.js"; import { + DIGEST, fakeRegistry, infraDeps, perfkit, @@ -53,6 +58,41 @@ describe("perfkit run", () => { expect(infra.account.calls).to.deep.equal([]); }); + it("should pin an image with a tagSuffix at the target version followed by the suffix", async () => { + // Arrange + const dir = await mkdtemp(join(tmpdir(), "perfkit-suffix-")); + await cp(dirname(ROUTR_CONFIG), dir, { recursive: true }); + const config = join(dir, "perfkit.yaml"); + const raw = parse(await readFile(config, "utf8")); + raw.images.edgeportGo = { + name: "fonoster/routr-edgeport", + tagSuffix: "-go-experimental" + }; + await writeFile(config, stringify(raw)); + const infra = infraDeps({ + registry: fakeRegistry({ + "fonoster/routr-edgeport:2.31.5": DIGEST.edgeport, + "fonoster/routr-echo:2.31.5": DIGEST.echo, + "fonoster/routr-one:2.31.5": DIGEST.one, + "fonoster/routr-edgeport:2.31.5-go-experimental": DIGEST.one + }) + }); + + // Act + const result = await perfkit( + ["run", "--config", config, "--target-version", "2.31.5", "--dry-run"], + infra + ); + + // Assert + expect(result.err).to.equal(""); + expect(result.code).to.equal(0); + expect(result.out).to.include( + " fonoster/routr-one:2.31.5@sha256:9b07…1d3c\n" + + " fonoster/routr-edgeport:2.31.5-go-experimental@sha256:9b07…1d3c\n" + ); + }); + it("should list only the benchmarks given with --benchmarks", async () => { // Act const result = await perfkit( diff --git a/test/orchestrate/buildComposeEnv.unit.test.ts b/test/orchestrate/buildComposeEnv.unit.test.ts new file mode 100644 index 0000000..629eb32 --- /dev/null +++ b/test/orchestrate/buildComposeEnv.unit.test.ts @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { buildComposeEnv, type RunState } from "../../src/orchestrate/index.js"; +import type { BenchmarkConfig } from "../../src/schemas/index.js"; + +const DIGEST = `sha256:${"a".repeat(64)}`; + +const image = (key: string, tag: string) => ({ + key, + name: "fonoster/routr-edgeport", + tag, + digest: DIGEST, + ref: `fonoster/routr-edgeport:${tag}@${DIGEST}` +}); + +const STATE = { + plan: { + targetVersion: "2.32.0", + images: [ + image("edgeport", "2.32.0"), + image("edgeportGo", "2.32.0-go-experimental"), + image("routr-one.v2", "2.32.0") + ] + }, + hosts: { + uutIp: "10.116.0.2", + loadgenIp: "10.116.0.3", + vpcCidr: "10.116.0.0/20" + } +} as unknown as RunState; + +const BENCHMARK = { + telemetry: { otlp: false, sampleRatio: 0.01 } +} as unknown as BenchmarkConfig; + +describe("buildComposeEnv", () => { + it("should name each image PERFKIT_IMAGE_, upper-cased with every other character turned into _", () => { + // Act + const env = buildComposeEnv(STATE, BENCHMARK); + + // Assert + expect(env).to.deep.equal({ + PERFKIT_IMAGE_EDGEPORT: `fonoster/routr-edgeport:2.32.0@${DIGEST}`, + PERFKIT_IMAGE_EDGEPORTGO: `fonoster/routr-edgeport:2.32.0-go-experimental@${DIGEST}`, + PERFKIT_IMAGE_ROUTR_ONE_V2: `fonoster/routr-edgeport:2.32.0@${DIGEST}`, + PERFKIT_UUT_IP: "10.116.0.2", + PERFKIT_LOADGEN_IP: "10.116.0.3", + PERFKIT_VPC_CIDR: "10.116.0.0/20", + PERFKIT_TARGET_VERSION: "2.32.0" + }); + }); +}); diff --git a/test/schemas/perfkitConfig.unit.test.ts b/test/schemas/perfkitConfig.unit.test.ts index 9c61564..4165141 100644 --- a/test/schemas/perfkitConfig.unit.test.ts +++ b/test/schemas/perfkitConfig.unit.test.ts @@ -239,6 +239,34 @@ describe("perfkitConfigSchema", () => { ]); }); + it("should accept an image entry with a name and a tagSuffix", () => { + const raw = loadConfig(); + raw.images.edgeportGo = { + name: "fonoster/routr-edgeport", + tagSuffix: "-go-experimental" + }; + const config = perfkitConfigSchema.parse(raw); + expect(config.images.edgeportGo).to.deep.equal({ + name: "fonoster/routr-edgeport", + tagSuffix: "-go-experimental" + }); + expect(config.images.edgeport).to.equal("fonoster/routr-edgeport"); + }); + + it("should reject an image entry with a tagged name, a bad tagSuffix or no tagSuffix", () => { + const raw = loadConfig(); + raw.images.tagged = { name: "fonoster/routr-edgeport:2.32.0", tagSuffix: "-go" }; + raw.images.slash = { name: "fonoster/routr-edgeport", tagSuffix: "/go" }; + raw.images.empty = { name: "fonoster/routr-edgeport", tagSuffix: "" }; + raw.images.bare = { name: "fonoster/routr-edgeport" }; + expect(errorsOf(perfkitConfigSchema, raw)).to.deep.equal([ + "images.bare: does not match any allowed shape", + `images.empty.tagSuffix: expected a tag suffix of letters, digits, '.', '_' or '-'; got ""`, + `images.slash.tagSuffix: expected a tag suffix of letters, digits, '.', '_' or '-'; got "/go"`, + "images.tagged.name: expected an image name without a tag or digest; the tag comes from --target-version" + ]); + }); + it("should reject a process match that is not a regex", () => { const raw = loadConfig(); raw.benchmarks[0].uut.processes[0].match = "io\\.routr("; diff --git a/test/schemas/report.unit.test.ts b/test/schemas/report.unit.test.ts index f194b6b..7d33b40 100644 --- a/test/schemas/report.unit.test.ts +++ b/test/schemas/report.unit.test.ts @@ -200,6 +200,12 @@ describe("reportSchema", () => { ]); }); + it("should accept an image whose tag carries a suffix", () => { + const report = loadReport("passed"); + report.run.images.edgeportGo = `fonoster/routr-edgeport:2.32.0-go-experimental@sha256:${"a".repeat(64)}`; + expect(errorsOf(reportSchema, report)).to.deep.equal([]); + }); + it("should reject unknown keys", () => { const report = loadReport("passed"); report.run.infra.uut.disk = 100;