From 8c97d0af76841d4538d194f5af910b9144be1878 Mon Sep 17 00:00:00 2001 From: Pedro Sanders Date: Thu, 5 Nov 2026 08:32:36 -0400 Subject: [PATCH 1/2] feat: show a run in progress live on the benchmarks page The agent writes live.json (perfkit/live/v1) every 15 s and serves it read-only with CORS for the project's site and localhost, tunnelled with @ngrok/ngrok when NGROK_AUTHTOKEN is set. The latest page polls it and draws the live view; perfkit serve previews a site locally. --- .github/workflows/run.yaml | 7 +- package-lock.json | 230 +++++++ package.json | 1 + schemas/live-v1.schema.json | 614 ++++++++++++++++++ schemas/perfkit-v1.schema.json | 33 + schemas/report-v1.schema.json | 13 + src/analyze/createAnalyzeSoak.ts | 13 +- src/analyze/index.ts | 1 + src/analyze/memoryGrowthMBPerHour.ts | 30 + src/cli.ts | 96 ++- src/cli/createStartExecutor.ts | 6 +- src/cli/runCli.ts | 56 ++ src/cli/runCollectCommand.ts | 50 +- src/cli/runRunCommand.ts | 28 +- src/cli/types.ts | 31 +- src/live/buildLiveCapacity.ts | 76 +++ src/live/buildLiveSnapshot.ts | 63 ++ src/live/buildLiveSoak.ts | 91 +++ src/live/constants.ts | 11 + src/live/createHandleLiveRequest.ts | 62 ++ src/live/createLiveRecorder.ts | 50 ++ src/live/createNodeServeLive.ts | 34 + src/live/createServeLive.ts | 77 +++ src/live/createStartLiveTunnel.ts | 31 + src/live/createWriteLive.ts | 131 ++++ src/live/downsample.ts | 16 + src/live/estimateSchedule.ts | 44 ++ src/live/handleLiveRequest.schema.ts | 20 + src/live/isAllowedOrigin.ts | 18 + src/live/listenHttp.ts | 55 ++ src/live/liveStateOf.ts | 16 + src/live/ngrokBindingCommand.ts | 36 + src/live/ngrokVersion.ts | 13 + src/live/runWithLive.ts | 41 ++ src/live/serveLive.schema.ts | 17 + src/live/startLiveTunnel.schema.ts | 22 + src/live/types.ts | 51 ++ src/live/writeLive.schema.ts | 9 + src/orchestrate/buildReport.ts | 11 +- src/orchestrate/createAgent.ts | 4 + src/orchestrate/createRunBenchmark.ts | 5 + src/orchestrate/createStartRun.ts | 17 + src/orchestrate/evaluateCapacityStep.ts | 28 +- src/orchestrate/paths.ts | 1 + src/orchestrate/runState.schema.ts | 10 +- src/orchestrate/startRun.schema.ts | 10 +- src/orchestrate/toReportProject.ts | 28 + src/render/app/createDrawRun.ts | 7 +- src/render/app/createLiveController.ts | 97 +++ src/render/app/createLiveDomView.ts | 57 ++ src/render/app/liveController.types.ts | 29 + src/render/app/main.ts | 85 ++- src/render/app/resolveLiveUrl.ts | 25 + src/render/app/types.ts | 9 +- src/render/charts/renderLatencyChart.ts | 62 +- src/render/charts/renderMemoryChart.ts | 28 +- src/render/icons.ts | 4 + src/render/live/describeStep.ts | 43 ++ src/render/live/formatClock.ts | 8 + src/render/live/formatUpdated.ts | 28 + src/render/live/lastPublishedLimit.ts | 27 + src/render/live/livePhase.ts | 13 + src/render/live/liveSectionHead.ts | 21 + src/render/live/liveStyles.ts | 69 ++ src/render/live/renderLiveBanner.ts | 26 + src/render/live/renderLiveCapacity.ts | 121 ++++ src/render/live/renderLiveHeader.ts | 57 ++ src/render/live/renderLiveNav.ts | 50 ++ src/render/live/renderLiveNumbers.ts | 94 +++ src/render/live/renderLivePage.ts | 51 ++ src/render/live/renderLiveProgress.ts | 67 ++ src/render/live/renderLiveSoak.ts | 74 +++ src/render/live/renderLiveView.ts | 46 ++ src/render/live/types.ts | 24 + src/schemas/index.ts | 18 + src/schemas/live.schema.ts | 180 +++++ src/schemas/perfkitConfig.schema.ts | 27 +- src/schemas/report.schema.ts | 10 +- src/schemas/toJsonSchemaFiles.ts | 4 +- src/serve/contentTypeOf.ts | 17 + src/serve/createNodeServeSite.ts | 19 + src/serve/createServeSite.ts | 69 ++ .../memoryGrowthMBPerHour.unit.test.ts | 21 + test/cli/serve.unit.test.ts | 84 +++ test/live/buildLiveCapacity.unit.test.ts | 133 ++++ test/live/buildLiveSnapshot.unit.test.ts | 80 +++ test/live/buildLiveSoak.unit.test.ts | 114 ++++ .../live/createHandleLiveRequest.unit.test.ts | 88 +++ test/live/createLiveRecorder.unit.test.ts | 46 ++ test/live/createNodeServeLive.unit.test.ts | 86 +++ test/live/createServeLive.unit.test.ts | 101 +++ test/live/createStartLiveTunnel.unit.test.ts | 67 ++ test/live/createWriteLive.unit.test.ts | 126 ++++ test/live/downsample.unit.test.ts | 29 + test/live/estimateSchedule.unit.test.ts | 49 ++ test/live/fakeFs.ts | 40 ++ test/live/isAllowedOrigin.unit.test.ts | 26 + test/live/listenHttp.unit.test.ts | 84 +++ test/live/liveStateOf.unit.test.ts | 21 + test/live/ngrokBindingCommand.unit.test.ts | 38 ++ test/live/runWithLive.unit.test.ts | 85 +++ test/live/sample.ts | 38 ++ test/live/state.ts | 96 +++ test/orchestrate/local.unit.test.ts | 31 + test/orchestrate/run.unit.test.ts | 58 ++ test/orchestrate/setup.ts | 7 +- test/orchestrate/toReportProject.unit.test.ts | 31 + test/render/app/createDrawRun.unit.test.ts | 29 +- .../app/createLiveController.unit.test.ts | 148 +++++ .../render/app/createLiveDomView.unit.test.ts | 74 +++ test/render/app/resolveLiveUrl.unit.test.ts | 28 + test/render/live/describeStep.unit.test.ts | 43 ++ test/render/live/formatters.unit.test.ts | 50 ++ test/render/live/history.ts | 12 + .../live/lastPublishedLimit.unit.test.ts | 24 + test/render/live/renderLiveView.unit.test.ts | 197 ++++++ test/schemas/live.unit.test.ts | 29 + test/schemas/perfkitConfig.unit.test.ts | 30 + test/schemas/toJsonSchemaFiles.unit.test.ts | 3 +- test/serve/createServeSite.unit.test.ts | 105 +++ 120 files changed, 6126 insertions(+), 98 deletions(-) create mode 100644 schemas/live-v1.schema.json create mode 100644 src/analyze/memoryGrowthMBPerHour.ts create mode 100644 src/live/buildLiveCapacity.ts create mode 100644 src/live/buildLiveSnapshot.ts create mode 100644 src/live/buildLiveSoak.ts create mode 100644 src/live/constants.ts create mode 100644 src/live/createHandleLiveRequest.ts create mode 100644 src/live/createLiveRecorder.ts create mode 100644 src/live/createNodeServeLive.ts create mode 100644 src/live/createServeLive.ts create mode 100644 src/live/createStartLiveTunnel.ts create mode 100644 src/live/createWriteLive.ts create mode 100644 src/live/downsample.ts create mode 100644 src/live/estimateSchedule.ts create mode 100644 src/live/handleLiveRequest.schema.ts create mode 100644 src/live/isAllowedOrigin.ts create mode 100644 src/live/listenHttp.ts create mode 100644 src/live/liveStateOf.ts create mode 100644 src/live/ngrokBindingCommand.ts create mode 100644 src/live/ngrokVersion.ts create mode 100644 src/live/runWithLive.ts create mode 100644 src/live/serveLive.schema.ts create mode 100644 src/live/startLiveTunnel.schema.ts create mode 100644 src/live/types.ts create mode 100644 src/live/writeLive.schema.ts create mode 100644 src/orchestrate/toReportProject.ts create mode 100644 src/render/app/createLiveController.ts create mode 100644 src/render/app/createLiveDomView.ts create mode 100644 src/render/app/liveController.types.ts create mode 100644 src/render/app/resolveLiveUrl.ts create mode 100644 src/render/live/describeStep.ts create mode 100644 src/render/live/formatClock.ts create mode 100644 src/render/live/formatUpdated.ts create mode 100644 src/render/live/lastPublishedLimit.ts create mode 100644 src/render/live/livePhase.ts create mode 100644 src/render/live/liveSectionHead.ts create mode 100644 src/render/live/liveStyles.ts create mode 100644 src/render/live/renderLiveBanner.ts create mode 100644 src/render/live/renderLiveCapacity.ts create mode 100644 src/render/live/renderLiveHeader.ts create mode 100644 src/render/live/renderLiveNav.ts create mode 100644 src/render/live/renderLiveNumbers.ts create mode 100644 src/render/live/renderLivePage.ts create mode 100644 src/render/live/renderLiveProgress.ts create mode 100644 src/render/live/renderLiveSoak.ts create mode 100644 src/render/live/renderLiveView.ts create mode 100644 src/render/live/types.ts create mode 100644 src/schemas/live.schema.ts create mode 100644 src/serve/contentTypeOf.ts create mode 100644 src/serve/createNodeServeSite.ts create mode 100644 src/serve/createServeSite.ts create mode 100644 test/analyze/memoryGrowthMBPerHour.unit.test.ts create mode 100644 test/cli/serve.unit.test.ts create mode 100644 test/live/buildLiveCapacity.unit.test.ts create mode 100644 test/live/buildLiveSnapshot.unit.test.ts create mode 100644 test/live/buildLiveSoak.unit.test.ts create mode 100644 test/live/createHandleLiveRequest.unit.test.ts create mode 100644 test/live/createLiveRecorder.unit.test.ts create mode 100644 test/live/createNodeServeLive.unit.test.ts create mode 100644 test/live/createServeLive.unit.test.ts create mode 100644 test/live/createStartLiveTunnel.unit.test.ts create mode 100644 test/live/createWriteLive.unit.test.ts create mode 100644 test/live/downsample.unit.test.ts create mode 100644 test/live/estimateSchedule.unit.test.ts create mode 100644 test/live/fakeFs.ts create mode 100644 test/live/isAllowedOrigin.unit.test.ts create mode 100644 test/live/listenHttp.unit.test.ts create mode 100644 test/live/liveStateOf.unit.test.ts create mode 100644 test/live/ngrokBindingCommand.unit.test.ts create mode 100644 test/live/runWithLive.unit.test.ts create mode 100644 test/live/sample.ts create mode 100644 test/live/state.ts create mode 100644 test/orchestrate/toReportProject.unit.test.ts create mode 100644 test/render/app/createLiveController.unit.test.ts create mode 100644 test/render/app/createLiveDomView.unit.test.ts create mode 100644 test/render/app/resolveLiveUrl.unit.test.ts create mode 100644 test/render/live/describeStep.unit.test.ts create mode 100644 test/render/live/formatters.unit.test.ts create mode 100644 test/render/live/history.ts create mode 100644 test/render/live/lastPublishedLimit.unit.test.ts create mode 100644 test/render/live/renderLiveView.unit.test.ts create mode 100644 test/schemas/live.unit.test.ts create mode 100644 test/serve/createServeSite.unit.test.ts diff --git a/.github/workflows/run.yaml b/.github/workflows/run.yaml index 6aa61d4..85951f5 100644 --- a/.github/workflows/run.yaml +++ b/.github/workflows/run.yaml @@ -3,7 +3,7 @@ # # uses: fonoster/perfkit/.github/workflows/run.yaml@v0.1.0 # with: { perfkit-ref: v0.1.0, target-version: 2.31.5, commit: ${{ github.sha }} } -# secrets: { DIGITALOCEAN_TOKEN: ..., PERFKIT_SSH_KEY: ... } +# secrets: { DIGITALOCEAN_TOKEN: ..., PERFKIT_SSH_KEY: ..., NGROK_AUTHTOKEN: ... (optional) } # # Secrets come only through `secrets:` below, never `secrets: inherit`. name: perfkit run @@ -36,6 +36,9 @@ on: required: true PERFKIT_SSH_KEY: required: true + # Optional: opens the agent's ngrok tunnel to live.json (live.domain in perfkit.yaml). + NGROK_AUTHTOKEN: + required: false outputs: run-id: description: The run id. @@ -77,6 +80,8 @@ jobs: env: DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} PERFKIT_SSH_KEY: ${{ secrets.PERFKIT_SSH_KEY }} + # Goes into the agent container's environment on the load generator only. + NGROK_AUTHTOKEN: ${{ secrets.NGROK_AUTHTOKEN }} CONFIG: ${{ inputs.config }} TARGET_VERSION: ${{ inputs.target-version }} COMMIT: ${{ inputs.commit }} diff --git a/package-lock.json b/package-lock.json index 4fe3ad0..22b233c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.10", "license": "MIT", "dependencies": { + "@ngrok/ngrok": "1.7.0", "commander": "^14.0.3", "yaml": "^2.9.1", "zod": "^4.6.7" @@ -955,6 +956,235 @@ "node": ">=22" } }, + "node_modules/@ngrok/ngrok": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok/-/ngrok-1.7.0.tgz", + "integrity": "sha512-P06o9TpxrJbiRbHQkiwy/rUrlXRupc+Z8KT4MiJfmcdWxvIdzjCaJOdnNkcOTs6DMyzIOefG5tvk/HLdtjqr0g==", + "license": "(MIT OR Apache-2.0)", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@ngrok/ngrok-android-arm64": "1.7.0", + "@ngrok/ngrok-darwin-arm64": "1.7.0", + "@ngrok/ngrok-darwin-universal": "1.7.0", + "@ngrok/ngrok-darwin-x64": "1.7.0", + "@ngrok/ngrok-freebsd-x64": "1.7.0", + "@ngrok/ngrok-linux-arm-gnueabihf": "1.7.0", + "@ngrok/ngrok-linux-arm64-gnu": "1.7.0", + "@ngrok/ngrok-linux-arm64-musl": "1.7.0", + "@ngrok/ngrok-linux-x64-gnu": "1.7.0", + "@ngrok/ngrok-linux-x64-musl": "1.7.0", + "@ngrok/ngrok-win32-arm64-msvc": "1.7.0", + "@ngrok/ngrok-win32-ia32-msvc": "1.7.0", + "@ngrok/ngrok-win32-x64-msvc": "1.7.0" + } + }, + "node_modules/@ngrok/ngrok-android-arm64": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-android-arm64/-/ngrok-android-arm64-1.7.0.tgz", + "integrity": "sha512-8tco3ID6noSaNy+CMS7ewqPoIkIM6XO5COCzsUp3Wv3XEbMSyn65RN6cflX2JdqLfUCHcMyD0ahr9IEiHwqmbQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-darwin-arm64": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-darwin-arm64/-/ngrok-darwin-arm64-1.7.0.tgz", + "integrity": "sha512-+dmJSOzSO+MNDVrPOca2yYDP1W3KfP4qOlAkarIeFRIfqonQwq3QCBmcR7HAlZocLsSqEwyG6KP4RRvAuT0WGQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-darwin-universal": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-darwin-universal/-/ngrok-darwin-universal-1.7.0.tgz", + "integrity": "sha512-fDEfewyE2pWGFBhOSwQZObeHUkc65U1l+3HIgSOe094TMHsqmyJD0KTCgW9KSn0VP4OvDZbAISi1T3nvqgZYhQ==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-darwin-x64": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-darwin-x64/-/ngrok-darwin-x64-1.7.0.tgz", + "integrity": "sha512-+fwMi5uHd9G8BS42MMa9ye6exI5lwTcjUO6Ut497Vu0qgLONdVRenRqnEePV+Q3KtQR7NjqkMnomVfkr9MBjtw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-freebsd-x64": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-freebsd-x64/-/ngrok-freebsd-x64-1.7.0.tgz", + "integrity": "sha512-2OGgbrjy3yLRrqAz5N6hlUKIWIXSpR5RjQa2chtZMsSbszQ6c9dI+uVQfOKAeo05tHMUgrYAZ7FocC+ig0dzdQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-linux-arm-gnueabihf": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-linux-arm-gnueabihf/-/ngrok-linux-arm-gnueabihf-1.7.0.tgz", + "integrity": "sha512-SN9YIfEQiR9xN90QVNvdgvAemqMLoFVSeTWZs779145hQMhvF9Qd9rnWi6J+2uNNK10OczdV1oc/nq1es7u/3g==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-linux-arm64-gnu": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-linux-arm64-gnu/-/ngrok-linux-arm64-gnu-1.7.0.tgz", + "integrity": "sha512-KDMgzPKFU2kbpVSaA2RZBBia5IPdJEe063YlyVFnSMJmPYWCUnMwdybBsucXfV9u1Lw/ZjKTKotIlbTWGn3HGw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-linux-arm64-musl": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-linux-arm64-musl/-/ngrok-linux-arm64-musl-1.7.0.tgz", + "integrity": "sha512-e66vUdVrBlQ0lT9ZdamB4U604zt5Gualt8/WVcUGzbu8s5LajWd6g/mzZCUjK4UepjvMpfgmCp1/+rX7Rk8d5A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-linux-x64-gnu": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-linux-x64-gnu/-/ngrok-linux-x64-gnu-1.7.0.tgz", + "integrity": "sha512-M6gF0DyOEFqXLfWxObfL3bxYZ4+PnKBHuyLVaqNfFN9Y5utY2mdPOn5422Ppbk4XoIK5/YkuhRqPJl/9FivKEw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-linux-x64-musl": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-linux-x64-musl/-/ngrok-linux-x64-musl-1.7.0.tgz", + "integrity": "sha512-4Ijm0dKeoyzZTMaYxR2EiNjtlK81ebflg/WYIO1XtleFrVy4UJEGnxtxEidYoT4BfCqi4uvXiK2Mx216xXKvog==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-win32-arm64-msvc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-win32-arm64-msvc/-/ngrok-win32-arm64-msvc-1.7.0.tgz", + "integrity": "sha512-u7qyWIJI2/YG1HTBnHwUR1+Z2tyGfAsUAItJK/+N1G0FeWJhIWQvSIFJHlaPy4oW1Dc8mSDBX9qvVsiQgLaRFg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-win32-ia32-msvc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-win32-ia32-msvc/-/ngrok-win32-ia32-msvc-1.7.0.tgz", + "integrity": "sha512-/UdYUsLNv/Q8j9YJsyIfq/jLCoD8WP+NidouucTUzSoDtmOsXBBT3itLrmPiZTEdEgKiFYLuC1Zon8XQQvbVLA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@ngrok/ngrok-win32-x64-msvc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@ngrok/ngrok-win32-x64-msvc/-/ngrok-win32-x64-msvc-1.7.0.tgz", + "integrity": "sha512-UFJg/duEWzZlLkEs61Gz6/5nYhGaKI62I8dvUGdBR3NCtIMagehnFaFxmnXZldyHmCM8U0aCIFNpWRaKcrQkoA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@open-draft/until": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-3.0.1.tgz", diff --git a/package.json b/package.json index e1a8108..943d1a7 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "schemas": "tsx scripts/writeSchemas.ts" }, "dependencies": { + "@ngrok/ngrok": "1.7.0", "commander": "^14.0.3", "yaml": "^2.9.1", "zod": "^4.6.7" diff --git a/schemas/live-v1.schema.json b/schemas/live-v1.schema.json new file mode 100644 index 0000000..75622b7 --- /dev/null +++ b/schemas/live-v1.schema.json @@ -0,0 +1,614 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "schema": { + "type": "string", + "const": "perfkit/live/v1" + }, + "runId": { + "type": "string", + "pattern": "^\\d{8}T\\d{6}Z-[0-9a-f]{7}$" + }, + "version": { + "type": "string", + "minLength": 1 + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{7,40}$" + }, + "startedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "status": { + "type": "string", + "enum": [ + "running", + "finished", + "failed" + ] + }, + "reason": { + "type": [ + "string", + "null" + ] + }, + "estimatedEndAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "provider": { + "type": "string", + "enum": [ + "digitalocean", + "local" + ] + }, + "project": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "repo": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "siteUrl": { + "type": "string", + "format": "uri" + }, + "accent": { + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "logoSvg": { + "type": "string", + "pattern": "]" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "lead": { + "type": "string", + "minLength": 1 + }, + "live": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri" + } + }, + "required": [ + "url" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "repo", + "siteUrl", + "accent", + "title", + "lead" + ], + "additionalProperties": false + }, + "benchmarks": { + "minItems": 1, + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9-]+$" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "unit": { + "type": "string", + "pattern": "^[a-z]+$" + }, + "state": { + "type": "string", + "enum": [ + "queued", + "running", + "done", + "not-run" + ] + }, + "startedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "loadStartedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "estimatedStartAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "plannedS": { + "type": "number", + "minimum": 0 + }, + "kind": { + "type": "string", + "const": "capacity" + }, + "rate": { + "type": "object", + "properties": { + "start": { + "type": "number", + "exclusiveMinimum": 0 + }, + "step": { + "type": "number", + "exclusiveMinimum": 0 + }, + "everyS": { + "type": "number", + "exclusiveMinimum": 0 + }, + "max": { + "type": "number", + "exclusiveMinimum": 0 + } + }, + "required": [ + "start", + "step", + "everyS", + "max" + ], + "additionalProperties": false + }, + "thresholds": { + "type": "object", + "properties": { + "p99LatencyMs": { + "type": "number", + "exclusiveMinimum": 0 + }, + "failedCallsPct": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "p99LatencyMs", + "failedCallsPct" + ], + "additionalProperties": false + }, + "current": { + "anyOf": [ + { + "type": "object", + "properties": { + "index": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "total": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "offeredCps": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "index", + "total", + "offeredCps" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "points": { + "type": "array", + "items": { + "type": "object", + "properties": { + "offeredCps": { + "type": "number", + "minimum": 0 + }, + "achievedCps": { + "type": "number", + "minimum": 0 + }, + "calls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "p50Ms": { + "type": "number", + "minimum": 0 + }, + "p95Ms": { + "type": "number", + "minimum": 0 + }, + "p99Ms": { + "type": "number", + "minimum": 0 + }, + "failedPct": { + "type": "number", + "minimum": 0 + }, + "ok": { + "type": "boolean" + }, + "stopReason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "offeredCps", + "achievedCps", + "calls", + "failed", + "p50Ms", + "p95Ms", + "p99Ms", + "failedPct", + "ok", + "stopReason" + ], + "additionalProperties": false + } + }, + "heldCps": { + "anyOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, + "result": { + "anyOf": [ + { + "type": "object", + "properties": { + "limitCps": { + "anyOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, + "lowerBound": { + "type": "boolean" + }, + "stopReason": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "limitCps", + "lowerBound", + "stopReason" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "id", + "title", + "unit", + "state", + "startedAt", + "loadStartedAt", + "estimatedStartAt", + "plannedS", + "kind", + "rate", + "thresholds", + "current", + "points", + "heldCps", + "result" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9-]+$" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "unit": { + "type": "string", + "pattern": "^[a-z]+$" + }, + "state": { + "type": "string", + "enum": [ + "queued", + "running", + "done", + "not-run" + ] + }, + "startedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "loadStartedAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "estimatedStartAt": { + "anyOf": [ + { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + { + "type": "null" + } + ] + }, + "plannedS": { + "type": "number", + "minimum": 0 + }, + "kind": { + "type": "string", + "const": "soak" + }, + "cps": { + "type": "number", + "exclusiveMinimum": 0 + }, + "warmupS": { + "type": "number", + "minimum": 0 + }, + "durationS": { + "type": "number", + "exclusiveMinimum": 0 + }, + "elapsedS": { + "type": "number", + "minimum": 0 + }, + "thresholds": { + "type": "object", + "properties": { + "memoryGrowthMBPerHour": { + "type": "number", + "minimum": 0 + }, + "failedCallsPct": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "memoryGrowthMBPerHour", + "failedCallsPct" + ], + "additionalProperties": false + }, + "calls": { + "type": "object", + "properties": { + "placed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "placed", + "failed" + ], + "additionalProperties": false + }, + "memory": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "samples": { + "maxItems": 300, + "type": "array", + "items": { + "type": "object", + "properties": { + "t": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z))$" + }, + "rssMB": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "t", + "rssMB" + ], + "additionalProperties": false + } + }, + "growthMBPerHour": { + "type": [ + "number", + "null" + ] + } + }, + "required": [ + "name", + "samples", + "growthMBPerHour" + ], + "additionalProperties": false + } + } + }, + "required": [ + "id", + "title", + "unit", + "state", + "startedAt", + "loadStartedAt", + "estimatedStartAt", + "plannedS", + "kind", + "cps", + "warmupS", + "durationS", + "elapsedS", + "thresholds", + "calls", + "memory" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "schema", + "runId", + "version", + "commit", + "startedAt", + "updatedAt", + "status", + "reason", + "estimatedEndAt", + "provider", + "project", + "benchmarks" + ], + "additionalProperties": false, + "title": "live.json v1", + "description": "A preliminary snapshot of a perfkit run in progress (schema: perfkit/live/v1), rewritten every 15 seconds." +} diff --git a/schemas/perfkit-v1.schema.json b/schemas/perfkit-v1.schema.json index bb38789..9fbbba5 100644 --- a/schemas/perfkit-v1.schema.json +++ b/schemas/perfkit-v1.schema.json @@ -36,6 +36,19 @@ "lead": { "type": "string", "minLength": 1 + }, + "live": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri" + } + }, + "required": [ + "url" + ], + "additionalProperties": false } }, "required": [ @@ -99,6 +112,26 @@ ], "additionalProperties": false }, + "live": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$" + }, + "port": { + "default": 8787, + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "url": { + "type": "string", + "format": "uri" + } + }, + "additionalProperties": false + }, "images": { "type": "object", "propertyNames": { diff --git a/schemas/report-v1.schema.json b/schemas/report-v1.schema.json index 3db5caf..bdbf2e3 100644 --- a/schemas/report-v1.schema.json +++ b/schemas/report-v1.schema.json @@ -36,6 +36,19 @@ "lead": { "type": "string", "minLength": 1 + }, + "live": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri" + } + }, + "required": [ + "url" + ], + "additionalProperties": false } }, "required": [ diff --git a/src/analyze/createAnalyzeSoak.ts b/src/analyze/createAnalyzeSoak.ts index d83e3e7..2e5aeb6 100644 --- a/src/analyze/createAnalyzeSoak.ts +++ b/src/analyze/createAnalyzeSoak.ts @@ -11,8 +11,8 @@ import { import { computeHops } from "./computeHops.js"; import { countRestarts } from "./countRestarts.js"; import { durationToSeconds } from "./durationToSeconds.js"; -import { leastSquaresSlope } from "./leastSquaresSlope.js"; import { listRawDir } from "./listRawDir.js"; +import { memoryGrowthMBPerHour } from "./memoryGrowthMBPerHour.js"; import { nearestRank } from "./nearestRank.js"; import { parseJsonDocument } from "./parseJsonDocument.js"; import { parseOtlpSpanLines } from "./parseOtlpSpans.js"; @@ -148,16 +148,7 @@ export function createAnalyzeSoak(deps: AnalyzeDeps) { const growth = uutProcesses.map((name) => ({ name, - slope: roundTo( - leastSquaresSlope( - after - .filter((sample) => sample.name === name) - .map((sample) => ({ - x: (sample.tMs - warmupEndMs) / HOUR_MS, - y: sample.rssKB / 1024 - })) - ) - ) + slope: memoryGrowthMBPerHour(after, name, warmupEndMs) })); const rules: SoakRule[] = [ ...growth.map(({ name, slope }) => ({ diff --git a/src/analyze/index.ts b/src/analyze/index.ts index a3efa56..ae5e14b 100644 --- a/src/analyze/index.ts +++ b/src/analyze/index.ts @@ -28,6 +28,7 @@ export { export { createNodeAnalyzeDeps } from "./createNodeAnalyzeDeps.js"; export { durationToSeconds } from "./durationToSeconds.js"; export { leastSquaresSlope } from "./leastSquaresSlope.js"; +export { memoryGrowthMBPerHour } from "./memoryGrowthMBPerHour.js"; export { MAX_SIPP_CPU_PCT, measureCapacityStep, diff --git a/src/analyze/memoryGrowthMBPerHour.ts b/src/analyze/memoryGrowthMBPerHour.ts new file mode 100644 index 0000000..fc516c4 --- /dev/null +++ b/src/analyze/memoryGrowthMBPerHour.ts @@ -0,0 +1,30 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { leastSquaresSlope } from "./leastSquaresSlope.js"; +import { roundTo } from "./roundTo.js"; +import type { ProcessSample } from "./types.js"; + +const HOUR_MS = 3_600_000; + +/** + * One process's memory growth after the warm-up: the least-squares slope of + * its RSS in MB per hour over the samples from `warmupEndMs` on, rounded as + * the report rounds it. The soak verdict and the live page both use it. + */ +export function memoryGrowthMBPerHour( + samples: ReadonlyArray>, + name: string, + warmupEndMs: number +): number { + return roundTo( + leastSquaresSlope( + samples + .filter((sample) => sample.name === name && sample.tMs >= warmupEndMs) + .map((sample) => ({ + x: (sample.tMs - warmupEndMs) / HOUR_MS, + y: sample.rssKB / 1024 + })) + ) + ); +} diff --git a/src/cli.ts b/src/cli.ts index ac7acbe..77f2938 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -22,6 +22,18 @@ import { createDockerEngine } from "./orchestrate/createDockerEngine.js"; import { createNodeAgentFs } from "./orchestrate/createNodeAgentFs.js"; import { runPaths } from "./orchestrate/paths.js"; import { runStateSchema } from "./orchestrate/runState.schema.js"; +import { + LIVE_DEFAULT_HOST, + LIVE_DEFAULT_PORT, + LIVE_LINGER_MS +} from "./live/constants.js"; +import { createLiveRecorder } from "./live/createLiveRecorder.js"; +import { createNodeServeLive } from "./live/createNodeServeLive.js"; +import { createWriteLive } from "./live/createWriteLive.js"; +import { listenHttp } from "./live/listenHttp.js"; +import { runWithLive } from "./live/runWithLive.js"; +import { createNodeServeSite } from "./serve/createNodeServeSite.js"; +import { LIVE_INTERVAL_S } from "./schemas/index.js"; const require = createRequire(import.meta.url); const { version } = require("../package.json") as { version: string }; @@ -63,7 +75,9 @@ const code = await runCli(process.argv, { infra: { env: { DIGITALOCEAN_TOKEN: process.env.DIGITALOCEAN_TOKEN, - PERFKIT_SSH_KEY: process.env.PERFKIT_SSH_KEY + PERFKIT_SSH_KEY: process.env.PERFKIT_SSH_KEY, + NGROK_AUTHTOKEN: process.env.NGROK_AUTHTOKEN, + NGROK_DOMAIN: process.env.NGROK_DOMAIN }, now: () => Date.now(), sleep, @@ -88,25 +102,81 @@ const code = await runCli(process.argv, { const state = runStateSchema.parse( JSON.parse(await fs.readFile(paths.runJson)) ); - return createAgent({ - shell: createAgentShell({ - engine: createDockerEngine("/var/run/docker.sock"), - state - }), - fs, - now: () => Date.now(), - sleep, + const shell = createAgentShell({ + engine: createDockerEngine("/var/run/docker.sock"), + state + }); + const recorder = createLiveRecorder(); + const now = () => Date.now(); + const log = async (line: string) => { + process.stdout.write(`${line}\n`); + await fs.appendFile(paths.progress, `${line}\n`); + }; + const writeLive = createWriteLive({ fs, shell, now, state, recorder }); + // With --local the CLI serves live.json on this machine; the agent + // serves it on a droplet, where nothing else can. + const remote = state.mode === "digitalocean"; + return runWithLive( + { + writeLive: () => writeLive({}), + serve: async () => + remote + ? createNodeServeLive({ + file: paths.live, + siteUrl: state.config.project.siteUrl, + log: (line) => + log(line.replace(/^perfkit:/, "perfkit agent:")) + })({ + siteUrl: state.config.project.siteUrl, + port: state.live?.port ?? LIVE_DEFAULT_PORT, + host: LIVE_DEFAULT_HOST, + token: process.env.NGROK_AUTHTOKEN || null, + domain: state.live?.domain ?? null + }) + : { close: async () => {} }, + sleep, + intervalMs: LIVE_INTERVAL_S * 1000, + lingerMs: remote ? LIVE_LINGER_MS : 0 + }, + () => + createAgent({ shell, fs, now, sleep, log, live: recorder })({ + root, + runId + }) + ); + }, + serveLive: async ({ root, live, siteUrl }) => + createNodeServeLive({ + file: runPaths(root).live, + siteUrl, log: async (line) => { - process.stdout.write(`${line}\n`); - await fs.appendFile(paths.progress, `${line}\n`); + process.stderr.write(`${line}\n`); } - })({ root, runId }); - } + })({ + siteUrl, + port: live.port, + host: LIVE_DEFAULT_HOST, + token: process.env.NGROK_AUTHTOKEN || null, + domain: live.domain + }) }, writeFile: async (path, text) => { await mkdir(dirname(path), { recursive: true }); await writeFile(path, text, "utf8"); }, + serveSite: async ({ site, port, host }) => { + const server = await listenHttp(createNodeServeSite(site), { port, host }); + const stopped = new Promise((resolve) => { + const stop = () => { + process.off("SIGINT", stop); + process.off("SIGTERM", stop); + server.close().then(resolve); + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + }); + return { port: server.port, stopped }; + }, fetchText: async (url) => { const response = await fetch(url, { redirect: "follow" }); return { status: response.status, text: await response.text() }; diff --git a/src/cli/createStartExecutor.ts b/src/cli/createStartExecutor.ts index 1c26da0..4bd37ce 100644 --- a/src/cli/createStartExecutor.ts +++ b/src/cli/createStartExecutor.ts @@ -54,7 +54,11 @@ export function createStartExecutor(deps: CliDeps): BenchmarkExecutor { }, projectDir: dirname(options.configPath), out: options.out, - agentKey: options.agentKey + agentKey: options.agentKey, + ...(options.live ? { live: options.live } : {}), + ...(infra.env.NGROK_AUTHTOKEN + ? { ngrokToken: infra.env.NGROK_AUTHTOKEN } + : {}) }); ctx.handOff(); return { passed: true }; diff --git a/src/cli/runCli.ts b/src/cli/runCli.ts index 109cdcf..9e77d39 100644 --- a/src/cli/runCli.ts +++ b/src/cli/runCli.ts @@ -111,6 +111,19 @@ Exit codes: 2 the report or the history is invalid, unreadable or of another schema 3 the history URL could not be fetched (a 404 means no history)`; +const SERVE_HELP = ` +A read-only static server for the directory perfkit render wrote, as GitHub +Pages serves it, with nothing cached. To watch a run on this machine live: + + perfkit serve --site out/benchmarks --port 8080 + perfkit run --local --config perf/perfkit.yaml --target-version + open "http://localhost:8080/?live=http://localhost:8787/live.json" + +Exit codes: + 0 stopped with Ctrl-C + 1 usage error + 2 the site could not be served (the port is taken)`; + const SUMMARY_HELP = ` Prints the report as GitHub Markdown, for a workflow's run summary: perfkit summary --report .perfkit//report.json >> "$GITHUB_STEP_SUMMARY" @@ -205,6 +218,10 @@ export async function runCli(argv: string[], deps: CliDeps): Promise { "run on this machine's Docker instead of DigitalOcean; needs no token", false ) + .option( + "--live-port ", + "the port live.json is served on (default: live.port of perfkit.yaml, else 8787)" + ) .addHelpText("after", RUN_HELP) .action( async (options: { @@ -216,6 +233,7 @@ export async function runCli(argv: string[], deps: CliDeps): Promise { detach: boolean; dryRun: boolean; local: boolean; + livePort?: string; }) => { exitCode = await runRunCommand(options, deps); } @@ -312,6 +330,44 @@ export async function runCli(argv: string[], deps: CliDeps): Promise { } ); + program + .command("serve") + .description( + "Serve a rendered benchmarks site on this machine, to preview it or watch a --local run live." + ) + .requiredOption("--site ", "the site directory perfkit render wrote") + .option("--port ", "the port to listen on", "8080") + .option("--host
", "the address to listen on", "127.0.0.1") + .addHelpText("after", SERVE_HELP) + .action(async (options: { site: string; port: string; host: string }) => { + const port = Number(options.port); + if (!Number.isInteger(port) || port < 0 || port > 65535) { + deps.stderr("perfkit: --port: expected a port from 0 to 65535\n"); + exitCode = EXIT_USAGE; + return; + } + if (!deps.serveSite) { + deps.stderr("perfkit: serve is not available here\n"); + exitCode = EXIT_INVALID; + return; + } + const served = await deps + .serveSite({ site: options.site, port, host: options.host }) + .catch((err: unknown) => err as Error); + if (served instanceof Error) { + deps.stderr( + `perfkit: cannot serve ${options.site}: ${served.message}\n` + ); + exitCode = EXIT_INVALID; + return; + } + deps.stdout( + `serving ${options.site} at http://${options.host}:${served.port}/ (Ctrl-C stops)\n` + ); + await served.stopped; + exitCode = EXIT_VALID; + }); + program .command("summary") .description( diff --git a/src/cli/runCollectCommand.ts b/src/cli/runCollectCommand.ts index 1e88eac..0f5dd53 100644 --- a/src/cli/runCollectCommand.ts +++ b/src/cli/runCollectCommand.ts @@ -10,6 +10,7 @@ import { formatDeleteFailure, messageOf } from "../infra/index.js"; +import { LIVE_DEFAULT_PORT } from "../live/constants.js"; import { REMOTE_ROOT } from "../orchestrate/constants.js"; import { createCollect } from "../orchestrate/createCollect.js"; import { createLocalRemote } from "../orchestrate/createLocalRemote.js"; @@ -77,23 +78,42 @@ export async function runCollectCommand( const localText = await readFile(dirs.local, "utf8").catch(() => null); if (localText !== null) { const { root } = JSON.parse(localText) as { root: string }; - const config = await readFile(runPaths(root).runJson, "utf8") - .then((text) => runStateSchema.parse(JSON.parse(text)).config) + const state = await readFile(runPaths(root).runJson, "utf8") + .then((text) => runStateSchema.parse(JSON.parse(text))) .catch(() => null); - return createCollect({ - ...common, - remote: createLocalRemote({ run: infra.runProgram }), - root, - teardown: async () => { - const result = config - ? await localTeardown(options.runId, config, infra) - : { summary: "", failures: [] }; - if (result.failures.length === 0) { - await rm(dirs.dir, { recursive: true, force: true }).catch(() => {}); + const config = state?.config ?? null; + // The agent's files are on this machine, so this process serves + // live.json while it follows the run. + const live = + state && infra.serveLive && !options.wait + ? await infra + .serveLive({ + root, + live: state.live ?? { port: LIVE_DEFAULT_PORT, domain: null }, + siteUrl: state.config.project.siteUrl + }) + .catch(() => null) + : null; + try { + return await createCollect({ + ...common, + remote: createLocalRemote({ run: infra.runProgram }), + root, + teardown: async () => { + const result = config + ? await localTeardown(options.runId, config, infra) + : { summary: "", failures: [] }; + if (result.failures.length === 0) { + await rm(dirs.dir, { recursive: true, force: true }).catch( + () => {} + ); + } + return result; } - return result; - } - })(input); + })(input); + } finally { + await live?.close().catch(() => {}); + } } const token = infra.env.DIGITALOCEAN_TOKEN; diff --git a/src/cli/runRunCommand.ts b/src/cli/runRunCommand.ts index 62be34e..da76d5b 100644 --- a/src/cli/runRunCommand.ts +++ b/src/cli/runRunCommand.ts @@ -1,6 +1,8 @@ /** * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. */ +import { LIVE_DEFAULT_PORT } from "../live/constants.js"; +import type { LiveSettings } from "../orchestrate/runState.schema.js"; import { mkdir, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { ValidationError } from "../errors/index.js"; @@ -46,6 +48,8 @@ export interface RunCommandOptions { detach: boolean; dryRun: boolean; local?: boolean; + /** --live-port; default live.port of perfkit.yaml, else 8787. */ + livePort?: string; } const FLAG_OF: Record = { @@ -96,6 +100,18 @@ export async function runRunCommand( return EXIT_BAD_INPUT; } const local = options.local === true; + const livePort = + options.livePort === undefined ? null : Number(options.livePort); + if ( + livePort !== null && + !(Number.isInteger(livePort) && livePort >= 1 && livePort <= 65535) + ) { + return fail(EXIT_BAD_INPUT, "--live-port: expected a port from 1 to 65535"); + } + const live = { + port: livePort ?? loaded.config.live?.port ?? LIVE_DEFAULT_PORT, + domain: loaded.config.live?.domain ?? (infra.env.NGROK_DOMAIN || null) + }; const commit = options.commit ?? (await infra.gitHead(dirname(resolve(options.config)))); @@ -160,7 +176,10 @@ export async function runRunCommand( if (options.dryRun) return EXIT_OK; const command = describeCommand(options); if (local) { - return runLocal({ plan, config: loaded.config, command, options }, deps); + return runLocal( + { plan, config: loaded.config, command, options, live }, + deps + ); } const agentKey = await infra.ssh.generateKeyPair(); @@ -191,7 +210,8 @@ export async function runRunCommand( configPath: options.config, config: loaded.config, command, - agentKey: agentKey.privateKey + agentKey: agentKey.privateKey, + live }), { authorizedKeys: [agentKey.publicKey] } ); @@ -233,6 +253,7 @@ async function runLocal( config: PerfkitConfig; command: string; options: RunCommandOptions; + live: LiveSettings; }, deps: CliDeps ): Promise { @@ -269,7 +290,8 @@ async function runLocal( vpcCidr: "127.0.0.0/8" }, projectDir: dirname(resolve(options.config)), - out: options.out + out: options.out, + live: input.live }); } catch (err) { deps.stderr(`perfkit: ${messageOf(err)}\n`); diff --git a/src/cli/types.ts b/src/cli/types.ts index 4e34a0e..4c457bf 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -11,7 +11,9 @@ import type { Sleep, SshTools } from "../infra/index.js"; +import type { Closable } from "../live/types.js"; import type { AgentOutcome } from "../orchestrate/createAgent.js"; +import type { LiveSettings } from "../orchestrate/runState.schema.js"; import type { PerfkitConfig } from "../schemas/index.js"; import type { RenderSiteDeps } from "../render/types.js"; import type { ValidateFileDeps } from "../validate/types.js"; @@ -34,6 +36,8 @@ export interface RunOptions { command: string; /** The run-only private key the agent logs in to the UUT with. */ agentKey: string; + /** Where the agent serves live.json, and its ngrok domain. */ + live?: LiveSettings; } /** @@ -49,7 +53,14 @@ export type BenchmarkExecutor = ( /** Everything `run` and `sweep` reach outside the process, injected. */ export interface InfraCliDeps { - env: { DIGITALOCEAN_TOKEN?: string; PERFKIT_SSH_KEY?: string }; + env: { + DIGITALOCEAN_TOKEN?: string; + PERFKIT_SSH_KEY?: string; + /** Opens an ngrok tunnel to live.json; never printed or written. */ + NGROK_AUTHTOKEN?: string; + /** The static ngrok domain when perfkit.yaml has no live.domain. */ + NGROK_DOMAIN?: string; + }; now: Now; sleep: Sleep; signals?: SignalSource; @@ -69,12 +80,30 @@ export interface InfraCliDeps { namePrefix: string; /** The real executor when not set. */ executeBenchmarks?: BenchmarkExecutor; + /** + * Serves a --local run's live.json from this machine while `run` or + * `collect` follows it; absent, nothing is served. + */ + serveLive?: (input: { + root: string; + live: LiveSettings; + siteUrl: string; + }) => Promise; /** `perfkit agent`, on the load generator. */ runAgent: (input: { root: string; runId: string }) => Promise; } export interface CliDeps extends ValidateFileDeps, RenderSiteDeps { version: string; + /** + * `perfkit serve`: serves a site directory until Ctrl-C; `stopped` + * resolves once it has stopped. Rejects when it can't listen. + */ + serveSite?: (input: { + site: string; + port: number; + host: string; + }) => Promise<{ port: number; stopped: Promise }>; stdout: (text: string) => void; stderr: (text: string) => void; infra: InfraCliDeps; diff --git a/src/live/buildLiveCapacity.ts b/src/live/buildLiveCapacity.ts new file mode 100644 index 0000000..022a616 --- /dev/null +++ b/src/live/buildLiveCapacity.ts @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { durationToSeconds } from "../analyze/durationToSeconds.js"; +import type { + CapacityBenchmarkConfig, + LiveCapacity +} from "../schemas/index.js"; +import { liveStateOf } from "./liveStateOf.js"; +import type { LiveBenchmarkRecord } from "./types.js"; + +const iso = (ms: number | null | undefined) => + ms == null ? null : new Date(ms).toISOString(); + +/** + * A capacity benchmark's entry in live.json: the step SIPp offers now (from + * the clock, as SIPp steps it), the steps the agent evaluated with the + * same rules that stop the ramp, the last step that held, and once done + * where the ramp ended. Nothing here is a verdict. + */ +export function buildLiveCapacity(input: { + benchmark: CapacityBenchmarkConfig; + record: LiveBenchmarkRecord | undefined; + selected: boolean; + nowMs: number; + estimatedStartMs: number | null; +}): LiveCapacity { + const { benchmark, record, nowMs } = input; + const { start, step, every, max } = benchmark.load.rate; + const everyS = durationToSeconds(every); + const total = Math.floor((max - start) / step) + 1; + const state = liveStateOf(input.selected, record); + const points = record?.points ?? []; + const firstStop = points.findIndex((point) => !point.ok); + const passed = firstStop === -1 ? points : points.slice(0, firstStop); + const heldCps = passed.at(-1)?.offeredCps ?? null; + const loadStartedMs = record?.loadStartedMs ?? null; + const index = + firstStop !== -1 + ? firstStop + : Math.min( + total - 1, + Math.max( + 0, + Math.floor((nowMs - (loadStartedMs ?? nowMs)) / (everyS * 1000)) + ) + ); + return { + id: benchmark.id, + kind: "capacity", + title: benchmark.title, + unit: benchmark.load.unit, + state, + startedAt: iso(record?.startedMs), + loadStartedAt: iso(loadStartedMs), + estimatedStartAt: state === "queued" ? iso(input.estimatedStartMs) : null, + plannedS: total * everyS, + rate: { start, step, everyS, max }, + thresholds: { ...benchmark.thresholds }, + current: + state === "running" && loadStartedMs !== null + ? { index, total, offeredCps: start + index * step } + : null, + points, + heldCps, + result: + state === "done" + ? { + limitCps: heldCps, + lowerBound: points.length > 0 && firstStop === -1, + stopReason: + firstStop === -1 ? null : (points[firstStop]?.stopReason ?? null) + } + : null + }; +} diff --git a/src/live/buildLiveSnapshot.ts b/src/live/buildLiveSnapshot.ts new file mode 100644 index 0000000..e9d5d36 --- /dev/null +++ b/src/live/buildLiveSnapshot.ts @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { RunState } from "../orchestrate/runState.schema.js"; +import { toReportProject } from "../orchestrate/toReportProject.js"; +import { LIVE_SCHEMA_ID, type Live } from "../schemas/index.js"; +import { buildLiveCapacity } from "./buildLiveCapacity.js"; +import { buildLiveSoak } from "./buildLiveSoak.js"; +import { estimateSchedule } from "./estimateSchedule.js"; +import type { LiveRecord, SoakProgress } from "./types.js"; + +/** + * The run's live.json from its state, what the agent recorded so far and + * each soak's progress: every benchmark of perfkit.yaml in its order, one + * not selected as not-run, and the expected end while the run goes on. + */ +export function buildLiveSnapshot(input: { + state: RunState; + record: LiveRecord; + nowMs: number; + soaks: ReadonlyMap; +}): Live { + const { state, record, nowMs } = input; + const { plan, config } = state; + const selected = new Set(plan.benchmarks.map(({ id }) => id)); + const schedule = estimateSchedule({ + benchmarks: plan.benchmarks, + records: record.benchmarks, + nowMs + }); + const running = record.status === "running"; + const benchmarks = config.benchmarks.map((benchmark) => { + const common = { + record: record.benchmarks[benchmark.id], + selected: selected.has(benchmark.id), + nowMs, + estimatedStartMs: running + ? (schedule.startsMs.get(benchmark.id) ?? null) + : null + }; + return benchmark.kind === "capacity" + ? buildLiveCapacity({ ...common, benchmark }) + : buildLiveSoak({ + ...common, + benchmark, + progress: input.soaks.get(benchmark.id) ?? null + }); + }); + return { + schema: LIVE_SCHEMA_ID, + runId: plan.runId, + version: plan.targetVersion, + commit: plan.commit, + startedAt: plan.startedAt, + updatedAt: new Date(nowMs).toISOString(), + status: record.status, + reason: record.reason, + estimatedEndAt: running ? new Date(schedule.endMs).toISOString() : null, + provider: state.mode, + project: toReportProject(state), + benchmarks + }; +} diff --git a/src/live/buildLiveSoak.ts b/src/live/buildLiveSoak.ts new file mode 100644 index 0000000..b5d760a --- /dev/null +++ b/src/live/buildLiveSoak.ts @@ -0,0 +1,91 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { durationToSeconds } from "../analyze/durationToSeconds.js"; +import { memoryGrowthMBPerHour } from "../analyze/memoryGrowthMBPerHour.js"; +import { + LIVE_MAX_SAMPLES, + type LiveSoak, + type SoakBenchmarkConfig +} from "../schemas/index.js"; +import { downsample } from "./downsample.js"; +import { liveStateOf } from "./liveStateOf.js"; +import type { LiveBenchmarkRecord, SoakProgress } from "./types.js"; + +const HOUR_MS = 3_600_000; +const iso = (ms: number | null | undefined) => + ms == null ? null : new Date(ms).toISOString(); + +/** + * A soak's entry in live.json: how far it is, the calls so far, and each + * process's raw memory, at most 300 samples. Growth in MB/h, the same + * least-squares slope as the verdict, appears only once there is a full + * hour of samples after the warm-up: over minutes it overstates growth + * while the heap settles. + */ +export function buildLiveSoak(input: { + benchmark: SoakBenchmarkConfig; + record: LiveBenchmarkRecord | undefined; + selected: boolean; + nowMs: number; + estimatedStartMs: number | null; + progress: SoakProgress | null; +}): LiveSoak { + const { benchmark, record, progress } = input; + const warmupS = durationToSeconds(benchmark.warmup); + const durationS = durationToSeconds(benchmark.duration); + const plannedS = warmupS + durationS; + const state = liveStateOf(input.selected, record); + const loadStartedMs = record?.loadStartedMs ?? null; + const untilMs = record?.endedMs ?? input.nowMs; + const elapsedS = + loadStartedMs === null + ? 0 + : Math.min( + plannedS, + Math.max(0, Math.round((untilMs - loadStartedMs) / 1000)) + ); + const warmupEndMs = + loadStartedMs === null ? null : loadStartedMs + warmupS * 1000; + const samples = progress?.samples ?? []; + const memory = benchmark.uut.processes.map(({ name }) => { + const own = samples.filter((sample) => sample.name === name); + const last = own.at(-1); + const hourAfterWarmup = + warmupEndMs !== null && + last !== undefined && + last.tMs - warmupEndMs >= HOUR_MS; + return { + name, + samples: downsample(own, LIVE_MAX_SAMPLES).map((sample) => ({ + t: new Date(sample.tMs).toISOString(), + rssMB: Math.round((sample.rssKB / 1024) * 10) / 10 + })), + growthMBPerHour: + hourAfterWarmup && warmupEndMs !== null + ? memoryGrowthMBPerHour(own, name, warmupEndMs) + : null + }; + }); + return { + id: benchmark.id, + kind: "soak", + title: benchmark.title, + unit: benchmark.load.unit, + state, + startedAt: iso(record?.startedMs), + loadStartedAt: iso(loadStartedMs), + estimatedStartAt: state === "queued" ? iso(input.estimatedStartMs) : null, + plannedS, + cps: benchmark.load.rate.cps, + warmupS, + durationS, + elapsedS, + thresholds: { + memoryGrowthMBPerHour: benchmark.thresholds.memoryGrowthMBPerHour, + failedCallsPct: benchmark.thresholds.failedCallsPct + }, + calls: { placed: progress?.placed ?? 0, failed: progress?.failed ?? 0 }, + memory + }; +} diff --git a/src/live/constants.ts b/src/live/constants.ts new file mode 100644 index 0000000..6414267 --- /dev/null +++ b/src/live/constants.ts @@ -0,0 +1,11 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** The agent's live.json server listens on this port by default. */ +export const LIVE_DEFAULT_PORT = 8787; +/** And on this address: the tunnel reaches it on the same machine. */ +export const LIVE_DEFAULT_HOST = "127.0.0.1"; +/** A benchmark's start (compose up, healthy, setup) and end, estimated. */ +export const BENCHMARK_OVERHEAD_S = 90; +/** After the run ends the agent keeps serving this long, so pages see it. */ +export const LIVE_LINGER_MS = 30 * 1000; diff --git a/src/live/createHandleLiveRequest.ts b/src/live/createHandleLiveRequest.ts new file mode 100644 index 0000000..d1ea414 --- /dev/null +++ b/src/live/createHandleLiveRequest.ts @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { withErrorHandlingAndValidation } from "../errors/index.js"; +import { + type LiveRequest, + liveRequestSchema, + type LiveResponse +} from "./handleLiveRequest.schema.js"; +import { isAllowedOrigin } from "./isAllowedOrigin.js"; + +export const LIVE_PATH = "/live.json"; + +/** + * Creates the live.json server's request handler. `GET /live.json` answers + * the latest snapshot (503 before the first one), never cached; `OPTIONS` + * answers the CORS preflight. A page on the project's site or on this + * machine may read it, and may send `ngrok-skip-browser-warning`, which + * ngrok's free domains need. Anything else is 404. + * + * @param deps - Reads the latest live.json, null before the first one; + * and the project's site URL (injected) + */ +export function createHandleLiveRequest(deps: { + readLive: () => Promise; + siteUrl: string; +}) { + const fn = async (request: LiveRequest): Promise => { + const path = request.url.split("?")[0]; + const cors: Record = + request.origin !== null && isAllowedOrigin(request.origin, deps.siteUrl) + ? { + "Access-Control-Allow-Origin": request.origin, + "Access-Control-Allow-Methods": "GET, OPTIONS", + "Access-Control-Allow-Headers": "ngrok-skip-browser-warning", + "Access-Control-Max-Age": "600" + } + : {}; + const base = { "Cache-Control": "no-store", Vary: "Origin", ...cors }; + const text = (status: number, body: string) => ({ + status, + headers: { ...base, "Content-Type": "text/plain; charset=utf-8" }, + body + }); + if (path !== LIVE_PATH) return text(404, "not found\n"); + if (request.method === "OPTIONS") { + return { status: 204, headers: base, body: "" }; + } + if (request.method !== "GET" && request.method !== "HEAD") { + return text(404, "not found\n"); + } + const live = await deps.readLive().catch(() => null); + if (live === null) return text(503, "no snapshot yet\n"); + return { + status: 200, + headers: { ...base, "Content-Type": "application/json; charset=utf-8" }, + body: request.method === "HEAD" ? "" : live + }; + }; + + return withErrorHandlingAndValidation(fn, liveRequestSchema); +} diff --git a/src/live/createLiveRecorder.ts b/src/live/createLiveRecorder.ts new file mode 100644 index 0000000..e6ebc9f --- /dev/null +++ b/src/live/createLiveRecorder.ts @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveBenchmarkRecord, LiveRecord, LiveRecorder } from "./types.js"; + +/** + * Creates the in-memory recorder the agent fills as the run goes, for the + * live writer. It never fails and never blocks the run. + */ +export function createLiveRecorder(): LiveRecorder { + const record: LiveRecord = { + status: "running", + reason: null, + benchmarks: {} + }; + const of = (id: string): LiveBenchmarkRecord => { + const existing = record.benchmarks[id]; + if (existing) return existing; + const fresh: LiveBenchmarkRecord = { + startedMs: null, + loadStartedMs: null, + endedMs: null, + points: [] + }; + record.benchmarks[id] = fresh; + return fresh; + }; + return { + started: (id, ms) => { + of(id).startedMs = ms; + }, + loadStarted: (id, ms) => { + of(id).loadStartedMs = ms; + }, + point: (id, point) => { + of(id).points.push(point); + }, + done: (id, ms) => { + of(id).endedMs = ms; + }, + finished: () => { + record.status = "finished"; + }, + failed: (reason) => { + record.status = "failed"; + record.reason = reason; + }, + read: () => structuredClone(record) + }; +} diff --git a/src/live/createNodeServeLive.ts b/src/live/createNodeServeLive.ts new file mode 100644 index 0000000..fb1666f --- /dev/null +++ b/src/live/createNodeServeLive.ts @@ -0,0 +1,34 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { readFile } from "node:fs/promises"; +import { createHandleLiveRequest } from "./createHandleLiveRequest.js"; +import { createServeLive } from "./createServeLive.js"; +import { createStartLiveTunnel } from "./createStartLiveTunnel.js"; +import { listenHttp } from "./listenHttp.js"; +import type { NgrokForward } from "./startLiveTunnel.schema.js"; + +/** + * `serveLive` for a live.json file on this machine, with node:http and the + * @ngrok/ngrok package, loaded only when a tunnel is asked for: its native + * binding may be missing on the agent's platform, which means no tunnel. + */ +export function createNodeServeLive(input: { + file: string; + siteUrl: string; + log: (line: string) => Promise; +}) { + const forward: NgrokForward = async (config) => { + const ngrok = await import("@ngrok/ngrok"); + return ngrok.forward(config); + }; + return createServeLive({ + handle: createHandleLiveRequest({ + readLive: () => readFile(input.file, "utf8").catch(() => null), + siteUrl: input.siteUrl + }), + listen: listenHttp, + startTunnel: createStartLiveTunnel({ forward }), + log: input.log + }); +} diff --git a/src/live/createServeLive.ts b/src/live/createServeLive.ts new file mode 100644 index 0000000..0536fb6 --- /dev/null +++ b/src/live/createServeLive.ts @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { withErrorHandlingAndValidation } from "../errors/index.js"; +import type { HttpServer } from "./listenHttp.js"; +import type { LiveRequest, LiveResponse } from "./handleLiveRequest.schema.js"; +import { + type ServeLiveInput, + serveLiveInputSchema +} from "./serveLive.schema.js"; +import type { StartLiveTunnelInput } from "./startLiveTunnel.schema.js"; +import type { Closable } from "./types.js"; + +/** + * Creates `serveLive`: the read-only live.json server on `host:port` + * and, with an ngrok token, a tunnel to it, whose public URL it logs. A + * port that is taken or a tunnel that fails is logged and the run goes on: + * live.json is a convenience, never a reason to stop a benchmark. The token + * is never logged. + * + * @param deps - The request handler, the HTTP listener, the tunnel and the + * log (injected) + * @returns A validated function that returns what to close at the end + */ +export function createServeLive(deps: { + handle: (request: LiveRequest) => Promise; + listen: ( + handle: (request: LiveRequest) => Promise, + options: { port: number; host: string } + ) => Promise; + startTunnel: ( + input: StartLiveTunnelInput + ) => Promise<{ url: string } & Closable>; + log: (line: string) => Promise; +}) { + const fn = async (input: ServeLiveInput): Promise => { + const message = (err: unknown) => + err instanceof Error ? err.message : String(err); + const server = await deps + .listen(deps.handle, { port: input.port, host: input.host }) + .catch(async (err: unknown) => { + await deps.log( + `perfkit: live.json not served on ${input.host}:${input.port}: ${message(err)}` + ); + return null; + }); + if (server === null) return { close: async () => {} }; + await deps.log( + `perfkit: live.json at http://${input.host}:${server.port}/live.json` + ); + const tunnel = + input.token === null + ? null + : await deps + .startTunnel({ + token: input.token, + domain: input.domain, + port: server.port + }) + .catch(async (err: unknown) => { + await deps.log( + `perfkit: no live tunnel: ${message(err) + .split(input.token ?? "") + .join("***")}` + ); + return null; + }); + if (tunnel) await deps.log(`perfkit: live.json at ${tunnel.url}/live.json`); + return { + close: async () => { + await tunnel?.close().catch(() => {}); + await server.close(); + } + }; + }; + return withErrorHandlingAndValidation(fn, serveLiveInputSchema); +} diff --git a/src/live/createStartLiveTunnel.ts b/src/live/createStartLiveTunnel.ts new file mode 100644 index 0000000..ef10b30 --- /dev/null +++ b/src/live/createStartLiveTunnel.ts @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { withErrorHandlingAndValidation } from "../errors/index.js"; +import { + type NgrokForward, + type StartLiveTunnelInput, + startLiveTunnelInputSchema +} from "./startLiveTunnel.schema.js"; + +/** + * Creates `startLiveTunnel`: an ngrok tunnel, with the official + * @ngrok/ngrok package, from the static domain to the live.json server on + * this machine. Returns its public URL and a close. + * + * @param deps - ngrok's `forward` (injected; tests pass a fake) + */ +export function createStartLiveTunnel(deps: { forward: NgrokForward }) { + const fn = async ({ token, domain, port }: StartLiveTunnelInput) => { + const listener = await deps.forward({ + addr: `127.0.0.1:${port}`, + authtoken: token, + ...(domain ? { domain } : {}) + }); + return { + url: listener.url() ?? (domain ? `https://${domain}` : ""), + close: () => listener.close() + }; + }; + return withErrorHandlingAndValidation(fn, startLiveTunnelInputSchema); +} diff --git a/src/live/createWriteLive.ts b/src/live/createWriteLive.ts new file mode 100644 index 0000000..c5bca86 --- /dev/null +++ b/src/live/createWriteLive.ts @@ -0,0 +1,131 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { basename } from "node:path"; +import { parseSamplerJsonl } from "../analyze/parseSamplerJsonl.js"; +import { parseStatCsv } from "../analyze/parseStatCsv.js"; +import { sumStatRows } from "../analyze/sumStatRows.js"; +import { withErrorHandlingAndValidation } from "../errors/index.js"; +import type { Now } from "../infra/index.js"; +import { runPaths } from "../orchestrate/paths.js"; +import type { RunState } from "../orchestrate/runState.schema.js"; +import type { AgentFs, HostShell } from "../orchestrate/types.js"; +import { + type Live, + liveSchema, + type SoakBenchmarkConfig +} from "../schemas/index.js"; +import { buildLiveSnapshot } from "./buildLiveSnapshot.js"; +import type { LiveRecorder, SoakProgress } from "./types.js"; +import { + type WriteLiveInput, + writeLiveInputSchema +} from "./writeLive.schema.js"; + +const ALL_TIME = { fromMs: 0, toMs: Number.POSITIVE_INFINITY }; +const completeLines = (text: string) => + text.slice(0, text.lastIndexOf("\n") + 1); + +/** + * Creates `writeLive()`, which writes the run's live.json atomically (a + * temporary file, then a rename) from what the agent recorded. For a soak + * whose load has started it reads the UUT's sampler file (on this machine + * with --local; over SSH otherwise, only the bytes added since the last + * read) and the uac's SIPp counters. A file it can't read leaves the last + * numbers in place: live.json never stops the run. + * + * @param deps - The agent's file system and host shell, the clock, the run + * state and the recorder (injected) + * @returns A validated function that returns the snapshot it wrote + */ +export function createWriteLive(deps: { + fs: AgentFs; + shell: HostShell; + now: Now; + state: RunState; + recorder: LiveRecorder; +}) { + const { fs, shell, state } = deps; + const paths = runPaths(state.root); + const sampler = new Map(); + const progress = new Map(); + + const readSampler = async (id: string): Promise => { + const file = `${paths.rawOf(id)}/sampler-uut.jsonl`; + if (state.mode === "local") return fs.readFile(file); + const seen = sampler.get(id) ?? { offset: 0, text: "" }; + const out = await shell( + "uut", + `tail -c +${seen.offset + 1} ${file} 2>/dev/null; true`, + { timeoutMs: 60 * 1000 } + ); + const next = { + offset: seen.offset + Buffer.byteLength(out.stdout), + text: seen.text + out.stdout + }; + sampler.set(id, next); + return next.text; + }; + + const readCalls = async (benchmark: SoakBenchmarkConfig) => { + const rawDir = paths.rawOf(benchmark.id); + const base = basename(benchmark.load.uac.scenario).replace(/\.xml$/i, ""); + const pattern = new RegExp( + `^${base.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}_\\d+_\\.csv$` + ); + const names = (await fs.readdir(rawDir)).filter((name) => + pattern.test(name) + ); + const sums = await Promise.all( + names.map(async (name) => { + const text = completeLines(await fs.readFile(`${rawDir}/${name}`)); + return sumStatRows(parseStatCsv(text, name).rows, ALL_TIME); + }) + ); + return { + placed: sums.reduce((total, sum) => total + sum.attempted, 0), + failed: sums.reduce((total, sum) => total + sum.failed, 0) + }; + }; + + const readSoak = async (benchmark: SoakBenchmarkConfig) => { + const previous = progress.get(benchmark.id) ?? { + samples: [], + placed: 0, + failed: 0 + }; + const samples = await readSampler(benchmark.id) + .then((text) => + parseSamplerJsonl(completeLines(text), "sampler-uut.jsonl").filter( + (sample) => sample.host === "uut" + ) + ) + .catch(() => previous.samples); + const calls = await readCalls(benchmark).catch(() => previous); + progress.set(benchmark.id, { + samples, + placed: calls.placed, + failed: calls.failed + }); + }; + + const fn = async (_input: WriteLiveInput): Promise => { + const record = deps.recorder.read(); + const soaks = state.plan.benchmarks.filter( + (benchmark): benchmark is SoakBenchmarkConfig => + benchmark.kind === "soak" && + record.benchmarks[benchmark.id]?.loadStartedMs != null && + (record.benchmarks[benchmark.id]?.endedMs == null || + !progress.has(benchmark.id)) + ); + await Promise.all(soaks.map(readSoak)); + const snapshot = liveSchema.parse( + buildLiveSnapshot({ state, record, nowMs: deps.now(), soaks: progress }) + ); + await fs.writeFile(`${paths.live}.tmp`, `${JSON.stringify(snapshot)}\n`); + await fs.rename(`${paths.live}.tmp`, paths.live); + return snapshot; + }; + + return withErrorHandlingAndValidation(fn, writeLiveInputSchema); +} diff --git a/src/live/downsample.ts b/src/live/downsample.ts new file mode 100644 index 0000000..c4bf3ba --- /dev/null +++ b/src/live/downsample.ts @@ -0,0 +1,16 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** + * At most `max` items of `items`, evenly spaced, always keeping the first + * and the last, in order. Fewer items come back as they are. + */ +export function downsample(items: readonly T[], max: number): T[] { + if (items.length <= max) return [...items]; + if (max <= 1) return items.slice(-1); + const step = (items.length - 1) / (max - 1); + return Array.from( + { length: max }, + (_, i) => items[Math.round(i * step)] as T + ); +} diff --git a/src/live/estimateSchedule.ts b/src/live/estimateSchedule.ts new file mode 100644 index 0000000..00fd6cb --- /dev/null +++ b/src/live/estimateSchedule.ts @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { plannedSeconds } from "../orchestrate/createAgent.js"; +import type { BenchmarkConfig } from "../schemas/index.js"; +import { BENCHMARK_OVERHEAD_S } from "./constants.js"; +import type { LiveBenchmarkRecord } from "./types.js"; + +/** + * When each benchmark still to run is expected to start, and when the run + * is expected to end, from the clock: the running benchmark's planned load + * time left, then each queued one's planned time, each with a fixed + * allowance for compose up, setup and teardown. A capacity ramp that has + * stopped is about to end. + */ +export function estimateSchedule(input: { + /** The selected benchmarks, in the order the agent runs them. */ + benchmarks: readonly BenchmarkConfig[]; + records: Readonly>; + nowMs: number; +}): { startsMs: Map; endMs: number } { + const overheadMs = BENCHMARK_OVERHEAD_S * 1000; + return input.benchmarks.reduce( + (acc, benchmark) => { + const record = input.records[benchmark.id]; + const plannedMs = plannedSeconds(benchmark) * 1000; + if (record?.endedMs != null) return acc; + if (record?.startedMs != null) { + const stopped = record.points.some((point) => !point.ok); + const leftMs = + record.loadStartedMs == null + ? plannedMs + overheadMs + : stopped + ? overheadMs / 3 + : Math.max(0, record.loadStartedMs + plannedMs - input.nowMs) + + overheadMs / 3; + return { ...acc, endMs: acc.endMs + leftMs }; + } + acc.startsMs.set(benchmark.id, acc.endMs); + return { ...acc, endMs: acc.endMs + plannedMs + overheadMs }; + }, + { startsMs: new Map(), endMs: input.nowMs } + ); +} diff --git a/src/live/handleLiveRequest.schema.ts b/src/live/handleLiveRequest.schema.ts new file mode 100644 index 0000000..9950a6d --- /dev/null +++ b/src/live/handleLiveRequest.schema.ts @@ -0,0 +1,20 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { z } from "zod/v4"; + +export const liveRequestSchema = z.strictObject({ + method: z.string().min(1, { error: "expected an HTTP method" }), + /** The request target, like `/live.json?t=1`. */ + url: z.string().min(1, { error: "expected a request target" }), + /** The Origin header, or null without one. */ + origin: z.string().nullable() +}); + +export type LiveRequest = z.infer; + +export interface LiveResponse { + status: number; + headers: Record; + body: string; +} diff --git a/src/live/isAllowedOrigin.ts b/src/live/isAllowedOrigin.ts new file mode 100644 index 0000000..9bd34f2 --- /dev/null +++ b/src/live/isAllowedOrigin.ts @@ -0,0 +1,18 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +const LOCAL = /^http:\/\/(localhost|127\.0\.0\.1|\[::1\])(:\d{1,5})?$/; + +/** + * Whether a page at `origin` may read live.json: the project's site (the + * origin of `project.siteUrl`), or a page on this machine over http, for + * `perfkit serve`. + */ +export function isAllowedOrigin(origin: string, siteUrl: string): boolean { + if (LOCAL.test(origin)) return true; + try { + return new URL(siteUrl).origin === origin; + } catch { + return false; + } +} diff --git a/src/live/listenHttp.ts b/src/live/listenHttp.ts new file mode 100644 index 0000000..40b0022 --- /dev/null +++ b/src/live/listenHttp.ts @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { createServer } from "node:http"; +import type { LiveRequest, LiveResponse } from "./handleLiveRequest.schema.js"; + +export interface HttpServer { + /** The port it listens on: the one asked for, or a free one for 0. */ + port: number; + close(): Promise; +} + +/** + * Serves `handle` with node:http on `host:port`. A handler that throws + * answers 500. Resolves once it listens; rejects when it can't (the port + * is taken). + */ +export function listenHttp( + handle: (request: LiveRequest) => Promise, + options: { port: number; host: string } +): Promise { + const server = createServer((req, res) => { + const origin = req.headers.origin; + handle({ + method: req.method ?? "GET", + url: req.url ?? "/", + origin: typeof origin === "string" ? origin : null + }) + .catch(() => ({ + status: 500, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + body: "error\n" + })) + .then((response) => { + res.writeHead(response.status, response.headers); + res.end(response.body); + }); + }); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(options.port, options.host, () => { + server.off("error", reject); + const address = server.address(); + resolve({ + port: + typeof address === "object" && address ? address.port : options.port, + close: () => + new Promise((done) => { + server.closeAllConnections(); + server.close(() => done()); + }) + }); + }); + }); +} diff --git a/src/live/liveStateOf.ts b/src/live/liveStateOf.ts new file mode 100644 index 0000000..55d63e2 --- /dev/null +++ b/src/live/liveStateOf.ts @@ -0,0 +1,16 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveBenchmarkState } from "../schemas/index.js"; +import type { LiveBenchmarkRecord } from "./types.js"; + +/** not-run when not selected; else done, running or queued by the record. */ +export function liveStateOf( + selected: boolean, + record: LiveBenchmarkRecord | undefined +): LiveBenchmarkState { + if (!selected) return "not-run"; + if (record?.endedMs != null) return "done"; + if (record?.startedMs != null) return "running"; + return "queued"; +} diff --git a/src/live/ngrokBindingCommand.ts b/src/live/ngrokBindingCommand.ts new file mode 100644 index 0000000..6f76572 --- /dev/null +++ b/src/live/ngrokBindingCommand.ts @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { NODE_IMAGE } from "../tools.js"; +import { shellJoin } from "../orchestrate/shellQuote.js"; + +/** + * The command that adds @ngrok/ngrok's native binding for the agent's own + * platform (Linux, musl, in NODE_IMAGE) to the staged agent package: the + * CLI's machine has only its own binding. It installs the same version in + * a throwaway prefix in NODE_IMAGE and copies the `@ngrok` scope over. + * It always exits 0: without the binding the agent runs without a tunnel. + */ +export function ngrokBindingCommand(input: { + agentDir: string; + version: string; +}): string { + const script = [ + "set -e", + "npm install --silent --no-save --no-package-lock --ignore-scripts --no-audit --no-fund --prefix /tmp/ngrok " + + shellJoin([`@ngrok/ngrok@${input.version}`]), + `mkdir -p ${input.agentDir}/node_modules/@ngrok`, + `cp -R /tmp/ngrok/node_modules/@ngrok/. ${input.agentDir}/node_modules/@ngrok/` + ].join("; "); + return `${shellJoin([ + "docker", + "run", + "--rm", + "-v", + `${input.agentDir}:${input.agentDir}`, + NODE_IMAGE, + "sh", + "-c", + script + ])} >/dev/null 2>&1; true`; +} diff --git a/src/live/ngrokVersion.ts b/src/live/ngrokVersion.ts new file mode 100644 index 0000000..3763698 --- /dev/null +++ b/src/live/ngrokVersion.ts @@ -0,0 +1,13 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +/** The @ngrok/ngrok version perfkit's package.json depends on, exactly. */ +export async function ngrokVersion(packageRoot: string): Promise { + const pkg = JSON.parse( + await readFile(join(packageRoot, "package.json"), "utf8") + ) as { dependencies?: Record }; + return (pkg.dependencies?.["@ngrok/ngrok"] ?? "latest").replace(/^[\^~]/, ""); +} diff --git a/src/live/runWithLive.ts b/src/live/runWithLive.ts new file mode 100644 index 0000000..57f249f --- /dev/null +++ b/src/live/runWithLive.ts @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { Closable } from "./types.js"; + +/** + * Runs `body` (the agent) while live.json is written every `intervalMs` + * and served: one write right away, one per interval, and a last one when + * the body ends, after which the server stays up `lingerMs` so open pages + * see the run end, then closes. A failed write is skipped; the body's + * result or error is passed through. + */ +export async function runWithLive( + deps: { + writeLive: () => Promise; + serve: () => Promise; + sleep: (ms: number, signal?: AbortSignal) => Promise; + intervalMs: number; + lingerMs: number; + }, + body: () => Promise +): Promise { + const write = () => deps.writeLive().catch(() => undefined); + const stop = new AbortController(); + const loop = async (): Promise => { + await write(); + await deps.sleep(deps.intervalMs, stop.signal); + if (!stop.signal.aborted) return loop(); + }; + const service = await deps.serve().catch(() => ({ close: async () => {} })); + const writing = loop(); + try { + return await body(); + } finally { + stop.abort(); + await writing; + await write(); + await deps.sleep(deps.lingerMs); + await service.close().catch(() => {}); + } +} diff --git a/src/live/serveLive.schema.ts b/src/live/serveLive.schema.ts new file mode 100644 index 0000000..9f747a5 --- /dev/null +++ b/src/live/serveLive.schema.ts @@ -0,0 +1,17 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { z } from "zod/v4"; + +export const serveLiveInputSchema = z.strictObject({ + /** The project's site; its origin may read live.json. */ + siteUrl: z.string().min(1), + port: z.number().int().min(0).max(65535), + host: z.string().min(1), + /** NGROK_AUTHTOKEN, or null: no tunnel, this machine only. */ + token: z.string().min(1).nullable(), + /** The static ngrok domain, or null. */ + domain: z.string().min(1).nullable() +}); + +export type ServeLiveInput = z.infer; diff --git a/src/live/startLiveTunnel.schema.ts b/src/live/startLiveTunnel.schema.ts new file mode 100644 index 0000000..af7840c --- /dev/null +++ b/src/live/startLiveTunnel.schema.ts @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { z } from "zod/v4"; + +export const startLiveTunnelInputSchema = z.strictObject({ + /** NGROK_AUTHTOKEN; never logged. */ + token: z.string().min(1, { error: "expected an ngrok authtoken" }), + /** The static ngrok domain, like perfkit.ngrok.app; none for a random one. */ + domain: z.string().min(1).nullable(), + /** The local port of the live.json server. */ + port: z.number().int().min(1).max(65535) +}); + +export type StartLiveTunnelInput = z.infer; + +/** The part of @ngrok/ngrok the tunnel uses. */ +export type NgrokForward = (config: { + addr: string; + authtoken: string; + domain?: string; +}) => Promise<{ url(): string | null; close(): Promise }>; diff --git a/src/live/types.ts b/src/live/types.ts new file mode 100644 index 0000000..566aa68 --- /dev/null +++ b/src/live/types.ts @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LivePoint } from "../schemas/index.js"; + +/** What the agent has recorded about one benchmark so far. */ +export interface LiveBenchmarkRecord { + /** When the agent started it (compose up), in ms. */ + startedMs: number | null; + /** When SIPp started placing calls, in ms. */ + loadStartedMs: number | null; + /** When it ended, in ms. */ + endedMs: number | null; + /** The capacity steps evaluated so far. */ + points: LivePoint[]; +} + +/** What the agent has recorded about the run so far. */ +export interface LiveRecord { + status: "running" | "finished" | "failed"; + reason: string | null; + benchmarks: Record; +} + +/** + * The agent's notes for live.json: the agent and each benchmark call it as + * things happen; the live writer reads it every 15 seconds. + */ +export interface LiveRecorder { + started(id: string, ms: number): void; + loadStarted(id: string, ms: number): void; + point(id: string, point: LivePoint): void; + done(id: string, ms: number): void; + finished(): void; + failed(reason: string): void; + /** A copy of everything recorded so far. */ + read(): LiveRecord; +} + +/** A soak's progress read from its raw files. */ +export interface SoakProgress { + /** The UUT's samples, as parseSamplerJsonl returns them. */ + samples: ReadonlyArray<{ name: string; tMs: number; rssKB: number }>; + placed: number; + failed: number; +} + +/** A started server or tunnel, closed when the run ends. */ +export interface Closable { + close(): Promise; +} diff --git a/src/live/writeLive.schema.ts b/src/live/writeLive.schema.ts new file mode 100644 index 0000000..b10ee16 --- /dev/null +++ b/src/live/writeLive.schema.ts @@ -0,0 +1,9 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { z } from "zod/v4"; + +/** writeLive takes no options: it writes the run's live.json now. */ +export const writeLiveInputSchema = z.strictObject({}); + +export type WriteLiveInput = z.infer; diff --git a/src/orchestrate/buildReport.ts b/src/orchestrate/buildReport.ts index 03a9038..19a6737 100644 --- a/src/orchestrate/buildReport.ts +++ b/src/orchestrate/buildReport.ts @@ -9,6 +9,7 @@ import { } from "../schemas/index.js"; import { NOT_SELECTED } from "./constants.js"; import type { RunState } from "./runState.schema.js"; +import { toReportProject } from "./toReportProject.js"; /** * The run's report.json v1 from its state and the results of the selected @@ -42,15 +43,7 @@ export function buildReport( benchmarks.filter((benchmark) => benchmark.verdict === verdict).length; const report = { schema: SCHEMA_IDS.report, - project: { - name: config.project.name, - repo: config.project.repo, - siteUrl: config.project.siteUrl, - accent: config.project.accent, - ...(state.logoSvg ? { logoSvg: state.logoSvg } : {}), - title: config.project.title, - lead: config.project.lead - }, + project: toReportProject(state), run: { id: plan.runId, version: plan.targetVersion, diff --git a/src/orchestrate/createAgent.ts b/src/orchestrate/createAgent.ts index 127c9ce..31f5fc1 100644 --- a/src/orchestrate/createAgent.ts +++ b/src/orchestrate/createAgent.ts @@ -63,7 +63,9 @@ export function createAgent(deps: AgentDeps) { plannedS: plannedSeconds(benchmark) }; await save(); + deps.live?.started(benchmark.id, deps.now()); await runBenchmark(benchmark); + deps.live?.done(benchmark.id, deps.now()); status.benchmarks[benchmark.id] = { ...status.benchmarks[benchmark.id]!, state: "done" @@ -83,12 +85,14 @@ export function createAgent(deps: AgentDeps) { : `the agent failed: ${err instanceof Error ? err.message : String(err)}`; await deps.fs.writeFile(paths.failed, `${message}\n`); status.failed = message; + deps.live?.failed(message); status.endedAt = new Date(deps.now()).toISOString(); await save(); await deps.log(`perfkit agent: ${message}`); return { status: "failed", message }; } status.done = true; + deps.live?.finished(); status.endedAt = new Date(deps.now()).toISOString(); await save(); await deps.fs.writeFile(paths.done, `${status.endedAt}\n`); diff --git a/src/orchestrate/createRunBenchmark.ts b/src/orchestrate/createRunBenchmark.ts index 1b3263b..e2d4993 100644 --- a/src/orchestrate/createRunBenchmark.ts +++ b/src/orchestrate/createRunBenchmark.ts @@ -52,6 +52,7 @@ import { runPaths } from "./paths.js"; import type { RunState } from "./runState.schema.js"; import { buildOtelcolConfig, buildSamplerSpec } from "./sampler.js"; import { writeFileCommand } from "./shellQuote.js"; +import type { LiveRecorder } from "../live/types.js"; import type { AgentFs, Host, HostShell } from "./types.js"; export interface AgentDeps { @@ -61,6 +62,8 @@ export interface AgentDeps { sleep: Sleep; /** One progress line, to the agent's log and progress.log. */ log: (line: string) => Promise; + /** Notes for live.json; absent in tests that don't look at it. */ + live?: LiveRecorder; } const iso = (ms: number) => new Date(ms).toISOString(); @@ -142,6 +145,7 @@ export function createRunBenchmark(deps: AgentDeps, state: RunState) { registerUsers ); const startedMs = started.startedMs; + if (!started.aborted) deps.live?.loadStarted(benchmark.id, startedMs); const monitored = started.aborted ? { steps: 0, aborted: started.aborted } : await monitor(benchmark, rawDir, startedMs); @@ -481,6 +485,7 @@ export function createRunBenchmark(deps: AgentDeps, state: RunState) { startMs: startedMs }); await deps.log(evaluation.line); + if (evaluation.point) deps.live?.point(benchmark.id, evaluation.point); if (evaluation.stop || index + 1 >= total) { return { steps: index + 1, aborted: null }; } diff --git a/src/orchestrate/createStartRun.ts b/src/orchestrate/createStartRun.ts index 01c6534..facd991 100644 --- a/src/orchestrate/createStartRun.ts +++ b/src/orchestrate/createStartRun.ts @@ -13,6 +13,8 @@ import { SIPP_BINARY, SIPP_IMAGE } from "../tools.js"; +import { ngrokBindingCommand } from "../live/ngrokBindingCommand.js"; +import { ngrokVersion } from "../live/ngrokVersion.js"; import { nameOf, RUN_LABEL } from "./constants.js"; import { agentKeyPath } from "./createAgentShell.js"; import { lastLine } from "./errors.js"; @@ -145,6 +147,7 @@ export function createStartRun(deps: StartRunDeps) { config: input.config, ...(input.logoSvg ? { logoSvg: input.logoSvg } : {}), hosts: input.hosts, + ...(input.live ? { live: input.live } : {}), machines: { uut, loadgen }, tools: { sipp, ...(otelcol ? { otelcol } : {}) } }; @@ -185,6 +188,19 @@ export function createStartRun(deps: StartRunDeps) { ); } + const tunnel = input.mode === "digitalocean" && input.ngrokToken; + if (tunnel) { + // The tunnel's native binding for the agent's Linux; a failure only + // means no tunnel, so it never stops the run. + await remote.exec( + "loadgen", + ngrokBindingCommand({ + agentDir: paths.agent, + version: await ngrokVersion(deps.packageRoot) + }), + { timeoutMs: 5 * 60 * 1000 } + ); + } const agentName = nameOf(input.namePrefix, input.runId, "agent"); await exec( "loadgen", @@ -204,6 +220,7 @@ export function createStartRun(deps: StartRunDeps) { `${input.root}:${input.root}`, "-w", input.root, + ...(tunnel ? ["-e", `NGROK_AUTHTOKEN=${input.ngrokToken}`] : []), NODE_IMAGE, "node", `${paths.agent}/dist/cli.js`, diff --git a/src/orchestrate/evaluateCapacityStep.ts b/src/orchestrate/evaluateCapacityStep.ts index 2105813..cf9b983 100644 --- a/src/orchestrate/evaluateCapacityStep.ts +++ b/src/orchestrate/evaluateCapacityStep.ts @@ -9,7 +9,7 @@ import { parseUdpJsonl, readSippResults } from "../analyze/index.js"; -import type { CapacityBenchmarkConfig } from "../schemas/index.js"; +import type { CapacityBenchmarkConfig, LivePoint } from "../schemas/index.js"; import { liveAnalyzeDeps } from "./liveAnalyzeDeps.js"; import type { AgentFs } from "./types.js"; @@ -18,6 +18,8 @@ export interface StepEvaluation { line: string; /** True when the ramp ends here: a rule broken or the step invalid. */ stop: boolean; + /** The step's numbers for live.json; null when SIPp wrote nothing. */ + point: LivePoint | null; } /** @@ -47,7 +49,11 @@ export async function evaluateCapacityStep(input: { benchmark.load.uac.scenario ); if (uac === null) { - return { line: `${prefix} SIPp wrote no results; stopping`, stop: true }; + return { + line: `${prefix} SIPp wrote no results; stopping`, + stop: true, + point: null + }; } const read = async (file: string) => names.includes(file) ? deps.readFile(join(rawDir, file)) : ""; @@ -79,7 +85,23 @@ export async function evaluateCapacityStep(input: { : measured.failedPct >= failedCallsPct ? "over the failed limit; stopping" : "ok"; - return { line: `${numbers} ${outcome}`, stop: outcome !== "ok" }; + const stop = outcome !== "ok"; + return { + line: `${numbers} ${outcome}`, + stop, + point: { + offeredCps, + achievedCps: measured.step.achievedCps, + calls: measured.step.calls, + failed: measured.step.failed, + p50Ms: measured.step.p50Ms, + p95Ms: measured.step.p95Ms, + p99Ms: p99, + failedPct: measured.failedPct, + ok: !stop, + stopReason: stop ? outcome.replace(/; stopping$/, "") : null + } + }; } function formatRate(value: number): string { diff --git a/src/orchestrate/paths.ts b/src/orchestrate/paths.ts index 9ba5635..0949931 100644 --- a/src/orchestrate/paths.ts +++ b/src/orchestrate/paths.ts @@ -7,6 +7,7 @@ export function runPaths(root: string) { root, runJson: `${root}/run.json`, status: `${root}/status.json`, + live: `${root}/live.json`, progress: `${root}/progress.log`, done: `${root}/done`, failed: `${root}/failed`, diff --git a/src/orchestrate/runState.schema.ts b/src/orchestrate/runState.schema.ts index bb9a855..02e413b 100644 --- a/src/orchestrate/runState.schema.ts +++ b/src/orchestrate/runState.schema.ts @@ -29,6 +29,11 @@ const machineInfoSchema = z.strictObject({ docker: nonEmptyString }); +export const liveSettingsSchema = z.strictObject({ + port: z.number().int().min(1).max(65535), + domain: z.string().min(1).nullable() +}); + export const runModeSchema = z.enum(["digitalocean", "local"]); export const runStateSchema = z.strictObject({ @@ -57,7 +62,9 @@ export const runStateSchema = z.strictObject({ tools: z.strictObject({ sipp: nonEmptyString, otelcol: nonEmptyString.optional() - }) + }), + /** Where live.json is served, and the static ngrok domain, if any. */ + live: liveSettingsSchema.optional() }); const benchmarkStatusSchema = z.strictObject({ @@ -80,6 +87,7 @@ export const agentInputSchema = z.strictObject({ runId: runIdSchema }); +export type LiveSettings = z.infer; export type RunMode = z.infer; export type RunState = z.infer; export type RunStatus = z.infer; diff --git a/src/orchestrate/startRun.schema.ts b/src/orchestrate/startRun.schema.ts index 8f1e657..1889eda 100644 --- a/src/orchestrate/startRun.schema.ts +++ b/src/orchestrate/startRun.schema.ts @@ -15,14 +15,20 @@ export const startRunInputSchema = z.strictObject({ plan: true, config: true, logoSvg: true, - hosts: true + hosts: true, + live: true }).shape, /** The directory of perfkit.yaml. */ projectDir: z.string().min(1, { error: "expected the project directory" }), /** The run's --out, left out of the project copy. */ out: z.string().min(1, { error: "expected the output directory" }), /** The run-only private key for the agent; DigitalOcean runs only. */ - agentKey: z.string().optional() + agentKey: z.string().optional(), + /** + * NGROK_AUTHTOKEN for the agent's tunnel, DigitalOcean runs only. It goes + * into the agent container's environment, never into run.json or a log. + */ + ngrokToken: z.string().min(1).optional() }); export type StartRunInput = z.infer; diff --git a/src/orchestrate/toReportProject.ts b/src/orchestrate/toReportProject.ts new file mode 100644 index 0000000..f025ca0 --- /dev/null +++ b/src/orchestrate/toReportProject.ts @@ -0,0 +1,28 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { Report } from "../schemas/index.js"; +import type { RunState } from "./runState.schema.js"; + +/** + * The project as report.json and live.json carry it, from perfkit.yaml. Its + * live URL is `project.live.url`, else `live.url`, else the live.json of the static ngrok + * domain in `live.domain`, else there is none. + */ +export function toReportProject(state: RunState): Report["project"] { + const { project, live } = state.config; + const liveUrl = + project.live?.url ?? + live?.url ?? + (live?.domain ? `https://${live.domain}/live.json` : null); + return { + name: project.name, + repo: project.repo, + siteUrl: project.siteUrl, + accent: project.accent, + ...(state.logoSvg ? { logoSvg: state.logoSvg } : {}), + title: project.title, + lead: project.lead, + ...(liveUrl ? { live: { url: liveUrl } } : {}) + }; +} diff --git a/src/render/app/createDrawRun.ts b/src/render/app/createDrawRun.ts index 3fcc2b1..2dc2851 100644 --- a/src/render/app/createDrawRun.ts +++ b/src/render/app/createDrawRun.ts @@ -62,7 +62,12 @@ export function createDrawRun(deps: DrawRunDeps) { perfkitVersion: deps.version }) ); - return { ok: true, runId: current.id, html }; + return { + ok: true, + runId: current.id, + html, + liveUrl: report.value.project.live?.url ?? null + }; }; async function fetchValid( diff --git a/src/render/app/createLiveController.ts b/src/render/app/createLiveController.ts new file mode 100644 index 0000000..9fd0736 --- /dev/null +++ b/src/render/app/createLiveController.ts @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/// +import { type Live, liveSchema, type RunEntry } from "../../schemas/index.js"; +import type { LiveViewContext } from "../live/types.js"; +import type { LiveControllerDeps, LiveMode } from "./liveController.types.js"; + +/** + * Creates the latest page's live controller. Each `poll()` fetches + * live.json: a run in progress turns the page live (and redraws it); when + * that run finishes or stops, the page says so and waits for its report in + * data/runs.json, then draws it. A failed poll while live shows "connection + * lost, retrying" and keeps looking for the report. With no run in + * progress (fetch fails, 503, an invalid file, or a run that ended before + * the page saw it live) the published report stays as it is. + */ +export function createLiveController(deps: LiveControllerDeps) { + let mode: LiveMode = "final"; + let live: Live | null = null; + let runs: RunEntry[] = []; + let failing = false; + let mounted: string | null = null; + + const context = (): LiveViewContext => ({ + live: live as Live, + runs, + nowMs: deps.now(), + connection: failing ? "lost" : "ok", + perfkitVersion: deps.version + }); + + const draw = () => { + if (!live) return; + deps.view.showLive(context(), mounted !== live.runId); + mounted = live.runId; + }; + + const read = async (): Promise => { + try { + const response = await deps.fetchLive(deps.liveUrl); + if (response.status !== 200) return null; + const parsed = liveSchema.safeParse(JSON.parse(response.text)); + return parsed.success ? parsed.data : null; + } catch { + return null; + } + }; + + const refreshRuns = async () => { + const fresh = await deps.fetchRuns().catch(() => null); + if (fresh) runs = fresh; + return fresh; + }; + + /** Draws the final report once the run being watched is published. */ + const checkPublished = async (runId: string) => { + const fresh = await refreshRuns(); + if (!fresh?.some((run) => run.id === runId)) return false; + mode = "final"; + live = null; + mounted = null; + failing = false; + await deps.view.showFinal(); + return true; + }; + + const poll = async (): Promise => { + const next = await read(); + if (next) { + failing = false; + if (next.status === "running") { + if (mode === "final" || live?.runId !== next.runId) await refreshRuns(); + mode = "live"; + live = next; + draw(); + } else if (mode === "live") { + live = next; + draw(); + await checkPublished(next.runId); + } + return mode; + } + if (mode === "live" && live) { + failing = true; + draw(); + await checkPublished(live.runId); + } + return mode; + }; + + const tick = () => { + if (mode === "live" && live) deps.view.tick(context()); + }; + + return { poll, tick, mode: () => mode }; +} diff --git a/src/render/app/createLiveDomView.ts b/src/render/app/createLiveDomView.ts new file mode 100644 index 0000000..678f13d --- /dev/null +++ b/src/render/app/createLiveDomView.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/// +import { formatUpdated } from "../live/formatUpdated.js"; +import { renderLivePage } from "../live/renderLivePage.js"; +import { renderLiveView } from "../live/renderLiveView.js"; +import type { LiveViewContext } from "../live/types.js"; +import { mountPage } from "./mountPage.js"; + +/** + * The live view in a document: the first draw of a run mounts the whole + * page; later draws replace the banner, main and footer in place, keeping + * the scroll position and every open "Numbers behind this chart", so the + * page can be read while it updates. `tick` only rewrites "updated N s + * ago". + */ +export function createLiveDomView(doc: Document) { + const showLive = (ctx: LiveViewContext, first: boolean) => { + const main = doc.getElementById("live-main"); + if (first || !main) { + mountPage(doc, renderLivePage(ctx)); + return; + } + const parts = renderLiveView(ctx); + const win = doc.defaultView; + const y = win?.scrollY ?? 0; + const open = Array.from(main.querySelectorAll("details")).map( + (d) => d.open + ); + const set = (id: string, html: string) => { + const el = doc.getElementById(id); + if (el) el.innerHTML = html; + }; + set("live-banner", parts.banner); + main.innerHTML = parts.main; + set("live-footer", parts.footer); + Array.from(main.querySelectorAll("details")).forEach((d, i) => { + if (open[i]) d.open = true; + }); + if (win && win.scrollY !== y) win.scrollTo(0, y); + }; + + const tick = (ctx: LiveViewContext) => { + const el = doc.querySelector("[data-updated]"); + if (!el) return; + const updated = formatUpdated( + ctx.live.updatedAt, + ctx.nowMs, + ctx.connection + ); + if (el.textContent !== updated.text) el.textContent = updated.text; + el.className = `updated is-${updated.tone}`; + }; + + return { showLive, tick }; +} diff --git a/src/render/app/liveController.types.ts b/src/render/app/liveController.types.ts new file mode 100644 index 0000000..5298845 --- /dev/null +++ b/src/render/app/liveController.types.ts @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { RunEntry } from "../../schemas/index.js"; +import type { LiveViewContext } from "../live/types.js"; + +/** How the live controller reaches the page and the network, injected. */ +export interface LiveControllerDeps { + /** The live.json URL to poll. */ + liveUrl: string; + /** The perfkit version of the app, for the footer. */ + version: string; + /** Fetches live.json; rejects on a network error. */ + fetchLive: (url: string) => Promise<{ status: number; text: string }>; + /** The published history, newest first; null when it can't be read. */ + fetchRuns: () => Promise; + view: { + /** Draws the live view: a whole page when `first`, else in place. */ + showLive: (ctx: LiveViewContext, first: boolean) => void; + /** Refreshes "updated N s ago" only. */ + tick: (ctx: LiveViewContext) => void; + /** Draws the latest published report, as the page does without live. */ + showFinal: () => Promise; + }; + now: () => number; +} + +/** What the page shows: the published report, or a run in progress. */ +export type LiveMode = "final" | "live"; diff --git a/src/render/app/main.ts b/src/render/app/main.ts index 8b09de3..6ff027e 100644 --- a/src/render/app/main.ts +++ b/src/render/app/main.ts @@ -5,11 +5,19 @@ /** * The browser app, bundled into `app//app.js`. The shell loads it * with `defer`; it reads which run to draw from the page's URL, fetches - * the data next to it and replaces the shell with the page. + * the data next to it and replaces the shell with the page. The latest page + * also polls the project's live.json every 15 seconds and shows a run in + * progress live; a run page never does. */ +import { LIVE_INTERVAL_S, runsSchema } from "../../schemas/index.js"; import { createDrawRun } from "./createDrawRun.js"; +import { createLiveController } from "./createLiveController.js"; +import { createLiveDomView } from "./createLiveDomView.js"; import { mountPage } from "./mountPage.js"; +import { resolveLiveUrl } from "./resolveLiveUrl.js"; +import { resolveRunPath } from "./resolveRunPath.js"; import { showErrors } from "./showErrors.js"; +import { SITE_PATHS } from "./sitePaths.js"; import { toSitePath } from "./toSitePath.js"; declare const __PERFKIT_VERSION__: string; @@ -21,24 +29,69 @@ const scriptUrl = const root = new URL("../../", scriptUrl); const path = toSitePath(location.href, scriptUrl); -const drawRun = createDrawRun({ - version: __PERFKIT_VERSION__, - fetchText: async (file) => { - const response = await fetch(new URL(file, root), { cache: "no-cache" }); - return { status: response.status, text: await response.text() }; - } -}); +const fetchText = async (file: string) => { + const response = await fetch(new URL(file, root), { cache: "no-cache" }); + return { status: response.status, text: await response.text() }; +}; + +const drawRun = createDrawRun({ version: __PERFKIT_VERSION__, fetchText }); + +const draw = async (sitePath: string) => { + const result = await drawRun({ path: sitePath }).catch((err: unknown) => ({ + ok: false as const, + errors: [err instanceof Error ? err.message : String(err)] + })); + if (result.ok) mountPage(document, result.html); + else showErrors(document, result.errors); + return result; +}; + +const goLive = (liveUrl: string, sitePath: string) => { + const dom = createLiveDomView(document); + const controller = createLiveController({ + liveUrl, + version: __PERFKIT_VERSION__, + fetchLive: async (url) => { + const response = await fetch(url, { + cache: "no-store", + // ngrok's free domains answer browsers with a warning page without it. + headers: { "ngrok-skip-browser-warning": "1" } + }); + return { status: response.status, text: await response.text() }; + }, + fetchRuns: async () => { + const response = await fetchText(SITE_PATHS.runs); + if (response.status !== 200) return null; + const parsed = runsSchema.safeParse(JSON.parse(response.text)); + return parsed.success ? parsed.data.runs : null; + }, + view: { + showLive: dom.showLive, + tick: dom.tick, + showFinal: async () => { + await draw(sitePath); + } + }, + now: () => Date.now() + }); + const loop = () => { + controller.poll().finally(() => { + setTimeout(loop, LIVE_INTERVAL_S * 1000); + }); + }; + loop(); + setInterval(controller.tick, 1000); +}; if (path === null) { showErrors(document, [`perfkit: ${location.href} is outside the site`]); } else { - drawRun({ path }) - .then((result) => - result.ok - ? mountPage(document, result.html) - : showErrors(document, result.errors) - ) - .catch((err: unknown) => - showErrors(document, [err instanceof Error ? err.message : String(err)]) + draw(path).then((result) => { + if (resolveRunPath(path)?.page !== "latest") return; + const liveUrl = resolveLiveUrl( + location.href, + result.ok ? result.liveUrl : null ); + if (liveUrl) goLive(liveUrl, path); + }); } diff --git a/src/render/app/resolveLiveUrl.ts b/src/render/app/resolveLiveUrl.ts new file mode 100644 index 0000000..5c52671 --- /dev/null +++ b/src/render/app/resolveLiveUrl.ts @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** + * The live.json URL the latest page polls: `?live=` on the page (for + * a local run, like `?live=http://localhost:8787/live.json`), else the + * project's `live.url` from the latest report, else none. Only http and + * https URLs count. + */ +export function resolveLiveUrl( + pageUrl: string, + fromReport: string | null +): string | null { + const override = new URL(pageUrl).searchParams.get("live"); + const candidate = override ?? fromReport; + if (!candidate) return null; + try { + const url = new URL(candidate, pageUrl); + return url.protocol === "http:" || url.protocol === "https:" + ? url.href + : null; + } catch { + return null; + } +} diff --git a/src/render/app/types.ts b/src/render/app/types.ts index ab05a6c..0bfaa2c 100644 --- a/src/render/app/types.ts +++ b/src/render/app/types.ts @@ -13,4 +13,11 @@ export interface DrawRunDeps { } export type DrawRunResult = - { ok: true; runId: string; html: string } | { ok: false; errors: string[] }; + | { + ok: true; + runId: string; + html: string; + /** The project's live.json URL from the report, or null. */ + liveUrl: string | null; + } + | { ok: false; errors: string[] }; diff --git a/src/render/charts/renderLatencyChart.ts b/src/render/charts/renderLatencyChart.ts index 93df5d9..1912994 100644 --- a/src/render/charts/renderLatencyChart.ts +++ b/src/render/charts/renderLatencyChart.ts @@ -17,6 +17,12 @@ export interface LatencyChartInput { limit: { cps: number; lowerBound: boolean; p99Ms: number | null } | null; /** The run stopped early: the chart shows what was measured. */ partial: boolean; + /** + * A ramp in progress (the live page): the x axis spans the whole ramp, + * the step offered now is marked and the steps not run yet are shaded. + * Left out, the chart is the final report's, unchanged. + */ + live?: { maxCps: number; nowCps: number }; } const W = 820; @@ -39,8 +45,17 @@ const SERIES = [ */ export function renderLatencyChart(input: LatencyChartInput): string { const { steps, thresholdMs, limit } = input; - const maxCps = Math.max(...steps.map((s) => s.offeredCps), limit?.cps ?? 0); - const maxMs = Math.max(...steps.map((s) => s.p99Ms), thresholdMs ?? 0); + const { live } = input; + const maxCps = Math.max( + ...steps.map((s) => s.offeredCps), + limit?.cps ?? 0, + live?.maxCps ?? 0 + ); + const maxMs = Math.max( + ...steps.map((s) => s.p99Ms), + thresholdMs ?? 0, + live ? 1 : 0 + ); const xScale = niceScale(maxCps); const yScale = niceScale(maxMs * 1.05); const sx = (cps: number) => LEFT + (cps / xScale.max) * (RIGHT - LEFT); @@ -98,13 +113,17 @@ export function renderLatencyChart(input: LatencyChartInput): string { ms${grid} ${xTicks} offered ${esc(input.unit)}/s -${overLimit}${threshold}${lines}${limitMarker} +${live ? liveRegion(live, steps, sx) : ""}${overLimit}${threshold}${lines}${limitMarker}${live ? liveMarker(live, steps, input.unit, sx, sy) : ""} `; - const legend = SERIES.map( - (s) => - `
  • ${s.label}
    • ` - ).join(""); + const legend = + SERIES.map( + (s) => + `
  • ${s.label}
    • ` + ).join("") + + (live + ? '
  • live
    • ' + : ""); const rows = steps .map( @@ -128,3 +147,32 @@ ${overLimit}${threshold}${lines}${limitMarker} `; } + +/** The steps not run yet, shaded from the step offered now to the end. */ +function liveRegion( + live: { maxCps: number; nowCps: number }, + steps: ReadonlyArray, + sx: (cps: number) => number +): string { + const from = sx(Math.max(live.nowCps, steps.at(-1)?.offeredCps ?? 0)); + if (from >= RIGHT - 1) return ""; + return `steps not run yet`; +} + +/** "now ": a tag over the last measured p99, or the axis at first. */ +function liveMarker( + live: { maxCps: number; nowCps: number }, + steps: ReadonlyArray, + unit: string, + sx: (cps: number) => number, + sy: (ms: number) => number +): string { + const last = steps.at(-1); + const x = sx(last?.offeredCps ?? live.nowCps); + const y = last ? sy(last.p99Ms) : BOTTOM; + const text = `now ${formatRate(live.nowCps, unit)}`; + const width = text.length * 7.4 + 16; + const tagX = Math.min(Math.max(x - width / 2, LEFT), RIGHT - width); + const tagY = Math.max(y - 44, TOP); + return `${last ? `` : ""}${esc(text)}`; +} diff --git a/src/render/charts/renderMemoryChart.ts b/src/render/charts/renderMemoryChart.ts index f0f39e2..7f02b0a 100644 --- a/src/render/charts/renderMemoryChart.ts +++ b/src/render/charts/renderMemoryChart.ts @@ -4,6 +4,7 @@ import type { Resources } from "../../schemas/index.js"; import { formatDuration, formatGrowth, formatNumber } from "../format.js"; import { esc } from "../html.js"; +import { icon } from "../icons.js"; import { linePath } from "./linePath.js"; import { niceScale } from "./niceScale.js"; @@ -17,6 +18,12 @@ export interface MemoryChartInput { growth: ReadonlyMap; /** The run stopped early: the chart shows what was measured. */ partial: boolean; + /** + * A soak in progress (the live page): the x axis starts when the load + * started and spans `durationS`, with a "now" line at `nowS` and a note + * under the chart. Left out, the chart is the final report's, unchanged. + */ + live?: { startMs: number; nowS: number | null; note: string | null }; } const W = 820; @@ -39,8 +46,11 @@ export function renderMemoryChart(input: MemoryChartInput): string { const times = withSamples.flatMap((p) => p.samples.map((s) => Date.parse(s.t)) ); + const { live } = input; const end = Math.max(...times); - const start = Math.min(end - input.durationS * 1000, ...times); + const start = live + ? live.startMs + : Math.min(end - input.durationS * 1000, ...times); const domainS = Math.max((end - start) / 1000, input.durationS, 1); const maxMB = Math.max( ...withSamples.flatMap((p) => p.samples.map((s) => s.rssMB)) @@ -73,7 +83,7 @@ export function renderMemoryChart(input: MemoryChartInput): string { const warmup = input.warmupS && input.warmupS > 0 - ? `warm-up` + ? `${live ? "warm-up, not measured" : "warm-up"}` : ""; const lines = withSamples @@ -112,7 +122,9 @@ export function renderMemoryChart(input: MemoryChartInput): string { const svg = ` MB${grid} ${ticks} -${warmup}${lines}${labels}${partialMark} +${warmup}${lines}${labels}${partialMark}${ + live && live.nowS !== null ? nowLine(sx(Math.min(live.nowS, domainS))) : "" + } `; const legend = withSamples @@ -134,7 +146,7 @@ ${warmup}${lines}${labels}${partialMark} return `
    -

    Memory per process

    Resident memory per process in the container

    +

    ${live ? "Memory per process, raw" : "Memory per process"}

    Resident memory per process in the container

    ${input.partial ? 'Partial' : ""}
      ${legend}
    ${svg}
    @@ -144,7 +156,7 @@ ${warmup}${lines}${labels}${partialMark} ProcessFirst MBLast MBPeak MBGrowth after warm-upSamples ${rows} - +${live?.note ? `\n
    ${icon("info", 16)}

    ${esc(live.note)}

    ` : ""}
    `; } @@ -170,3 +182,9 @@ function spreadLabels(labels: Label[]): Label[] { return [...placed, { ...label, y: Math.max(y, TOP + 12) }]; }, []); } + +/** The live chart's "now" line, its label inside the plot at the right edge. */ +function nowLine(x: number): string { + const nearEdge = x > RIGHT - 40; + return `now`; +} diff --git a/src/render/icons.ts b/src/render/icons.ts index 7c827ce..3756619 100644 --- a/src/render/icons.ts +++ b/src/render/icons.ts @@ -43,6 +43,10 @@ const PATHS = { '', link: '', "arrow-right": '', + radio: + '', + "loader-circle": '', + flag: '', sprout: '' } as const; diff --git a/src/render/live/describeStep.ts b/src/render/live/describeStep.ts new file mode 100644 index 0000000..a8f4a9b --- /dev/null +++ b/src/render/live/describeStep.ts @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveBenchmark } from "../../schemas/index.js"; +import { + formatCount, + formatDuration, + formatNumber, + formatRate, + formatTime +} from "../format.js"; + +/** + * A progress card's second line: `held 1,700 messages/s · preliminary`, + * `step 14 of 37 · 4,250 messages/s`, `2 h 14 min of 8 h`, `starts about + * 14:55 UTC`. + */ +export function describeStep(benchmark: LiveBenchmark): string { + if (benchmark.state === "queued") { + return benchmark.estimatedStartAt + ? `starts about ${formatTime(benchmark.estimatedStartAt)} UTC` + : "queued"; + } + if (benchmark.kind === "capacity") { + if (benchmark.state === "done") { + const result = benchmark.result; + if (!result || result.limitCps === null) + return "no step held · preliminary"; + return `held ${result.lowerBound ? "at least " : ""}${formatRate(result.limitCps, benchmark.unit)} · preliminary`; + } + return benchmark.current + ? `step ${benchmark.current.index + 1} of ${benchmark.current.total} · ${formatRate(benchmark.current.offeredCps, benchmark.unit)}` + : "starting the server under test"; + } + if (benchmark.state === "done") { + return `${formatDuration(benchmark.elapsedS)} · ${formatCount(benchmark.calls.placed)} ${benchmark.unit} · preliminary`; + } + if (benchmark.loadStartedAt === null) return "starting the server under test"; + const ofTotal = `${formatDuration(benchmark.elapsedS)} of ${formatDuration(benchmark.plannedS)}`; + return benchmark.elapsedS < benchmark.warmupS + ? `warm-up · ${ofTotal}` + : `${ofTotal} · ${formatNumber(benchmark.cps)} ${benchmark.unit}/s`; +} diff --git a/src/render/live/formatClock.ts b/src/render/live/formatClock.ts new file mode 100644 index 0000000..c1f8b54 --- /dev/null +++ b/src/render/live/formatClock.ts @@ -0,0 +1,8 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** `0:52`, `12:05`: hours and minutes of a span in seconds. */ +export function formatClock(seconds: number): string { + const minutes = Math.max(0, Math.floor(seconds / 60)); + return `${Math.floor(minutes / 60)}:${String(minutes % 60).padStart(2, "0")}`; +} diff --git a/src/render/live/formatUpdated.ts b/src/render/live/formatUpdated.ts new file mode 100644 index 0000000..b8f9e9e --- /dev/null +++ b/src/render/live/formatUpdated.ts @@ -0,0 +1,28 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** The snapshot counts as stale after this, and the link as lost after that. */ +export const STALE_AFTER_MS = 60 * 1000; +export const LOST_AFTER_MS = 3 * 60 * 1000; + +/** + * The banner's "updated N s ago": warning-coloured once the snapshot is a + * minute old, and "connection lost, retrying" once the last poll failed or + * the snapshot is three minutes old. + */ +export function formatUpdated( + updatedAt: string, + nowMs: number, + connection: "ok" | "lost" +): { text: string; tone: "ok" | "stale" | "lost" } { + const ageMs = Math.max(0, nowMs - Date.parse(updatedAt)); + if (connection === "lost" || ageMs >= LOST_AFTER_MS) { + return { text: "connection lost, retrying", tone: "lost" }; + } + const seconds = Math.round(ageMs / 1000); + const text = + seconds < 60 + ? `updated ${seconds} s ago` + : `updated ${Math.floor(seconds / 60)} min ago`; + return { text, tone: ageMs >= STALE_AFTER_MS ? "stale" : "ok" }; +} diff --git a/src/render/live/lastPublishedLimit.ts b/src/render/live/lastPublishedLimit.ts new file mode 100644 index 0000000..34ed84b --- /dev/null +++ b/src/render/live/lastPublishedLimit.ts @@ -0,0 +1,27 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { RunEntry } from "../../schemas/index.js"; + +/** + * The newest published capacity limit of a benchmark, from data/runs.json, + * leaving out the run in progress; null when no published run has one. + */ +export function lastPublishedLimit( + runs: readonly RunEntry[], + benchmarkId: string, + exceptRunId: string +): { limitCps: number; version: string } | null { + return runs.reduce<{ limitCps: number; version: string } | null>( + (found, run) => { + if (found || run.id === exceptRunId) return found; + const entry = run.benchmarks.find((b) => b.id === benchmarkId); + return entry?.kind === "capacity" && + entry.headline !== null && + entry.headline.limitCps !== null + ? { limitCps: entry.headline.limitCps, version: run.version } + : null; + }, + null + ); +} diff --git a/src/render/live/livePhase.ts b/src/render/live/livePhase.ts new file mode 100644 index 0000000..f072fb9 --- /dev/null +++ b/src/render/live/livePhase.ts @@ -0,0 +1,13 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { Live } from "../../schemas/index.js"; + +/** + * running while the run goes on; publishing once it finished, until the + * report is on the site; stopped when the agent stopped it. + */ +export function livePhase(live: Live): "running" | "publishing" | "stopped" { + if (live.status === "running") return "running"; + return live.status === "finished" ? "publishing" : "stopped"; +} diff --git a/src/render/live/liveSectionHead.ts b/src/render/live/liveSectionHead.ts new file mode 100644 index 0000000..1fa3742 --- /dev/null +++ b/src/render/live/liveSectionHead.ts @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveBenchmark } from "../../schemas/index.js"; +import { esc } from "../html.js"; + +/** `02 · CAPACITY · RUNNING`, the title and a meta line. */ +export function liveSectionHead( + benchmark: LiveBenchmark, + index: number, + meta: string +): string { + const number = String(index + 1).padStart(2, "0"); + return `
    +
    +

    ${number} · ${esc(benchmark.kind.toUpperCase())} · ${esc(benchmark.state.toUpperCase())}

    +

    ${esc(benchmark.title)}

    +
    +${meta ? `

    ${esc(meta)}

    ` : ""} +
    `; +} diff --git a/src/render/live/liveStyles.ts b/src/render/live/liveStyles.ts new file mode 100644 index 0000000..0ca1b7c --- /dev/null +++ b/src/render/live/liveStyles.ts @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** + * The live view's CSS, added after the report's own (`pageStyles`), which + * it leaves untouched: the final report stays byte for byte the same. + * Tokens and sizes are the "State / Live (preliminary)" frame of + * `designs/benchmarks.pen`. Motion stops under prefers-reduced-motion. + */ +export function liveStyles(): string { + return `.live-dot { display: inline-block; flex: none; width: 8px; height: 8px; border-radius: 50%; background: var(--danger); animation: live-pulse 2s ease-in-out infinite; } +.badge-live { background: #F970661F; color: var(--danger); } +.legend-live { color: var(--danger); } +.legend-live .live-dot { animation: none; } +.live-banner { background: #60A5FA14; border-bottom: 1px solid #60A5FA40; } +.live-banner .wrap { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: center; gap: 8px 16px; padding-top: 14px; padding-bottom: 14px; } +.live-banner p { display: flex; align-items: center; gap: 10px; color: var(--text-secondary); font-size: 14px; } +.live-banner .icon { color: var(--info); } +.updated { color: var(--text-muted); font: 12px var(--font-mono); white-space: nowrap; } +.updated.is-stale { color: var(--warning); } +.updated.is-lost { color: var(--danger); } +.eyebrow-live { color: var(--danger); } +.hero-live .wrap { padding-top: 64px; padding-bottom: 40px; } +.progress .wrap { display: flex; flex-direction: column; gap: 16px; padding-bottom: 56px; } +.steps { margin: 0; padding: 0; list-style: none; display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 12px; } +.step { display: flex; flex-direction: column; gap: 6px; padding: 20px; border: 1px solid var(--border); border-radius: 14px; min-width: 0; } +.step-title { display: flex; align-items: center; gap: 8px; font-size: 15px; font-weight: 600; } +.step-sub { color: var(--text-muted); font: 12px var(--font-mono); } +.step-done .step-title .icon { color: var(--success); } +.step-running { background: var(--accent-subtle); border-color: var(--accent); } +.step-running .step-sub { color: var(--accent-text); } +.step-running .step-title .icon { color: var(--accent); animation: live-spin 1.2s linear infinite; } +.step-queued .step-title { color: var(--text-secondary); } +.step-queued .step-title .icon { color: var(--text-muted); } +.overall { display: flex; align-items: center; gap: 16px; } +.track { flex: 1; height: 6px; border-radius: 3px; background: var(--surface); overflow: hidden; } +.track-done { height: 100%; border-radius: 3px; background: var(--accent); transition: width .6s ease; } +.overall-text { color: var(--text-muted); font: 12px var(--font-mono); white-space: nowrap; } +.stat-sub { color: var(--text-muted); font: 12px var(--font-mono); } +.bench-live .wrap { gap: 32px; padding-top: 72px; padding-bottom: 72px; } +.queued-note { color: var(--text-muted); font-size: 14px; } +.so-far { gap: 14px; } +.so-far .so-far-value { color: var(--text-primary); font-size: 28px; font-weight: 600; letter-spacing: -0.5px; line-height: 1.2; } +.so-far p { color: var(--text-secondary); font-size: 14px; line-height: 1.5; } +.baseline { display: flex; gap: 10px; padding: 14px; border-radius: 10px; background: var(--accent-subtle); color: var(--accent-text); font-size: 13px; } +.baseline .icon { margin-top: 2px; } +.chart .not-run { fill: #FFFFFF05; } +.chart .not-run-label, .chart .now-label { fill: var(--text-muted); font-size: 11px; } +.chart .now-halo { fill: var(--accent); opacity: .2; } +.chart .now-dot { fill: var(--accent); stroke: var(--surface); stroke-width: 2.5; } +.chart .now-tag { fill: var(--accent); } +.chart .now-tag-text { fill: var(--on-accent); font-size: 12px; font-weight: 500; } +.chart .now-line { stroke: var(--text-muted); stroke-width: 1; } +.rule-note { display: flex; gap: 10px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; color: var(--text-secondary); font-size: 13px; line-height: 1.5; } +.rule-note .icon { margin-top: 2px; color: var(--text-muted); } +.empty-chart { color: var(--text-muted); font-size: 14px; } +@keyframes live-pulse { 50% { opacity: .35; } } +@keyframes live-spin { to { transform: rotate(360deg); } } +@media (prefers-reduced-motion: reduce) { + .live-dot, .step-running .step-title .icon { animation: none; } + .track-done { transition: none; } +} +@media (max-width: 720px) { + .hero-live .wrap { padding-top: 32px; padding-bottom: 24px; } + .progress .wrap { padding-bottom: 32px; } + .steps { grid-template-columns: minmax(0, 1fr); } + .bench-live .wrap { gap: 24px; padding-top: 40px; padding-bottom: 40px; } +}`; +} diff --git a/src/render/live/renderLiveBanner.ts b/src/render/live/renderLiveBanner.ts new file mode 100644 index 0000000..d2b4e9b --- /dev/null +++ b/src/render/live/renderLiveBanner.ts @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { esc } from "../html.js"; +import { icon } from "../icons.js"; +import { formatUpdated } from "./formatUpdated.js"; +import { livePhase } from "./livePhase.js"; +import type { LiveViewContext } from "./types.js"; + +/** + * The banner under the nav: the numbers are preliminary, and how fresh + * they are ("updated N s ago", which the app keeps ticking); once the run + * ends, that its report is being published, or why it stopped. + */ +export function renderLiveBanner(ctx: LiveViewContext): string { + const { live } = ctx; + const phase = livePhase(live); + const sentence = + phase === "running" + ? "Running now. These numbers are preliminary and change every 15 seconds; the verdict appears when the run ends." + : phase === "publishing" + ? "Run finished; the report is being published. This page shows it as soon as it is on the site." + : `The run stopped: ${live.reason ?? "the agent failed"}. Any results are published once the run is collected.`; + const updated = formatUpdated(live.updatedAt, ctx.nowMs, ctx.connection); + return `

    ${icon("radio", 16)}${esc(sentence)}

    ${esc(updated.text)}
    `; +} diff --git a/src/render/live/renderLiveCapacity.ts b/src/render/live/renderLiveCapacity.ts new file mode 100644 index 0000000..ff1534f --- /dev/null +++ b/src/render/live/renderLiveCapacity.ts @@ -0,0 +1,121 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveCapacity, Step } from "../../schemas/index.js"; +import { renderLatencyChart } from "../charts/renderLatencyChart.js"; +import { formatDuration, formatNumber, formatRate } from "../format.js"; +import { esc } from "../html.js"; +import { icon } from "../icons.js"; +import { lastPublishedLimit } from "./lastPublishedLimit.js"; +import { liveSectionHead } from "./liveSectionHead.js"; +import type { LiveViewContext } from "./types.js"; + +/** + * A capacity benchmark on the live page: the final report's "Latency as + * load rises" chart drawn up to the step offered now, the steps not run + * yet shaded, and "So far": what held, why the limit is not known yet, and + * the last published result to compare with. + */ +export function renderLiveCapacity( + benchmark: LiveCapacity, + index: number, + ctx: LiveViewContext +): string { + const { unit, rate } = benchmark; + const rateOf = (value: number) => formatRate(value, unit); + const meta = `+${formatNumber(rate.step)} ${unit}/s every ${formatDuration(rate.everyS)} · ${rateOf(rate.start)} to ${rateOf(rate.max)}`; + const head = liveSectionHead(benchmark, index, meta); + if (benchmark.state === "queued") { + return section( + benchmark, + `${head} +

    Queued: this ramp starts when the benchmarks before it end.

    ` + ); + } + const steps: Step[] = benchmark.points.map((point) => ({ + offeredCps: point.offeredCps, + achievedCps: point.achievedCps, + calls: point.calls, + p50Ms: point.p50Ms, + p95Ms: point.p95Ms, + p99Ms: point.p99Ms, + failed: point.failed, + failuresByReason: {}, + retransmissions: 0, + sippCpuPct: 0, + uutCpuPct: 0, + loadgenUdpDrops: 0, + uutUdpDrops: 0, + valid: point.ok || point.stopReason === null + })); + const done = benchmark.state === "done"; + const held = benchmark.heldCps; + const heldPoint = benchmark.points.find((p) => p.offeredCps === held); + const chart = renderLatencyChart({ + steps, + unit, + thresholdMs: benchmark.thresholds.p99LatencyMs, + limit: + done && held !== null + ? { + cps: held, + lowerBound: benchmark.result?.lowerBound ?? false, + p99Ms: heldPoint?.p99Ms ?? null + } + : null, + partial: false, + ...(done + ? {} + : { + live: { + maxCps: rate.max, + nowCps: benchmark.current?.offeredCps ?? rate.start + } + }) + }); + + const passed = benchmark.points.filter((p) => p.ok).length; + const value = done + ? held === null + ? "No step held" + : `Held ${benchmark.result?.lowerBound ? "at least " : ""}${rateOf(held)}` + : held === null + ? "Nothing held yet" + : `Held ${rateOf(held)} so far`; + const sentence = done + ? benchmark.result?.stopReason + ? `The ramp stopped at ${rateOf(benchmark.points.find((p) => !p.ok)?.offeredCps ?? 0)}: ${benchmark.result.stopReason}. The verdict comes from the full analysis when the run is collected.` + : "Every step passed up to the highest rate offered. The verdict comes from the full analysis when the run is collected." + : `${passed === 0 ? "No step has finished yet." : `${passed} ${passed === 1 ? "step" : "steps"} passed both rules.`} The limit is the last step that passes before one fails; it is not known until the ramp stops.`; + const last = lastPublishedLimit(ctx.runs, benchmark.id, ctx.live.runId); + const baseline = last + ? `Last published result: ${rateOf(last.limitCps)} on v${last.version}.${ + held === null + ? "" + : held >= last.limitCps + ? " Already past it." + : ` ${rateOf(last.limitCps - held)} to go to match it.` + }` + : null; + return section( + benchmark, + `${head} +
    +${chart} +
    +

    ${done ? "Where the ramp ended" : "So far"}

    +

    ${esc(value)}

    +

    ${esc(sentence)}

    +${baseline ? `
    ${icon("flag", 16)}

    ${esc(baseline)}

    ` : ""} +
    +
    ` + ); +} + +function section(benchmark: LiveCapacity, body: string): string { + return `
    +
    +${body} +
    +
    `; +} diff --git a/src/render/live/renderLiveHeader.ts b/src/render/live/renderLiveHeader.ts new file mode 100644 index 0000000..bf78529 --- /dev/null +++ b/src/render/live/renderLiveHeader.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { esc } from "../html.js"; +import { livePhase } from "./livePhase.js"; +import type { LiveViewContext } from "./types.js"; + +const WORDS = [ + "No", + "One", + "Two", + "Three", + "Four", + "Five", + "Six", + "Seven", + "Eight", + "Nine", + "Ten" +]; + +/** + * The live header: "LIVE · RUN ", " is under test + * right now." and a lead that says how many benchmarks run, where, and + * that nothing is final until the run ends. + */ +export function renderLiveHeader(ctx: LiveViewContext): string { + const { live } = ctx; + const phase = livePhase(live); + const count = live.benchmarks.filter((b) => b.state !== "not-run").length; + const where = + live.provider === "local" + ? "this machine's Docker" + : "two dedicated DigitalOcean servers"; + const many = WORDS[count] ?? String(count); + const lead = + phase !== "running" + ? count === 1 + ? `One benchmark ran on ${where}. The numbers below are still preliminary; the verdict comes with the published report.` + : `${many} benchmarks ran one after another on ${where}. The numbers below are still preliminary; the verdicts come with the published report.` + : count === 1 + ? `One benchmark runs on ${where}. Watch it as it happens; nothing on this page is final until the run ends.` + : `${many} benchmarks run one after another on ${where}. Watch each one as it happens; nothing on this page is final until the run ends.`; + const title = + phase === "running" + ? `${live.project.name} ${live.version} is under test right now.` + : phase === "publishing" + ? `${live.project.name} ${live.version} has finished its run.` + : `${live.project.name} ${live.version}: the run stopped.`; + return `
    +
    +

    ${phase === "running" ? "LIVE" : "ENDED"} · RUN ${esc(live.runId.replace(/-[0-9a-f]+$/, ""))}

    +

    ${esc(title)}

    +

    ${esc(lead)}

    +
    +
    `; +} diff --git a/src/render/live/renderLiveNav.ts b/src/render/live/renderLiveNav.ts new file mode 100644 index 0000000..6813a23 --- /dev/null +++ b/src/render/live/renderLiveNav.ts @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { formatDate, formatTime, plural } from "../format.js"; +import { esc } from "../html.js"; +import { icon } from "../icons.js"; +import { PICKER_RUNS } from "../sections/renderNav.js"; +import { renderLogo } from "../sections/renderLogo.js"; +import { summaryText, summaryTone } from "../verdicts.js"; +import type { LiveViewContext } from "./types.js"; + +/** + * The live page's nav: the logo and "/ Benchmarks", and the run picker + * showing the run in progress, "Running now · v" with a live dot + * and badge. Its list holds the run in progress and the published runs. + */ +export function renderLiveNav(ctx: LiveViewContext): string { + const { live, runs } = ctx; + const version = `v${live.version}`; + const items = runs.slice(0, PICKER_RUNS).map( + (run) => `
  • +${esc(formatDate(run.startedAt))} ${esc(formatTime(run.startedAt))}v${esc(run.version)} +${esc(summaryText(run.verdict, true))} +
    • ` + ); + const seeAll = + runs.length > PICKER_RUNS + ? `

    ${esc(plural(runs.length, "published run"))}

    ` + : ""; + return `
    +
    +
    ${renderLogo(live.project)}Benchmarks
    +
    +
    + + +
    +
    +
    +
    `; +} diff --git a/src/render/live/renderLiveNumbers.ts b/src/render/live/renderLiveNumbers.ts new file mode 100644 index 0000000..29cd71b --- /dev/null +++ b/src/render/live/renderLiveNumbers.ts @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { + formatCount, + formatDuration, + formatMs, + formatNumber, + formatPct +} from "../format.js"; +import { esc } from "../html.js"; +import { formatClock } from "./formatClock.js"; +import type { LiveViewContext } from "./types.js"; + +type Cell = [value: string, label: string, sub: string]; + +/** + * The live numbers: for the benchmark running now, the rate offered and + * placed, p99 of the last step (a soak: the calls placed), failed so far; + * and always how long the run has gone and how long it has to go. + */ +export function renderLiveNumbers(ctx: LiveViewContext): string { + const { live, nowMs } = ctx; + const running = live.benchmarks.find((b) => b.state === "running"); + const endMs = live.estimatedEndAt ? Date.parse(live.estimatedEndAt) : null; + const elapsedS = + ((live.status === "running" ? nowMs : Date.parse(live.updatedAt)) - + Date.parse(live.startedAt)) / + 1000; + const elapsed: Cell = [ + formatClock(elapsedS), + live.status === "running" + ? "hours:minutes into the run" + : "hours:minutes the run took", + endMs !== null && live.status === "running" + ? `about ${formatDuration(Math.max(60, (endMs - nowMs) / 1000))} to go` + : "the run has ended" + ]; + const cells: Cell[] = []; + if (running?.kind === "capacity" && running.current) { + const last = running.points.at(-1); + cells.push( + [ + formatNumber(running.current.offeredCps), + `${running.unit}/s offered now, ${running.title}`, + last + ? `placed ${formatNumber(last.achievedCps)}/s · ${Math.round((last.achievedCps / Math.max(last.offeredCps, 1)) * 100)}% (last step)` + : "first step in progress" + ], + [ + last ? formatMs(last.p99Ms) : "--", + "p99 latency, last step", + `limit ${formatMs(running.thresholds.p99LatencyMs)}` + ], + [ + last ? formatPct(last.failedPct) : "--", + `failed ${running.unit}, last step`, + `limit ${formatPct(running.thresholds.failedCallsPct)}` + ] + ); + } else if (running?.kind === "soak" && running.loadStartedAt !== null) { + const { placed, failed } = running.calls; + cells.push( + [ + formatNumber(running.cps), + `${running.unit}/s offered now, ${running.title}`, + running.elapsedS < running.warmupS + ? "warm-up, not measured" + : "steady rate" + ], + [ + formatCount(placed), + `${running.unit} placed so far`, + "since the load started" + ], + [ + formatPct(placed > 0 ? (failed / placed) * 100 : 0), + `failed ${running.unit} so far`, + `limit ${formatPct(running.thresholds.failedCallsPct)}` + ] + ); + } + const all = [...cells, elapsed]; + return `
    +
    +
    ${all + .map( + ([value, label, sub]) => + `

    ${esc(value)}

    ${esc(label)}

    ${esc(sub)}

    ` + ) + .join("")}
    +
    +
    `; +} diff --git a/src/render/live/renderLivePage.ts b/src/render/live/renderLivePage.ts new file mode 100644 index 0000000..7919caa --- /dev/null +++ b/src/render/live/renderLivePage.ts @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { esc } from "../html.js"; +import { pageStyles } from "../pageStyles.js"; +import { PICKER_SCRIPT } from "../pickerScript.js"; +import { liveStyles } from "./liveStyles.js"; +import { renderLiveView } from "./renderLiveView.js"; +import type { LiveViewContext } from "./types.js"; + +const FONTS = + "https://fonts.googleapis.com/css2?family=Poppins:wght@400;500;600&family=Roboto+Mono:wght@400;500&display=swap"; + +/** + * The whole live page, the report page's head and styles plus the live + * view's: the app mounts it once per run, then replaces the banner, main + * and footer (`#live-banner`, `#live-main`, `#live-footer`) on each poll. + */ +export function renderLivePage(ctx: LiveViewContext): string { + const { live } = ctx; + const parts = renderLiveView(ctx); + return ` + + + + +${esc(`${live.project.name} benchmarks · running now · v${live.version}`)} + + + + + + + + +${parts.nav} +
    ${parts.banner}
    +
    +${parts.main} +
    + + + + +`; +} diff --git a/src/render/live/renderLiveProgress.ts b/src/render/live/renderLiveProgress.ts new file mode 100644 index 0000000..cc0200e --- /dev/null +++ b/src/render/live/renderLiveProgress.ts @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { formatDuration, formatTime } from "../format.js"; +import { esc } from "../html.js"; +import { icon, type IconName } from "../icons.js"; +import { describeStep } from "./describeStep.js"; +import type { LiveViewContext } from "./types.js"; + +type StepState = "done" | "running" | "queued"; + +const ICONS: Record = { + done: "circle-check", + running: "loader-circle", + queued: "circle-dashed" +}; + +/** + * The progress cards (setup, then each benchmark of the run: done, running + * or queued) and the overall bar with when the run should end. + */ +export function renderLiveProgress(ctx: LiveViewContext): string { + const { live, nowMs } = ctx; + const selected = live.benchmarks.filter((b) => b.state !== "not-run"); + const firstStart = + selected.find((b) => b.startedAt !== null)?.startedAt ?? null; + const setupState: StepState = + firstStart === null && live.status === "running" ? "running" : "done"; + const machines = live.provider === "local" ? "containers up" : "servers up"; + const setupSub = + firstStart === null + ? `${machines}, images pinned` + : `${machines}, images pinned · ${formatDuration((Date.parse(firstStart) - Date.parse(live.startedAt)) / 1000)}`; + const card = (state: StepState, title: string, sub: string) => + `

  • ${icon(ICONS[state], 16)}${esc(title)}

    ${esc(sub)}

    • `; + const cards = [ + card(setupState, "Setup", setupSub), + ...selected.map((b) => + card(b.state === "not-run" ? "queued" : b.state, b.title, describeStep(b)) + ) + ].join(""); + + const startMs = Date.parse(live.startedAt); + const endMs = live.estimatedEndAt + ? Date.parse(live.estimatedEndAt) + : Date.parse(live.updatedAt); + const pct = + live.status === "running" + ? Math.min( + 99, + Math.max( + 0, + Math.floor(((nowMs - startMs) / Math.max(1, endMs - startMs)) * 100) + ) + ) + : 100; + const text = + live.status === "running" + ? `${pct}% · ends about ${formatTime(new Date(endMs).toISOString())} UTC` + : `ended ${formatTime(live.updatedAt)} UTC`; + return `
    +
    +
      ${cards}
    +

    ${esc(text)}

    +
    +
    `; +} diff --git a/src/render/live/renderLiveSoak.ts b/src/render/live/renderLiveSoak.ts new file mode 100644 index 0000000..3c226f4 --- /dev/null +++ b/src/render/live/renderLiveSoak.ts @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveSoak } from "../../schemas/index.js"; +import { renderMemoryChart } from "../charts/renderMemoryChart.js"; +import { formatCount, formatDuration, formatNumber } from "../format.js"; +import { esc } from "../html.js"; +import { liveSectionHead } from "./liveSectionHead.js"; +import type { LiveViewContext } from "./types.js"; + +export const GROWTH_NOTE = + "Memory growth per hour appears once there is a full hour of data after the warm-up. Until then the page shows the raw curve only, because a slope over a few minutes overstates growth while the heap settles."; + +/** + * A soak on the live page: how far it is and its calls so far, and the + * raw memory per process with the warm-up shaded and a "now" line. MB/h + * shows only once a full hour after the warm-up is in. + */ +export function renderLiveSoak( + benchmark: LiveSoak, + index: number, + _ctx: LiveViewContext +): string { + const { unit } = benchmark; + const meta = + benchmark.state === "queued" + ? `${formatDuration(benchmark.durationS)} + ${formatDuration(benchmark.warmupS)} warm-up · steady ${formatNumber(benchmark.cps)} ${unit}/s` + : `${formatDuration(benchmark.elapsedS)} of ${formatDuration(benchmark.plannedS)} · ${formatCount(benchmark.calls.placed)} ${unit} · ${formatCount(benchmark.calls.failed)} failed`; + const head = liveSectionHead(benchmark, index, meta); + const processes = benchmark.memory + .filter((p) => p.samples.length > 0) + .map((p) => ({ + name: p.name, + samples: p.samples.map((s) => ({ + t: s.t, + rssMB: s.rssMB, + cpuPct: 0, + fds: 0, + pid: 1 + })) + })); + const growth = new Map( + benchmark.memory.flatMap((p) => + p.growthMBPerHour === null + ? [] + : [[p.name, p.growthMBPerHour] as [string, number]] + ) + ); + const body = + benchmark.state === "queued" + ? `

    Queued: this soak starts when the benchmarks before it end.

    ` + : processes.length === 0 || benchmark.loadStartedAt === null + ? `

    Memory per process, raw

    The first memory samples appear within a minute of the load starting.

    ` + : renderMemoryChart({ + processes, + durationS: benchmark.plannedS, + warmupS: benchmark.warmupS, + growth, + partial: false, + live: { + startMs: Date.parse(benchmark.loadStartedAt), + nowS: benchmark.state === "done" ? null : benchmark.elapsedS, + note: benchmark.memory.some((p) => p.growthMBPerHour === null) + ? GROWTH_NOTE + : null + } + }); + return `
    +
    +${head} +${body} +
    +
    `; +} diff --git a/src/render/live/renderLiveView.ts b/src/render/live/renderLiveView.ts new file mode 100644 index 0000000..e934579 --- /dev/null +++ b/src/render/live/renderLiveView.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { esc } from "../html.js"; +import { renderLogo } from "../sections/renderLogo.js"; +import { renderLiveBanner } from "./renderLiveBanner.js"; +import { renderLiveCapacity } from "./renderLiveCapacity.js"; +import { renderLiveHeader } from "./renderLiveHeader.js"; +import { renderLiveNav } from "./renderLiveNav.js"; +import { renderLiveNumbers } from "./renderLiveNumbers.js"; +import { renderLiveProgress } from "./renderLiveProgress.js"; +import { renderLiveSoak } from "./renderLiveSoak.js"; +import type { LiveViewContext, LiveViewParts } from "./types.js"; + +/** + * The live view's parts: the nav (drawn once per run), and the banner, + * main and footer, which the app replaces on every poll. + */ +export function renderLiveView(ctx: LiveViewContext): LiveViewParts { + const { live } = ctx; + const sections = live.benchmarks + .map((benchmark, index) => + benchmark.state === "not-run" + ? "" + : benchmark.kind === "capacity" + ? renderLiveCapacity(benchmark, index, ctx) + : renderLiveSoak(benchmark, index, ctx) + ) + .filter(Boolean) + .join("\n"); + return { + nav: renderLiveNav(ctx), + banner: renderLiveBanner(ctx), + main: [ + renderLiveHeader(ctx), + renderLiveProgress(ctx), + renderLiveNumbers(ctx), + sections + ].join("\n"), + footer: `
    +
    +
    ${renderLogo(live.project)}

    Generated by perfkit ${esc(ctx.perfkitVersion)} from live.json · schema ${esc(live.schema)} · preliminary

    +
    +
    ` + }; +} diff --git a/src/render/live/types.ts b/src/render/live/types.ts new file mode 100644 index 0000000..1f5c039 --- /dev/null +++ b/src/render/live/types.ts @@ -0,0 +1,24 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { Live, RunEntry } from "../../schemas/index.js"; + +/** Everything the live view is drawn from. */ +export interface LiveViewContext { + live: Live; + /** The published history, newest first; empty when there is none. */ + runs: RunEntry[]; + nowMs: number; + /** lost: the last poll failed, or the snapshot is over 3 minutes old. */ + connection: "ok" | "lost"; + /** The perfkit version of the app, for the footer. */ + perfkitVersion: string; +} + +/** The parts the app replaces on each poll, apart from the nav. */ +export interface LiveViewParts { + nav: string; + banner: string; + main: string; + footer: string; +} diff --git a/src/schemas/index.ts b/src/schemas/index.ts index 9df20ed..b42e4a5 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -41,6 +41,8 @@ export { capacityHeadlineSchema, capacityRuleSchema, hopsSchema, + projectLiveSchema, + reportProjectSchema, reasonSchema, reportSchema, resourcesSchema, @@ -64,6 +66,22 @@ export { type Step, type StoppedBy } from "./report.schema.js"; +export { + LIVE_INTERVAL_S, + LIVE_MAX_SAMPLES, + LIVE_SCHEMA_ID, + liveBenchmarkSchema, + liveCapacitySchema, + livePointSchema, + liveSchema, + liveSoakSchema, + type Live, + type LiveBenchmark, + type LiveBenchmarkState, + type LiveCapacity, + type LivePoint, + type LiveSoak +} from "./live.schema.js"; export { runBenchmarkSchema, runEntrySchema, diff --git a/src/schemas/live.schema.ts b/src/schemas/live.schema.ts new file mode 100644 index 0000000..be84deb --- /dev/null +++ b/src/schemas/live.schema.ts @@ -0,0 +1,180 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +/** + * `live.json`, `schema: perfkit/live/v1`: a snapshot of a run in progress, + * which the agent rewrites every 15 seconds and serves to the benchmarks + * page. Every number in it is preliminary: the verdicts come from the + * analyzer once the run is collected, never from this file. + */ +import { z } from "zod/v4"; +import { + benchmarkIdSchema, + commitSchema, + count, + nonEmptyString, + nonNegativeNumber, + positiveNumber, + runIdSchema, + timestampSchema +} from "./primitives.schema.js"; +import { reportProjectSchema } from "./report.schema.js"; + +export const LIVE_SCHEMA_ID = "perfkit/live/v1"; + +/** How often the agent rewrites live.json, and the page polls it. */ +export const LIVE_INTERVAL_S = 15; + +export const liveBenchmarkStateSchema = z.enum([ + "queued", + "running", + "done", + "not-run" +]); + +const unitSchema = z + .string() + .regex(/^[a-z]+$/, { error: "expected a lowercase plural noun" }); + +/** One evaluated capacity step, as the agent judged it when it ended. */ +export const livePointSchema = z.strictObject({ + offeredCps: nonNegativeNumber, + achievedCps: nonNegativeNumber, + calls: count, + failed: count, + p50Ms: nonNegativeNumber, + p95Ms: nonNegativeNumber, + p99Ms: nonNegativeNumber, + failedPct: nonNegativeNumber, + /** True when the step met both rules and its measurement counts. */ + ok: z.boolean(), + /** Why the ramp stopped at this step, or null. */ + stopReason: z.string().nullable() +}); + +const baseFields = { + id: benchmarkIdSchema, + title: nonEmptyString, + unit: unitSchema, + state: liveBenchmarkStateSchema, + /** When the agent started the benchmark (compose up), or null. */ + startedAt: timestampSchema.nullable(), + /** When SIPp started placing calls, or null. */ + loadStartedAt: timestampSchema.nullable(), + /** When it is expected to start, for a queued benchmark; else null. */ + estimatedStartAt: timestampSchema.nullable(), + /** The longest it runs once the load starts, in seconds. */ + plannedS: nonNegativeNumber +}; + +export const liveCapacitySchema = z.strictObject({ + ...baseFields, + kind: z.literal("capacity"), + rate: z.strictObject({ + start: positiveNumber, + step: positiveNumber, + everyS: positiveNumber, + max: positiveNumber + }), + thresholds: z.strictObject({ + p99LatencyMs: positiveNumber, + failedCallsPct: nonNegativeNumber + }), + /** The step SIPp is offering now, while running; else null. */ + current: z + .strictObject({ + index: count, + total: positiveNumber.int({ error: "expected a whole number" }), + offeredCps: nonNegativeNumber + }) + .nullable(), + /** The steps evaluated so far, in order. */ + points: z.array(livePointSchema), + /** The offered rate of the last step that passed, or null. */ + heldCps: nonNegativeNumber.nullable(), + /** Once done: where the ramp ended, still preliminary. */ + result: z + .strictObject({ + limitCps: nonNegativeNumber.nullable(), + /** True when no step broke a rule: the limit is a lower bound. */ + lowerBound: z.boolean(), + stopReason: z.string().nullable() + }) + .nullable() +}); + +const liveSampleSchema = z.strictObject({ + t: timestampSchema, + rssMB: nonNegativeNumber +}); + +/** At most this many samples per process in live.json. */ +export const LIVE_MAX_SAMPLES = 300; + +export const liveSoakSchema = z.strictObject({ + ...baseFields, + kind: z.literal("soak"), + cps: positiveNumber, + warmupS: nonNegativeNumber, + durationS: positiveNumber, + /** Seconds since the load started, while running or done. */ + elapsedS: nonNegativeNumber, + thresholds: z.strictObject({ + memoryGrowthMBPerHour: nonNegativeNumber, + failedCallsPct: nonNegativeNumber + }), + /** Calls the uac started and failed so far (SIPp's counters). */ + calls: z.strictObject({ placed: count, failed: count }), + memory: z.array( + z.strictObject({ + name: nonEmptyString, + samples: z + .array(liveSampleSchema) + .max(LIVE_MAX_SAMPLES, { + error: `expected at most ${LIVE_MAX_SAMPLES} samples` + }), + /** + * The least-squares growth after the warm-up, only once there is a + * full hour of data after it; null before. + */ + growthMBPerHour: z.number().nullable() + }) + ) +}); + +export const liveBenchmarkSchema = z.discriminatedUnion("kind", [ + liveCapacitySchema, + liveSoakSchema +]); + +export const liveSchema = z + .strictObject({ + schema: z.literal(LIVE_SCHEMA_ID), + runId: runIdSchema, + version: nonEmptyString, + commit: commitSchema, + startedAt: timestampSchema, + updatedAt: timestampSchema, + status: z.enum(["running", "finished", "failed"]), + /** Why the agent stopped the run, when status is failed; else null. */ + reason: z.string().nullable(), + /** When the run is expected to end; null once it has. */ + estimatedEndAt: timestampSchema.nullable(), + provider: z.enum(["digitalocean", "local"]), + project: reportProjectSchema, + benchmarks: z + .array(liveBenchmarkSchema) + .min(1, { error: "expected at least one benchmark" }) + }) + .meta({ + title: "live.json v1", + description: + "A preliminary snapshot of a perfkit run in progress (schema: perfkit/live/v1), rewritten every 15 seconds." + }); + +export type Live = z.infer; +export type LiveBenchmark = z.infer; +export type LiveCapacity = z.infer; +export type LiveSoak = z.infer; +export type LivePoint = z.infer; +export type LiveBenchmarkState = z.infer; diff --git a/src/schemas/perfkitConfig.schema.ts b/src/schemas/perfkitConfig.schema.ts index 2f0f5d6..ecbc80b 100644 --- a/src/schemas/perfkitConfig.schema.ts +++ b/src/schemas/perfkitConfig.schema.ts @@ -9,6 +9,7 @@ */ import { z } from "zod/v4"; import { checkUniqueIds } from "./checkUniqueIds.js"; +import { projectLiveSchema } from "./report.schema.js"; import { checkWarmupShorterThanStep } from "./checkWarmupShorterThanStep.js"; import { benchmarkIdSchema, @@ -40,7 +41,30 @@ const configProjectSchema = z.strictObject({ }) .optional(), title: nonEmptyString, - lead: nonEmptyString + lead: nonEmptyString, + live: projectLiveSchema.optional() +}); + +/** + * The agent's live.json server: its port on the load generator, the static + * ngrok domain its tunnel uses when NGROK_AUTHTOKEN is set, and the URL the + * benchmarks page polls (`project.live.url` is the same and wins). + */ +const configLiveSchema = z.strictObject({ + domain: z + .string() + .regex(/^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/, { + error: "expected a hostname like perfkit.ngrok.app" + }) + .optional(), + port: z + .number() + .int({ error: "expected a whole number" }) + .min(1, { error: "expected a port from 1 to 65535" }) + .max(65535, { error: "expected a port from 1 to 65535" }) + .default(8787), + /** Where the page polls live.json; default https:///live.json. */ + url: siteUrlSchema.optional() }); const machineSizeSchema = z.strictObject({ @@ -231,6 +255,7 @@ export const perfkitConfigSchema = z apiVersion: z.literal(SCHEMA_IDS.config), project: configProjectSchema, infra: configInfraSchema, + live: configLiveSchema.optional(), images: configImagesSchema, benchmarks: z .array(benchmarkConfigSchema) diff --git a/src/schemas/report.schema.ts b/src/schemas/report.schema.ts index a445c12..2b47acf 100644 --- a/src/schemas/report.schema.ts +++ b/src/schemas/report.schema.ts @@ -32,7 +32,12 @@ export const reasonSchema = z.string().regex(/^[a-z](?:.*[^.])?$/s, { error: "expected a lowercase clause with no final period" }); -const reportProjectSchema = z.strictObject({ +/** Where the benchmarks page polls a run in progress: its live.json URL. */ +export const projectLiveSchema = z.strictObject({ + url: siteUrlSchema +}); + +export const reportProjectSchema = z.strictObject({ name: nonEmptyString, repo: repoSchema, siteUrl: siteUrlSchema, @@ -42,7 +47,8 @@ const reportProjectSchema = z.strictObject({ .regex(/]/, { error: "expected inline SVG markup" }) .optional(), title: nonEmptyString, - lead: nonEmptyString + lead: nonEmptyString, + live: projectLiveSchema.optional() }); const machineSchema = z.strictObject({ diff --git a/src/schemas/toJsonSchemaFiles.ts b/src/schemas/toJsonSchemaFiles.ts index 0ba5efd..0cdff40 100644 --- a/src/schemas/toJsonSchemaFiles.ts +++ b/src/schemas/toJsonSchemaFiles.ts @@ -2,6 +2,7 @@ * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. */ import { z } from "zod/v4"; +import { liveSchema } from "./live.schema.js"; import { perfkitConfigSchema } from "./perfkitConfig.schema.js"; import { reportSchema } from "./report.schema.js"; import { runsSchema } from "./runs.schema.js"; @@ -18,7 +19,8 @@ export function toJsonSchemaFiles(): Record { io: "input" }), "report-v1.schema.json": z.toJSONSchema(reportSchema), - "runs-v1.schema.json": z.toJSONSchema(runsSchema) + "runs-v1.schema.json": z.toJSONSchema(runsSchema), + "live-v1.schema.json": z.toJSONSchema(liveSchema) }; return Object.fromEntries( Object.entries(files).map(([name, schema]) => [ diff --git a/src/serve/contentTypeOf.ts b/src/serve/contentTypeOf.ts new file mode 100644 index 0000000..a8d8857 --- /dev/null +++ b/src/serve/contentTypeOf.ts @@ -0,0 +1,17 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +const TYPES: Record = { + html: "text/html; charset=utf-8", + js: "text/javascript; charset=utf-8", + json: "application/json; charset=utf-8", + css: "text/css; charset=utf-8", + svg: "image/svg+xml", + txt: "text/plain; charset=utf-8" +}; + +/** The Content-Type of a site file by its extension. */ +export function contentTypeOf(path: string): string { + const ext = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase() ?? ""; + return TYPES[ext] ?? "application/octet-stream"; +} diff --git a/src/serve/createNodeServeSite.ts b/src/serve/createNodeServeSite.ts new file mode 100644 index 0000000..bf0e60c --- /dev/null +++ b/src/serve/createNodeServeSite.ts @@ -0,0 +1,19 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { createServeSite } from "./createServeSite.js"; + +/** `createServeSite` over a directory on this machine. */ +export function createNodeServeSite(siteDir: string) { + const at = (path: string) => join(siteDir, ...path.split("/")); + return createServeSite({ + readSiteFile: async (path) => { + const stats = await stat(at(path)).catch(() => null); + return stats?.isFile() ? readFile(at(path), "utf8") : null; + }, + isSiteDir: async (path) => + (await stat(at(path)).catch(() => null))?.isDirectory() ?? false + }); +} diff --git a/src/serve/createServeSite.ts b/src/serve/createServeSite.ts new file mode 100644 index 0000000..79aeabf --- /dev/null +++ b/src/serve/createServeSite.ts @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { posix } from "node:path"; +import { withErrorHandlingAndValidation } from "../errors/index.js"; +import { + type LiveRequest, + liveRequestSchema, + type LiveResponse +} from "../live/handleLiveRequest.schema.js"; +import { contentTypeOf } from "./contentTypeOf.js"; + +/** + * Creates the request handler of `perfkit serve`: a read-only static + * server for a rendered benchmarks site, as GitHub Pages serves it. A path + * that ends in `/` serves its `index.html`; a directory without the + * slash redirects to it; anything outside the site, missing, or not GET or + * HEAD is 404. Nothing is cached, so a re-render shows at once. + * + * @param deps - Reads a site file by its path relative to the site root; + * null when it is missing or a directory (injected) + */ +export function createServeSite(deps: { + readSiteFile: (path: string) => Promise; + isSiteDir: (path: string) => Promise; +}) { + const fn = async (request: LiveRequest): Promise => { + const notFound = { + status: 404, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + body: "not found\n" + }; + if (request.method !== "GET" && request.method !== "HEAD") return notFound; + let pathname: string; + try { + pathname = decodeURIComponent( + new URL(request.url, "http://site").pathname + ); + } catch { + return notFound; + } + const normal = posix.normalize(pathname); + if (!normal.startsWith("/") || normal.split("/").includes("..")) + return notFound; + const relative = normal.slice(1); + if ( + relative !== "" && + !relative.endsWith("/") && + (await deps.isSiteDir(relative)) + ) { + return { status: 301, headers: { Location: `${pathname}/` }, body: "" }; + } + const file = + relative === "" || relative.endsWith("/") + ? `${relative}index.html` + : relative; + const text = await deps.readSiteFile(file).catch(() => null); + if (text === null) return notFound; + return { + status: 200, + headers: { + "Content-Type": contentTypeOf(file), + "Cache-Control": "no-cache" + }, + body: request.method === "HEAD" ? "" : text + }; + }; + return withErrorHandlingAndValidation(fn, liveRequestSchema); +} diff --git a/test/analyze/memoryGrowthMBPerHour.unit.test.ts b/test/analyze/memoryGrowthMBPerHour.unit.test.ts new file mode 100644 index 0000000..f3535a5 --- /dev/null +++ b/test/analyze/memoryGrowthMBPerHour.unit.test.ts @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { memoryGrowthMBPerHour } from "../../src/analyze/index.js"; + +describe("memoryGrowthMBPerHour", () => { + it("should give one process's least-squares MB/h after the warm-up", () => { + // Arrange: 1 MB per hour after t=1000, a spike before it + const samples = [ + { name: "one", tMs: 0, rssKB: 999_999 }, + { name: "one", tMs: 1000, rssKB: 1024 }, + { name: "one", tMs: 1000 + 3_600_000, rssKB: 2048 }, + { name: "two", tMs: 2000, rssKB: 9 } + ]; + + // Act & Assert + expect(memoryGrowthMBPerHour(samples, "one", 1000)).to.equal(1); + expect(memoryGrowthMBPerHour(samples, "two", 1000)).to.equal(0); + }); +}); diff --git a/test/cli/serve.unit.test.ts b/test/cli/serve.unit.test.ts new file mode 100644 index 0000000..3de7945 --- /dev/null +++ b/test/cli/serve.unit.test.ts @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { runCli } from "../../src/cli/runCli.js"; +import type { CliDeps } from "../../src/cli/types.js"; +import { infraDeps } from "../infra/fakes.js"; + +async function serve(args: string[], serveSite?: CliDeps["serveSite"]) { + const out: string[] = []; + const err: string[] = []; + const code = await runCli(["node", "perfkit", "serve", ...args], { + version: "9.8.7", + appBundle: async () => "", + stdout: (line) => out.push(line), + stderr: (line) => err.push(line), + readFile: async () => "", + isFile: async () => true, + infra: infraDeps(), + writeFile: async () => {}, + fetchText: async () => ({ status: 404, text: "" }), + ...(serveSite ? { serveSite } : {}) + }); + return { code, out: out.join(""), err: err.join("") }; +} + +describe("perfkit serve", () => { + it("should serve the site on the port asked for until it is stopped", async () => { + // Arrange + const calls: unknown[] = []; + + // Act + const result = await serve( + ["--site", "out/benchmarks", "--port", "8095"], + async (input) => { + calls.push(input); + return { port: input.port, stopped: Promise.resolve() }; + } + ); + + // Assert + expect(calls).to.deep.equal([ + { site: "out/benchmarks", port: 8095, host: "127.0.0.1" } + ]); + expect(result).to.deep.equal({ + code: 0, + out: "serving out/benchmarks at http://127.0.0.1:8095/ (Ctrl-C stops)\n", + err: "" + }); + }); + + it("should default to port 8080", async () => { + let port = 0; + await serve(["--site", "s"], async (input) => { + port = input.port; + return { port: input.port, stopped: Promise.resolve() }; + }); + expect(port).to.equal(8080); + }); + + it("should exit 1 for a bad port and 2 when it can't listen", async () => { + const bad = await serve(["--site", "s", "--port", "x"], async () => { + throw new Error("never"); + }); + const taken = await serve(["--site", "s"], async () => { + throw new Error( + "listen EADDRINUSE: address already in use 127.0.0.1:8080" + ); + }); + expect(bad.code).to.equal(1); + expect(bad.err).to.equal( + "perfkit: --port: expected a port from 0 to 65535\n" + ); + expect(taken.code).to.equal(2); + expect(taken.err).to.equal( + "perfkit: cannot serve s: listen EADDRINUSE: address already in use 127.0.0.1:8080\n" + ); + }); + + it("should exit 1 without --site", async () => { + const result = await serve([]); + expect(result.code).to.equal(1); + }); +}); diff --git a/test/live/buildLiveCapacity.unit.test.ts b/test/live/buildLiveCapacity.unit.test.ts new file mode 100644 index 0000000..cda0b82 --- /dev/null +++ b/test/live/buildLiveCapacity.unit.test.ts @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { buildLiveCapacity } from "../../src/live/buildLiveCapacity.js"; +import { CAPACITY, MIN, point, record, STARTED_MS } from "./state.js"; + +const at = (minutes: number) => STARTED_MS + minutes * MIN; + +describe("buildLiveCapacity", () => { + it("should be queued with its estimated start before it starts", () => { + const live = buildLiveCapacity({ + benchmark: CAPACITY, + record: undefined, + selected: true, + nowMs: at(1), + estimatedStartMs: at(5) + }); + expect(live).to.deep.include({ + state: "queued", + current: null, + heldCps: null, + result: null, + estimatedStartAt: new Date(at(5)).toISOString(), + plannedS: 100 * 60 + }); + }); + + it("should give the step offered now from the clock and what held so far", () => { + // Arrange + const rec = record({ + [CAPACITY.id]: { + startedMs: at(0), + loadStartedMs: at(1), + points: [point(50), point(100)] + } + }).benchmarks[CAPACITY.id]; + + // Act: 2.5 steps of 60 s after the load started + const live = buildLiveCapacity({ + benchmark: CAPACITY, + record: rec, + selected: true, + nowMs: at(3.5), + estimatedStartMs: null + }); + + // Assert + expect(live.state).to.equal("running"); + expect(live.current).to.deep.equal({ + index: 2, + total: 100, + offeredCps: 150 + }); + expect(live.heldCps).to.equal(100); + expect(live.points).to.have.length(2); + expect(live.rate).to.deep.equal({ + start: 50, + step: 50, + everyS: 60, + max: 5000 + }); + }); + + it("should stay on the step that broke a rule, and end there once done", () => { + // Arrange + const points = [point(50), point(100), point(150, false)]; + const running = { + startedMs: at(0), + loadStartedMs: at(1), + endedMs: null, + points + }; + + // Act + const live = buildLiveCapacity({ + benchmark: CAPACITY, + record: running, + selected: true, + nowMs: at(30), + estimatedStartMs: null + }); + const done = buildLiveCapacity({ + benchmark: CAPACITY, + record: { ...running, endedMs: at(5) }, + selected: true, + nowMs: at(30), + estimatedStartMs: null + }); + + // Assert + expect(live.current?.offeredCps).to.equal(150); + expect(done.state).to.equal("done"); + expect(done.current).to.equal(null); + expect(done.result).to.deep.equal({ + limitCps: 100, + lowerBound: false, + stopReason: "over the p99 limit" + }); + }); + + it("should call a ramp that never broke a rule a lower bound", () => { + const live = buildLiveCapacity({ + benchmark: CAPACITY, + record: { + startedMs: 1, + loadStartedMs: 2, + endedMs: 3, + points: [point(50)] + }, + selected: true, + nowMs: 4, + estimatedStartMs: null + }); + expect(live.result).to.deep.equal({ + limitCps: 50, + lowerBound: true, + stopReason: null + }); + }); + + it("should be not-run when not selected", () => { + const live = buildLiveCapacity({ + benchmark: CAPACITY, + record: undefined, + selected: false, + nowMs: 0, + estimatedStartMs: 5 + }); + expect(live.state).to.equal("not-run"); + expect(live.estimatedStartAt).to.equal(null); + }); +}); diff --git a/test/live/buildLiveSnapshot.unit.test.ts b/test/live/buildLiveSnapshot.unit.test.ts new file mode 100644 index 0000000..bf613e5 --- /dev/null +++ b/test/live/buildLiveSnapshot.unit.test.ts @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { buildLiveSnapshot } from "../../src/live/buildLiveSnapshot.js"; +import { LIVE_SCHEMA_ID, liveSchema } from "../../src/schemas/index.js"; +import { + CAPACITY, + MIN, + point, + record, + RUN_ID, + runState, + SOAK, + STARTED_MS +} from "./state.js"; + +describe("buildLiveSnapshot", () => { + it("should build a valid perfkit/live/v1 snapshot of a run in progress", () => { + // Arrange + const rec = record({ + [CAPACITY.id]: { + startedMs: STARTED_MS + MIN, + loadStartedMs: STARTED_MS + 2 * MIN, + points: [point(50)] + } + }); + + // Act + const live = buildLiveSnapshot({ + state: runState(), + record: rec, + nowMs: STARTED_MS + 4 * MIN, + soaks: new Map() + }); + + // Assert + expect(liveSchema.safeParse(live).success).to.equal(true); + expect(live).to.deep.include({ + schema: LIVE_SCHEMA_ID, + runId: RUN_ID, + version: "2.32.2", + status: "running", + reason: null, + provider: "digitalocean", + updatedAt: new Date(STARTED_MS + 4 * MIN).toISOString() + }); + expect(live.project.name).to.equal("Routr"); + expect(live.benchmarks.map((b) => [b.id, b.state])).to.deep.equal([ + [CAPACITY.id, "running"], + [SOAK.id, "queued"] + ]); + expect(live.estimatedEndAt).to.be.a("string"); + expect(live.benchmarks[1]?.estimatedStartAt).to.be.a("string"); + }); + + it("should list a benchmark not selected as not-run, and have no end once finished", () => { + const live = buildLiveSnapshot({ + state: runState({ selected: [SOAK.id], mode: "local" }), + record: record({}, "finished"), + nowMs: STARTED_MS, + soaks: new Map() + }); + expect(live.benchmarks[0]?.state).to.equal("not-run"); + expect(live.estimatedEndAt).to.equal(null); + expect(live.provider).to.equal("local"); + expect(live.status).to.equal("finished"); + }); + + it("should carry why a failed run stopped", () => { + const live = buildLiveSnapshot({ + state: runState(), + record: record({}, "failed"), + nowMs: STARTED_MS, + soaks: new Map() + }); + expect(live.status).to.equal("failed"); + expect(live.reason).to.equal("the agent failed: boom"); + }); +}); diff --git a/test/live/buildLiveSoak.unit.test.ts b/test/live/buildLiveSoak.unit.test.ts new file mode 100644 index 0000000..2decc10 --- /dev/null +++ b/test/live/buildLiveSoak.unit.test.ts @@ -0,0 +1,114 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { buildLiveSoak } from "../../src/live/buildLiveSoak.js"; +import { MIN, SOAK, STARTED_MS } from "./state.js"; + +const samples = ( + name: string, + fromMs: number, + minutes: number, + mbPerHour: number +) => + Array.from({ length: minutes * 6 + 1 }, (_, i) => ({ + name, + tMs: fromMs + i * 10_000, + rssKB: (100 + (mbPerHour * i * 10) / 3600) * 1024 + })); + +describe("buildLiveSoak", () => { + const loadStartedMs = STARTED_MS; + + it("should show raw memory and no growth before an hour after the warm-up", () => { + // Arrange: 30 min warm-up + 40 min + const progress = { + samples: samples("one", loadStartedMs, 70, 2), + placed: 420_000, + failed: 3 + }; + + // Act + const live = buildLiveSoak({ + benchmark: SOAK, + record: { + startedMs: loadStartedMs - MIN, + loadStartedMs, + endedMs: null, + points: [] + }, + selected: true, + nowMs: loadStartedMs + 70 * MIN, + estimatedStartMs: null, + progress + }); + + // Assert + expect(live).to.deep.include({ + state: "running", + warmupS: 1800, + durationS: 21600, + plannedS: 23400, + elapsedS: 4200, + calls: { placed: 420_000, failed: 3 } + }); + const one = live.memory.find((p) => p.name === "one"); + expect(one?.growthMBPerHour).to.equal(null); + expect(one?.samples.length).to.be.at.most(300); + expect(one?.samples[0]).to.deep.equal({ + t: new Date(loadStartedMs).toISOString(), + rssMB: 100 + }); + expect(live.memory.map((p) => p.name)).to.deep.equal([ + "one", + "edgeport", + "postgres" + ]); + }); + + it("should give the verdict's growth once a full hour after the warm-up is in", () => { + // Arrange: 30 min warm-up + 61 min + const progress = { + samples: samples("one", loadStartedMs, 91, 2), + placed: 1, + failed: 0 + }; + + // Act + const live = buildLiveSoak({ + benchmark: SOAK, + record: { + startedMs: loadStartedMs, + loadStartedMs, + endedMs: null, + points: [] + }, + selected: true, + nowMs: loadStartedMs + 91 * MIN, + estimatedStartMs: null, + progress + }); + + // Assert + expect(live.memory[0]?.growthMBPerHour).to.be.closeTo(2, 0.01); + expect(live.memory[1]?.growthMBPerHour).to.equal(null); + }); + + it("should be queued with nothing measured before it starts", () => { + const live = buildLiveSoak({ + benchmark: SOAK, + record: undefined, + selected: true, + nowMs: loadStartedMs, + estimatedStartMs: loadStartedMs + MIN, + progress: null + }); + expect(live).to.deep.include({ + state: "queued", + elapsedS: 0, + calls: { placed: 0, failed: 0 }, + loadStartedAt: null + }); + expect(live.memory.every((p) => p.samples.length === 0)).to.equal(true); + }); +}); diff --git a/test/live/createHandleLiveRequest.unit.test.ts b/test/live/createHandleLiveRequest.unit.test.ts new file mode 100644 index 0000000..a4646ac --- /dev/null +++ b/test/live/createHandleLiveRequest.unit.test.ts @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { ValidationError } from "../../src/errors/index.js"; +import { createHandleLiveRequest } from "../../src/live/createHandleLiveRequest.js"; + +const SITE = "https://fonoster.github.io/routr/benchmarks"; + +const handlerWith = (live: string | null) => + createHandleLiveRequest({ readLive: async () => live, siteUrl: SITE }); + +describe("createHandleLiveRequest", () => { + it("should answer GET /live.json with the snapshot, never cached, with CORS for the site", async () => { + // Act + const response = await handlerWith('{"a":1}')({ + method: "GET", + url: "/live.json?t=1", + origin: "https://fonoster.github.io" + }); + + // Assert + expect(response.status).to.equal(200); + expect(response.body).to.equal('{"a":1}'); + expect(response.headers).to.deep.include({ + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + "Access-Control-Allow-Origin": "https://fonoster.github.io", + "Access-Control-Allow-Headers": "ngrok-skip-browser-warning" + }); + }); + + it("should answer the preflight with 204 and the allowed header", async () => { + const response = await handlerWith(null)({ + method: "OPTIONS", + url: "/live.json", + origin: "http://localhost:8080" + }); + expect(response.status).to.equal(204); + expect(response.headers).to.deep.include({ + "Access-Control-Allow-Origin": "http://localhost:8080", + "Access-Control-Allow-Methods": "GET, OPTIONS", + "Access-Control-Allow-Headers": "ngrok-skip-browser-warning" + }); + }); + + it("should send no CORS headers to another origin", async () => { + const response = await handlerWith("{}")({ + method: "GET", + url: "/live.json", + origin: "https://evil.example" + }); + expect(response.status).to.equal(200); + expect(response.headers).to.not.have.property( + "Access-Control-Allow-Origin" + ); + }); + + it("should answer 503 before the first snapshot", async () => { + const response = await handlerWith(null)({ + method: "GET", + url: "/live.json", + origin: null + }); + expect(response.status).to.equal(503); + expect(response.headers["Cache-Control"]).to.equal("no-store"); + }); + + it("should answer 404 for any other path or method", async () => { + const handle = handlerWith("{}"); + const statuses = await Promise.all([ + handle({ method: "GET", url: "/", origin: null }), + handle({ method: "GET", url: "/status.json", origin: null }), + handle({ method: "POST", url: "/live.json", origin: null }) + ]); + expect(statuses.map((r) => r.status)).to.deep.equal([404, 404, 404]); + }); + + it("should throw ValidationError for a request without a method", async () => { + let error: unknown; + await handlerWith("{}")({ url: "/live.json", origin: null }).catch( + (err: unknown) => { + error = err; + } + ); + expect(error).to.be.instanceOf(ValidationError); + }); +}); diff --git a/test/live/createLiveRecorder.unit.test.ts b/test/live/createLiveRecorder.unit.test.ts new file mode 100644 index 0000000..e75b07a --- /dev/null +++ b/test/live/createLiveRecorder.unit.test.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { createLiveRecorder } from "../../src/live/createLiveRecorder.js"; +import { point } from "./state.js"; + +describe("createLiveRecorder", () => { + it("should record each benchmark's times and points, and the run's end", () => { + // Arrange + const recorder = createLiveRecorder(); + + // Act + recorder.started("a", 1); + recorder.loadStarted("a", 2); + recorder.point("a", point(50)); + recorder.done("a", 3); + recorder.finished(); + + // Assert + expect(recorder.read()).to.deep.equal({ + status: "finished", + reason: null, + benchmarks: { + a: { startedMs: 1, loadStartedMs: 2, endedMs: 3, points: [point(50)] } + } + }); + }); + + it("should record why the run failed", () => { + const recorder = createLiveRecorder(); + recorder.failed("the agent failed: boom"); + expect(recorder.read()).to.include({ + status: "failed", + reason: "the agent failed: boom" + }); + }); + + it("should hand out copies that later notes don't change", () => { + const recorder = createLiveRecorder(); + recorder.started("a", 1); + const before = recorder.read(); + recorder.point("a", point(50)); + expect(before.benchmarks.a?.points).to.deep.equal([]); + }); +}); diff --git a/test/live/createNodeServeLive.unit.test.ts b/test/live/createNodeServeLive.unit.test.ts new file mode 100644 index 0000000..d0f8330 --- /dev/null +++ b/test/live/createNodeServeLive.unit.test.ts @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { request } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import nock from "nock"; +import { createNodeServeLive } from "../../src/live/createNodeServeLive.js"; + +function get( + url: string, + method = "GET", + headers: Record = {} +) { + return new Promise<{ + status: number; + body: string; + headers: Record; + }>((resolve, reject) => { + const req = request(url, { method, headers }, (res) => { + let body = ""; + res.on("data", (chunk) => (body += chunk)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body, headers: res.headers }) + ); + }); + req.on("error", reject); + req.end(); + }); +} + +describe("createNodeServeLive", () => { + before(() => nock.enableNetConnect("127.0.0.1")); + after(() => nock.disableNetConnect()); + + it("should serve a live.json file on this machine, 503 before it exists, and log where", async () => { + // Arrange + const dir = await mkdtemp(join(tmpdir(), "perfkit-live-")); + const file = join(dir, "live.json"); + const log: string[] = []; + const serveLive = createNodeServeLive({ + file, + siteUrl: "https://fonoster.github.io/routr/benchmarks", + log: async (line) => { + log.push(line); + } + }); + + // Act + const service = await serveLive({ + siteUrl: "https://fonoster.github.io/routr/benchmarks", + port: 0, + host: "127.0.0.1", + token: null, + domain: null + }); + const port = /:(\d+)\/live\.json$/.exec(log[0] ?? "")?.[1]; + const before = await get(`http://127.0.0.1:${port}/live.json`); + await writeFile(file, '{"schema":"perfkit/live/v1"}'); + const after = await get(`http://127.0.0.1:${port}/live.json`, "GET", { + Origin: "https://fonoster.github.io" + }); + const preflight = await get( + `http://127.0.0.1:${port}/live.json`, + "OPTIONS", + { + Origin: "http://localhost:8080" + } + ); + await service.close(); + + // Assert + expect(before.status).to.equal(503); + expect(after.status).to.equal(200); + expect(after.body).to.equal('{"schema":"perfkit/live/v1"}'); + expect(after.headers["access-control-allow-origin"]).to.equal( + "https://fonoster.github.io" + ); + expect(preflight.status).to.equal(204); + expect(preflight.headers["access-control-allow-headers"]).to.equal( + "ngrok-skip-browser-warning" + ); + }); +}); diff --git a/test/live/createServeLive.unit.test.ts b/test/live/createServeLive.unit.test.ts new file mode 100644 index 0000000..8c76848 --- /dev/null +++ b/test/live/createServeLive.unit.test.ts @@ -0,0 +1,101 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { ValidationError } from "../../src/errors/index.js"; +import { createServeLive } from "../../src/live/createServeLive.js"; + +function setup(options: { listenFails?: boolean; tunnelFails?: boolean } = {}) { + const log: string[] = []; + const closed: string[] = []; + const tunnels: unknown[] = []; + const serveLive = createServeLive({ + handle: async () => ({ status: 200, headers: {}, body: "" }), + listen: async (_handle, { port }) => { + if (options.listenFails) throw new Error("EADDRINUSE"); + return { + port, + close: async () => { + closed.push("server"); + } + }; + }, + startTunnel: async (input) => { + tunnels.push(input); + if (options.tunnelFails) throw new Error(`bad token ${input.token}`); + return { + url: "https://routr-benchmarks.ngrok-free.dev", + close: async () => { + closed.push("tunnel"); + } + }; + }, + log: async (line) => { + log.push(line); + } + }); + return { serveLive, log, closed, tunnels }; +} + +const INPUT = { + siteUrl: "https://fonoster.github.io/routr/benchmarks", + port: 8787, + host: "127.0.0.1", + token: "secret-token", + domain: "routr-benchmarks.ngrok-free.dev" +}; + +describe("createServeLive", () => { + it("should serve live.json, open the tunnel and log its public URL, never the token", async () => { + // Arrange + const { serveLive, log, closed, tunnels } = setup(); + + // Act + const service = await serveLive(INPUT); + await service.close(); + + // Assert + expect(tunnels).to.deep.equal([ + { + token: "secret-token", + domain: "routr-benchmarks.ngrok-free.dev", + port: 8787 + } + ]); + expect(log).to.deep.equal([ + "perfkit: live.json at http://127.0.0.1:8787/live.json", + "perfkit: live.json at https://routr-benchmarks.ngrok-free.dev/live.json" + ]); + expect(closed).to.deep.equal(["tunnel", "server"]); + }); + + it("should open no tunnel without a token", async () => { + const { serveLive, tunnels } = setup(); + await serveLive({ ...INPUT, token: null }); + expect(tunnels).to.deep.equal([]); + }); + + it("should go on without a tunnel when it fails, and keep the token out of the log", async () => { + const { serveLive, log } = setup({ tunnelFails: true }); + await serveLive(INPUT); + expect(log[1]).to.equal("perfkit: no live tunnel: bad token ***"); + }); + + it("should go on when the port is taken", async () => { + const { serveLive, log } = setup({ listenFails: true }); + const service = await serveLive(INPUT); + await service.close(); + expect(log).to.deep.equal([ + "perfkit: live.json not served on 127.0.0.1:8787: EADDRINUSE" + ]); + }); + + it("should throw ValidationError for a bad port", async () => { + const { serveLive } = setup(); + let error: unknown; + await serveLive({ ...INPUT, port: 70000 }).catch((err: unknown) => { + error = err; + }); + expect(error).to.be.instanceOf(ValidationError); + }); +}); diff --git a/test/live/createStartLiveTunnel.unit.test.ts b/test/live/createStartLiveTunnel.unit.test.ts new file mode 100644 index 0000000..9f2c498 --- /dev/null +++ b/test/live/createStartLiveTunnel.unit.test.ts @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { ValidationError } from "../../src/errors/index.js"; +import { createStartLiveTunnel } from "../../src/live/createStartLiveTunnel.js"; + +describe("createStartLiveTunnel", () => { + it("should forward the static domain to the local server with the token", async () => { + // Arrange + const calls: unknown[] = []; + let closed = false; + const start = createStartLiveTunnel({ + forward: async (config) => { + calls.push(config); + return { + url: () => "https://routr-benchmarks.ngrok-free.dev", + close: async () => { + closed = true; + } + }; + } + }); + + // Act + const tunnel = await start({ + token: "tok", + domain: "routr-benchmarks.ngrok-free.dev", + port: 8787 + }); + await tunnel.close(); + + // Assert + expect(calls).to.deep.equal([ + { + addr: "127.0.0.1:8787", + authtoken: "tok", + domain: "routr-benchmarks.ngrok-free.dev" + } + ]); + expect(tunnel.url).to.equal("https://routr-benchmarks.ngrok-free.dev"); + expect(closed).to.equal(true); + }); + + it("should ask for no domain without one", async () => { + const calls: unknown[] = []; + const start = createStartLiveTunnel({ + forward: async (config) => { + calls.push(config); + return { url: () => "https://random.ngrok.app", close: async () => {} }; + } + }); + await start({ token: "tok", domain: null, port: 1 }); + expect(calls).to.deep.equal([{ addr: "127.0.0.1:1", authtoken: "tok" }]); + }); + + it("should throw ValidationError without a token", async () => { + const start = createStartLiveTunnel({ + forward: async () => ({ url: () => null, close: async () => {} }) + }); + let error: unknown; + await start({ token: "", domain: null, port: 1 }).catch((err: unknown) => { + error = err; + }); + expect(error).to.be.instanceOf(ValidationError); + }); +}); diff --git a/test/live/createWriteLive.unit.test.ts b/test/live/createWriteLive.unit.test.ts new file mode 100644 index 0000000..bdd081d --- /dev/null +++ b/test/live/createWriteLive.unit.test.ts @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseStatCsv, sumStatRows } from "../../src/analyze/index.js"; +import { ValidationError } from "../../src/errors/index.js"; +import { createLiveRecorder } from "../../src/live/createLiveRecorder.js"; +import { createWriteLive } from "../../src/live/createWriteLive.js"; +import type { HostShell } from "../../src/orchestrate/types.js"; +import { liveSchema } from "../../src/schemas/index.js"; +import { ROOT } from "../fixtures.js"; +import { fakeFs } from "./fakeFs.js"; +import { runState, SOAK, STARTED_MS } from "./state.js"; + +const RAW = `/opt/perfkit/raw/${SOAK.id}`; +const STAT = readFileSync( + join(ROOT, "test/fixtures/sipp-edgeport-echo/uac_invite_echo_1_.csv"), + "utf8" +); +const line = (tMs: number, name: string, rssKB: number) => + `${JSON.stringify({ t: new Date(tMs).toISOString().replace(/\.\d{3}Z$/, "Z"), host: "uut", name, pid: 7, rssKB, cpuTicks: 1, clkTck: 100, fds: 3 })}\n`; + +function setup(mode: "local" | "digitalocean", files: Record) { + const fs = fakeFs(files); + const commands: string[] = []; + const chunks: string[] = []; + const shell: HostShell = async (_host, command) => { + commands.push(command); + return { code: 0, stdout: chunks.shift() ?? "", stderr: "" }; + }; + const recorder = createLiveRecorder(); + recorder.started(SOAK.id, STARTED_MS); + recorder.loadStarted(SOAK.id, STARTED_MS); + const writeLive = createWriteLive({ + fs, + shell, + now: () => STARTED_MS + 60_000, + state: runState({ mode, selected: [SOAK.id] }), + recorder + }); + return { fs, commands, chunks, writeLive, recorder }; +} + +describe("createWriteLive", () => { + it("should write a valid live.json atomically with a soak's memory and calls (--local)", async () => { + // Arrange + const { fs, writeLive } = setup("local", { + [`${RAW}/sampler-uut.jsonl`]: + line(STARTED_MS, "one", 102400) + + line(STARTED_MS + 10_000, "one", 103424) + + '{"t":"half a line', + [`${RAW}/uac_invite_42_.csv`]: STAT + }); + + // Act + const live = await writeLive({}); + + // Assert + const written = JSON.parse(fs.files.get("/opt/perfkit/live.json") ?? "{}"); + expect(liveSchema.safeParse(written).success).to.equal(true); + expect(fs.files.has("/opt/perfkit/live.json.tmp")).to.equal(false); + const soak = live.benchmarks.find((b) => b.id === SOAK.id); + expect(soak?.kind === "soak" && soak.memory[0]?.samples).to.deep.equal([ + { t: new Date(STARTED_MS).toISOString(), rssMB: 100 }, + { t: new Date(STARTED_MS + 10_000).toISOString(), rssMB: 101 } + ]); + const sums = sumStatRows(parseStatCsv(STAT, "stat").rows, { + fromMs: 0, + toMs: Infinity + }); + expect(soak?.kind === "soak" && soak.calls).to.deep.equal({ + placed: sums.attempted, + failed: sums.failed + }); + }); + + it("should read the UUT's sampler over SSH, only the bytes added since the last read", async () => { + // Arrange + const { commands, chunks, writeLive } = setup("digitalocean", {}); + const first = line(STARTED_MS, "one", 102400); + chunks.push(first, line(STARTED_MS + 10_000, "one", 104448)); + + // Act + await writeLive({}); + const live = await writeLive({}); + + // Assert + expect(commands[0]).to.equal( + `tail -c +1 ${RAW}/sampler-uut.jsonl 2>/dev/null; true` + ); + expect(commands[1]).to.equal( + `tail -c +${Buffer.byteLength(first) + 1} ${RAW}/sampler-uut.jsonl 2>/dev/null; true` + ); + const soak = live.benchmarks.find((b) => b.id === SOAK.id); + expect( + soak?.kind === "soak" && soak.memory[0]?.samples.map((s) => s.rssMB) + ).to.deep.equal([100, 102]); + }); + + it("should keep the last numbers when a file can't be read", async () => { + // Arrange + const { fs, writeLive } = setup("local", { + [`${RAW}/sampler-uut.jsonl`]: line(STARTED_MS, "one", 102400) + }); + await writeLive({}); + fs.files.delete(`${RAW}/sampler-uut.jsonl`); + + // Act + const live = await writeLive({}); + + // Assert + const soak = live.benchmarks.find((b) => b.id === SOAK.id); + expect(soak?.kind === "soak" && soak.memory[0]?.samples).to.have.length(1); + }); + + it("should throw ValidationError for options it doesn't take", async () => { + const { writeLive } = setup("local", {}); + let error: unknown; + await writeLive({ now: 1 }).catch((err: unknown) => { + error = err; + }); + expect(error).to.be.instanceOf(ValidationError); + }); +}); diff --git a/test/live/downsample.unit.test.ts b/test/live/downsample.unit.test.ts new file mode 100644 index 0000000..ddab7a5 --- /dev/null +++ b/test/live/downsample.unit.test.ts @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { downsample } from "../../src/live/downsample.js"; + +describe("downsample", () => { + it("should keep a short series as it is", () => { + expect(downsample([1, 2, 3], 5)).to.deep.equal([1, 2, 3]); + }); + + it("should keep at most max items, evenly spaced, first and last kept", () => { + // Arrange + const items = Array.from({ length: 1000 }, (_, i) => i); + + // Act + const result = downsample(items, 300); + + // Assert + expect(result).to.have.length(300); + expect(result[0]).to.equal(0); + expect(result.at(-1)).to.equal(999); + expect([...result].sort((a, b) => a - b)).to.deep.equal(result); + }); + + it("should keep only the last item when max is 1", () => { + expect(downsample([1, 2, 3], 1)).to.deep.equal([3]); + }); +}); diff --git a/test/live/estimateSchedule.unit.test.ts b/test/live/estimateSchedule.unit.test.ts new file mode 100644 index 0000000..46b311a --- /dev/null +++ b/test/live/estimateSchedule.unit.test.ts @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { BENCHMARK_OVERHEAD_S } from "../../src/live/constants.js"; +import { estimateSchedule } from "../../src/live/estimateSchedule.js"; +import { CAPACITY, MIN, point, record, SOAK, STARTED_MS } from "./state.js"; + +const OVERHEAD = BENCHMARK_OVERHEAD_S * 1000; + +describe("estimateSchedule", () => { + it("should queue every benchmark after the one running, with its planned time", () => { + // Arrange: the ramp's load started 10 min ago, of 100 min + const records = record({ + [CAPACITY.id]: { startedMs: STARTED_MS, loadStartedMs: STARTED_MS } + }).benchmarks; + const nowMs = STARTED_MS + 10 * MIN; + + // Act + const schedule = estimateSchedule({ + benchmarks: [CAPACITY, SOAK], + records, + nowMs + }); + + // Assert + const soakStart = nowMs + 90 * MIN + OVERHEAD / 3; + expect(schedule.startsMs.get(SOAK.id)).to.equal(soakStart); + expect(schedule.endMs).to.equal(soakStart + 390 * MIN + OVERHEAD); + }); + + it("should end a ramp that stopped soon, and skip what is done", () => { + const records = record({ + [CAPACITY.id]: { + startedMs: 1, + loadStartedMs: 1, + points: [point(50, false)] + }, + [SOAK.id]: { startedMs: 1, endedMs: 2 } + }).benchmarks; + const schedule = estimateSchedule({ + benchmarks: [CAPACITY, SOAK], + records, + nowMs: 1000 + }); + expect(schedule.endMs).to.equal(1000 + OVERHEAD / 3); + expect(schedule.startsMs.size).to.equal(0); + }); +}); diff --git a/test/live/fakeFs.ts b/test/live/fakeFs.ts new file mode 100644 index 0000000..f8d5b0c --- /dev/null +++ b/test/live/fakeFs.ts @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { AgentFs } from "../../src/orchestrate/types.js"; + +/** An in-memory AgentFs; readdir lists the files right under a directory. */ +export function fakeFs(files: Record = {}): AgentFs & { + files: Map; +} { + const map = new Map(Object.entries(files)); + const missing = (path: string) => + Object.assign(new Error(`ENOENT: ${path}`), { code: "ENOENT" }); + return { + files: map, + readFile: async (path) => { + const text = map.get(path); + if (text === undefined) throw missing(path); + return text; + }, + writeFile: async (path, text) => { + map.set(path, text); + }, + appendFile: async (path, text) => { + map.set(path, (map.get(path) ?? "") + text); + }, + readdir: async (dir) => + [...map.keys()] + .filter((path) => path.startsWith(`${dir}/`)) + .map((path) => path.slice(dir.length + 1)) + .filter((name) => !name.includes("/")), + mkdir: async () => {}, + rename: async (from, to) => { + const text = map.get(from); + if (text === undefined) throw missing(from); + map.set(to, text); + map.delete(from); + }, + exists: async (path) => map.has(path) + }; +} diff --git a/test/live/isAllowedOrigin.unit.test.ts b/test/live/isAllowedOrigin.unit.test.ts new file mode 100644 index 0000000..368d89d --- /dev/null +++ b/test/live/isAllowedOrigin.unit.test.ts @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { isAllowedOrigin } from "../../src/live/isAllowedOrigin.js"; + +const SITE = "https://fonoster.github.io/routr/benchmarks"; + +describe("isAllowedOrigin", () => { + it("should allow the project's site and pages on this machine", () => { + expect(isAllowedOrigin("https://fonoster.github.io", SITE)).to.equal(true); + expect(isAllowedOrigin("http://localhost:8080", SITE)).to.equal(true); + expect(isAllowedOrigin("http://127.0.0.1:3000", SITE)).to.equal(true); + expect(isAllowedOrigin("http://localhost", SITE)).to.equal(true); + }); + + it("should refuse any other origin", () => { + expect(isAllowedOrigin("https://evil.example", SITE)).to.equal(false); + expect(isAllowedOrigin("http://fonoster.github.io", SITE)).to.equal(false); + expect(isAllowedOrigin("http://localhost.evil.example", SITE)).to.equal( + false + ); + expect(isAllowedOrigin("https://localhost:8080", SITE)).to.equal(false); + expect(isAllowedOrigin("https://x", "not a url")).to.equal(false); + }); +}); diff --git a/test/live/listenHttp.unit.test.ts b/test/live/listenHttp.unit.test.ts new file mode 100644 index 0000000..8307f3b --- /dev/null +++ b/test/live/listenHttp.unit.test.ts @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { request } from "node:http"; +import nock from "nock"; +import { listenHttp } from "../../src/live/listenHttp.js"; + +function get(port: number, path: string, headers: Record = {}) { + return new Promise<{ + status: number; + headers: Record; + body: string; + }>((resolve, reject) => { + const req = request({ host: "127.0.0.1", port, path, headers }, (res) => { + let body = ""; + res.on("data", (chunk) => (body += chunk)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, headers: res.headers, body }) + ); + }); + req.on("error", reject); + req.end(); + }); +} + +describe("listenHttp", () => { + before(() => nock.enableNetConnect("127.0.0.1")); + after(() => nock.disableNetConnect()); + + it("should serve the handler's responses on a local port and close", async () => { + // Arrange + const server = await listenHttp( + async (req) => ({ + status: 200, + headers: { "X-Origin": req.origin ?? "none" }, + body: `${req.method} ${req.url}` + }), + { port: 0, host: "127.0.0.1" } + ); + + // Act + const response = await get(server.port, "/live.json", { + Origin: "http://localhost:1" + }); + await server.close(); + + // Assert + expect(response.status).to.equal(200); + expect(response.body).to.equal("GET /live.json"); + expect(response.headers["x-origin"]).to.equal("http://localhost:1"); + }); + + it("should answer 500 when the handler throws", async () => { + const server = await listenHttp( + async () => { + throw new Error("boom"); + }, + { port: 0, host: "127.0.0.1" } + ); + const response = await get(server.port, "/"); + await server.close(); + expect(response.status).to.equal(500); + }); + + it("should reject when the port is taken", async () => { + const first = await listenHttp( + async () => ({ status: 200, headers: {}, body: "" }), + { + port: 0, + host: "127.0.0.1" + } + ); + let error: unknown; + await listenHttp(async () => ({ status: 200, headers: {}, body: "" }), { + port: first.port, + host: "127.0.0.1" + }).catch((err: unknown) => { + error = err; + }); + await first.close(); + expect(error).to.be.instanceOf(Error); + }); +}); diff --git a/test/live/liveStateOf.unit.test.ts b/test/live/liveStateOf.unit.test.ts new file mode 100644 index 0000000..ea6e001 --- /dev/null +++ b/test/live/liveStateOf.unit.test.ts @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { liveStateOf } from "../../src/live/liveStateOf.js"; + +const rec = (startedMs: number | null, endedMs: number | null) => ({ + startedMs, + loadStartedMs: null, + endedMs, + points: [] +}); + +describe("liveStateOf", () => { + it("should tell not-run, queued, running and done apart", () => { + expect(liveStateOf(false, undefined)).to.equal("not-run"); + expect(liveStateOf(true, undefined)).to.equal("queued"); + expect(liveStateOf(true, rec(1, null))).to.equal("running"); + expect(liveStateOf(true, rec(1, 2))).to.equal("done"); + }); +}); diff --git a/test/live/ngrokBindingCommand.unit.test.ts b/test/live/ngrokBindingCommand.unit.test.ts new file mode 100644 index 0000000..b6cc6e0 --- /dev/null +++ b/test/live/ngrokBindingCommand.unit.test.ts @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { writeFile, mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ngrokBindingCommand } from "../../src/live/ngrokBindingCommand.js"; +import { ngrokVersion } from "../../src/live/ngrokVersion.js"; +import { NODE_IMAGE } from "../../src/tools.js"; + +describe("ngrokBindingCommand", () => { + it("should install the pinned @ngrok/ngrok in NODE_IMAGE and copy its scope into the agent, never failing", () => { + // Act + const command = ngrokBindingCommand({ + agentDir: "/opt/perfkit/agent", + version: "1.7.0" + }); + + // Assert + expect(command).to.contain(NODE_IMAGE); + expect(command).to.contain("@ngrok/ngrok@1.7.0"); + expect(command).to.contain("--ignore-scripts"); + expect(command).to.contain("/opt/perfkit/agent/node_modules/@ngrok/"); + expect(command.endsWith("; true")).to.equal(true); + }); +}); + +describe("ngrokVersion", () => { + it("should read the exact @ngrok/ngrok version from package.json", async () => { + const dir = await mkdtemp(join(tmpdir(), "perfkit-ngrok-")); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ dependencies: { "@ngrok/ngrok": "^1.7.0" } }) + ); + expect(await ngrokVersion(dir)).to.equal("1.7.0"); + }); +}); diff --git a/test/live/runWithLive.unit.test.ts b/test/live/runWithLive.unit.test.ts new file mode 100644 index 0000000..5cc3a5e --- /dev/null +++ b/test/live/runWithLive.unit.test.ts @@ -0,0 +1,85 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { runWithLive } from "../../src/live/runWithLive.js"; + +/** A sleep that waits for the body, or an abort, instead of time. */ +function setup() { + const events: string[] = []; + let release: () => void = () => {}; + const bodyGate = new Promise((resolve) => { + release = resolve; + }); + let writes = 0; + const deps = { + writeLive: async () => { + writes += 1; + events.push(`write ${writes}`); + if (writes === 2) throw new Error("disk full"); + }, + serve: async () => { + events.push("serve"); + return { + close: async () => { + events.push("close"); + } + }; + }, + sleep: async (ms: number, signal?: AbortSignal) => { + events.push(`sleep ${ms}`); + if (!signal) return; + if (writes >= 3) { + release(); + await new Promise((resolve) => + signal.addEventListener("abort", () => resolve()) + ); + } + }, + intervalMs: 15_000, + lingerMs: 30_000 + }; + return { deps, events, bodyGate }; +} + +describe("runWithLive", () => { + it("should write every interval, skip a failed write, write once more at the end and linger", async () => { + // Arrange + const { deps, events, bodyGate } = setup(); + + // Act + const result = await runWithLive(deps, async () => { + await bodyGate; + events.push("body done"); + return 42; + }); + + // Assert + expect(result).to.equal(42); + expect(events).to.deep.equal([ + "serve", + "write 1", + "sleep 15000", + "write 2", + "sleep 15000", + "write 3", + "sleep 15000", + "body done", + "write 4", + "sleep 30000", + "close" + ]); + }); + + it("should pass the body's error through after closing", async () => { + const { deps, events } = setup(); + let error: unknown; + await runWithLive(deps, async () => { + throw new Error("agent crashed"); + }).catch((err: unknown) => { + error = err; + }); + expect((error as Error).message).to.equal("agent crashed"); + expect(events.at(-1)).to.equal("close"); + }); +}); diff --git a/test/live/sample.ts b/test/live/sample.ts new file mode 100644 index 0000000..cb5236f --- /dev/null +++ b/test/live/sample.ts @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { buildLiveSnapshot } from "../../src/live/buildLiveSnapshot.js"; +import type { Live } from "../../src/schemas/index.js"; +import { + CAPACITY, + MIN, + point, + record, + runState, + SOAK, + STARTED_MS +} from "./state.js"; + +/** A snapshot of the Routr example: the ramp done, the soak 40 min in. */ +export function sampleLive(nowMs = STARTED_MS + 150 * MIN): Live { + const soakLoad = STARTED_MS + 105 * MIN; + const samples = Array.from({ length: 241 }, (_, i) => ({ + name: "one", + tMs: soakLoad + i * 10_000, + rssKB: 200 * 1024 + i * 10 + })); + return buildLiveSnapshot({ + state: runState(), + record: record({ + [CAPACITY.id]: { + startedMs: STARTED_MS + 2 * MIN, + loadStartedMs: STARTED_MS + 3 * MIN, + endedMs: STARTED_MS + 100 * MIN, + points: [point(50), point(100), point(150, false)] + }, + [SOAK.id]: { startedMs: STARTED_MS + 101 * MIN, loadStartedMs: soakLoad } + }), + nowMs, + soaks: new Map([[SOAK.id, { samples, placed: 270_000, failed: 12 }]]) + }); +} diff --git a/test/live/state.ts b/test/live/state.ts new file mode 100644 index 0000000..3b678c1 --- /dev/null +++ b/test/live/state.ts @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import type { LiveBenchmarkRecord, LiveRecord } from "../../src/live/types.js"; +import type { RunState } from "../../src/orchestrate/runState.schema.js"; +import { + type CapacityBenchmarkConfig, + type LivePoint, + perfkitConfigSchema, + type SoakBenchmarkConfig +} from "../../src/schemas/index.js"; +import { loadConfig } from "../fixtures.js"; + +export const RUN_ID = "20261106T140212Z-bd3254a"; +export const STARTED_MS = Date.parse("2026-11-06T14:02:12Z"); +export const MIN = 60_000; + +const config = perfkitConfigSchema.parse(loadConfig()); +export const CAPACITY = config.benchmarks[0] as CapacityBenchmarkConfig; +export const SOAK = config.benchmarks[1] as SoakBenchmarkConfig; + +/** The Routr example's run state, both benchmarks selected unless told. */ +export function runState( + overrides: { mode?: "digitalocean" | "local"; selected?: string[] } = {} +): RunState { + const selected = overrides.selected ?? [CAPACITY.id, SOAK.id]; + return { + runId: RUN_ID, + mode: overrides.mode ?? "digitalocean", + root: "/opt/perfkit", + namePrefix: "perfkit", + perfkitVersion: "0.2.0", + command: "perfkit run", + plan: { + runId: RUN_ID, + project: "Routr", + targetVersion: "2.32.2", + commit: "bd3254a4f0000000000000000000000000000000", + startedAt: "2026-11-06T14:02:12Z", + region: "nyc3", + resourceName: `perfkit-${RUN_ID}`, + runTag: `perfkit-run-${RUN_ID}`, + tags: ["perfkit"], + uut: { name: "uut", size: "c-8", image: "ubuntu-22-04-x64" }, + loadgen: { name: "loadgen", size: "c-16", image: "ubuntu-22-04-x64" }, + sshSource: "203.0.113.7/32", + images: [], + benchmarks: config.benchmarks.filter(({ id }) => selected.includes(id)), + maxLifetime: "9h" + }, + config, + hosts: { uutIp: "10.0.0.2", loadgenIp: "10.0.0.3", vpcCidr: "10.0.0.0/24" }, + machines: { + uut: { vcpus: 8, memoryMB: 16000, os: "Ubuntu", docker: "27" }, + loadgen: { vcpus: 16, memoryMB: 32000, os: "Ubuntu", docker: "27" } + }, + tools: { sipp: "3.7.3" } + } as unknown as RunState; +} + +export function point(offeredCps: number, ok = true): LivePoint { + return { + offeredCps, + achievedCps: offeredCps, + calls: offeredCps * 60, + failed: 0, + p50Ms: 2, + p95Ms: 4, + p99Ms: ok ? 8 : 62, + failedPct: 0, + ok, + stopReason: ok ? null : "over the p99 limit" + }; +} + +export function record( + benchmarks: Record> = {}, + status: LiveRecord["status"] = "running" +): LiveRecord { + return { + status, + reason: status === "failed" ? "the agent failed: boom" : null, + benchmarks: Object.fromEntries( + Object.entries(benchmarks).map(([id, value]) => [ + id, + { + startedMs: null, + loadStartedMs: null, + endedMs: null, + points: [], + ...value + } + ]) + ) + }; +} diff --git a/test/orchestrate/local.unit.test.ts b/test/orchestrate/local.unit.test.ts index fb1dd14..6315787 100644 --- a/test/orchestrate/local.unit.test.ts +++ b/test/orchestrate/local.unit.test.ts @@ -94,4 +94,35 @@ describe("perfkit run --local", () => { expect(hosts.commands).to.deep.equal([]); expect(infra.clients).to.deep.equal([]); }); + + it("should serve the run's live.json from this machine while it follows the run, then close it", async () => { + // Arrange + const { infra, out } = await localSetup(); + const served: Array<{ root: string; live: unknown; siteUrl: string }> = []; + let closed = 0; + infra.serveLive = async (input) => { + served.push(input); + return { + close: async () => { + closed += 1; + } + }; + }; + + // Act + const result = await perfkit( + [...MINIMAL, "--local", "--out", out, "--live-port", "8799"], + infra + ); + + // Assert + expect(result.code).to.equal(0); + expect(served).to.have.length(1); + expect(served[0]?.live).to.deep.equal({ port: 8799, domain: null }); + expect(served[0]?.siteUrl).to.equal( + "https://fonoster.github.io/perfkit/benchmarks" + ); + expect(served[0]?.root).to.match(/\.perfkit\/runs\/[^/]+\/root$/); + expect(closed).to.equal(1); + }); }); diff --git a/test/orchestrate/run.unit.test.ts b/test/orchestrate/run.unit.test.ts index 5900d84..5557fb0 100644 --- a/test/orchestrate/run.unit.test.ts +++ b/test/orchestrate/run.unit.test.ts @@ -339,4 +339,62 @@ describe("perfkit run (orchestration)", () => { }); expect(setup.account.calls).to.deep.equal([]); }); + + it("should hand NGROK_AUTHTOKEN to the agent's environment only, add ngrok's binding, and keep the live settings in run.json", async () => { + // Arrange + const setup = await doSetup({ + env: { + DIGITALOCEAN_TOKEN: TOKEN, + NGROK_AUTHTOKEN: "ngrok-secret-123", + NGROK_DOMAIN: "routr-benchmarks.ngrok-free.dev" + } + }); + + // Act + const result = await perfkit( + [...MINIMAL, "--out", setup.out, "--live-port", "9876"], + setup.infra + ); + + // Assert + expect(result.code).to.equal(0); + expect(result.out + result.err).to.not.include("ngrok-secret-123"); + const commands = allCommands(setup); + const agentStart = commands.find((command) => + command.includes(" agent --run ") + ); + expect(agentStart).to.include("-e NGROK_AUTHTOKEN=ngrok-secret-123"); + expect( + commands.some((command) => command.includes("@ngrok/ngrok@")) + ).to.equal(true); + const runJson = await readFile( + join(setup.hosts.roots.loadgen, "run.json"), + "utf8" + ); + expect(runJson).to.not.include("ngrok-secret-123"); + expect(JSON.parse(runJson).live).to.deep.equal({ + port: 9876, + domain: "routr-benchmarks.ngrok-free.dev" + }); + }); + + it("should start no tunnel and add no binding without NGROK_AUTHTOKEN", async () => { + const setup = await doSetup(); + await perfkit([...MINIMAL, "--out", setup.out], setup.infra); + const commands = allCommands(setup); + expect(commands.some((c) => c.includes("NGROK_AUTHTOKEN"))).to.equal(false); + expect(commands.some((c) => c.includes("@ngrok/ngrok@"))).to.equal(false); + }); + + it("should exit 2 for a --live-port that is not a port", async () => { + const setup = await doSetup(); + const result = await perfkit( + [...MINIMAL, "--out", setup.out, "--live-port", "http"], + setup.infra + ); + expect(result.code).to.equal(2); + expect(result.err).to.equal( + "perfkit: --live-port: expected a port from 1 to 65535\n" + ); + }); }); diff --git a/test/orchestrate/setup.ts b/test/orchestrate/setup.ts index e2172e3..b20b8ff 100644 --- a/test/orchestrate/setup.ts +++ b/test/orchestrate/setup.ts @@ -148,7 +148,12 @@ export function agentRunner(hosts: FakeHosts, root = REMOTE_ROOT) { export async function doSetup( options: { files?: RawFiles; - env?: { DIGITALOCEAN_TOKEN?: string; PERFKIT_SSH_KEY?: string }; + env?: { + DIGITALOCEAN_TOKEN?: string; + PERFKIT_SSH_KEY?: string; + NGROK_AUTHTOKEN?: string; + NGROK_DOMAIN?: string; + }; agentOnStart?: boolean; } = {} ) { diff --git a/test/orchestrate/toReportProject.unit.test.ts b/test/orchestrate/toReportProject.unit.test.ts new file mode 100644 index 0000000..4513e05 --- /dev/null +++ b/test/orchestrate/toReportProject.unit.test.ts @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { toReportProject } from "../../src/orchestrate/toReportProject.js"; +import { runState } from "../live/state.js"; + +describe("toReportProject", () => { + it("should carry the project without a live URL when none is configured", () => { + const project = toReportProject(runState()); + expect(project.name).to.equal("Routr"); + expect(project).to.not.have.property("live"); + }); + + it("should take project.live.url first, then live.url, then the live domain's live.json", () => { + const state = runState(); + const config = state.config; + config.live = { port: 8787, domain: "routr-benchmarks.ngrok-free.dev" }; + expect(toReportProject(state).live?.url).to.equal( + "https://routr-benchmarks.ngrok-free.dev/live.json" + ); + config.live = { ...config.live, url: "https://b.example/live.json" }; + expect(toReportProject(state).live?.url).to.equal( + "https://b.example/live.json" + ); + config.project.live = { url: "https://a.example/live.json" }; + expect(toReportProject(state).live?.url).to.equal( + "https://a.example/live.json" + ); + }); +}); diff --git a/test/render/app/createDrawRun.unit.test.ts b/test/render/app/createDrawRun.unit.test.ts index 2ae3dba..5f4ed49 100644 --- a/test/render/app/createDrawRun.unit.test.ts +++ b/test/render/app/createDrawRun.unit.test.ts @@ -60,7 +60,8 @@ describe("createDrawRun", () => { expect(result).to.deep.equal({ ok: true, runId: ID, - html: expected("latest") + html: expected("latest"), + liveUrl: null }); expect(fetched).to.deep.equal([ "data/runs.json", @@ -82,10 +83,34 @@ describe("createDrawRun", () => { expect(result).to.deep.equal({ ok: true, runId: ID, - html: expected("run") + html: expected("run"), + liveUrl: null }); }); + it("should return the project's live.json URL from the report", async () => { + // Arrange + const withLive = { + ...report, + project: { + ...(report.project as object), + live: { url: "https://perfkit.ngrok.app/live.json" } + } + }; + const { drawRun } = drawWith({ + "data/runs.json": runs, + [`data/runs/${ID}/report.json`]: withLive + }); + + // Act + const result = await drawRun({ path: "" }); + + // Assert + expect(result.ok && result.liveUrl).to.equal( + "https://perfkit.ngrok.app/live.json" + ); + }); + it("should read a report of the old layout from runs//", async () => { // Arrange const { drawRun, fetched } = drawWith({ diff --git a/test/render/app/createLiveController.unit.test.ts b/test/render/app/createLiveController.unit.test.ts new file mode 100644 index 0000000..a9c448d --- /dev/null +++ b/test/render/app/createLiveController.unit.test.ts @@ -0,0 +1,148 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { createLiveController } from "../../../src/render/app/createLiveController.js"; +import type { LiveViewContext } from "../../../src/render/live/types.js"; +import type { Live, RunEntry } from "../../../src/schemas/index.js"; +import { sampleLive } from "../../live/sample.js"; +import { HISTORY } from "../live/history.js"; + +type Reply = { status: number; text: string } | Error; + +function setup(replies: Reply[], published: RunEntry[][] = []) { + const events: string[] = []; + const shown: LiveViewContext[] = []; + let now = Date.parse("2026-11-06T16:32:12Z"); + const controller = createLiveController({ + liveUrl: "https://x.ngrok-free.dev/live.json", + version: "0.2.0", + fetchLive: async () => { + const reply = replies.shift() ?? new Error("offline"); + if (reply instanceof Error) throw reply; + return reply; + }, + fetchRuns: async () => published.shift() ?? HISTORY, + view: { + showLive: (ctx, first) => { + shown.push(ctx); + events.push( + `live ${first ? "mount" : "update"} ${ctx.live.status} ${ctx.connection}` + ); + }, + tick: (ctx) => { + events.push(`tick ${ctx.connection}`); + }, + showFinal: async () => { + events.push("final"); + } + }, + now: () => (now += 1000) + }); + return { controller, events, shown }; +} + +const ok = (live: Live) => ({ status: 200, text: JSON.stringify(live) }); +const running = sampleLive(); +const finished: Live = { ...running, status: "finished", estimatedEndAt: null }; +const publishedRun = { ...HISTORY[0]!, id: running.runId }; + +describe("createLiveController", () => { + it("should leave the published report when no run is in progress", async () => { + // Arrange: nothing served, then 503, then an invalid file, then a run already finished + const { controller, events } = setup([ + new Error("offline"), + { status: 503, text: "no snapshot yet" }, + { status: 200, text: "{}" }, + ok(finished) + ]); + + // Act + const modes = [ + await controller.poll(), + await controller.poll(), + await controller.poll(), + await controller.poll() + ]; + controller.tick(); + + // Assert + expect(modes).to.deep.equal(["final", "final", "final", "final"]); + expect(events).to.deep.equal([]); + }); + + it("should go live for a run in progress, mount once, then update in place", async () => { + const { controller, events, shown } = setup([ok(running), ok(running)]); + expect(await controller.poll()).to.equal("live"); + await controller.poll(); + controller.tick(); + expect(events).to.deep.equal([ + "live mount running ok", + "live update running ok", + "tick ok" + ]); + expect(shown[0]?.runs).to.deep.equal(HISTORY); + }); + + it("should say the connection is lost when a poll fails, and come back", async () => { + const { controller, events } = setup([ + ok(running), + new Error("offline"), + ok(running) + ]); + await controller.poll(); + expect(await controller.poll()).to.equal("live"); + controller.tick(); + await controller.poll(); + expect(events).to.deep.equal([ + "live mount running ok", + "live update running lost", + "tick lost", + "live update running ok" + ]); + }); + + it("should wait for the report once the run finished, then show it", async () => { + // Arrange: runs.json first without the run, then with it + const { controller, events } = setup( + [ok(running), ok(finished), ok(finished)], + [HISTORY, HISTORY, [publishedRun, ...HISTORY]] + ); + + // Act + await controller.poll(); + const waiting = await controller.poll(); + const after = await controller.poll(); + + // Assert + expect(waiting).to.equal("live"); + expect(after).to.equal("final"); + expect(events).to.deep.equal([ + "live mount running ok", + "live update finished ok", + "live update finished ok", + "final" + ]); + }); + + it("should show the report when the server is gone but the run was published", async () => { + const { controller, events } = setup( + [ok(running), new Error("offline")], + [HISTORY, [publishedRun, ...HISTORY]] + ); + await controller.poll(); + expect(await controller.poll()).to.equal("final"); + expect(events.at(-1)).to.equal("final"); + }); + + it("should mount again when another run starts", async () => { + const other = { ...running, runId: "20261107T140212Z-bd3254a" }; + const { controller, events } = setup([ok(running), ok(other)]); + await controller.poll(); + await controller.poll(); + expect(events).to.deep.equal([ + "live mount running ok", + "live mount running ok" + ]); + }); +}); diff --git a/test/render/app/createLiveDomView.unit.test.ts b/test/render/app/createLiveDomView.unit.test.ts new file mode 100644 index 0000000..d3f8d4f --- /dev/null +++ b/test/render/app/createLiveDomView.unit.test.ts @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { JSDOM } from "jsdom"; +import { createLiveDomView } from "../../../src/render/app/createLiveDomView.js"; +import type { LiveViewContext } from "../../../src/render/live/types.js"; +import { MIN, STARTED_MS } from "../../live/state.js"; +import { sampleLive } from "../../live/sample.js"; +import { HISTORY } from "../live/history.js"; + +const NOW = STARTED_MS + 150 * MIN; +const ctx = (overrides: Partial = {}): LiveViewContext => ({ + live: sampleLive(NOW - 5_000), + runs: HISTORY, + nowMs: NOW, + connection: "ok", + perfkitVersion: "0.2.0", + ...overrides +}); + +describe("createLiveDomView", () => { + it("should mount the live page over the report, then update it in place", () => { + // Arrange + const dom = new JSDOM( + "Report
    report
    " + ); + const doc = dom.window.document; + const view = createLiveDomView(doc); + + // Act + view.showLive(ctx(), true); + const nav = doc.querySelector(".nav"); + doc.querySelector("details")!.open = true; + view.showLive(ctx({ nowMs: NOW + 70_000 }), false); + + // Assert + expect(doc.title).to.equal("Routr benchmarks · running now · v2.32.2"); + expect(doc.querySelector(".nav")).to.equal(nav); + expect(doc.querySelector("[data-updated]")?.className).to.equal( + "updated is-stale" + ); + expect(doc.querySelector("details")?.open).to.equal(true); + }); + + it("should tick the updated text and its tone only", () => { + const dom = new JSDOM("

    shell

    "); + const doc = dom.window.document; + const view = createLiveDomView(doc); + view.showLive(ctx(), true); + const main = doc.getElementById("live-main")!.innerHTML; + + view.tick(ctx({ nowMs: NOW + 3_000 })); + expect(doc.querySelector("[data-updated]")?.textContent).to.equal( + "updated 8 s ago" + ); + view.tick(ctx({ connection: "lost" })); + + expect(doc.querySelector("[data-updated]")?.textContent).to.equal( + "connection lost, retrying" + ); + expect(doc.querySelector("[data-updated]")?.className).to.equal( + "updated is-lost" + ); + expect(doc.getElementById("live-main")!.innerHTML).to.equal(main); + }); + + it("should mount the whole page when the live regions are missing", () => { + const dom = new JSDOM("

    shell

    "); + const view = createLiveDomView(dom.window.document); + view.showLive(ctx(), false); + expect(dom.window.document.getElementById("live-main")).to.not.equal(null); + }); +}); diff --git a/test/render/app/resolveLiveUrl.unit.test.ts b/test/render/app/resolveLiveUrl.unit.test.ts new file mode 100644 index 0000000..894cbda --- /dev/null +++ b/test/render/app/resolveLiveUrl.unit.test.ts @@ -0,0 +1,28 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { resolveLiveUrl } from "../../../src/render/app/resolveLiveUrl.js"; + +const PAGE = "https://fonoster.github.io/routr/benchmarks/"; + +describe("resolveLiveUrl", () => { + it("should take ?live= first, then the report's live URL", () => { + expect( + resolveLiveUrl( + `${PAGE}?live=http://localhost:8787/live.json`, + "https://a.example/live.json" + ) + ).to.equal("http://localhost:8787/live.json"); + expect(resolveLiveUrl(PAGE, "https://a.example/live.json")).to.equal( + "https://a.example/live.json" + ); + }); + + it("should be null without one, or for a URL that is not http or https", () => { + expect(resolveLiveUrl(PAGE, null)).to.equal(null); + expect(resolveLiveUrl(`${PAGE}?live=javascript:alert(1)`, null)).to.equal( + null + ); + }); +}); diff --git a/test/render/live/describeStep.unit.test.ts b/test/render/live/describeStep.unit.test.ts new file mode 100644 index 0000000..f87ea3f --- /dev/null +++ b/test/render/live/describeStep.unit.test.ts @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { describeStep } from "../../../src/render/live/describeStep.js"; +import type { LiveCapacity, LiveSoak } from "../../../src/schemas/index.js"; +import { sampleLive } from "../../live/sample.js"; + +describe("describeStep", () => { + const live = sampleLive(); + const capacity = live.benchmarks[0] as LiveCapacity; + const soak = live.benchmarks[1] as LiveSoak; + + it("should say what a done ramp held, preliminary", () => { + expect(describeStep(capacity)).to.equal( + "held 100 messages/s · preliminary" + ); + }); + + it("should say which step a running ramp is on", () => { + expect( + describeStep({ + ...capacity, + state: "running", + current: { index: 13, total: 37, offeredCps: 4250 } + }) + ).to.equal("step 14 of 37 · 4,250 messages/s"); + }); + + it("should say how far a soak is, and when a queued one starts", () => { + expect(describeStep(soak)).to.equal("45 min of 6 h 30 min · 100 calls/s"); + expect(describeStep({ ...soak, elapsedS: 600 })).to.equal( + "warm-up · 10 min of 6 h 30 min" + ); + expect( + describeStep({ + ...soak, + state: "queued", + estimatedStartAt: "2026-11-06T14:55:00Z" + }) + ).to.equal("starts about 14:55 UTC"); + }); +}); diff --git a/test/render/live/formatters.unit.test.ts b/test/render/live/formatters.unit.test.ts new file mode 100644 index 0000000..a166044 --- /dev/null +++ b/test/render/live/formatters.unit.test.ts @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { formatClock } from "../../../src/render/live/formatClock.js"; +import { formatUpdated } from "../../../src/render/live/formatUpdated.js"; +import { livePhase } from "../../../src/render/live/livePhase.js"; +import { sampleLive } from "../../live/sample.js"; + +const AT = "2026-11-06T14:00:00Z"; +const ms = Date.parse(AT); + +describe("formatUpdated", () => { + it("should say how old the snapshot is", () => { + expect(formatUpdated(AT, ms + 8_000, "ok")).to.deep.equal({ + text: "updated 8 s ago", + tone: "ok" + }); + }); + + it("should turn warning-coloured after a minute", () => { + expect(formatUpdated(AT, ms + 61_000, "ok")).to.deep.equal({ + text: "updated 1 min ago", + tone: "stale" + }); + }); + + it("should say the connection is lost after 3 minutes or a failed poll", () => { + const lost = { text: "connection lost, retrying", tone: "lost" }; + expect(formatUpdated(AT, ms + 180_000, "ok")).to.deep.equal(lost); + expect(formatUpdated(AT, ms + 1_000, "lost")).to.deep.equal(lost); + }); +}); + +describe("formatClock", () => { + it("should write hours and minutes", () => { + expect(formatClock(52 * 60 + 30)).to.equal("0:52"); + expect(formatClock(12 * 3600 + 5 * 60)).to.equal("12:05"); + expect(formatClock(-5)).to.equal("0:00"); + }); +}); + +describe("livePhase", () => { + it("should map the status to running, publishing or stopped", () => { + const live = sampleLive(); + expect(livePhase(live)).to.equal("running"); + expect(livePhase({ ...live, status: "finished" })).to.equal("publishing"); + expect(livePhase({ ...live, status: "failed" })).to.equal("stopped"); + }); +}); diff --git a/test/render/live/history.ts b/test/render/live/history.ts new file mode 100644 index 0000000..21e2e3f --- /dev/null +++ b/test/render/live/history.ts @@ -0,0 +1,12 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { type RunEntry, runsSchema } from "../../../src/schemas/index.js"; +import { EXAMPLES } from "../../fixtures.js"; + +/** The 14-run Routr history; its newest run held edgeport-echo at 1,240/s. */ +export const HISTORY: RunEntry[] = runsSchema.parse( + JSON.parse(readFileSync(join(EXAMPLES, "history", "runs-14.json"), "utf8")) +).runs; diff --git a/test/render/live/lastPublishedLimit.unit.test.ts b/test/render/live/lastPublishedLimit.unit.test.ts new file mode 100644 index 0000000..423fbd4 --- /dev/null +++ b/test/render/live/lastPublishedLimit.unit.test.ts @@ -0,0 +1,24 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { lastPublishedLimit } from "../../../src/render/live/lastPublishedLimit.js"; +import { HISTORY } from "./history.js"; + +describe("lastPublishedLimit", () => { + it("should find the newest published limit of a benchmark", () => { + expect(lastPublishedLimit(HISTORY, "edgeport-echo", "x")).to.deep.equal({ + limitCps: 1240, + version: "2.31.5" + }); + }); + + it("should leave out the run in progress and know nothing of other benchmarks", () => { + const newest = HISTORY[0]!.id; + expect( + lastPublishedLimit(HISTORY, "edgeport-echo", newest)?.version + ).to.not.equal(undefined); + expect(lastPublishedLimit(HISTORY, "nope", "x")).to.equal(null); + expect(lastPublishedLimit([], "edgeport-echo", "x")).to.equal(null); + }); +}); diff --git a/test/render/live/renderLiveView.unit.test.ts b/test/render/live/renderLiveView.unit.test.ts new file mode 100644 index 0000000..d200b3d --- /dev/null +++ b/test/render/live/renderLiveView.unit.test.ts @@ -0,0 +1,197 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import * as cheerio from "cheerio"; +import { renderLivePage } from "../../../src/render/live/renderLivePage.js"; +import { GROWTH_NOTE } from "../../../src/render/live/renderLiveSoak.js"; +import { renderLiveView } from "../../../src/render/live/renderLiveView.js"; +import type { LiveViewContext } from "../../../src/render/live/types.js"; +import type { LiveCapacity } from "../../../src/schemas/index.js"; +import { + CAPACITY, + MIN, + point, + record, + runState, + STARTED_MS +} from "../../live/state.js"; +import { buildLiveSnapshot } from "../../../src/live/buildLiveSnapshot.js"; +import { sampleLive } from "../../live/sample.js"; +import { HISTORY } from "./history.js"; + +const NOW = STARTED_MS + 150 * MIN; + +function ctx(overrides: Partial = {}): LiveViewContext { + return { + live: sampleLive(NOW - 8_000), + runs: HISTORY, + nowMs: NOW, + connection: "ok", + perfkitVersion: "0.2.0", + ...overrides + }; +} + +const text = (html: string) => + cheerio.load(html).root().text().replace(/\s+/g, " "); + +describe("renderLiveView", () => { + it("should draw the nav's live picker, the preliminary banner and the header", () => { + // Act + const parts = renderLiveView(ctx()); + + // Assert + expect(text(parts.nav)) + .to.contain("Running now · v2.32.2") + .and.contain("live"); + expect(text(parts.banner)).to.contain( + "Running now. These numbers are preliminary and change every 15 seconds; the verdict appears when the run ends." + ); + expect(text(parts.banner)).to.contain("updated 8 s ago"); + const $ = cheerio.load(parts.main); + expect($(".eyebrow-live").text()).to.equal("LIVE · RUN 20261106T140212Z"); + expect($("h1").text()).to.equal("Routr 2.32.2 is under test right now."); + expect($(".lead").text()).to.equal( + "Two benchmarks run one after another on two dedicated DigitalOcean servers. Watch each one as it happens; nothing on this page is final until the run ends." + ); + }); + + it("should draw the progress cards and the overall bar", () => { + const $ = cheerio.load(renderLiveView(ctx()).main); + const steps = $(".step") + .map((_, el) => $(el).attr("class")) + .get(); + expect(steps).to.deep.equal([ + "step step-done", + "step step-done", + "step step-running" + ]); + expect($(".step-sub").first().text()).to.equal( + "servers up, images pinned · 2 min" + ); + expect($(".overall-text").text()).to.match( + /^\d+% · ends about \d\d:\d\d UTC$/ + ); + }); + + it("should draw a done ramp's chart with its preliminary limit and the last published result", () => { + const $ = cheerio.load(renderLiveView(ctx()).main); + const section = $(`#${CAPACITY.id}`); + expect(section.find(".eyebrow").text()).to.equal("01 · CAPACITY · DONE"); + expect(section.find(".so-far-value").text()).to.equal( + "Held 100 messages/s" + ); + expect(section.find(".baseline").text()).to.equal( + "Last published result: 1,240 messages/s on v2.31.5. 1,140 messages/s to go to match it." + ); + expect(section.find(".limit-line")).to.have.length(1); + }); + + it("should draw a running ramp up to the step offered now, with the rest shaded", () => { + // Arrange + const live = buildLiveSnapshot({ + state: runState({ selected: [CAPACITY.id] }), + record: record({ + [CAPACITY.id]: { + startedMs: STARTED_MS, + loadStartedMs: STARTED_MS, + points: [point(50), point(100)] + } + }), + nowMs: STARTED_MS + 2.5 * MIN, + soaks: new Map() + }); + + // Act + const $ = cheerio.load( + renderLiveView(ctx({ live, nowMs: STARTED_MS + 2.5 * MIN })).main + ); + + // Assert + expect($(".so-far-value").text()).to.equal("Held 100 messages/s so far"); + expect($(".so-far p").eq(1).text()).to.match( + /^2 steps passed both rules\./ + ); + expect($(".not-run-label").text()).to.equal("steps not run yet"); + expect($(".now-tag-text").text()).to.equal("now 150 messages/s"); + expect($(".legend-live").text()).to.equal("live"); + expect( + $(".stat-value") + .map((_, el) => $(el).text()) + .get() + ).to.deep.equal(["150", "8 ms", "0%", "0:02"]); + expect((live.benchmarks[0] as LiveCapacity).current?.index).to.equal(2); + }); + + it("should draw a soak's raw memory with the warm-up, a now line and the growth note", () => { + const $ = cheerio.load(renderLiveView(ctx()).main); + const soak = $("#one-soak"); + expect(soak.find(".eyebrow").text()).to.equal("02 · SOAK · RUNNING"); + expect(soak.find(".section-meta").text()).to.equal( + "45 min of 6 h 30 min · 270K calls · 12 failed" + ); + expect(soak.find(".warmup-label").text()).to.equal("warm-up, not measured"); + expect(soak.find(".now-label").text()).to.equal("now"); + expect(soak.find(".rule-note").text()).to.equal(GROWTH_NOTE); + expect(soak.find(".card-title").first().text()).to.equal( + "Memory per process, raw" + ); + }); + + it("should turn the updated text warning-coloured, then lost", () => { + const stale = cheerio.load( + renderLiveView(ctx({ nowMs: NOW + 70_000 })).banner + ); + expect(stale(".updated").attr("class")).to.equal("updated is-stale"); + const lost = cheerio.load( + renderLiveView(ctx({ connection: "lost" })).banner + ); + expect(lost(".updated").text()).to.equal("connection lost, retrying"); + expect(lost(".updated").attr("class")).to.equal("updated is-lost"); + }); + + it("should say the report is being published once the run finished", () => { + const live = { + ...sampleLive(), + status: "finished" as const, + estimatedEndAt: null + }; + const parts = renderLiveView(ctx({ live })); + expect(text(parts.banner)).to.contain( + "Run finished; the report is being published." + ); + expect(cheerio.load(parts.main)(".eyebrow-live").text()).to.match(/^ENDED/); + }); + + it("should leave a benchmark not in this run out", () => { + const live = buildLiveSnapshot({ + state: runState({ selected: [CAPACITY.id], mode: "local" }), + record: record({}), + nowMs: STARTED_MS, + soaks: new Map() + }); + const $ = cheerio.load(renderLiveView(ctx({ live, runs: [] })).main); + expect($("#one-soak")).to.have.length(0); + expect($(".lead").text()).to.match( + /^One benchmark runs on this machine's Docker\./ + ); + expect($(".step-running .step-title").text()).to.contain("Setup"); + }); +}); + +describe("renderLivePage", () => { + it("should be a whole page with the report's styles, the live styles and the regions the app updates", () => { + const html = renderLivePage(ctx()); + const $ = cheerio.load(html); + expect($("title").text()).to.equal( + "Routr benchmarks · running now · v2.32.2" + ); + expect($("#live-banner, #live-main, #live-footer")).to.have.length(3); + expect($("body").attr("data-live-run")).to.equal( + "20261106T140212Z-bd3254a" + ); + expect($("style").text()).to.contain("prefers-reduced-motion"); + expect($("[data-picker]")).to.have.length(1); + }); +}); diff --git a/test/schemas/live.unit.test.ts b/test/schemas/live.unit.test.ts new file mode 100644 index 0000000..3715e59 --- /dev/null +++ b/test/schemas/live.unit.test.ts @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { liveSchema } from "../../src/schemas/index.js"; +import { sampleLive } from "../live/sample.js"; +import { errorsOf } from "./check.js"; + +describe("liveSchema", () => { + it("should accept a snapshot the agent builds", () => { + expect(errorsOf(liveSchema, sampleLive())).to.deep.equal([]); + }); + + it("should reject another schema id, a bad status and unknown fields", () => { + const live = sampleLive() as unknown as Record; + expect(errorsOf(liveSchema, { ...live, schema: "perfkit/live/v2" })[0]).to.match(/^schema: /); + expect(errorsOf(liveSchema, { ...live, status: "paused" })[0]).to.match(/^status: /); + expect(errorsOf(liveSchema, { ...live, extra: 1 })).to.have.length(1); + }); + + it("should reject more than 300 memory samples per process", () => { + const live = sampleLive(); + const soak = live.benchmarks[1]; + if (soak?.kind !== "soak") throw new Error("expected the soak"); + const sample = soak.memory[0]!.samples[0]!; + soak.memory[0]!.samples = Array.from({ length: 301 }, () => sample); + expect(errorsOf(liveSchema, live)[0]).to.match(/samples: expected at most 300 samples/); + }); +}); diff --git a/test/schemas/perfkitConfig.unit.test.ts b/test/schemas/perfkitConfig.unit.test.ts index 4165141..11370f8 100644 --- a/test/schemas/perfkitConfig.unit.test.ts +++ b/test/schemas/perfkitConfig.unit.test.ts @@ -52,6 +52,36 @@ describe("perfkitConfigSchema", () => { ); }); + it("should accept a live section and default its port to 8787", () => { + // Arrange + const raw = loadConfig(); + raw.live = { domain: "routr-benchmarks.ngrok-free.dev" }; + raw.project.live = { + url: "https://routr-benchmarks.ngrok-free.dev/live.json" + }; + + // Act + const config = perfkitConfigSchema.parse(raw); + + // Assert + expect(config.live).to.deep.equal({ + domain: "routr-benchmarks.ngrok-free.dev", + port: 8787 + }); + expect(config.project.live?.url).to.equal( + "https://routr-benchmarks.ngrok-free.dev/live.json" + ); + }); + + it("should reject a live domain that is not a hostname, and a bad port", () => { + const raw = loadConfig(); + raw.live = { domain: "https://x.ngrok.app", port: 0 }; + expect(errorsOf(perfkitConfigSchema, raw)).to.deep.equal([ + 'live.domain: expected a hostname like perfkit.ngrok.app; got "https://x.ngrok.app"', + "live.port: expected a port from 1 to 65535; got 0" + ]); + }); + it("should accept a project without a logo", () => { const raw = loadConfig(); delete raw.project.logo; diff --git a/test/schemas/toJsonSchemaFiles.unit.test.ts b/test/schemas/toJsonSchemaFiles.unit.test.ts index e948d1d..a042e14 100644 --- a/test/schemas/toJsonSchemaFiles.unit.test.ts +++ b/test/schemas/toJsonSchemaFiles.unit.test.ts @@ -10,8 +10,9 @@ import { ROOT } from "../fixtures.js"; describe("toJsonSchemaFiles", () => { const files = toJsonSchemaFiles(); - it("should write the three v1 schema files", () => { + it("should write the four v1 schema files", () => { expect(Object.keys(files).sort()).to.deep.equal([ + "live-v1.schema.json", "perfkit-v1.schema.json", "report-v1.schema.json", "runs-v1.schema.json" diff --git a/test/serve/createServeSite.unit.test.ts b/test/serve/createServeSite.unit.test.ts new file mode 100644 index 0000000..364bfc4 --- /dev/null +++ b/test/serve/createServeSite.unit.test.ts @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2026 by Fonoster Inc (https://fonoster.com). MIT License. + */ +import { expect } from "chai"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ValidationError } from "../../src/errors/index.js"; +import { contentTypeOf } from "../../src/serve/contentTypeOf.js"; +import { createNodeServeSite } from "../../src/serve/createNodeServeSite.js"; +import { createServeSite } from "../../src/serve/createServeSite.js"; + +const SITE: Record = { + "index.html": "

    latest

    ", + "runs/abc/index.html": "

    run

    ", + "app/1.0.0/app.js": "app()", + "data/runs.json": "{}" +}; +const DIRS = new Set(["runs", "runs/abc", "app", "app/1.0.0", "data"]); + +const serve = createServeSite({ + readSiteFile: async (path) => SITE[path] ?? null, + isSiteDir: async (path) => DIRS.has(path) +}); +const get = (url: string, method = "GET") => + serve({ method, url, origin: null }); + +describe("createServeSite", () => { + it("should serve the site root and a run's page from their index.html", async () => { + const root = await get("/?live=http://localhost:8787/live.json"); + const run = await get("/runs/abc/"); + expect([root.status, root.body]).to.deep.equal([200, "

    latest

    "]); + expect(root.headers).to.deep.include({ + "Content-Type": "text/html; charset=utf-8", + "Cache-Control": "no-cache" + }); + expect(run.body).to.equal("

    run

    "); + }); + + it("should serve files with their content type, and HEAD without a body", async () => { + expect((await get("/app/1.0.0/app.js")).headers["Content-Type"]).to.equal( + "text/javascript; charset=utf-8" + ); + const head = await get("/data/runs.json", "HEAD"); + expect([head.status, head.body]).to.deep.equal([200, ""]); + }); + + it("should redirect a directory without its slash", async () => { + const response = await get("/runs/abc"); + expect(response.status).to.equal(301); + expect(response.headers.Location).to.equal("/runs/abc/"); + }); + + it("should answer 404 for a missing file, a path out of the site or another method", async () => { + const responses = await Promise.all([ + get("/nope.html"), + get("/../secret"), + get("/runs/%2e%2e/%2e%2e/secret"), + get("/index.html", "POST") + ]); + expect(responses.map((r) => r.status)).to.deep.equal([404, 404, 404, 404]); + }); + + it("should throw ValidationError for a request without a URL", async () => { + let error: unknown; + await serve({ method: "GET", url: "", origin: null }).catch( + (err: unknown) => { + error = err; + } + ); + expect(error).to.be.instanceOf(ValidationError); + }); +}); + +describe("createNodeServeSite", () => { + it("should serve a directory on disk", async () => { + // Arrange + const dir = await mkdtemp(join(tmpdir(), "perfkit-serve-")); + await mkdir(join(dir, "runs", "abc"), { recursive: true }); + await writeFile(join(dir, "index.html"), "root"); + await writeFile(join(dir, "runs", "abc", "index.html"), "run"); + const handle = createNodeServeSite(dir); + + // Act + const [root, run, redirect, missing] = await Promise.all( + ["/", "/runs/abc/", "/runs/abc", "/x.json"].map((url) => + handle({ method: "GET", url, origin: null }) + ) + ); + + // Assert + expect(root?.body).to.equal("root"); + expect(run?.body).to.equal("run"); + expect(redirect?.status).to.equal(301); + expect(missing?.status).to.equal(404); + }); +}); + +describe("contentTypeOf", () => { + it("should know the site's file types", () => { + expect(contentTypeOf("a.json")).to.equal("application/json; charset=utf-8"); + expect(contentTypeOf("a.svg")).to.equal("image/svg+xml"); + expect(contentTypeOf("a.bin")).to.equal("application/octet-stream"); + }); +}); From 96eada72ba9b4a1ded4a6a46aa323bc9d2987d3f Mon Sep 17 00:00:00 2001 From: Pedro Sanders Date: Thu, 5 Nov 2026 08:32:36 -0400 Subject: [PATCH 2/2] docs: live results, perfkit serve and the live.json format --- README.md | 57 ++++++++++++++++++++++++++++++++++++++++- docs/github-workflow.md | 2 ++ docs/perfkit-yaml.md | 20 +++++++++++++++ docs/report-page.md | 25 ++++++++++++++++++ docs/report-v1.md | 32 +++++++++++++++++++++++ 5 files changed, 135 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e33732b..f9258cf 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ perfkit version. ```sh perfkit run --config --target-version [--commit ] [--benchmarks ] - [--out ] [--detach] [--dry-run] [--local] + [--out ] [--detach] [--dry-run] [--local] [--live-port ] ``` Creates the run's infrastructure on DigitalOcean, runs the benchmarks on it, and deletes it at @@ -146,11 +146,14 @@ digest its tag had when the run started. | `--detach` | off | Return once the agent on the load generator has started; `perfkit collect` finishes the run. | | `--dry-run` | off | Print the plan and create nothing. Needs no token. | | `--local` | off | Run on this machine's Docker instead of DigitalOcean. Needs no token. | +| `--live-port ` | `live.port` of perfkit.yaml, else `8787` | Where live.json is served during the run (see [Live results](#live-results)). | | Environment | Meaning | | :--- | :--- | | `DIGITALOCEAN_TOKEN` | Required, except with `--dry-run`. Never printed, never written to a file, `report.json` or a droplet. | | `PERFKIT_NAME_PREFIX` | Optional; containers and compose projects are named `--`, `perfkit` by default. | +| `NGROK_AUTHTOKEN` | Optional. Opens an ngrok tunnel to the run's live.json (see [Live results](#live-results)). On a DigitalOcean run it goes into the agent container's environment on the load generator only; never into run.json, a log or the report. | +| `NGROK_DOMAIN` | Optional static ngrok domain when perfkit.yaml has no `live.domain`. | | `PERFKIT_SSH_KEY` | Optional ed25519 private key, in OpenSSH format, without a passphrase. When set, the run looks its public key up in the account by fingerprint and reuses it, or registers it as `perfkit-shared`. That key is never deleted, by teardown or by sweep, so parallel runs share it. Without it, the run creates `perfkit-` for this run only and deletes it at teardown. | `--dry-run` resolves the digests on Docker Hub and prints the plan: @@ -440,6 +443,58 @@ The verdict is **failed** when any rule failed and **passed** otherwise. Also re part of the verdict: p99 in the first and the last hour after the warm-up, the most open file descriptors per process, the most concurrent calls, the registrations and their failures, and, with telemetry, each service's p50 and p99 self time (a span's time minus its direct children's). +## Live results + +While a run goes on, the benchmarks page shows it live. Every run, published config or not: + +- **live.json.** The agent writes `/live.json` (`schema: perfkit/live/v1`, see + [docs/report-v1.md](docs/report-v1.md#livejson)) every 15 seconds: which benchmark runs, each + capacity step as the agent judged it, what held so far, a soak's calls and raw memory per + process, and when the run should end. Nothing in it is a verdict. +- **Served read-only.** `GET /live.json` on `127.0.0.1:` (8787 by default), never + cached, with CORS for the origin of `project.siteUrl` and for `http://localhost:*`; 503 before + the first snapshot, 404 for anything else. On a DigitalOcean run the agent serves it on the load + generator; with `--local` the `run` (or `collect`) process serves it on this machine. +- **ngrok.** With `NGROK_AUTHTOKEN` set, the server is tunnelled through the static domain in + `live.domain` (or `NGROK_DOMAIN`) with the `@ngrok/ngrok` package, and the public URL is logged. + Without it there is no tunnel. The tunnel and the server close when the run ends. +- **The page.** `project.live.url` (default `https:///live.json`) goes into + report.json, so the latest page (the site root) knows where to look. It polls every 15 + seconds; while a run is in progress it shows it live, marked preliminary. When the run ends it + says the report is being published and shows it as soon as `data/runs.json` lists the run. + With no run in progress the page is the latest report, unchanged. Run pages never go live. + `?live=` on the page overrides the URL. + +### Watch a local run live + +From a perfkit checkout (`npm ci && npm run build`); the minimal example takes about 4 minutes: + +```sh +# 1. A site to watch: render any report into a directory (an example works). +node dist/cli.js render --report examples/reports/passed.json \ + --history examples/history/runs-14.json --out /tmp/perfkit-site +# 2. Serve it (leave it running). +node dist/cli.js serve --site /tmp/perfkit-site --port 8080 +# 3. In another terminal, a short local run; it serves live.json on http://127.0.0.1:8787. +node dist/cli.js run --local --config examples/minimal/perfkit.yaml --target-version 3.7.3 +# 4. Open the page with the local live URL. +open "http://localhost:8080/?live=http://localhost:8787/live.json" +# 5. When the run ends, publish its report; the page switches to it. +node dist/cli.js render --report .perfkit//report.json \ + --history /tmp/perfkit-site/data/runs.json --out /tmp/perfkit-site +``` + +## perfkit serve + +```sh +perfkit serve --site [--port 8080] [--host 127.0.0.1] +``` + +A read-only static server for a directory `perfkit render` wrote, as GitHub Pages serves it, +with nothing cached: to preview a site, or to watch a `--local` run live (above). It prints +`serving at http://127.0.0.1:8080/ (Ctrl-C stops)` and runs until Ctrl-C. It exits 0 when +stopped, 1 for a usage error and 2 when it can't listen (the port is taken). + ## perfkit render ```sh diff --git a/docs/github-workflow.md b/docs/github-workflow.md index afa7a6e..120a5df 100644 --- a/docs/github-workflow.md +++ b/docs/github-workflow.md @@ -44,6 +44,7 @@ adds that machine's IP to the run's firewall before it logs in, with `PERFKIT_SS | :--- | :--- | | `DIGITALOCEAN_TOKEN` | The DigitalOcean token, with the scopes in the README. It never reaches a droplet, a log or the report. | | `PERFKIT_SSH_KEY` | An ed25519 private key in OpenSSH format, without a passphrase. Every job logs in to the droplets with it, registered in the account as `perfkit-shared`. Make one with `ssh-keygen -t ed25519 -N "" -f perfkit`. | +| `NGROK_AUTHTOKEN` | Optional. The agent opens an ngrok tunnel from `live.domain` of perfkit.yaml to its live.json server, so the benchmarks page shows the run live (see "Live results" in the README). It goes into the agent container's environment on the load generator and nowhere else: not run.json, a log or the report. Without it the run has no tunnel. | The workflow takes its secrets only through `secrets:`, so a caller has to name them: `secrets: inherit` would hand it every secret of the repository. @@ -85,6 +86,7 @@ jobs: secrets: DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }} PERFKIT_SSH_KEY: ${{ secrets.PERFKIT_SSH_KEY }} + NGROK_AUTHTOKEN: ${{ secrets.NGROK_AUTHTOKEN }} # optional: live results publish: needs: run diff --git a/docs/perfkit-yaml.md b/docs/perfkit-yaml.md index cb5090f..f3b19b3 100644 --- a/docs/perfkit-yaml.md +++ b/docs/perfkit-yaml.md @@ -19,6 +19,7 @@ Rules for the whole file: | `apiVersion` | `perfkit/v1` | yes | | Tells perfkit which format this is. | | `project` | object | yes | | See [project](#project). | | `infra` | object | yes | | See [infra](#infra). | +| `live` | object | no | none | Where the agent serves live.json during a run. See [live](#live). | | `images` | map of name to image | yes | | At least one. See [images](#images). | | `benchmarks` | list | yes | | At least one. See [benchmarks](#benchmarks). | @@ -33,6 +34,7 @@ Rules for the whole file: | `project.logo` | path | no | none | An `.svg` file. Without it, the nav shows `project.name`. | | `project.title` | string | yes | | The page's headline. | | `project.lead` | string | yes | | The paragraph under the headline. | +| `project.live.url` | URL | no | `live.url`, else `https:///live.json` | Where the benchmarks page polls a run in progress. Goes into report.json, so the published page knows it. | ## infra @@ -44,6 +46,24 @@ Rules for the whole file: | `infra.loadgen.size` | string | yes | | Droplet size slug of the load generator, like `c-16`. | | `infra.maxLifetime` | duration | no | `9h` | The sweep deletes this run's droplets after this long. | +## live + +Every run writes `live.json` (`schema: perfkit/live/v1`, see [report-v1.md](report-v1.md#livejson)) +every 15 seconds and serves it read-only on the load generator; with `NGROK_AUTHTOKEN` set, an +ngrok tunnel makes it public. The page polls it; see "Live results" in the README. + +| Field | Type | Required | Default | Notes | +| :--- | :--- | :--- | :--- | :--- | +| `live.port` | number | no | `8787` | The port of the live.json server. `perfkit run --live-port` overrides it. | +| `live.domain` | hostname | no | `NGROK_DOMAIN`, else none | The static ngrok domain of the tunnel, like `routr-benchmarks.ngrok-free.dev`. | +| `live.url` | URL | no | `https:///live.json` | Where the page polls it; `project.live.url` wins. | + +```yaml +live: + domain: routr-benchmarks.ngrok-free.dev + url: https://routr-benchmarks.ngrok-free.dev/live.json +``` + ## images A map from a short name (the key) to an image. At least one image is required. Each entry is diff --git a/docs/report-page.md b/docs/report-page.md index 28b59ba..1e4d701 100644 --- a/docs/report-page.md +++ b/docs/report-page.md @@ -91,3 +91,28 @@ N runs" shows the rest. The droplets and region, each image with its digest, the tools, and the command to run the same benchmarks yourself. + +## The live view + +While a run is in progress, the latest page (the site root, never a run page) shows it live from +the agent's live.json, polled every 15 seconds; see "Live results" in the README. The design is +the "State / Live (preliminary)" frame of `designs/benchmarks.pen`. + +- **Nav and banner.** The picker reads "Running now · v" with a live dot and badge. + Under the nav: "Running now. These numbers are preliminary and change every 15 seconds; the + verdict appears when the run ends." and "updated N s ago", which turns warning-coloured after a + minute and reads "connection lost, retrying" after 3 minutes or a failed poll. +- **Header and progress.** "LIVE · RUN ", " is under test right now.", + one card per step (setup, then each benchmark: done, running or queued with its expected start) + and a bar with when the run should end. +- **Live numbers.** For the benchmark running now: the rate offered (and placed, last step), + p99 and failed % of the last step (a soak: calls placed and failed % so far), and the time into + the run with the time to go. +- **Capacity.** The same "Latency as load rises" chart as the report, drawn up to the step + offered now, marked "now ", the steps not run yet shaded; and "So far": what held, why the + limit is not known yet, and the last published limit from `data/runs.json`. +- **Soak.** Raw memory per process with the warm-up shaded and a "now" line. Growth in MB/h shows + only once there is a full hour of data after the warm-up. + +When the run ends the page says the report is being published, and draws it once +`data/runs.json` lists the run. Animations stop under `prefers-reduced-motion`. diff --git a/docs/report-v1.md b/docs/report-v1.md index d08c240..81d4fcd 100644 --- a/docs/report-v1.md +++ b/docs/report-v1.md @@ -41,6 +41,7 @@ Rules for both files: | :--- | :--- | :--- | | `project.name`, `repo`, `siteUrl`, `accent`, `title`, `lead` | string | Copied from perfkit.yaml. | | `project.logoSvg` | string | The SVG markup of `project.logo`, inlined. Present only when perfkit.yaml has `logo`. | +| `project.live.url` | URL | Where the latest page polls live.json: `project.live.url`, else `live.url`, else `https:///live.json` of perfkit.yaml. Absent without them. | ### run @@ -149,3 +150,34 @@ A benchmark's `headline` in runs.json is: - `passed` or `failed`, capacity: `{ limitCps, p99AtLimitMs }`, both `null` when no step passed; - `passed` or `failed`, soak: `{ memoryGrowthMBPerHour }`; - `not-run`, `inconclusive` and `aborted`: `null`. + +## live.json + +`schema: perfkit/live/v1`, JSON Schema in [schemas/live-v1.schema.json](../schemas/live-v1.schema.json). +The agent rewrites it every 15 seconds for the whole run (a temporary file, then a rename) and +serves it; nothing is kept after the run. Every number in it is preliminary: verdicts come only +from report.json. + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `runId`, `version`, `commit`, `startedAt` | | As in report.json. | +| `updatedAt` | timestamp | When this snapshot was written. | +| `status` | `running`, `finished` or `failed` | `failed` when the agent stopped the run; `reason` says why. | +| `estimatedEndAt` | timestamp or null | While running: the planned time left of the running benchmark and every queued one, plus a fixed allowance per benchmark. | +| `provider` | `digitalocean` or `local` | | +| `project` | object | As report.json's `project`. | +| `benchmarks` | list | Every benchmark of perfkit.yaml, in order. | + +Every benchmark has `id`, `title`, `unit`, `kind`, `state` (`queued`, `running`, `done` or +`not-run`, the last for one not selected), `startedAt` (compose up), `loadStartedAt` (SIPp's +start), `estimatedStartAt` (queued only) and `plannedS`. Then by kind: + +- **capacity:** `rate` (`start`, `step`, `everyS`, `max`), `thresholds`, `current` (`index`, + `total`, `offeredCps` of the step offered now, from the clock), `points` (each step as the + agent judged it when it ended: `offeredCps`, `achievedCps`, `calls`, `failed`, `p50Ms`, + `p95Ms`, `p99Ms`, `failedPct`, `ok`, `stopReason`), `heldCps` (the last step that passed) and, + once done, `result` (`limitCps`, `lowerBound`, `stopReason`). +- **soak:** `cps`, `warmupS`, `durationS`, `elapsedS`, `thresholds`, `calls` (`placed`, `failed` + so far, SIPp's counters since the load started) and `memory`: per process, its raw RSS + (`samples` of `t`, `rssMB`, at most 300, evenly spaced) and `growthMBPerHour`, the verdict's + least-squares slope, `null` until there is a full hour of samples after the warm-up.