From e3151e90a12db03e050b4f0c187135e2b7c02ee1 Mon Sep 17 00:00:00 2001 From: SandroMaglione Date: Thu, 13 Aug 2026 10:20:17 +0200 Subject: [PATCH 1/3] Reduce pull request performance checks --- .changeset/calm-ravens-check.md | 6 + .github/pull_request_template.md | 6 +- .github/workflows/ci.yml | 19 +- .github/workflows/release.yml | 5 + .../workflows/runtime-performance-comment.yml | 6 +- .github/workflows/runtime-performance.yml | 55 ++- .../workflows/type-performance-comment.yml | 4 + .github/workflows/type-performance.yml | 41 ++- package.json | 7 +- perf/runtime/README.md | 66 +--- perf/runtime/effect-runtime.mjs | 198 ----------- perf/runtime/implementations.mjs | 35 +- perf/runtime/xstate.mjs | 319 ------------------ perf/types/handle-depth-12-control.ts | 92 ----- perf/types/handle-depth-12.ts | 73 ---- perf/types/handle-depth-16-control.ts | 116 ------- perf/types/handle-depth-16.ts | 89 ----- perf/types/handle-depth-8-control.ts | 68 ---- perf/types/handle-depth-8.ts | 57 ---- pnpm-lock.yaml | 17 - scripts/ci-changes.mjs | 152 +++++++++ scripts/ci-changes.test.mjs | 84 +++++ scripts/compare-runtime-performance.mjs | 35 +- scripts/runtime-performance-regression.mjs | 68 ++++ .../runtime-performance-regression.test.mjs | 70 ++++ scripts/type-performance.mjs | 42 --- 26 files changed, 557 insertions(+), 1173 deletions(-) create mode 100644 .changeset/calm-ravens-check.md delete mode 100644 perf/runtime/effect-runtime.mjs delete mode 100644 perf/runtime/xstate.mjs delete mode 100644 perf/types/handle-depth-12-control.ts delete mode 100644 perf/types/handle-depth-12.ts delete mode 100644 perf/types/handle-depth-16-control.ts delete mode 100644 perf/types/handle-depth-16.ts delete mode 100644 perf/types/handle-depth-8-control.ts delete mode 100644 perf/types/handle-depth-8.ts create mode 100644 scripts/ci-changes.mjs create mode 100644 scripts/ci-changes.test.mjs create mode 100644 scripts/runtime-performance-regression.mjs create mode 100644 scripts/runtime-performance-regression.test.mjs diff --git a/.changeset/calm-ravens-check.md b/.changeset/calm-ravens-check.md new file mode 100644 index 0000000..dc87d76 --- /dev/null +++ b/.changeset/calm-ravens-check.md @@ -0,0 +1,6 @@ +--- +"@typeonce/effect-machine": patch +--- + +Reduce pull request performance-check latency while preserving focused type, +runtime, and memory regression coverage. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6e584c0..752c404 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,10 +11,10 @@ - [ ] `pnpm check` - [ ] Relevant example checks, when examples changed -- [ ] Reviewed the automated type-performance report, when the public TypeScript API or inference changed -- [ ] Reviewed the automated runtime-performance report, when runtime behavior changed +- [ ] Automated type-performance measurement passed or was not required +- [ ] Automated runtime- and memory-performance measurement passed or was not required diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c22681..648130f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,6 @@ name: CI on: pull_request: - push: - branches: [main] concurrency: group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -50,10 +48,17 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.examples.outputs.matrix }} + required: ${{ steps.examples.outputs.examples_required }} steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - id: examples - run: echo "matrix=$(node scripts/list-examples.mjs)" >> "$GITHUB_OUTPUT" + run: >- + node scripts/ci-changes.mjs + --base ${{ github.event.pull_request.base.sha }} + --head ${{ github.event.pull_request.head.sha }} + --github-output "$GITHUB_OUTPUT" example: needs: discover-examples @@ -92,7 +97,13 @@ jobs: - name: Require every example check to pass env: DISCOVERY_RESULT: ${{ needs.discover-examples.result }} + EXAMPLES_REQUIRED: ${{ needs.discover-examples.outputs.required }} EXAMPLE_RESULT: ${{ needs.example.result }} run: | test "$DISCOVERY_RESULT" = "success" - test "$EXAMPLE_RESULT" = "success" + if test "$EXAMPLES_REQUIRED" = "true"; then + test "$EXAMPLE_RESULT" = "success" + else + test "$EXAMPLE_RESULT" = "skipped" + echo "Examples are not affected by this pull request." >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index afdf647..defe8c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,11 @@ name: Release on: push: branches: [main] + paths: + - ".changeset/**" + - "package.json" + - "pnpm-lock.yaml" + - "src/**" concurrency: group: release-${{ github.ref }} diff --git a/.github/workflows/runtime-performance-comment.yml b/.github/workflows/runtime-performance-comment.yml index 42db330..d5b5f67 100644 --- a/.github/workflows/runtime-performance-comment.yml +++ b/.github/workflows/runtime-performance-comment.yml @@ -19,13 +19,15 @@ jobs: comment: if: >- github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' + github.event.workflow_run.conclusion != 'cancelled' runs-on: ubuntu-latest steps: - name: Check out trusted reporting code uses: actions/checkout@v7 - name: Download performance report + id: download + continue-on-error: true uses: actions/download-artifact@v8 with: name: runtime-performance-report @@ -34,6 +36,7 @@ jobs: run-id: ${{ github.event.workflow_run.id }} - name: Render report from validated benchmark data + if: steps.download.outcome == 'success' run: >- node scripts/compare-runtime-performance.mjs report/base @@ -41,6 +44,7 @@ jobs: > "$RUNNER_TEMP/report.md" - name: Create or update pull request comment + if: steps.download.outcome == 'success' uses: actions/github-script@v9 env: REPORT_PATH: ${{ runner.temp }}/report.md diff --git a/.github/workflows/runtime-performance.yml b/.github/workflows/runtime-performance.yml index 1460389..98dbc4c 100644 --- a/.github/workflows/runtime-performance.yml +++ b/.github/workflows/runtime-performance.yml @@ -11,8 +11,25 @@ permissions: contents: read jobs: - runtime-performance: - name: runtime-performance + changes: + runs-on: ubuntu-latest + outputs: + required: ${{ steps.classify.outputs.runtime_performance }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - id: classify + run: >- + node scripts/ci-changes.mjs + --base ${{ github.event.pull_request.base.sha }} + --head ${{ github.event.pull_request.head.sha }} + --github-output "$GITHUB_OUTPUT" + + measure: + needs: changes + if: needs.changes.outputs.required == 'true' + name: runtime-performance measurement runs-on: ubuntu-latest timeout-minutes: 25 steps: @@ -52,6 +69,7 @@ jobs: pnpm --dir base build - name: Measure base and pull request + id: benchmark shell: bash run: | set -euo pipefail @@ -79,16 +97,41 @@ jobs: measure base "$reports/base/5.json" measure head "$reports/head/5.json" - node head/scripts/compare-runtime-performance.mjs \ - "$reports/base" \ - "$reports/head" \ - > "$RUNNER_TEMP/runtime-performance-report.md" + if ! node head/scripts/compare-runtime-performance.mjs \ + --check \ + "$reports/base" \ + "$reports/head" \ + > "$RUNNER_TEMP/runtime-performance-report.md"; then + cat "$RUNNER_TEMP/runtime-performance-report.md" >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi cat "$RUNNER_TEMP/runtime-performance-report.md" >> "$GITHUB_STEP_SUMMARY" - name: Upload report for the comment workflow + if: always() && steps.benchmark.outcome != 'skipped' uses: actions/upload-artifact@v7 with: name: runtime-performance-report path: ${{ runner.temp }}/runtime-performance if-no-files-found: error retention-days: 7 + + runtime-performance: + if: always() + needs: [changes, measure] + name: runtime-performance + runs-on: ubuntu-latest + steps: + - name: Require relevant runtime performance to pass + env: + CHANGE_RESULT: ${{ needs.changes.result }} + MEASUREMENT_REQUIRED: ${{ needs.changes.outputs.required }} + MEASUREMENT_RESULT: ${{ needs.measure.result }} + run: | + test "$CHANGE_RESULT" = "success" + if test "$MEASUREMENT_REQUIRED" = "true"; then + test "$MEASUREMENT_RESULT" = "success" + else + test "$MEASUREMENT_RESULT" = "skipped" + echo "Runtime performance is not affected by this pull request." >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/type-performance-comment.yml b/.github/workflows/type-performance-comment.yml index 4b7c2b0..69015a3 100644 --- a/.github/workflows/type-performance-comment.yml +++ b/.github/workflows/type-performance-comment.yml @@ -26,6 +26,8 @@ jobs: uses: actions/checkout@v7 - name: Download performance report + id: download + continue-on-error: true uses: actions/download-artifact@v8 with: name: type-performance-report @@ -34,6 +36,7 @@ jobs: run-id: ${{ github.event.workflow_run.id }} - name: Render report from validated benchmark data + if: steps.download.outcome == 'success' run: >- node scripts/compare-type-performance.mjs report/before.json @@ -41,6 +44,7 @@ jobs: > "$RUNNER_TEMP/report.md" - name: Create or update pull request comment + if: steps.download.outcome == 'success' uses: actions/github-script@v9 env: REPORT_PATH: ${{ runner.temp }}/report.md diff --git a/.github/workflows/type-performance.yml b/.github/workflows/type-performance.yml index b6be4d7..a33bf9e 100644 --- a/.github/workflows/type-performance.yml +++ b/.github/workflows/type-performance.yml @@ -11,8 +11,25 @@ permissions: contents: read jobs: - type-performance: - name: type-performance + changes: + runs-on: ubuntu-latest + outputs: + required: ${{ steps.classify.outputs.type_performance }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - id: classify + run: >- + node scripts/ci-changes.mjs + --base ${{ github.event.pull_request.base.sha }} + --head ${{ github.event.pull_request.head.sha }} + --github-output "$GITHUB_OUTPUT" + + measure: + needs: changes + if: needs.changes.outputs.required == 'true' + name: type-performance measurement runs-on: ubuntu-latest steps: - name: Check out base @@ -66,3 +83,23 @@ jobs: ${{ runner.temp }}/after.json if-no-files-found: error retention-days: 7 + + type-performance: + if: always() + needs: [changes, measure] + name: type-performance + runs-on: ubuntu-latest + steps: + - name: Require relevant type performance to pass + env: + CHANGE_RESULT: ${{ needs.changes.result }} + MEASUREMENT_REQUIRED: ${{ needs.changes.outputs.required }} + MEASUREMENT_RESULT: ${{ needs.measure.result }} + run: | + test "$CHANGE_RESULT" = "success" + if test "$MEASUREMENT_REQUIRED" = "true"; then + test "$MEASUREMENT_RESULT" = "success" + else + test "$MEASUREMENT_RESULT" = "skipped" + echo "Type performance is not affected by this pull request." >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/package.json b/package.json index b9b46ee..3c0bef6 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "test": "vitest run", "test:types": "tstyche", "check:architecture": "node --test scripts/check-architecture.test.mjs && node scripts/check-architecture.mjs", + "check:ci": "node --test scripts/ci-changes.test.mjs scripts/runtime-performance-regression.test.mjs", "typecheck": "tsc -p tsconfig.json --noEmit", "perf:types": "pnpm build && node scripts/type-performance.mjs", "perf:runtime": "pnpm build && node --expose-gc scripts/runtime-performance.mjs", @@ -63,7 +64,7 @@ "docs:site:serve": "node scripts/api-reference-site/serve.mjs", "test:consumer": "node scripts/test-consumer.mjs", "pack:check": "node scripts/pack-check.mjs", - "check": "pnpm format:check && pnpm check:architecture && pnpm docs:api:check && pnpm docs:site:check && pnpm typecheck && pnpm build && pnpm test && pnpm test:types && pnpm test:consumer && pnpm pack:check", + "check": "pnpm format:check && pnpm check:architecture && pnpm check:ci && pnpm docs:api:check && pnpm docs:site:check && pnpm typecheck && pnpm build && pnpm test && pnpm test:types && pnpm test:consumer && pnpm pack:check", "changeset": "changeset", "version-packages": "changeset version", "release": "pnpm build && changeset publish" @@ -82,9 +83,7 @@ "tstyche": "7.2.1", "typedoc": "0.28.20", "typescript": "6.0.3", - "vitest": "4.1.10", - "xstate-v5": "npm:xstate@5.32.5", - "xstate-v6": "npm:xstate@6.0.0-alpha.36" + "vitest": "4.1.10" }, "packageManager": "pnpm@10.17.1", "engines": { diff --git a/perf/runtime/README.md b/perf/runtime/README.md index 2e35f71..b15c4c0 100644 --- a/perf/runtime/README.md +++ b/perf/runtime/README.md @@ -1,7 +1,6 @@ # Runtime performance -Run the local runtime benchmark suite against the compiled package and the -pinned XState comparison versions: +Run the local runtime benchmark suite against the compiled package: ```sh pnpm perf:runtime @@ -15,29 +14,12 @@ The command reports: - repeated child lookup and delivery to one running child; - machine start-and-stop throughput; - parent-with-child start-and-stop throughput; -- Effect-only lifecycle reference points for a suspended fiber, a queue worker, - a minimal actor shell, and a two-shell family; -- Effect-only coordination reference points for an owner-only mutable snapshot, - a synchronized snapshot, and a terminal `Deferred` latch; +- generic and compiled raw-process lifecycle throughput; - heap and resident-memory growth at 100, 500, and 1,000 live units, including a raw generic process, a raw compiled process, an idle statechart, two independent statecharts, a parent with one child, that relationship with child-registry observation active, and an invoked child whose active snapshots - are observed; -- lower-bound memory profiles for Effect itself: a suspended fiber, a queue - with a waiting fiber, a minimal mailbox/state/completion actor shell, and a - minimal two-shell family. - -The comparison dependencies use package aliases, so XState 5 and 6 can be -loaded by the same process: - -- `xstate-v5`: `xstate@5.32.5`, the stable v5 baseline; -- `xstate-v6`: `xstate@6.0.0-alpha.31`, the latest published v6 alpha available - when the harness was added. - -All implementations use the same flat counter topology, immutable events, and -terminal fence. The XState adapter uses `assign` in v5 and the v6 transition -function API because `assign` is not exported by that alpha. + are observed. The burst benchmark reports useful counter increments per second. It enqueues one final fence event after all counter events and awaits the machine's terminal @@ -45,20 +27,11 @@ output, so the measured duration also amortizes that fence and terminal cleanup. This measures complete queue drainage, not only the enqueue time returned by `MachineRef.send`. -Results are informational. Compare runs on the same machine while it is idle, -using the same Node.js and dependency versions. Tinybench warms each scenario -before collecting samples, and the memory measurements force garbage -collection before every observation. Each implementation's memory curve runs -in a fresh child process so garbage from one library cannot distort another -library's baseline. - -These scenarios compare observable work, not identical internals. Effect -Machine plans transitions synchronously and validates schema-backed state and -events, while its running machine provisions Effect queues, fibers, -synchronization, change publication, and child/invoke lifecycle machinery. -XState's counter is a smaller synchronous actor. Treat the comparison as an -application-level cost baseline, not a claim that the libraries provide the -same runtime guarantees. +Compare runs on the same machine while it is idle, using the same Node.js and +dependency versions. Tinybench warms each scenario before collecting samples, +and the memory measurements force garbage collection before every observation. +Each memory profile runs in a fresh child process so garbage from one profile +cannot distort another profile's baseline. The fitted heap slope is the primary idle-capacity metric. Compare adjacent profiles to attribute retained memory: raw process to idle statechart isolates @@ -66,13 +39,8 @@ statechart machinery, two independent machines to parent-with-child isolates relationship bookkeeping, while the two observed parent-child profiles isolate registry and invoked-snapshot observation. Invoked snapshot mapping uses a direct, state-scoped delivery path; its profile measures the retained callback -and mapping state rather than a general `changes` stream subscription. The -Effect profiles are primitive lower bounds, not feature-equivalent competitors. -The Effect throughput reference points similarly bound individual runtime -operations rather than predicting a complete machine by themselves. In -particular, the owner-only mutable snapshot is safe only when one process fiber -owns active state writes; terminal arbitration and externally visible -observation still require separate coordination. +and mapping state rather than a general `changes` stream subscription. + Resident memory is reported as a raw diagnostic because V8 and the operating-system allocator can reuse already committed pages. The capacity-per-GiB value is a linear estimate that excludes shared process @@ -105,9 +73,11 @@ benchmark workflow has read-only repository access; a separate trusted `workflow_run` workflow validates the uploaded JSON before receiving permission to update the comment. -The implementation lives in `scripts/runtime-performance.mjs`; the Effect -Machine fixture is in `perf/runtime/counter.mjs`, and the comparison adapter is -in `perf/runtime/xstate.mjs`. Effect runtime reference fixtures are in -`perf/runtime/effect-runtime.mjs`. Add cross-library scenarios only when every -implementation performs equivalent observable work and the result is consumed -and checked so the JavaScript engine cannot discard it. +The required pull request check rejects throughput decreases above both 15% +and three times the observed process-level median absolute deviation. Heap per +unit uses the same variability rule with a 20% floor. RSS remains informational +because hosted-runner and allocator behavior makes it substantially noisier. + +The implementation lives in `scripts/runtime-performance.mjs`, and the Effect +Machine fixture is in `perf/runtime/counter.mjs`. Every scenario consumes and +checks its result so the JavaScript engine cannot discard the measured work. diff --git a/perf/runtime/effect-runtime.mjs b/perf/runtime/effect-runtime.mjs deleted file mode 100644 index e62d55b..0000000 --- a/perf/runtime/effect-runtime.mjs +++ /dev/null @@ -1,198 +0,0 @@ -import { createRequire } from "node:module" -import { readFileSync } from "node:fs" -import { dirname, join, resolve } from "node:path" -import { fileURLToPath, pathToFileURL } from "node:url" - -const implementationRoot = resolve( - process.env.EFFECT_MACHINE_BENCHMARK_ROOT ?? fileURLToPath(new URL("../..", import.meta.url)) -) -const implementationRequire = createRequire(pathToFileURL(join(implementationRoot, "package.json"))) -const effectPackagePath = implementationRequire.resolve("effect/package.json") -const effectPackage = JSON.parse(readFileSync(effectPackagePath, "utf8")) -const effect = await import(pathToFileURL(resolve(dirname(effectPackagePath), effectPackage.exports["."])).href) -const { Deferred, Effect, Fiber, MutableRef, Queue, Ref, SynchronizedRef } = effect - -const startUnits = (count, make) => - Effect.runPromise( - Effect.forEach(Array.from({ length: count }), make, { concurrency: 1 }) - ) - -const stopUnits = (units) => - Effect.runPromise( - Effect.forEach( - units, - (unit) => - Fiber.interruptAll(unit.fibers).pipe( - Effect.andThen(Effect.forEach(unit.queues, Queue.shutdown, { discard: true })) - ), - { concurrency: "unbounded", discard: true } - ) - ) - -const makeFiber = () => - Effect.map(Effect.forkDetach(Effect.never), (fiber) => ({ - fibers: [fiber], - queues: [] - })) - -const makeMailbox = () => - Effect.gen(function*() { - const queue = yield* Queue.unbounded() - const fiber = yield* Effect.forkDetach(Effect.forever(Queue.take(queue))) - return { fibers: [fiber], queues: [queue] } - }) - -const makeActorShell = () => - Effect.gen(function*() { - const queue = yield* Queue.unbounded() - const state = yield* Ref.make(0) - const done = yield* Deferred.make() - const fiber = yield* Effect.forkDetach( - Effect.forever( - Queue.take(queue).pipe( - Effect.flatMap((update) => Ref.update(state, update)) - ) - ) - ) - return { fibers: [fiber], queues: [queue], retained: [state, done] } - }) - -const makeActorFamily = () => - Effect.all([makeActorShell(), makeActorShell()], { concurrency: 1 }).pipe( - Effect.map((units) => ({ - fibers: units.flatMap((unit) => unit.fibers), - queues: units.flatMap((unit) => unit.queues), - retained: units - })) - ) - -const runLifecycle = async (make, expectedUnits = 1) => { - const units = await startUnits(expectedUnits, make) - try { - if (units.length !== expectedUnits) { - throw new Error(`Effect runtime lifecycle started ${units.length} units, expected ${expectedUnits}`) - } - return 1 - } finally { - await stopUnits(units) - } -} - -const mutableSnapshot = MutableRef.make(0) -const runMutableSnapshotBatch = (size) => { - const before = MutableRef.get(mutableSnapshot) - for (let index = 0; index < size; index += 1) { - MutableRef.update(mutableSnapshot, (value) => value + 1) - } - return MutableRef.get(mutableSnapshot) - before -} - -const synchronizedSnapshot = Effect.runSync(SynchronizedRef.make(0)) -const runSynchronizedSnapshotBatch = (size) => - Effect.runPromise( - Effect.gen(function*() { - const before = yield* SynchronizedRef.get(synchronizedSnapshot) - for (let index = 0; index < size; index += 1) { - yield* SynchronizedRef.update(synchronizedSnapshot, (value) => value + 1) - } - return (yield* SynchronizedRef.get(synchronizedSnapshot)) - before - }) - ) - -const runTerminalLatchBatch = (size) => - Effect.runPromise( - Effect.gen(function*() { - for (let index = 0; index < size; index += 1) { - const latch = yield* Deferred.make() - yield* Deferred.succeed(latch, undefined) - yield* Deferred.await(latch) - } - return size - }) - ) - -export const makeEffectRuntimeAdapter = (version) => ({ - implementation: "effect-runtime", - label: "Effect runtime primitives", - version, - runtimeBenchmarks: [ - { - id: "effect-fiber-start-stop", - label: "Start and interrupt a suspended fiber", - unit: "fibers/s", - async: true, - operations: () => 1, - run: () => runLifecycle(makeFiber) - }, - { - id: "effect-mailbox-start-stop", - label: "Start and stop a queue worker", - unit: "workers/s", - async: true, - operations: () => 1, - run: () => runLifecycle(makeMailbox) - }, - { - id: "effect-actor-shell-start-stop", - label: "Start and stop an actor shell", - unit: "actors/s", - async: true, - operations: () => 1, - run: () => runLifecycle(makeActorShell) - }, - { - id: "effect-actor-family-start-stop", - label: "Start and stop two actor shells", - unit: "families/s", - async: true, - operations: () => 1, - run: () => runLifecycle(makeActorFamily) - }, - { - id: "effect-mutable-snapshot-update", - label: "Update an owner-only mutable snapshot", - unit: "updates/s", - async: false, - operations: (configuration) => configuration.primitiveBatchSize, - run: runMutableSnapshotBatch - }, - { - id: "effect-synchronized-snapshot-update", - label: "Update a synchronized snapshot", - unit: "updates/s", - async: true, - operations: (configuration) => configuration.primitiveBatchSize, - run: runSynchronizedSnapshotBatch - }, - { - id: "effect-terminal-latch", - label: "Create, resolve, and await a terminal latch", - unit: "latches/s", - async: true, - operations: (configuration) => configuration.primitiveBatchSize, - run: runTerminalLatchBatch - } - ], - memoryProfiles: { - "effect-fiber": { - label: "Suspended Effect fiber", - start: (count) => startUnits(count, makeFiber), - stop: stopUnits - }, - "effect-mailbox": { - label: "Effect queue with waiting fiber", - start: (count) => startUnits(count, makeMailbox), - stop: stopUnits - }, - "effect-actor-shell": { - label: "Effect mailbox actor shell", - start: (count) => startUnits(count, makeActorShell), - stop: stopUnits - }, - "effect-actor-family": { - label: "Two Effect actor shells", - start: (count) => startUnits(count, makeActorFamily), - stop: stopUnits - } - } -}) diff --git a/perf/runtime/implementations.mjs b/perf/runtime/implementations.mjs index 486da1b..8b52080 100644 --- a/perf/runtime/implementations.mjs +++ b/perf/runtime/implementations.mjs @@ -1,11 +1,7 @@ import { readFileSync } from "node:fs" import { resolve } from "node:path" import { fileURLToPath } from "node:url" -import * as XStateV5 from "xstate-v5" -import * as XStateV6 from "xstate-v6" import { effectMachineAdapter } from "./counter.mjs" -import { makeEffectRuntimeAdapter } from "./effect-runtime.mjs" -import { makeXStateAdapter } from "./xstate.mjs" const readPackageVersion = (path) => JSON.parse(readFileSync(path, "utf8")).version const implementationRoot = resolve( @@ -15,34 +11,15 @@ const implementationRoot = resolve( export const packageVersions = { effectMachine: readPackageVersion(resolve(implementationRoot, "package.json")), effect: readPackageVersion(resolve(implementationRoot, "node_modules/effect/package.json")), - tinybench: readPackageVersion(new URL("../../node_modules/tinybench/package.json", import.meta.url)), - xstateV5: readPackageVersion(new URL("../../node_modules/xstate-v5/package.json", import.meta.url)), - xstateV6: readPackageVersion(new URL("../../node_modules/xstate-v6/package.json", import.meta.url)) + tinybench: readPackageVersion(new URL("../../node_modules/tinybench/package.json", import.meta.url)) } export const implementations = [ - { ...effectMachineAdapter, version: packageVersions.effectMachine }, - makeXStateAdapter({ - implementation: "xstate-v5", - label: "XState 5", - version: packageVersions.xstateV5, - xstate: XStateV5 - }), - makeXStateAdapter({ - implementation: "xstate-v6", - label: "XState 6 alpha", - version: packageVersions.xstateV6, - xstate: XStateV6 - }) + { ...effectMachineAdapter, version: packageVersions.effectMachine } ] -export const effectRuntimeImplementation = makeEffectRuntimeAdapter(packageVersions.effect) -export const runtimeReferenceImplementations = [ - ...implementations.filter((implementation) => implementation.runtimeBenchmarks !== undefined), - effectRuntimeImplementation -] +export const runtimeReferenceImplementations = implementations.filter( + (implementation) => implementation.runtimeBenchmarks !== undefined +) -export const memoryImplementations = [ - ...implementations, - effectRuntimeImplementation -] +export const memoryImplementations = implementations diff --git a/perf/runtime/xstate.mjs b/perf/runtime/xstate.mjs deleted file mode 100644 index 2473ce7..0000000 --- a/perf/runtime/xstate.mjs +++ /dev/null @@ -1,319 +0,0 @@ -const incrementEvent = Object.freeze({ type: "Increment" }) -const incrementLeftEvent = Object.freeze({ type: "IncrementLeft" }) -const incrementRightEvent = Object.freeze({ type: "IncrementRight" }) -const finishEvent = Object.freeze({ type: "Finish" }) - -export const makeXStateAdapter = (options) => { - const { implementation, label, version, xstate } = options - const increment = typeof xstate.assign === "function" - ? { - actions: xstate.assign({ - value: ({ context }) => context.value + 1 - }) - } - : ({ context }) => ({ - context: { value: context.value + 1 } - }) - - const machine = xstate.createMachine({ - id: `RuntimeBenchmarkCounter-${implementation}`, - context: { value: 0 }, - initial: "counting", - states: { - counting: { - on: { - Increment: increment, - Finish: { target: "done" } - } - }, - done: { type: "final" } - } - }) - const parentMachine = xstate.createMachine({ - id: `RuntimeBenchmarkCounterParent-${implementation}`, - initial: "active", - states: { - active: { - invoke: { - id: "counter", - src: machine - } - } - } - }) - const hierarchicalMachine = xstate.createMachine({ - id: `RuntimeBenchmarkHierarchicalCounter-${implementation}`, - context: { value: 0 }, - initial: "active", - states: { - active: { - initial: "counting", - states: { - counting: { - on: { - Increment: increment, - Finish: { target: "#complete" } - } - } - } - }, - complete: { id: "complete", type: "final" } - } - }) - const parallelIncrement = (key) => - typeof xstate.assign === "function" - ? { - actions: xstate.assign({ - [key]: ({ context }) => context[key] + 1 - }) - } - : ({ context }) => ({ - context: { ...context, [key]: context[key] + 1 } - }) - const parallelMachine = xstate.createMachine({ - id: `RuntimeBenchmarkParallelCounter-${implementation}`, - context: { left: 0, right: 0 }, - initial: "active", - states: { - active: { - type: "parallel", - on: { Finish: { target: "complete" } }, - states: { - left: { - on: { IncrementLeft: parallelIncrement("left") } - }, - right: { - on: { IncrementRight: parallelIncrement("right") } - } - } - }, - complete: { type: "final" } - } - }) - const initialSnapshot = xstate.initialTransition(machine)[0] - - const planCounterBatch = (size) => { - let snapshot = initialSnapshot - for (let index = 0; index < size; index += 1) { - snapshot = xstate.transition(machine, snapshot, incrementEvent)[0] - } - return snapshot.context.value - } - - const startCounter = () => xstate.createActor(machine).start() - const stopCounter = (actor) => actor.stop() - - const startObservedCounter = () => { - const actor = xstate.createActor(machine) - let observed - let complete - const completion = new Promise((resolve) => { - complete = resolve - }) - const subscription = actor.subscribe({ - next: (snapshot) => { - observed = snapshot - }, - complete - }) - actor.start() - return { actor, completion, getObserved: () => observed, subscription } - } - const stopObservedCounter = ({ actor, subscription }) => { - actor.stop() - subscription.unsubscribe() - } - - const startChildCounter = () => { - const parent = xstate.createActor(parentMachine).start() - if (parent.getSnapshot().children.counter === undefined) { - parent.stop() - throw new Error(`${label} child did not become ready`) - } - return parent - } - const stopChildCounter = (parent) => parent.stop() - - const runCounterBurst = (actor, size) => { - for (let index = 0; index < size; index += 1) { - actor.send(incrementEvent) - } - actor.send(finishEvent) - const snapshot = actor.getSnapshot() - if (snapshot.status !== "done") { - throw new Error(`${label} terminal fence produced status ${snapshot.status}`) - } - return snapshot.context.value - } - - const runMachineBurst = (actor, size, readValue, eventAt = () => incrementEvent) => { - for (let index = 0; index < size; index += 1) { - actor.send(eventAt(index)) - } - actor.send(finishEvent) - const snapshot = actor.getSnapshot() - if (snapshot.status !== "done") { - throw new Error(`${label} hierarchical terminal fence produced status ${snapshot.status}`) - } - return readValue(snapshot.context) - } - - const runObservedCounterBurst = async ({ actor, completion, getObserved }, size) => { - for (let index = 0; index < size; index += 1) { - actor.send(incrementEvent) - } - actor.send(finishEvent) - await completion - const snapshot = getObserved() - if (snapshot?.status !== "done") { - throw new Error(`${label} observed terminal fence produced status ${snapshot?.status}`) - } - return snapshot.context.value - } - - const runChildCounterBurst = (parent, size) => { - for (let index = 0; index < size; index += 1) { - const child = parent.getSnapshot().children.counter - if (child === undefined) { - throw new Error(`${label} child disappeared during the benchmark`) - } - child.send(incrementEvent) - } - const child = parent.getSnapshot().children.counter - if (child === undefined) { - throw new Error(`${label} child disappeared before the terminal fence`) - } - child.send(finishEvent) - const snapshot = child.getSnapshot() - if (snapshot.status !== "done") { - throw new Error(`${label} child terminal fence produced status ${snapshot.status}`) - } - return snapshot.context.value - } - - const runLifecycle = () => { - const actor = startCounter() - try { - const snapshot = actor.getSnapshot() - if (snapshot.status !== "active" || snapshot.context.value !== 0) { - throw new Error(`${label} lifecycle benchmark produced an invalid initial snapshot`) - } - } finally { - stopCounter(actor) - } - } - - const runChildLifecycle = () => { - const parent = startChildCounter() - try { - const child = parent.getSnapshot().children.counter - if (child === undefined || child.getSnapshot().status !== "active") { - throw new Error(`${label} child lifecycle benchmark produced an invalid initial snapshot`) - } - } finally { - stopChildCounter(parent) - } - } - - const startCounters = (count) => { - const actors = [] - for (let index = 0; index < count; index += 1) { - actors.push(startCounter()) - } - return actors - } - - const stopCounters = (actors) => { - for (const actor of actors) { - stopCounter(actor) - } - } - - const startChildCounters = (count) => { - const actors = [] - for (let index = 0; index < count; index += 1) { - actors.push(startChildCounter()) - } - return actors - } - - const stopChildCounters = stopCounters - - const startIndependentCounterPairs = (count) => { - const pairs = [] - for (let index = 0; index < count; index += 1) { - pairs.push([startCounter(), startCounter()]) - } - return pairs - } - - const stopIndependentCounterPairs = (pairs) => stopCounters(pairs.flat()) - - return { - implementation, - label, - version, - async: false, - planCounterBatch, - runCounterBurst, - runChildCounterBurst, - runChildLifecycle, - runLifecycle, - runObservedCounterBurst, - startCounter, - startChildCounter, - startCounters, - startChildCounters, - startObservedCounter, - stopCounter, - stopChildCounter, - stopChildCounters, - stopCounters, - stopObservedCounter, - additionalMachineBenchmarks: [ - { - id: "hierarchical-runtime-burst", - label: "Drain burst through a compound state", - unit: "events/s", - operations: ({ burstBatchSize }) => burstBatchSize, - expected: (operations) => operations, - start: () => xstate.createActor(hierarchicalMachine).start(), - run: (actor, size) => runMachineBurst(actor, size, (context) => context.value), - stop: stopCounter - }, - { - id: "parallel-runtime-burst", - label: "Drain burst through two parallel regions", - unit: "events/s", - operations: ({ burstBatchSize }) => burstBatchSize, - expected: (operations) => operations, - start: () => xstate.createActor(parallelMachine).start(), - run: (actor, size) => - runMachineBurst( - actor, - size, - (context) => context.left + context.right, - (index) => index % 2 === 0 ? incrementLeftEvent : incrementRightEvent - ), - stop: stopCounter - } - ], - memoryProfiles: { - idle: { - label: "Idle machine", - start: startCounters, - stop: stopCounters - }, - "two-independent": { - label: "Two independent idle machines", - start: startIndependentCounterPairs, - stop: stopIndependentCounterPairs - }, - "parent-with-child": { - label: "Idle parent with one child", - start: startChildCounters, - stop: stopChildCounters - } - } - } -} diff --git a/perf/types/handle-depth-12-control.ts b/perf/types/handle-depth-12-control.ts deleted file mode 100644 index b23cd36..0000000 --- a/perf/types/handle-depth-12-control.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { Machine } from "@typeonce/effect-machine" -import { Schema } from "effect" - -export const NodeState = Schema.TaggedStruct("Node", {}) - -export const States = Machine.defineStates({ - n0: { - schema: NodeState, - initial: "n1", - states: { - n1: { - schema: NodeState, - initial: "n2", - states: { - n2: { - schema: NodeState, - initial: "n3", - states: { - n3: { - schema: NodeState, - initial: "n4", - states: { - n4: { - schema: NodeState, - initial: "n5", - states: { - n5: { - schema: NodeState, - initial: "n6", - states: { - n6: { - schema: NodeState, - initial: "n7", - states: { - n7: { - schema: NodeState, - initial: "n8", - states: { - n8: { - schema: NodeState, - initial: "n9", - states: { - n9: { - schema: NodeState, - initial: "n10", - states: { - n10: { - schema: NodeState, - initial: "n11", - states: { - n11: { - schema: NodeState, - initial: "n12", - states: { - n12: { - schema: NodeState, - type: "final", - output: Schema.String - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -}) - -export const machine = Machine.make({ - states: States.states, - events: [], - initial: (): never => { - throw new Error("type-performance fixture") - } -}) diff --git a/perf/types/handle-depth-12.ts b/perf/types/handle-depth-12.ts deleted file mode 100644 index 3fa55cc..0000000 --- a/perf/types/handle-depth-12.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Machine } from "@typeonce/effect-machine" -import { Context, Data, Effect } from "effect" -import { machine } from "./handle-depth-12-control.js" - -type Equal = (() => Type extends Left ? 1 : 2) extends (() => Type extends Right ? 1 : 2) ? - true : - false -type Expect = Value - -class DeepService extends Context.Service()("perf/depth-12/DeepService") {} -class DeepFailure extends Data.TaggedError("DeepFailure")<{}> {} - -const handled = machine.handle({ - n0: { - states: { - n1: { - states: { - n2: { - states: { - n3: { - states: { - n4: { - states: { - n5: { - states: { - n6: { - states: { - n7: { - states: { - n8: { - states: { - n9: { - states: { - n10: { - states: { - n11: { - states: { - n12: { - entry: () => {}, - output: () => "done" - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -}) - -type ErrorIsExact = Expect, never>> -type ServicesAreExact = Expect, never>> -type EveryStateIsHandled = Expect, never>> - -void Machine.planInitial(handled) -export type { ErrorIsExact, EveryStateIsHandled, ServicesAreExact } diff --git a/perf/types/handle-depth-16-control.ts b/perf/types/handle-depth-16-control.ts deleted file mode 100644 index 4febb84..0000000 --- a/perf/types/handle-depth-16-control.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { Machine } from "@typeonce/effect-machine" -import { Schema } from "effect" - -export const NodeState = Schema.TaggedStruct("Node", {}) - -export const States = Machine.defineStates({ - n0: { - schema: NodeState, - initial: "n1", - states: { - n1: { - schema: NodeState, - initial: "n2", - states: { - n2: { - schema: NodeState, - initial: "n3", - states: { - n3: { - schema: NodeState, - initial: "n4", - states: { - n4: { - schema: NodeState, - initial: "n5", - states: { - n5: { - schema: NodeState, - initial: "n6", - states: { - n6: { - schema: NodeState, - initial: "n7", - states: { - n7: { - schema: NodeState, - initial: "n8", - states: { - n8: { - schema: NodeState, - initial: "n9", - states: { - n9: { - schema: NodeState, - initial: "n10", - states: { - n10: { - schema: NodeState, - initial: "n11", - states: { - n11: { - schema: NodeState, - initial: "n12", - states: { - n12: { - schema: NodeState, - initial: "n13", - states: { - n13: { - schema: NodeState, - initial: "n14", - states: { - n14: { - schema: NodeState, - initial: "n15", - states: { - n15: { - schema: NodeState, - initial: "n16", - states: { - n16: { - schema: NodeState, - type: "final", - output: Schema.String - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -}) - -export const machine = Machine.make({ - states: States.states, - events: [], - initial: (): never => { - throw new Error("type-performance fixture") - } -}) diff --git a/perf/types/handle-depth-16.ts b/perf/types/handle-depth-16.ts deleted file mode 100644 index 195473c..0000000 --- a/perf/types/handle-depth-16.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Machine } from "@typeonce/effect-machine" -import { Context, Data, Effect } from "effect" -import { machine } from "./handle-depth-16-control.js" - -type Equal = (() => Type extends Left ? 1 : 2) extends (() => Type extends Right ? 1 : 2) ? - true : - false -type Expect = Value - -class DeepService extends Context.Service()("perf/depth-16/DeepService") {} -class DeepFailure extends Data.TaggedError("DeepFailure")<{}> {} - -const handled = machine.handle({ - n0: { - states: { - n1: { - states: { - n2: { - states: { - n3: { - states: { - n4: { - states: { - n5: { - states: { - n6: { - states: { - n7: { - states: { - n8: { - states: { - n9: { - states: { - n10: { - states: { - n11: { - states: { - n12: { - states: { - n13: { - states: { - n14: { - states: { - n15: { - states: { - n16: { - entry: () => {}, - output: () => "done" - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -}) - -type ErrorIsExact = Expect, never>> -type ServicesAreExact = Expect, never>> -type EveryStateIsHandled = Expect, never>> - -void Machine.planInitial(handled) -export type { ErrorIsExact, EveryStateIsHandled, ServicesAreExact } diff --git a/perf/types/handle-depth-8-control.ts b/perf/types/handle-depth-8-control.ts deleted file mode 100644 index e432813..0000000 --- a/perf/types/handle-depth-8-control.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { Machine } from "@typeonce/effect-machine" -import { Schema } from "effect" - -export const NodeState = Schema.TaggedStruct("Node", {}) - -export const States = Machine.defineStates({ - n0: { - schema: NodeState, - initial: "n1", - states: { - n1: { - schema: NodeState, - initial: "n2", - states: { - n2: { - schema: NodeState, - initial: "n3", - states: { - n3: { - schema: NodeState, - initial: "n4", - states: { - n4: { - schema: NodeState, - initial: "n5", - states: { - n5: { - schema: NodeState, - initial: "n6", - states: { - n6: { - schema: NodeState, - initial: "n7", - states: { - n7: { - schema: NodeState, - initial: "n8", - states: { - n8: { - schema: NodeState, - type: "final", - output: Schema.String - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -}) - -export const machine = Machine.make({ - states: States.states, - events: [], - initial: (): never => { - throw new Error("type-performance fixture") - } -}) diff --git a/perf/types/handle-depth-8.ts b/perf/types/handle-depth-8.ts deleted file mode 100644 index d3457bc..0000000 --- a/perf/types/handle-depth-8.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Machine } from "@typeonce/effect-machine" -import { Context, Data, Effect } from "effect" -import { machine } from "./handle-depth-8-control.js" - -type Equal = (() => Type extends Left ? 1 : 2) extends (() => Type extends Right ? 1 : 2) ? - true : - false -type Expect = Value - -class DeepService extends Context.Service()("perf/depth-8/DeepService") {} -class DeepFailure extends Data.TaggedError("DeepFailure")<{}> {} - -const handled = machine.handle({ - n0: { - states: { - n1: { - states: { - n2: { - states: { - n3: { - states: { - n4: { - states: { - n5: { - states: { - n6: { - states: { - n7: { - states: { - n8: { - entry: () => {}, - output: () => "done" - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -}) - -type ErrorIsExact = Expect, never>> -type ServicesAreExact = Expect, never>> -type EveryStateIsHandled = Expect, never>> - -void Machine.planInitial(handled) -export type { ErrorIsExact, EveryStateIsHandled, ServicesAreExact } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 428334d..6ba77ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,12 +45,6 @@ importers: vitest: specifier: 4.1.10 version: 4.1.10(@types/node@25.7.0)(vite@8.1.5(@types/node@25.7.0)(yaml@2.9.0)) - xstate-v5: - specifier: npm:xstate@5.32.5 - version: xstate@5.32.5 - xstate-v6: - specifier: npm:xstate@6.0.0-alpha.36 - version: xstate@6.0.0-alpha.36 packages: @@ -1128,13 +1122,6 @@ packages: engines: {node: '>=8'} hasBin: true - xstate@5.32.5: - resolution: {integrity: sha512-ULazi1oe6wGrXl0Frb6otSlkm5HLifbbVTkMk5kkSKqz4TkxJaVpnl6jOJwKeid3ORPxYyZQgNLUSYX9q65SIA==} - - xstate@6.0.0-alpha.36: - resolution: {integrity: sha512-8zXcB9BU/o0j3NTrOLovqMr9h0Ni0Y06Dz1BW6g7ZHTg/C64RcTqZkkLdR1KywhMdTN44DzTbNwqgdY3rQz6DA==} - hasBin: true - yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -2117,8 +2104,4 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 - xstate@5.32.5: {} - - xstate@6.0.0-alpha.36: {} - yaml@2.9.0: {} diff --git a/scripts/ci-changes.mjs b/scripts/ci-changes.mjs new file mode 100644 index 0000000..4a56712 --- /dev/null +++ b/scripts/ci-changes.mjs @@ -0,0 +1,152 @@ +import { execFileSync } from "node:child_process" +import { appendFileSync, readdirSync, readFileSync } from "node:fs" +import { resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const performanceDependencies = new Set(["effect", "tinybench", "typescript"]) + +const dependencyVersions = (packageJson) => ({ + ...packageJson.dependencies, + ...packageJson.devDependencies, + ...packageJson.optionalDependencies, + ...packageJson.peerDependencies +}) + +const relevantDependenciesChanged = (beforePackageJson, afterPackageJson) => { + const before = dependencyVersions(beforePackageJson) + const after = dependencyVersions(afterPackageJson) + return [...performanceDependencies].some((dependency) => before[dependency] !== after[dependency]) +} + +const changedExampleNames = (changedFiles, availableExamples) => { + const available = new Set(availableExamples) + return [...new Set( + changedFiles + .map((path) => /^examples\/([^/]+)\//.exec(path)?.[1]) + .filter((name) => name !== undefined && available.has(name)) + )].sort() +} + +export const classifyChanges = ({ + afterPackageJson, + availableExamples, + beforePackageJson, + changedFiles +}) => { + const sourceChanged = changedFiles.some((path) => path.startsWith("src/")) + const dependencyChanged = relevantDependenciesChanged(beforePackageJson, afterPackageJson) + const classifierChanged = changedFiles.includes("scripts/ci-changes.mjs") + const typePerformance = sourceChanged || + dependencyChanged || + classifierChanged || + changedFiles.some((path) => + path.startsWith("perf/types/") || + path === "scripts/type-performance.mjs" || + path === "scripts/compare-type-performance.mjs" || + path === "tsconfig.json" || + path === "tsconfig.build.json" || + path === ".github/workflows/type-performance.yml" + ) + const runtimePerformance = dependencyChanged || + classifierChanged || + changedFiles.some((path) => + path.startsWith("src/internal/machine/") || + path === "src/Machine.ts" || + path.startsWith("perf/runtime/") || + path === "scripts/runtime-performance.mjs" || + path === "scripts/compare-runtime-performance.mjs" || + path === "tsconfig.build.json" || + path === ".github/workflows/runtime-performance.yml" + ) + const allExamples = sourceChanged || + classifierChanged || + changedFiles.some((path) => + path === "package.json" || + path === "pnpm-workspace.yaml" || + path === "tsconfig.build.json" || + path === "scripts/list-examples.mjs" || + path === ".github/workflows/ci.yml" + ) + const examples = allExamples + ? [...availableExamples].sort() + : changedExampleNames(changedFiles, availableExamples) + + return { + examples, + runtimePerformance, + typePerformance + } +} + +const git = (...args) => execFileSync("git", args, { encoding: "utf8" }) + +const readPackageJsonAt = (revision) => JSON.parse(git("show", `${revision}:package.json`)) + +const readAvailableExamples = () => { + const examplesRoot = resolve(import.meta.dirname, "..", "examples") + return readdirSync(examplesRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const packageJson = JSON.parse(readFileSync(resolve(examplesRoot, entry.name, "package.json"), "utf8")) + if (typeof packageJson.scripts?.check !== "string") { + throw new Error(`examples/${entry.name}/package.json must define a check script`) + } + return entry.name + }) +} + +const main = () => { + const options = { + base: undefined, + githubOutput: undefined, + head: undefined + } + for (let index = 2; index < process.argv.length; index += 1) { + const argument = process.argv[index] + if (argument === "--base" || argument === "--head" || argument === "--github-output") { + const value = process.argv[index + 1] + if (value === undefined) { + throw new Error(`${argument} requires a value`) + } + options[argument === "--github-output" ? "githubOutput" : argument.slice(2)] = value + index += 1 + } else { + throw new Error(`Unknown argument: ${argument}`) + } + } + if (options.base === undefined || options.head === undefined) { + throw new Error("Usage: node scripts/ci-changes.mjs --base --head [--github-output ]") + } + + const changedFiles = git( + "diff", + "--name-only", + "--diff-filter=ACDMRTUXB", + `${options.base}...${options.head}` + ).trim().split("\n").filter(Boolean) + const result = classifyChanges({ + afterPackageJson: readPackageJsonAt(options.head), + availableExamples: readAvailableExamples(), + beforePackageJson: readPackageJsonAt(options.base), + changedFiles + }) + const output = [ + `type_performance=${result.typePerformance}`, + `runtime_performance=${result.runtimePerformance}`, + `examples_required=${result.examples.length > 0}`, + `examples=${JSON.stringify(result.examples.map((example) => ({ + example, + directory: `examples/${example}` + })))}` + ].join("\n") + + if (options.githubOutput === undefined) { + console.log(JSON.stringify(result, null, 2)) + } else { + appendFileSync(options.githubOutput, `${output}\n`) + } +} + +if (process.argv[1] !== undefined && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + main() +} diff --git a/scripts/ci-changes.test.mjs b/scripts/ci-changes.test.mjs new file mode 100644 index 0000000..62bac0e --- /dev/null +++ b/scripts/ci-changes.test.mjs @@ -0,0 +1,84 @@ +import { strict as assert } from "node:assert" +import { test } from "node:test" +import { classifyChanges } from "./ci-changes.mjs" + +const packageJson = { + devDependencies: { + dprint: "1.0.0", + effect: "4.0.0", + tinybench: "2.0.0", + typescript: "6.0.0" + } +} + +const classify = (changedFiles, afterPackageJson = packageJson) => + classifyChanges({ + afterPackageJson, + availableExamples: ["platformer", "playground", "pokemon"], + beforePackageJson: packageJson, + changedFiles + }) + +test("skips performance and examples for documentation changes", () => { + assert.deepEqual(classify(["README.md", "docs/agent-guide.md"]), { + examples: [], + runtimePerformance: false, + typePerformance: false + }) +}) + +test("selects only the changed example", () => { + assert.deepEqual(classify(["examples/pokemon/src/machine.ts"]), { + examples: ["pokemon"], + runtimePerformance: false, + typePerformance: false + }) +}) + +test("runs type performance and every example for public source changes", () => { + assert.deepEqual(classify(["src/index.ts"]), { + examples: ["platformer", "playground", "pokemon"], + runtimePerformance: false, + typePerformance: true + }) +}) + +test("adds runtime performance for machine implementation changes", () => { + assert.deepEqual(classify(["src/internal/machine/runtime.ts"]), { + examples: ["platformer", "playground", "pokemon"], + runtimePerformance: true, + typePerformance: true + }) +}) + +test("distinguishes performance dependencies from unrelated tooling", () => { + assert.deepEqual(classify(["package.json"], { + ...packageJson, + devDependencies: { ...packageJson.devDependencies, dprint: "2.0.0" } + }), { + examples: ["platformer", "playground", "pokemon"], + runtimePerformance: false, + typePerformance: false + }) + assert.deepEqual(classify(["package.json"], { + ...packageJson, + devDependencies: { ...packageJson.devDependencies, effect: "4.1.0" } + }), { + examples: ["platformer", "playground", "pokemon"], + runtimePerformance: true, + typePerformance: true + }) +}) + +test("classifies performance harness and classifier changes", () => { + assert.deepEqual(classify(["scripts/type-performance.mjs"]), { + examples: [], + runtimePerformance: false, + typePerformance: true + }) + assert.deepEqual(classify(["scripts/ci-changes.mjs"]), { + examples: ["platformer", "playground", "pokemon"], + runtimePerformance: true, + typePerformance: true + }) +}) diff --git a/scripts/compare-runtime-performance.mjs b/scripts/compare-runtime-performance.mjs index 5cfbbbf..36e07a9 100644 --- a/scripts/compare-runtime-performance.mjs +++ b/scripts/compare-runtime-performance.mjs @@ -1,11 +1,14 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs" import { join } from "node:path" +import { findRuntimePerformanceRegressions } from "./runtime-performance-regression.mjs" -const [baseDirectory, pullRequestDirectory] = process.argv.slice(2) +const arguments_ = process.argv.slice(2) +const check = arguments_.includes("--check") +const [baseDirectory, pullRequestDirectory] = arguments_.filter((argument) => argument !== "--check") if (baseDirectory === undefined || pullRequestDirectory === undefined) { throw new Error( - "Usage: node scripts/compare-runtime-performance.mjs " + "Usage: node scripts/compare-runtime-performance.mjs [--check] " ) } @@ -240,6 +243,8 @@ const baseReports = readReports(baseDirectory, false) const pullRequestReports = readReports(pullRequestDirectory, true) const base = baseReports.length === 0 ? undefined : aggregate(baseReports, "base") const pullRequest = aggregate(pullRequestReports, "pull request") +const comparable = base !== undefined && JSON.stringify(base.configuration) === JSON.stringify(pullRequest.configuration) +const regressions = comparable ? findRuntimePerformanceRegressions(base, pullRequest) : [] const integer = new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 }) const decimal = new Intl.NumberFormat("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 }) @@ -324,7 +329,7 @@ for (const scenario of machineScenarios) { if (runtimeScenarios.length > 0) { lines.push( "", - "### Effect runtime reference points", + "### Process runtime reference points", "", `| Scenario | ${runtimeImplementations.map((implementation) => escapeCell(implementation.label)).join(" | ")} |`, `| --- | ${runtimeImplementations.map(() => "---:").join(" | ")} |` @@ -423,7 +428,7 @@ if (base === undefined) { if (baseRuntime.size > 0 && pullRequestRuntime.size > 0) { lines.push( "", - "### Effect runtime reference change from base", + "### Process runtime reference change from base", "", "| Metric | Base | Base variability | PR | PR variability | Difference |", "| --- | ---: | ---: | ---: | ---: | ---: |" @@ -440,6 +445,23 @@ if (base === undefined) { } } +lines.push( + "", + "### Regression guard", + "", + comparable + ? regressions.length === 0 + ? "No large, noise-adjusted throughput or heap regressions detected." + : "The required regression guard detected:" + : "The regression guard was not evaluated because the base and pull request configurations are not comparable." +) + +for (const regression of regressions) { + lines.push( + `- ${escapeCell(regression.label)} ${regression.kind === "throughput" ? "decreased" : "increased"} by ${decimal.format(regression.changePercent)}% (allowed ${decimal.format(regression.thresholdPercent)}%).` + ) +} + lines.push( "", "
", @@ -449,9 +471,12 @@ lines.push( `- ${escapeCell(implementation.label)}: ${code(implementation.version)}` ), "", - "Higher throughput is better; lower heap is better. Variability is the median absolute deviation across independent processes, relative to their median. Runtime measurements on shared GitHub-hosted hardware remain informational, so small differences should be confirmed across multiple workflow runs.", + "Higher throughput is better; lower heap is better. Variability is the median absolute deviation across independent processes, relative to their median. Small differences on shared GitHub-hosted hardware remain informational; the required guard rejects only large changes beyond the measured noise allowance.", "", "
" ) console.log(lines.join("\n")) +if (check && regressions.length > 0) { + process.exitCode = 1 +} diff --git a/scripts/runtime-performance-regression.mjs b/scripts/runtime-performance-regression.mjs new file mode 100644 index 0000000..9594eac --- /dev/null +++ b/scripts/runtime-performance-regression.mjs @@ -0,0 +1,68 @@ +const regressionThreshold = (floorPercent, beforeVariability, afterVariability) => + Math.max(floorPercent, 3 * Math.max(beforeVariability, afterVariability)) + +export const findRuntimePerformanceRegressions = ( + base, + pullRequest, + { heapFloorPercent = 20, throughputFloorPercent = 15 } = {} +) => { + const regressions = [] + const baseBenchmarks = new Map( + base.benchmarks + .filter((benchmark) => benchmark.implementation === "effect-machine") + .map((benchmark) => [benchmark.id, benchmark]) + ) + for (const after of pullRequest.benchmarks.filter( + (benchmark) => benchmark.implementation === "effect-machine" + )) { + const before = baseBenchmarks.get(after.id) + if (before === undefined || before.unit !== after.unit || before.medianThroughput <= 0) { + continue + } + const decreasePercent = (before.medianThroughput - after.medianThroughput) / before.medianThroughput * 100 + const thresholdPercent = regressionThreshold( + throughputFloorPercent, + before.processRelativeMad, + after.processRelativeMad + ) + if (decreasePercent > thresholdPercent) { + regressions.push({ + changePercent: decreasePercent, + id: after.id, + kind: "throughput", + label: after.label, + thresholdPercent + }) + } + } + + const beforeMemory = base.memory.find((measurement) => measurement.implementation === "effect-machine") + const afterMemory = pullRequest.memory.find((measurement) => measurement.implementation === "effect-machine") + if (beforeMemory === undefined || afterMemory === undefined) { + return regressions + } + for (const after of afterMemory.profiles ?? []) { + const before = beforeMemory.profiles?.find((profile) => profile.id === after.id) + if (before === undefined || before.heapBytesPerIdleMachine <= 0) { + continue + } + const increasePercent = (after.heapBytesPerIdleMachine - before.heapBytesPerIdleMachine) / + before.heapBytesPerIdleMachine * 100 + const thresholdPercent = regressionThreshold( + heapFloorPercent, + before.processRelativeMad, + after.processRelativeMad + ) + if (increasePercent > thresholdPercent) { + regressions.push({ + changePercent: increasePercent, + id: after.id, + kind: "heap", + label: after.label, + thresholdPercent + }) + } + } + + return regressions +} diff --git a/scripts/runtime-performance-regression.test.mjs b/scripts/runtime-performance-regression.test.mjs new file mode 100644 index 0000000..d6d0961 --- /dev/null +++ b/scripts/runtime-performance-regression.test.mjs @@ -0,0 +1,70 @@ +import { strict as assert } from "node:assert" +import { test } from "node:test" +import { findRuntimePerformanceRegressions } from "./runtime-performance-regression.mjs" + +const report = ({ heap = 1_000, heapMad = 2, throughput = 1_000, throughputMad = 2 } = {}) => ({ + benchmarks: [{ + id: "runtime-burst", + implementation: "effect-machine", + label: "Drain burst", + medianThroughput: throughput, + processRelativeMad: throughputMad, + unit: "events/s" + }], + memory: [{ + implementation: "effect-machine", + profiles: [{ + heapBytesPerIdleMachine: heap, + id: "idle", + label: "Idle machine", + processRelativeMad: heapMad + }] + }] +}) + +test("accepts changes within the fixed regression floors", () => { + assert.deepEqual( + findRuntimePerformanceRegressions(report(), report({ heap: 1_190, throughput: 860 })), + [] + ) +}) + +test("rejects large throughput and heap regressions", () => { + assert.deepEqual( + findRuntimePerformanceRegressions(report(), report({ heap: 1_250, throughput: 800 })), + [ + { + changePercent: 20, + id: "runtime-burst", + kind: "throughput", + label: "Drain burst", + thresholdPercent: 15 + }, + { + changePercent: 25, + id: "idle", + kind: "heap", + label: "Idle machine", + thresholdPercent: 20 + } + ] + ) +}) + +test("uses process variability when it exceeds the fixed floor", () => { + assert.deepEqual( + findRuntimePerformanceRegressions( + report({ heapMad: 10, throughputMad: 10 }), + report({ heap: 1_250, heapMad: 10, throughput: 800, throughputMad: 10 }) + ), + [] + ) +}) + +test("ignores unrelated implementations", () => { + const base = report() + const pullRequest = report({ heap: 1_500, throughput: 500 }) + pullRequest.benchmarks[0].implementation = "reference" + pullRequest.memory[0].implementation = "reference" + assert.deepEqual(findRuntimePerformanceRegressions(base, pullRequest), []) +}) diff --git a/scripts/type-performance.mjs b/scripts/type-performance.mjs index bec67fa..6a0d549 100644 --- a/scripts/type-performance.mjs +++ b/scripts/type-performance.mjs @@ -74,48 +74,6 @@ const scenarios = [ maxInstantiations: 30_000, maxMarginalInstantiations: 19_000 }, - { - id: "handle-depth-8-control", - label: "machine.handle depth 8 control", - file: "handle-depth-8-control.ts", - hidden: true - }, - { - id: "handle-depth-8", - label: "machine.handle (depth 8)", - file: "handle-depth-8.ts", - control: "handle-depth-8-control", - maxInstantiations: 145_000, - maxMarginalInstantiations: 136_000 - }, - { - id: "handle-depth-12-control", - label: "machine.handle depth 12 control", - file: "handle-depth-12-control.ts", - hidden: true - }, - { - id: "handle-depth-12", - label: "machine.handle (depth 12)", - file: "handle-depth-12.ts", - control: "handle-depth-12-control", - maxInstantiations: 160_000, - maxMarginalInstantiations: 150_000 - }, - { - id: "handle-depth-16-control", - label: "machine.handle depth 16 control", - file: "handle-depth-16-control.ts", - hidden: true - }, - { - id: "handle-depth-16", - label: "machine.handle (depth 16)", - file: "handle-depth-16.ts", - control: "handle-depth-16-control", - maxInstantiations: 180_000, - maxMarginalInstantiations: 168_000 - }, { id: "handle-depth-24-control", label: "machine.handle depth 24 control", From 068944b10f8d5fd17cb2ab5672aa67b8e6187058 Mon Sep 17 00:00:00 2001 From: SandroMaglione Date: Thu, 13 Aug 2026 10:22:43 +0200 Subject: [PATCH 2/3] Ignore non-package example directories --- scripts/ci-changes.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci-changes.mjs b/scripts/ci-changes.mjs index 4a56712..fa1ef69 100644 --- a/scripts/ci-changes.mjs +++ b/scripts/ci-changes.mjs @@ -1,5 +1,5 @@ import { execFileSync } from "node:child_process" -import { appendFileSync, readdirSync, readFileSync } from "node:fs" +import { appendFileSync, existsSync, readdirSync, readFileSync } from "node:fs" import { resolve } from "node:path" import { fileURLToPath } from "node:url" @@ -86,6 +86,7 @@ const readAvailableExamples = () => { const examplesRoot = resolve(import.meta.dirname, "..", "examples") return readdirSync(examplesRoot, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) + .filter((entry) => existsSync(resolve(examplesRoot, entry.name, "package.json"))) .map((entry) => { const packageJson = JSON.parse(readFileSync(resolve(examplesRoot, entry.name, "package.json"), "utf8")) if (typeof packageJson.scripts?.check !== "string") { From cd626d6e62e520ac6d0f626ae666221585256b10 Mon Sep 17 00:00:00 2001 From: SandroMaglione Date: Thu, 13 Aug 2026 10:28:56 +0200 Subject: [PATCH 3/3] Forward the selected example matrix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 648130f..d018710 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: discover-examples: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.examples.outputs.matrix }} + matrix: ${{ steps.examples.outputs.examples }} required: ${{ steps.examples.outputs.examples_required }} steps: - uses: actions/checkout@v7