diff --git a/.github/workflows/benchmark-trigger.yml b/.github/workflows/benchmark-trigger.yml new file mode 100644 index 0000000..2cbfe4c --- /dev/null +++ b/.github/workflows/benchmark-trigger.yml @@ -0,0 +1,38 @@ +name: Benchmark + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [opened, synchronize, reopened, labeled, unlabeled, edited, ready_for_review] + +permissions: + contents: read + pull-requests: read + issues: read + +jobs: + workload: + concurrency: + group: benchmark-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + uses: ./.github/workflows/benchmark.yml + + benchmark-status: + name: Benchmark Status + if: ${{ always() && github.event_name == 'pull_request' }} + needs: workload + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v7 + with: + node-version: '24' + - name: Require the benchmark workload to succeed + env: + BENCHMARK_JOB_RESULTS: ${{ toJSON(needs) }} + run: node scripts/check-ci-status.ts \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9bfa77e..1f836f7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,21 +1,13 @@ -name: Benchmark +name: Benchmark Workload on: - push: - branches: [main] - pull_request: - branches: [main] - types: [opened, synchronize, reopened, labeled, unlabeled, edited, ready_for_review] + workflow_call: permissions: contents: read pull-requests: read issues: read -concurrency: - group: benchmark-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - env: CARGO_TERM_COLOR: always @@ -291,7 +283,7 @@ jobs: # Provide one required check that also reports failed, skipped, or cancelled stages. benchmark-status: - name: Benchmark Status + name: Workload Status if: ${{ always() && github.event_name == 'pull_request' }} needs: [eligibility, configure, producer, smoke, prepare, measure, collect] runs-on: ubuntu-latest diff --git a/docs/README.md b/docs/README.md index 1e2f97d..664a417 100644 --- a/docs/README.md +++ b/docs/README.md @@ -108,9 +108,9 @@ The preview URL then returns 404. * `benchmarks: skip` replaces publishable measurements with one-request smoke coverage on KVM and MSHV. Smoke results are not published. Dependabot applies the label automatically. * Required checks evaluate the current PR policy on every subscribed event. Label and description edits restart the benchmark workflow. Measurements run unless `benchmarks: skip` is present. * Each of the 36 measurement jobs runs all three strategies sequentially with a fresh server process for each. Retrying a job repeats its three strategies. -* Preview builds run independently of benchmarks. Draft and Dependabot PRs have no preview. +* Preview builds run independently of benchmarks. Draft PRs have no preview. * Production uses published history. Previews include matching pending results when available. -* Benchmark and preview workflow YAML must match `main` before publishers accept their artifacts. +* Benchmark trigger, workload, and preview workflow YAML must match `main` before publishers accept their artifacts. * Merge requires `Benchmark Policy`, `Benchmark Status`, an up-to-date branch, and squash merging. Required PR review count is zero. Preview and publication success do not block merge. Publication calls Pages after a main push or a data change. Pages also runs for diff --git a/scripts/check-ci-status.ts b/scripts/check-ci-status.ts index 0f38f10..730daec 100644 --- a/scripts/check-ci-status.ts +++ b/scripts/check-ci-status.ts @@ -1,9 +1,11 @@ import { appendFileSync } from 'node:fs' const results = JSON.parse(process.env.BENCHMARK_JOB_RESULTS ?? '{}') as Record -const required = process.env.BENCHMARK_MODE === 'skip' - ? ['eligibility', 'configure', 'producer', 'smoke'] - : ['eligibility', 'configure', 'producer', 'prepare', 'measure', 'collect'] +const required = 'workload' in results + ? ['workload'] + : process.env.BENCHMARK_MODE === 'skip' + ? ['eligibility', 'configure', 'producer', 'smoke'] + : ['eligibility', 'configure', 'producer', 'prepare', 'measure', 'collect'] const failed = required.filter(job => results[job]?.result !== 'success') const summary = required.map(job => `* ${job}: ${results[job]?.result ?? 'missing'}`).join('\n') console.log(summary) diff --git a/scripts/publication.test.ts b/scripts/publication.test.ts index cd82f7f..19690e4 100644 --- a/scripts/publication.test.ts +++ b/scripts/publication.test.ts @@ -62,6 +62,8 @@ test('benchmark status requires smoke only for skipped measurements', () => { assert.equal(check('required', { ...shared, smoke: { result: 'skipped' }, prepare: { result: 'success' }, measure: { result: 'success' }, collect: { result: 'success' }, }).status, 0) + assert.equal(check('skip', { workload: { result: 'success' } }).status, 0) + assert.notEqual(check('skip', { workload: { result: 'cancelled' } }).status, 0) }) test('partial retries retain successful measurements from the same source revision', () => { @@ -414,7 +416,7 @@ test('workflows must match trusted main', async context => { assert.ok(path.startsWith('contents/'), `Unexpected GitHub request: ${path}`) return Response.json({ encoding: 'base64', content: Buffer.from(content).toString('base64') }) }) - for (const workflow of ['benchmark', 'preview']) { + for (const workflow of ['benchmark', 'benchmark-trigger', 'preview']) { const path = `.github/workflows/${workflow}.yml` writeFileSync(path, 'trusted workflow') content = 'trusted workflow' @@ -511,8 +513,9 @@ test('CI archival and promotion with simulated GitHub and Git', async context => id: 12345, run_attempt: 1, event: 'pull_request', conclusion: 'success', head_sha: 'a'.repeat(40), pull_requests: [{ number: 7 }], } } writeJson(eventPath, notification, false) - const workflow = readFileSync(resolve(root, '.github/workflows/benchmark.yml'), 'utf8') - let workflowContent = workflow + const triggerWorkflow = readFileSync(resolve(root, '.github/workflows/benchmark-trigger.yml'), 'utf8') + let triggerWorkflowContent = triggerWorkflow + const workloadWorkflow = readFileSync(resolve(root, '.github/workflows/benchmark.yml'), 'utf8') const bundle = fixture() const pr = { number: 7, base: { ref: 'main', repo: { full_name: repository }, sha: bundle.run.pullRequest!.base }, @@ -524,9 +527,10 @@ test('CI archival and promotion with simulated GitHub and Git', async context => let failedJob = false let pushes = 0 let downloads = 0 - const jobs = ['eligibility', 'configure', 'producer', 'collect', 'Benchmark Status', - ...Array.from({ length: 2 }, (_, index) => `prepare (${index})`), - ...Array.from({ length: 36 }, (_, index) => `measure (${index})`), + const jobs = ['workload / eligibility', 'workload / configure', 'workload / producer', + 'workload / collect', 'workload / Workload Status', 'Benchmark Status', + ...Array.from({ length: 2 }, (_, index) => `workload / prepare (${index})`), + ...Array.from({ length: 36 }, (_, index) => `workload / measure (${index})`), ].map(name => ({ name, conclusion: 'success' })) let retryJobs: typeof jobs = [] const statuses: any[] = [] @@ -552,9 +556,10 @@ test('CI archival and promotion with simulated GitHub and Git', async context => 'pulls/7': pr, [`commits/${pr.merge_commit_sha}/pulls`]: [pr], [`commits/${'d'.repeat(40)}/pulls`]: [pr], - 'actions/runs/12345/attempts/1': { id: 12345, event: 'pull_request', conclusion: 'success', path: '.github/workflows/benchmark.yml', head_sha: workflowHead, created_at: bundle.run.createdAt }, - 'actions/runs/12345/attempts/2': { id: 12345, event: 'pull_request', conclusion: 'success', path: '.github/workflows/benchmark.yml', head_sha: workflowHead, created_at: bundle.run.createdAt }, - 'contents/.github/workflows/benchmark.yml': { encoding: 'base64', content: Buffer.from(workflowContent).toString('base64') }, + 'actions/runs/12345/attempts/1': { id: 12345, event: 'pull_request', conclusion: 'success', path: '.github/workflows/benchmark-trigger.yml', head_sha: workflowHead, created_at: bundle.run.createdAt }, + 'actions/runs/12345/attempts/2': { id: 12345, event: 'pull_request', conclusion: 'success', path: '.github/workflows/benchmark-trigger.yml', head_sha: workflowHead, created_at: bundle.run.createdAt }, + 'contents/.github/workflows/benchmark-trigger.yml': { encoding: 'base64', content: Buffer.from(triggerWorkflowContent).toString('base64') }, + 'contents/.github/workflows/benchmark.yml': { encoding: 'base64', content: Buffer.from(workloadWorkflow).toString('base64') }, [`git/commits/${bundle.run.commit.sha}`]: { tree: { sha: bundle.run.commit.tree }, parents: [{ sha: bundle.run.pullRequest!.base }, { sha: bundle.run.pullRequest!.head }] }, [`git/commits/${pr.merge_commit_sha}`]: { tree: { sha: mergedTree }, message: 'Benchmark results (#7)' }, } @@ -575,7 +580,8 @@ test('CI archival and promotion with simulated GitHub and Git', async context => }) syncBuiltinESMExports() mkdirSync(resolve(temporary, '.github/workflows'), { recursive: true }) - writeFileSync(resolve(temporary, '.github/workflows/benchmark.yml'), workflow) + writeFileSync(resolve(temporary, '.github/workflows/benchmark-trigger.yml'), triggerWorkflow) + writeFileSync(resolve(temporary, '.github/workflows/benchmark.yml'), workloadWorkflow) const { main } = await import('./publish-ci.ts') const clearStore = () => { rmSync(resolve(temporary, 'data-store'), { recursive: true, force: true }) @@ -643,7 +649,8 @@ test('CI archival and promotion with simulated GitHub and Git', async context => const originalUrl = bundle.run.workflow.url bundle.run.attempt = 2 bundle.run.workflow.url = `https://github.com/${repository}/actions/runs/12345/attempts/2` - retryJobs = jobs.filter(job => ['eligibility', 'measure (0)', 'collect', 'Benchmark Status'].includes(job.name)) + retryJobs = jobs.filter(job => ['eligibility', 'measure (0)', 'collect', 'Workload Status', 'Benchmark Status'] + .includes(job.name.split(' / ').at(-1)!)) failedJob = true writeJson(eventPath, { workflow_run: { ...notification.workflow_run, run_attempt: 2 } }, false) try { @@ -653,10 +660,10 @@ test('CI archival and promotion with simulated GitHub and Git', async context => assert.deepEqual(readJson(resolve(temporary, 'data-store/index.json')), { schemaVersion: 1, runs: [{ id: '12345', attempt: 2 }] }) const stored = validateCompleteRun(readJson(resolve(temporary, 'data-store/runs/12345/2.json'))) assert.deepEqual(stored.measurements, bundle.measurements) - retryJobs = retryJobs.map(job => ({ ...job, conclusion: job.name === 'measure (0)' ? 'failure' : 'success' })) + retryJobs = retryJobs.map(job => ({ ...job, conclusion: job.name.endsWith('measure (0)') ? 'failure' : 'success' })) await assert.rejects(main(), /successful measure/) assert.equal(downloads, 1, 'A failed retry must not use an earlier successful job') - retryJobs = retryJobs.filter(job => job.name !== 'eligibility') + retryJobs = retryJobs.filter(job => !job.name.endsWith('eligibility')) await assert.rejects(main(), /successful eligibility/) assert.equal(downloads, 1) } finally { @@ -700,16 +707,16 @@ test('CI archival and promotion with simulated GitHub and Git', async context => }) await context.test('changed workflows cannot archive before matching main', async () => { clearStore() - workflowContent = `${workflow}\n` + triggerWorkflowContent = `${triggerWorkflow}\n` pr.merged = false pr.state = 'open' try { - await assert.rejects(main(), /workflow must match current main/) + await assert.rejects(main(), /workflow files must match current main/) assert.equal(downloads, 0) assert.equal(pushes, 0) assert.equal(statuses.at(-1).state, 'failure') } finally { - workflowContent = workflow + triggerWorkflowContent = triggerWorkflow pr.merged = true pr.state = 'closed' clearStore() diff --git a/scripts/publish-ci.ts b/scripts/publish-ci.ts index e4335b6..0f443ca 100644 --- a/scripts/publish-ci.ts +++ b/scripts/publish-ci.ts @@ -42,11 +42,13 @@ function pushStore() { async function verifiedRun(runId: number, attempt: number) { const run = await github(`actions/runs/${runId}/attempts/${attempt}`) - if (run.event !== 'pull_request' || run.conclusion !== 'success' || run.path !== '.github/workflows/benchmark.yml') { + if (run.event !== 'pull_request' || run.conclusion !== 'success' || run.path !== '.github/workflows/benchmark-trigger.yml') { throw new Error('Expected a successful PR Benchmark workflow attempt') } - if (!await workflowTrusted('.github/workflows/benchmark.yml', run.head_sha)) { - throw new Error('Benchmark workflow must match current main. After merge, rerun this publication workflow.') + for (const path of ['.github/workflows/benchmark-trigger.yml', '.github/workflows/benchmark.yml']) { + if (!await workflowTrusted(path, run.head_sha)) { + throw new Error('Benchmark workflow files must match current main. After merge, rerun this publication workflow.') + } } const latestJobs = new Map() for (let currentAttempt = attempt; currentAttempt >= 1; currentAttempt--) { @@ -64,10 +66,14 @@ async function verifiedRun(runId: number, attempt: number) { const jobs = [...latestJobs.values()] const expected = new Map([ ['eligibility', 1], ['configure', 1], ['producer', 1], ['prepare', 2], - ['measure', publicationPolicy.catalog.platforms.length * publicationPolicy.catalog.runtimes.length], ['collect', 1], ['Benchmark Status', 1], + ['measure', publicationPolicy.catalog.platforms.length * publicationPolicy.catalog.runtimes.length], + ['collect', 1], ['Workload Status', 1], ['Benchmark Status', 1], ]) for (const [name, count] of expected) { - const matching = jobs.filter(job => job.name === name || job.name.startsWith(`${name} (`)) + const matching = jobs.filter(job => { + const leaf = job.name.split(' / ').at(-1) + return leaf === name || leaf?.startsWith(`${name} (`) + }) if (matching.length !== count || matching.some(job => job.conclusion !== 'success')) { throw new Error(`Run ${runId} through attempt ${attempt} requires ${count} successful ${name} jobs. Rerun the failed jobs.`) }