Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 43 additions & 12 deletions .github/e2e-shard-capacity.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,38 @@
"url": "https://github.com/msrivas-7/CodeTutor-AI/actions/runs/33385421742",
"headSha": "ced40c1b465cfeddba37c6299e4668a38d235139",
"topologies": [16, 20]
},
{
"runId": 34688798759,
"url": "https://github.com/msrivas-7/CodeTutor-AI/actions/runs/34688798759",
"headSha": "144335888eb1129dfd8e4bc6f0fbc47644308af8",
"topologies": [16, 20]
}
],
"method": "After the account moved to GitHub Pro, 16 and 20 shards ran sequentially on the same stable commit with two workers per shard and no retries. All jobs started without an account queue and every shard passed; 16 had both the faster retry-free test critical path and the faster topology completion.",
"totalTests": 439,
"method": "On the exact PR head, 16 and 20 shards ran sequentially against one frozen 484-test inventory with two workers per shard and no retries. All 16-shard jobs passed. Six of 20-shard jobs failed after the shared development database reached its 200-client connection ceiling, so 20 is unreliable and 16 remains selected.",
"totalTests": 484,
"topologies": [
{ "shards": 16, "modeledTestCriticalPathSeconds": 160, "topologyReadySeconds": 379, "reliable": true },
{ "shards": 20, "modeledTestCriticalPathSeconds": 198, "topologyReadySeconds": 416, "reliable": true }
{
"shards": 16,
"modeledTestCriticalPathSeconds": 160,
"topologyReadySeconds": 289,
"reliable": true
},
{
"shards": 20,
"modeledTestCriticalPathSeconds": 151,
"topologyReadySeconds": 287,
"reliable": false
}
],
"selectedModeledTestCriticalPathSeconds": 160,
"selectedTopologyReadySeconds": 379,
"selectedAverageTestsPerShard": 27.4,
"measuredAt": "2026-08-31T11:20:21Z"
"selectedTopologyReadySeconds": 289,
"selectedAverageTestsPerShard": 30.3,
"measuredAt": "2026-09-12T10:44:41Z"
},
"rebenchmark": {
"atOrBelowTests": 411,
"atOrAboveTests": 467,
"atOrBelowTests": 453,
"atOrAboveTests": 515,
"rule": "Rebenchmark when the suite changes by one selected shard's measured average workload."
},
"selectionPolicy": {
Expand All @@ -48,9 +64,24 @@
"selected": true
},
"workerCandidates": [
{ "workers": 2, "testCriticalPathSeconds": 190, "reliable": true, "selected": true },
{ "workers": 3, "testCriticalPathSeconds": 160, "reliable": false, "selected": false },
{ "workers": 4, "testCriticalPathSeconds": null, "reliable": false, "selected": false }
{
"workers": 2,
"testCriticalPathSeconds": 190,
"reliable": true,
"selected": true
},
{
"workers": 3,
"testCriticalPathSeconds": 160,
"reliable": false,
"selected": false
},
{
"workers": 4,
"testCriticalPathSeconds": null,
"reliable": false,
"selected": false
}
],
"maximumChromiumShards": 20,
"method": "Three images prepared in parallel, then local-build and digest-pinned 16-shard stages ran sequentially on the same commit with zero retries. Higher worker counts were eligible only when every shard passed."
Expand Down
15 changes: 15 additions & 0 deletions .github/scripts/e2e-shard-benchmark.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,21 @@ test("workflow benchmarks the complete measured range under the account ceiling"
assert.match(topologyWorkflow, /--shard=\$\{\{ matrix\.shard }}\/\$\{\{ inputs\.total }}/);
});

test("capacity candidates share one frozen history and use the normal duration planner", () => {
assert.match(benchmarkWorkflow, /for total in 16 20; do/);
assert.match(benchmarkWorkflow, /node \.github\/scripts\/e2e-duration-plan\.mjs/);
assert.match(benchmarkWorkflow, /--history e2e\/shard-capacity-plans\/history\.json/);
assert.match(benchmarkWorkflow, /needs: plan/);
assert.match(benchmarkWorkflow, /needs: \[plan, benchmark-sixteen\]/);
assert.match(benchmarkWorkflow, /needs\.plan\.result == 'success'/);
assert.equal((benchmarkWorkflow.match(/duration_plan_artifact: shard-capacity-plans/g) ?? []).length, 2);
assert.match(topologyWorkflow, /if: inputs\.duration_plan_artifact != ''/);
assert.match(topologyWorkflow, /test -s "\$plan"/);
assert.match(topologyWorkflow, /--test-list="\$plan" --workers=\$\{\{ inputs\.workers }} --retries=0/);
// Older worker/image experiments retain their explicit native-shard control.
assert.match(topologyWorkflow, /else\n\s+npx playwright test --project=chromium --shard=/);
});

test("does not treat preceding-experiment wait time as test work", () => {
const result = compareShardTopologies({
benchmarkRun,
Expand Down
152 changes: 117 additions & 35 deletions .github/scripts/e2e-shard-capacity.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,40 +7,73 @@ import {
evaluateShardCapacity,
} from "./e2e-shard-capacity.mjs";

const record = JSON.parse(readFileSync(new URL("../e2e-shard-capacity.json", import.meta.url), "utf8"));
const workflow = readFileSync(new URL("../workflows/e2e.yml", import.meta.url), "utf8");
const record = JSON.parse(
readFileSync(new URL("../e2e-shard-capacity.json", import.meta.url), "utf8"),
);
const workflow = readFileSync(
new URL("../workflows/e2e.yml", import.meta.url),
"utf8",
);

test("derives a one-shard-workload rebenchmark band", () => {
assert.deepEqual(deriveRebenchmarkBounds(439, 16), {
atOrBelowTests: 411,
atOrAboveTests: 467,
assert.deepEqual(deriveRebenchmarkBounds(484, 16), {
atOrBelowTests: 453,
atOrAboveTests: 515,
});
});

test("tracked decision preserves the clean controlled benchmark evidence", () => {
assert.deepEqual(
record.benchmark.runs.map(({ runId, headSha, topologies }) => ({ runId, headSha, topologies })),
record.benchmark.runs.map(({ runId, headSha, topologies }) => ({
runId,
headSha,
topologies,
})),
[
{
runId: 33385421742,
headSha: "ced40c1b465cfeddba37c6299e4668a38d235139",
topologies: [16, 20],
},
{
runId: 34688798759,
headSha: "144335888eb1129dfd8e4bc6f0fbc47644308af8",
topologies: [16, 20],
},
],
);
assert.deepEqual(
record.benchmark.topologies.map(({ shards, modeledTestCriticalPathSeconds, topologyReadySeconds, reliable }) => ({
shards,
modeledTestCriticalPathSeconds,
topologyReadySeconds,
reliable,
})),
record.benchmark.topologies.map(
({
shards,
modeledTestCriticalPathSeconds,
topologyReadySeconds,
reliable,
}) => ({
shards,
modeledTestCriticalPathSeconds,
topologyReadySeconds,
reliable,
}),
),
[
{ shards: 16, modeledTestCriticalPathSeconds: 160, topologyReadySeconds: 379, reliable: true },
{ shards: 20, modeledTestCriticalPathSeconds: 198, topologyReadySeconds: 416, reliable: true },
{
shards: 16,
modeledTestCriticalPathSeconds: 160,
topologyReadySeconds: 289,
reliable: true,
},
{
shards: 20,
modeledTestCriticalPathSeconds: 151,
topologyReadySeconds: 287,
reliable: false,
},
],
);
assert.equal(record.benchmark.totalTests, 484);
assert.equal(record.benchmark.selectedModeledTestCriticalPathSeconds, 160);
assert.equal(record.benchmark.selectedAverageTestsPerShard, 30.3);
assert.deepEqual(record.runtimeOptimization.imageReuse, {
localBuildEndToEndSeconds: 369,
prebuiltEndToEndSecondsIncludingPreparation: 338,
Expand All @@ -63,26 +96,42 @@ test("tracked decision preserves the clean controlled benchmark evidence", () =>
});

test("blocking workflow uses the selected matrix and derives its denominator", () => {
const exhaustiveJob = workflow.match(/\n e2e:\n([\s\S]+?)\n cross-browser-core:/)?.[1] ?? "";
const shardMatrix = exhaustiveJob.match(/matrix:\n\s+shard: \[([^\]]+)]/)?.[1]
const exhaustiveJob =
workflow.match(/\n e2e:\n([\s\S]+?)\n cross-browser-core:/)?.[1] ?? "";
const shardMatrix = exhaustiveJob
.match(/matrix:\n\s+shard: \[([^\]]+)]/)?.[1]
.split(",")
.map((value) => Number(value.trim()));
assert.deepEqual(shardMatrix, Array.from({ length: record.selectedShards }, (_, index) => index + 1));
assert.deepEqual(
shardMatrix,
Array.from({ length: record.selectedShards }, (_, index) => index + 1),
);
assert.match(exhaustiveJob, /--active-shards "\$\{\{ strategy\.job-total }}/);
assert.match(workflow, /--output e2e\/duration-plan\/full[\s\S]+--shards 16/);
assert.match(exhaustiveJob, /--test-list=duration-plan-artifact\/full\/shard-\$\{\{ matrix\.shard }}\.txt/);
assert.match(
exhaustiveJob,
/--test-list=duration-plan-artifact\/full\/shard-\$\{\{ matrix\.shard }}\.txt/,
);
assert.match(exhaustiveJob, /name: Upload test-duration evidence/);
});

test("advisory critical coverage is split across two isolated duration-balanced jobs", () => {
assert.match(workflow, /--output e2e\/duration-plan\/critical[\s\S]+--shards 2[\s\S]+--tag lane:critical/);
assert.match(
workflow,
/--output e2e\/duration-plan\/critical[\s\S]+--shards 2[\s\S]+--tag lane:critical/,
);
assert.match(workflow, /critical-shadow:[\s\S]+matrix:\n\s+shard: \[1, 2]/);
assert.doesNotMatch(workflow, /critical-shadow-summary:/);
assert.match(workflow, /shadow-evidence:[\s\S]+needs: \[duration-plan, critical-shadow, e2e, cross-browser-core][\s\S]+files\.length!==2/);
assert.match(
workflow,
/shadow-evidence:[\s\S]+needs: \[duration-plan, critical-shadow, e2e, cross-browser-core][\s\S]+files\.length!==2/,
);
});

test("duration planning receives the authenticated fixture environment required for discovery", () => {
const planningJob = workflow.match(/\n duration-plan:\n([\s\S]+?)\n prepare-backend:/)?.[1] ?? "";
const planningJob =
workflow.match(/\n duration-plan:\n([\s\S]+?)\n prepare-backend:/)?.[1] ??
"";
for (const variable of [
"SUPABASE_URL",
"SUPABASE_ANON_KEY",
Expand All @@ -92,21 +141,50 @@ test("duration planning receives the authenticated fixture environment required
"DATABASE_URL",
"BYOK_ENCRYPTION_KEY",
]) {
assert.match(planningJob, new RegExp(`${variable}: \\$\\{\\{ secrets\\.${variable} }}`));
assert.match(
planningJob,
new RegExp(`${variable}: \\$\\{\\{ secrets\\.${variable} }}`),
);
}
});

test("accepts the measured inventory and normal growth", () => {
assert.equal(evaluateShardCapacity({ record, totalTests: 439, activeShards: 16 }).eligible, true);
assert.equal(evaluateShardCapacity({ record, totalTests: 466, activeShards: 16 }).eligible, true);
assert.equal(evaluateShardCapacity({ record, totalTests: 412, activeShards: 16 }).eligible, true);
assert.equal(
evaluateShardCapacity({ record, totalTests: 484, activeShards: 16 })
.eligible,
true,
);
assert.equal(
evaluateShardCapacity({ record, totalTests: 514, activeShards: 16 })
.eligible,
true,
);
assert.equal(
evaluateShardCapacity({ record, totalTests: 454, activeShards: 16 })
.eligible,
true,
);
});

test("requires a new benchmark at either capacity boundary", () => {
const upper = evaluateShardCapacity({ record, totalTests: 467, activeShards: 16 });
const lower = evaluateShardCapacity({ record, totalTests: 411, activeShards: 16 });
assert.deepEqual({ eligible: upper.eligible, direction: upper.direction }, { eligible: false, direction: "upper" });
assert.deepEqual({ eligible: lower.eligible, direction: lower.direction }, { eligible: false, direction: "lower" });
const upper = evaluateShardCapacity({
record,
totalTests: 515,
activeShards: 16,
});
const lower = evaluateShardCapacity({
record,
totalTests: 453,
activeShards: 16,
});
assert.deepEqual(
{ eligible: upper.eligible, direction: upper.direction },
{ eligible: false, direction: "upper" },
);
assert.deepEqual(
{ eligible: lower.eligible, direction: lower.direction },
{ eligible: false, direction: "lower" },
);
});

test("fails closed when workflow topology drifts from the measured record", () => {
Expand All @@ -118,11 +196,15 @@ test("fails closed when workflow topology drifts from the measured record", () =

test("fails closed when recorded boundaries are stale or hand-edited", () => {
assert.throws(
() => evaluateShardCapacity({
record: { ...record, rebenchmark: { atOrBelowTests: 1, atOrAboveTests: 999 } },
totalTests: 439,
activeShards: 16,
}),
/capacity record bounds must be 411\/467/,
() =>
evaluateShardCapacity({
record: {
...record,
rebenchmark: { atOrBelowTests: 1, atOrAboveTests: 999 },
},
totalTests: 484,
activeShards: 16,
}),
/capacity record bounds must be 453\/515/,
);
});
58 changes: 56 additions & 2 deletions .github/workflows/e2e-shard-benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,23 +26,77 @@ permissions:
pull-requests: read

jobs:
plan:
name: Freeze duration-balanced benchmark plans
if: github.event.label.name == 'ci-shard-benchmark'
runs-on: ubuntu-latest
env:
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }}
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: '20'
cache: npm
cache-dependency-path: e2e/package-lock.json
- name: Restore one timing baseline for both candidates
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
with:
path: .e2e-duration
key: e2e-duration-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
e2e-duration-${{ runner.os }}-
- name: Install e2e deps
working-directory: e2e
run: npm ci
- name: Plan every test with the normal CI algorithm
run: |
set -euo pipefail
mkdir -p e2e/shard-capacity-plans .e2e-duration
if [ ! -s .e2e-duration/history.json ]; then
cp .github/e2e-duration-seed.json .e2e-duration/history.json
fi
cp .e2e-duration/history.json e2e/shard-capacity-plans/history.json
cd e2e
npx playwright test --list --project=chromium --reporter=json > shard-capacity-plans/inventory.json
cd ..
for total in 16 20; do
node .github/scripts/e2e-duration-plan.mjs \
--inventory e2e/shard-capacity-plans/inventory.json \
--history e2e/shard-capacity-plans/history.json \
--output "e2e/shard-capacity-plans/$total" \
--shards "$total"
done
- name: Preserve frozen plans and timing provenance
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: shard-capacity-plans
path: e2e/shard-capacity-plans
retention-days: 90
if-no-files-found: error

benchmark-sixteen:
if: github.event.label.name == 'ci-shard-benchmark'
needs: plan
uses: ./.github/workflows/e2e-shard-topology.yml
with:
total: 16
shards: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]'
max_parallel: 16
duration_plan_artifact: shard-capacity-plans
secrets: inherit

benchmark-twenty:
if: always() && github.event.label.name == 'ci-shard-benchmark'
needs: benchmark-sixteen
if: always() && github.event.label.name == 'ci-shard-benchmark' && needs.plan.result == 'success'
needs: [plan, benchmark-sixteen]
uses: ./.github/workflows/e2e-shard-topology.yml
with:
total: 20
shards: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]'
max_parallel: 20
duration_plan_artifact: shard-capacity-plans
secrets: inherit

compare:
Expand Down
Loading
Loading