diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d88906c01b..f2fe2ca454 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -231,7 +231,7 @@ jobs: matrix: # Linux coverage for these files comes from the integration job, which # runs the whole stack integration project. - os: [macos-latest, windows-latest] + os: [blacksmith-6vcpu-macos-latest, windows-latest] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -381,7 +381,8 @@ jobs: - { suite: cli, shard: 3, shards: 5, name: cli 3/5 } - { suite: cli, shard: 4, shards: 5, name: cli 4/5 } - { suite: cli, shard: 5, shards: 5, name: cli 5/5 } - - { suite: cli-e2e, name: cli-e2e } + - { suite: cli-e2e, shard: 1, shards: 2, name: cli-e2e 1/2 } + - { suite: cli-e2e, shard: 2, shards: 2, name: cli-e2e 2/2 } steps: - name: Checkout uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta @@ -395,19 +396,6 @@ jobs: sticky-disks: "true" docker-images: "true" - - name: Cache Go CLI binary - id: cache-go-binary - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: apps/cli-go/supabase-go - key: go-cli-${{ runner.os }}-${{ hashFiles('apps/cli-go/**/*.go', - 'apps/cli-go/go.mod', 'apps/cli-go/go.sum') }} - - - name: Build Go CLI - if: steps.cache-go-binary.outputs.cache-hit != 'true' - run: go build -o supabase-go . - working-directory: apps/cli-go - # The e2e harness invokes `node apps/cli/dist/supabase.js` with # `SUPABASE_CLI_BINARY_OVERRIDE` pointing at the compiled binary in # `apps/cli/dist/`. Build the CLI explicitly before invoking every @@ -424,7 +412,7 @@ jobs: - name: Run cli-e2e end-to-end tests if: matrix.suite == 'cli-e2e' - run: pnpm exec turbo run test:e2e:run --only --filter=@supabase/cli-e2e + run: pnpm exec turbo run test:e2e:run --only --filter=@supabase/cli-e2e -- --shard=${{ matrix.shard }}/${{ matrix.shards }} env: SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go diff --git a/apps/cli/src/commands/gen/types/types.e2e.test.ts b/apps/cli/src/commands/gen/types/types.e2e.test.ts index 78997f9979..002cf43717 100644 --- a/apps/cli/src/commands/gen/types/types.e2e.test.ts +++ b/apps/cli/src/commands/gen/types/types.e2e.test.ts @@ -167,6 +167,9 @@ const waitForLocalPostgres = Effect.fnUntraced(function* (containerName: string) "PGPASSWORD=postgres", containerName, "psql", + // The entrypoint's init server listens only on the Unix socket, so TCP reaches the final server. + "-h", + "127.0.0.1", "-U", "postgres", "-d", diff --git a/apps/cli/tests/e2e-sequencer.ts b/apps/cli/tests/e2e-sequencer.ts index 0ce50c1f43..aec487fedb 100644 --- a/apps/cli/tests/e2e-sequencer.ts +++ b/apps/cli/tests/e2e-sequencer.ts @@ -4,7 +4,9 @@ import { BaseSequencer, type TestSpecification } from "vitest/node"; /** Bun `--timings` file of measured e2e file durations, refreshed from CI artifacts. */ const TIMINGS_FILE = "tests/e2e-timings.json"; -const E2E_PROJECTS = new Set(["e2e", "e2e-exclusive"]); +const E2E_PROJECT = "e2e"; +const E2E_EXCLUSIVE_PROJECT = "e2e-exclusive"; +const E2E_PROJECTS = new Set([E2E_PROJECT, E2E_EXCLUSIVE_PROJECT]); /** Duration assumed for every file while no timing has been measured yet. */ export const DEFAULT_DURATION_MS = 30_000; @@ -28,31 +30,70 @@ function compareKeys(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } +/** An e2e file to shard; exclusive files run one at a time after the concurrent ones. */ +export interface ShardFile { + readonly key: string; + readonly exclusive: boolean; +} + +interface ShardLoad { + readonly longestConcurrent: number; + readonly concurrentTotal: number; + readonly exclusiveTotal: number; +} + +function withFile(load: ShardLoad, duration: number, exclusive: boolean): ShardLoad { + return exclusive + ? { ...load, exclusiveTotal: load.exclusiveTotal + duration } + : { + ...load, + longestConcurrent: Math.max(load.longestConcurrent, duration), + concurrentTotal: load.concurrentTotal + duration, + }; +} + +function estimatedWall(load: ShardLoad, workers: number): number { + return Math.max(load.longestConcurrent, load.concurrentTotal / workers) + load.exclusiveTotal; +} + /** - * Splits keys into `count` shards by measured duration, longest first into the emptiest shard. - * The result depends only on the set of keys, so every shard process derives the same partition. + * Splits files into `count` shards by estimated wall time, longest first into the shard whose + * estimate stays lowest, the less loaded one on ties. Concurrent files share `workers` slots; + * exclusive files add their full duration. + * The result depends only on the set of files, so every shard process derives the same partition. */ export function packShards( - keys: readonly string[], + files: readonly ShardFile[], timings: Readonly>, count: number, + workers: number, ): string[][] { const fallback = fallbackDuration(timings); - const durations = keys - .map((key) => ({ key, duration: timings[key] ?? fallback })) + const durations = files + .map((file) => ({ ...file, duration: timings[file.key] ?? fallback })) .sort((a, b) => b.duration - a.duration || compareKeys(a.key, b.key)); - const shards = Array.from({ length: count }, () => ({ load: 0, keys: new Array() })); - for (const { key, duration } of durations) { - let target = shards[0]; + const empty: ShardLoad = { longestConcurrent: 0, concurrentTotal: 0, exclusiveTotal: 0 }; + const shards = Array.from({ length: count }, () => ({ keys: new Array(), load: empty })); + for (const { key, exclusive, duration } of durations) { + let target: + | { shard: (typeof shards)[number]; load: ShardLoad; wall: number; before: number } + | undefined; for (const shard of shards) { - if (target === undefined || shard.load < target.load) { - target = shard; + const load = withFile(shard.load, duration, exclusive); + const wall = estimatedWall(load, workers); + const before = estimatedWall(shard.load, workers); + if ( + target === undefined || + wall < target.wall || + (wall === target.wall && before < target.before) + ) { + target = { shard, load, wall, before }; } } if (target !== undefined) { - target.load += duration; - target.keys.push(key); + target.shard.load = target.load; + target.shard.keys.push(key); } } return shards.map((shard) => shard.keys); @@ -109,7 +150,15 @@ export class E2eSequencer extends BaseSequencer { const byKey = new Map( files.map((spec) => [relative(root, spec.moduleId).replaceAll("\\", "/"), spec]), ); - const assigned = packShards([...byKey.keys()], timings, shard.count)[shard.index - 1] ?? []; + const shardFiles = [...byKey].map(([key, spec]) => ({ + key, + exclusive: spec.project.name === E2E_EXCLUSIVE_PROJECT, + })); + const maxWorkers = + files.find((spec) => spec.project.name === E2E_PROJECT)?.project.config.maxWorkers ?? 1; + // Concurrent files share one Docker daemon and slow each other, so pack against half the slots. + const workers = Math.max(1, Math.floor(maxWorkers / 2)); + const assigned = packShards(shardFiles, timings, shard.count, workers)[shard.index - 1] ?? []; return assigned.flatMap((key) => { const spec = byKey.get(key); return spec === undefined ? [] : [spec]; diff --git a/apps/cli/tests/e2e-sequencer.unit.test.ts b/apps/cli/tests/e2e-sequencer.unit.test.ts index 1826cab568..d90a7e3813 100644 --- a/apps/cli/tests/e2e-sequencer.unit.test.ts +++ b/apps/cli/tests/e2e-sequencer.unit.test.ts @@ -9,11 +9,13 @@ const files = ["src/a.e2e.test.ts", "src/b.e2e.test.ts", "src/c.e2e.test.ts", "s const sortedShards = (shards: readonly (readonly string[])[]) => shards.map((shard) => [...shard].sort()); +const concurrent = (keys: readonly string[]) => keys.map((key) => ({ key, exclusive: false })); + describe("packShards", () => { test.each([1, files.length, files.length + 3])( "partitions every file into exactly one of %i shards", (count) => { - const shards = packShards(files, { "src/a.e2e.test.ts": 100 }, count); + const shards = packShards(concurrent(files), { "src/a.e2e.test.ts": 100 }, count, 1); expect(shards).toHaveLength(count); expect(shards.flat().sort()).toEqual([...files].sort()); @@ -21,7 +23,7 @@ describe("packShards", () => { ); test("leaves trailing shards empty when there are more shards than files", () => { - const shards = packShards(["src/a.e2e.test.ts", "src/b.e2e.test.ts"], {}, 4); + const shards = packShards(concurrent(["src/a.e2e.test.ts", "src/b.e2e.test.ts"]), {}, 4, 1); expect(shards).toEqual([["src/a.e2e.test.ts"], ["src/b.e2e.test.ts"], [], []]); }); @@ -33,8 +35,8 @@ describe("packShards", () => { "src/c.e2e.test.ts": 900, }; - const forward = packShards(files, timings, 2); - const reversed = packShards([...files].reverse(), timings, 2); + const forward = packShards(concurrent(files), timings, 2, 1); + const reversed = packShards(concurrent([...files].reverse()), timings, 2, 1); expect(reversed).toEqual(forward); }); @@ -47,7 +49,7 @@ describe("packShards", () => { "src/d.e2e.test.ts": 10, }; - const shards = packShards(files, timings, 2); + const shards = packShards(concurrent(files), timings, 2, 1); expect(sortedShards(shards)).toEqual([ ["src/b.e2e.test.ts"], @@ -55,6 +57,27 @@ describe("packShards", () => { ]); }); + test("packs concurrent files beside a long one while an exclusive file runs alone", () => { + const timings = { + "src/long.e2e.test.ts": 90, + "src/exclusive.e2e.test.ts": 80, + "src/a.e2e.test.ts": 20, + "src/b.e2e.test.ts": 20, + "src/c.e2e.test.ts": 20, + }; + const shardFiles = Object.keys(timings).map((key) => ({ + key, + exclusive: key === "src/exclusive.e2e.test.ts", + })); + + const shards = packShards(shardFiles, timings, 2, 4); + + expect(sortedShards(shards)).toEqual([ + ["src/a.e2e.test.ts", "src/b.e2e.test.ts", "src/c.e2e.test.ts", "src/long.e2e.test.ts"], + ["src/exclusive.e2e.test.ts"], + ]); + }); + test("weights an unmeasured file at the median of the measured ones", () => { const timings = { "src/a.e2e.test.ts": 100, @@ -63,7 +86,7 @@ describe("packShards", () => { }; // d is unmeasured; at the median (300) it ties with b and follows it into the second shard. - const shards = packShards(files, timings, 2); + const shards = packShards(concurrent(files), timings, 2, 1); expect(sortedShards(shards)).toEqual([ ["src/a.e2e.test.ts", "src/c.e2e.test.ts"], @@ -74,13 +97,15 @@ describe("packShards", () => { test("weights every file at the default duration when nothing has been measured", () => { const allDefault = Object.fromEntries(files.map((file) => [file, DEFAULT_DURATION_MS])); - expect(packShards(files, {}, 3)).toEqual(packShards(files, allDefault, 3)); + expect(packShards(concurrent(files), {}, 3, 1)).toEqual( + packShards(concurrent(files), allDefault, 3, 1), + ); }); test("breaks duration ties by key so equal files spread deterministically", () => { const timings = { "src/c.e2e.test.ts": 50, "src/a.e2e.test.ts": 50, "src/b.e2e.test.ts": 50 }; - const shards = packShards(Object.keys(timings), timings, 3); + const shards = packShards(concurrent(Object.keys(timings)), timings, 3, 1); expect(shards).toEqual([["src/a.e2e.test.ts"], ["src/b.e2e.test.ts"], ["src/c.e2e.test.ts"]]); });