ci: shorten the Test workflow e2e critical path

.github/workflows/test.yml

@@ -231,7 +231,7 @@ jobs:
       matrix:
         # Linux coverage for these files comes from the integration job, which
         # runs the whole stack integration project.
-        os: [macos-latest, windows-latest]
+        os: [blacksmith-6vcpu-macos-latest, windows-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -381,7 +381,8 @@ jobs:
           - { suite: cli, shard: 3, shards: 5, name: cli 3/5 }
           - { suite: cli, shard: 4, shards: 5, name: cli 4/5 }
           - { suite: cli, shard: 5, shards: 5, name: cli 5/5 }
-          - { suite: cli-e2e, name: cli-e2e }
+          - { suite: cli-e2e, shard: 1, shards: 2, name: cli-e2e 1/2 }
+          - { suite: cli-e2e, shard: 2, shards: 2, name: cli-e2e 2/2 }
     steps:
       - name: Checkout
         uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta
@@ -395,19 +396,6 @@ jobs:
           sticky-disks: "true"
           docker-images: "true"
 
-      - name: Cache Go CLI binary
-        id: cache-go-binary
-        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
-        with:
-          path: apps/cli-go/supabase-go
-          key: go-cli-${{ runner.os }}-${{ hashFiles('apps/cli-go/**/*.go',
-            'apps/cli-go/go.mod', 'apps/cli-go/go.sum') }}
-
-      - name: Build Go CLI
-        if: steps.cache-go-binary.outputs.cache-hit != 'true'
-        run: go build -o supabase-go .
-        working-directory: apps/cli-go
-
       # The e2e harness invokes `node apps/cli/dist/supabase.js` with
       # `SUPABASE_CLI_BINARY_OVERRIDE` pointing at the compiled binary in
       # `apps/cli/dist/`. Build the CLI explicitly before invoking every
@@ -424,7 +412,7 @@ jobs:
 
       - name: Run cli-e2e end-to-end tests
         if: matrix.suite == 'cli-e2e'
-        run: pnpm exec turbo run test:e2e:run --only --filter=@supabase/cli-e2e
+        run: pnpm exec turbo run test:e2e:run --only --filter=@supabase/cli-e2e -- --shard=${{ matrix.shard }}/${{ matrix.shards }}
         env:
           SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go
 

apps/cli/src/commands/gen/types/types.e2e.test.ts

@@ -167,6 +167,9 @@ const waitForLocalPostgres = Effect.fnUntraced(function* (containerName: string)
         "PGPASSWORD=postgres",
         containerName,
         "psql",
+        // The entrypoint's init server listens only on the Unix socket, so TCP reaches the final server.
+        "-h",
+        "127.0.0.1",
         "-U",
         "postgres",
         "-d",

apps/cli/tests/e2e-sequencer.ts

@@ -4,7 +4,9 @@ import { BaseSequencer, type TestSpecification } from "vitest/node";
 
 /** Bun `--timings` file of measured e2e file durations, refreshed from CI artifacts. */
 const TIMINGS_FILE = "tests/e2e-timings.json";
-const E2E_PROJECTS = new Set(["e2e", "e2e-exclusive"]);
+const E2E_PROJECT = "e2e";
+const E2E_EXCLUSIVE_PROJECT = "e2e-exclusive";
+const E2E_PROJECTS = new Set([E2E_PROJECT, E2E_EXCLUSIVE_PROJECT]);
 
 /** Duration assumed for every file while no timing has been measured yet. */
 export const DEFAULT_DURATION_MS = 30_000;
@@ -28,31 +30,70 @@ function compareKeys(a: string, b: string): number {
   return a < b ? -1 : a > b ? 1 : 0;
 }
 
+/** An e2e file to shard; exclusive files run one at a time after the concurrent ones. */
+export interface ShardFile {
+  readonly key: string;
+  readonly exclusive: boolean;
+}
+
+interface ShardLoad {
+  readonly longestConcurrent: number;
+  readonly concurrentTotal: number;
+  readonly exclusiveTotal: number;
+}
+
+function withFile(load: ShardLoad, duration: number, exclusive: boolean): ShardLoad {
+  return exclusive
+    ? { ...load, exclusiveTotal: load.exclusiveTotal + duration }
+    : {
+        ...load,
+        longestConcurrent: Math.max(load.longestConcurrent, duration),
+        concurrentTotal: load.concurrentTotal + duration,
+      };
+}
+
+function estimatedWall(load: ShardLoad, workers: number): number {
+  return Math.max(load.longestConcurrent, load.concurrentTotal / workers) + load.exclusiveTotal;
+}
+
 /**
- * Splits keys into `count` shards by measured duration, longest first into the emptiest shard.
- * The result depends only on the set of keys, so every shard process derives the same partition.
+ * Splits files into `count` shards by estimated wall time, longest first into the shard whose
+ * estimate stays lowest, the less loaded one on ties. Concurrent files share `workers` slots;
+ * exclusive files add their full duration.
+ * The result depends only on the set of files, so every shard process derives the same partition.
  */
 export function packShards(
-  keys: readonly string[],
+  files: readonly ShardFile[],
   timings: Readonly<Record<string, number>>,
   count: number,
+  workers: number,
 ): string[][] {
   const fallback = fallbackDuration(timings);
-  const durations = keys
-    .map((key) => ({ key, duration: timings[key] ?? fallback }))
+  const durations = files
+    .map((file) => ({ ...file, duration: timings[file.key] ?? fallback }))
     .sort((a, b) => b.duration - a.duration || compareKeys(a.key, b.key));
 
-  const shards = Array.from({ length: count }, () => ({ load: 0, keys: new Array<string>() }));
-  for (const { key, duration } of durations) {
-    let target = shards[0];
+  const empty: ShardLoad = { longestConcurrent: 0, concurrentTotal: 0, exclusiveTotal: 0 };
+  const shards = Array.from({ length: count }, () => ({ keys: new Array<string>(), load: empty }));
+  for (const { key, exclusive, duration } of durations) {
+    let target:
+      | { shard: (typeof shards)[number]; load: ShardLoad; wall: number; before: number }
+      | undefined;
     for (const shard of shards) {
-      if (target === undefined || shard.load < target.load) {
-        target = shard;
+      const load = withFile(shard.load, duration, exclusive);
+      const wall = estimatedWall(load, workers);
+      const before = estimatedWall(shard.load, workers);
+      if (
+        target === undefined ||
+        wall < target.wall ||
+        (wall === target.wall && before < target.before)
+      ) {
+        target = { shard, load, wall, before };
       }
     }
     if (target !== undefined) {
-      target.load += duration;
-      target.keys.push(key);
+      target.shard.load = target.load;
+      target.shard.keys.push(key);
     }
   }
   return shards.map((shard) => shard.keys);
@@ -109,7 +150,15 @@ export class E2eSequencer extends BaseSequencer {
     const byKey = new Map(
       files.map((spec) => [relative(root, spec.moduleId).replaceAll("\\", "/"), spec]),
     );
-    const assigned = packShards([...byKey.keys()], timings, shard.count)[shard.index - 1] ?? [];
+    const shardFiles = [...byKey].map(([key, spec]) => ({
+      key,
+      exclusive: spec.project.name === E2E_EXCLUSIVE_PROJECT,
+    }));
+    const maxWorkers =
+      files.find((spec) => spec.project.name === E2E_PROJECT)?.project.config.maxWorkers ?? 1;
+    // Concurrent files share one Docker daemon and slow each other, so pack against half the slots.
+    const workers = Math.max(1, Math.floor(maxWorkers / 2));
+    const assigned = packShards(shardFiles, timings, shard.count, workers)[shard.index - 1] ?? [];
     return assigned.flatMap((key) => {
       const spec = byKey.get(key);
       return spec === undefined ? [] : [spec];

apps/cli/tests/e2e-sequencer.unit.test.ts

@@ -9,19 +9,21 @@ const files = ["src/a.e2e.test.ts", "src/b.e2e.test.ts", "src/c.e2e.test.ts", "s
 const sortedShards = (shards: readonly (readonly string[])[]) =>
   shards.map((shard) => [...shard].sort());
 
+const concurrent = (keys: readonly string[]) => keys.map((key) => ({ key, exclusive: false }));
+
 describe("packShards", () => {
   test.each([1, files.length, files.length + 3])(
     "partitions every file into exactly one of %i shards",
     (count) => {
-      const shards = packShards(files, { "src/a.e2e.test.ts": 100 }, count);
+      const shards = packShards(concurrent(files), { "src/a.e2e.test.ts": 100 }, count, 1);
 
       expect(shards).toHaveLength(count);
       expect(shards.flat().sort()).toEqual([...files].sort());
     },
   );
 
   test("leaves trailing shards empty when there are more shards than files", () => {
-    const shards = packShards(["src/a.e2e.test.ts", "src/b.e2e.test.ts"], {}, 4);
+    const shards = packShards(concurrent(["src/a.e2e.test.ts", "src/b.e2e.test.ts"]), {}, 4, 1);
 
     expect(shards).toEqual([["src/a.e2e.test.ts"], ["src/b.e2e.test.ts"], [], []]);
   });
@@ -33,8 +35,8 @@ describe("packShards", () => {
       "src/c.e2e.test.ts": 900,
     };
 
-    const forward = packShards(files, timings, 2);
-    const reversed = packShards([...files].reverse(), timings, 2);
+    const forward = packShards(concurrent(files), timings, 2, 1);
+    const reversed = packShards(concurrent([...files].reverse()), timings, 2, 1);
 
     expect(reversed).toEqual(forward);
   });
@@ -47,14 +49,35 @@ describe("packShards", () => {
       "src/d.e2e.test.ts": 10,
     };
 
-    const shards = packShards(files, timings, 2);
+    const shards = packShards(concurrent(files), timings, 2, 1);
 
     expect(sortedShards(shards)).toEqual([
       ["src/b.e2e.test.ts"],
       ["src/a.e2e.test.ts", "src/c.e2e.test.ts", "src/d.e2e.test.ts"],
     ]);
   });
 
+  test("packs concurrent files beside a long one while an exclusive file runs alone", () => {
+    const timings = {
+      "src/long.e2e.test.ts": 90,
+      "src/exclusive.e2e.test.ts": 80,
+      "src/a.e2e.test.ts": 20,
+      "src/b.e2e.test.ts": 20,
+      "src/c.e2e.test.ts": 20,
+    };
+    const shardFiles = Object.keys(timings).map((key) => ({
+      key,
+      exclusive: key === "src/exclusive.e2e.test.ts",
+    }));
+
+    const shards = packShards(shardFiles, timings, 2, 4);
+
+    expect(sortedShards(shards)).toEqual([
+      ["src/a.e2e.test.ts", "src/b.e2e.test.ts", "src/c.e2e.test.ts", "src/long.e2e.test.ts"],
+      ["src/exclusive.e2e.test.ts"],
+    ]);
+  });
+
   test("weights an unmeasured file at the median of the measured ones", () => {
     const timings = {
       "src/a.e2e.test.ts": 100,
@@ -63,7 +86,7 @@ describe("packShards", () => {
     };
 
     // d is unmeasured; at the median (300) it ties with b and follows it into the second shard.
-    const shards = packShards(files, timings, 2);
+    const shards = packShards(concurrent(files), timings, 2, 1);
 
     expect(sortedShards(shards)).toEqual([
       ["src/a.e2e.test.ts", "src/c.e2e.test.ts"],
@@ -74,13 +97,15 @@ describe("packShards", () => {
   test("weights every file at the default duration when nothing has been measured", () => {
     const allDefault = Object.fromEntries(files.map((file) => [file, DEFAULT_DURATION_MS]));
 
-    expect(packShards(files, {}, 3)).toEqual(packShards(files, allDefault, 3));
+    expect(packShards(concurrent(files), {}, 3, 1)).toEqual(
+      packShards(concurrent(files), allDefault, 3, 1),
+    );
   });
 
   test("breaks duration ties by key so equal files spread deterministically", () => {
     const timings = { "src/c.e2e.test.ts": 50, "src/a.e2e.test.ts": 50, "src/b.e2e.test.ts": 50 };
 
-    const shards = packShards(Object.keys(timings), timings, 3);
+    const shards = packShards(concurrent(Object.keys(timings)), timings, 3, 1);
 
     expect(shards).toEqual([["src/a.e2e.test.ts"], ["src/b.e2e.test.ts"], ["src/c.e2e.test.ts"]]);
   });