diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6e8e4fed34..1bfebaa530 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -14,20 +14,134 @@ inputs: fails with a path validation error. required: false default: "true" + tools: + description: >- + Space-separated mise tools to install instead of every tool in + mise.toml. + required: false + default: "" + pnpm-store-cache: + description: >- + Whether to restore the pnpm store through actions/cache. Where the store + archive restores slower than the registry downloads it, disable this. + required: false + default: "true" + sticky-disks: + description: >- + Whether to mount dependency caches from Blacksmith sticky disks on + Blacksmith Linux runners. Disks are read-only unless commit-sticky-disks + is set. + required: false + default: "false" + commit-sticky-disks: + description: >- + Whether this job writes its sticky disks back. Only the fill workflow + sets it, so test jobs never race it with partial snapshots. + required: false + default: "false" + docker-images: + description: >- + Whether to keep Docker's image store on a Blacksmith sticky disk so + stack images are not pulled again on every job. Requires sticky-disks. + Callers must stop Docker in a final if: always() step so the disk + unmounts cleanly. + required: false + default: "false" + native-artifacts: + description: >- + Whether to keep the stack tests' native service artifacts on a + Blacksmith sticky disk. Requires sticky-disks. + required: false + default: "false" runs: using: "composite" steps: + # Sticky disks are shared by every run in the repository, so only the fill + # job on trusted events (pushes, schedules, manual runs) commits; pull request + # and merge queue code could otherwise plant dependencies or images that + # trusted runs later execute. + # Committing runs prune what the current lockfile no longer uses; bump the + # -vN key segment to reset a disk outright. + - name: Detect sticky disks + id: sticky + if: inputs.sticky-disks == 'true' + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + COMMIT: ${{ inputs.commit-sticky-disks }} + run: | + if [ "$RUNNER_OS" != Linux ]; then + exit 0 + fi + if [ -z "${BLACKSMITH_RUNNER_MESSAGE_WAIT_MS:-}" ]; then + echo "::notice::Sticky disks requested but this is not a Blacksmith runner; using actions/cache." + exit 0 + fi + echo "enabled=true" >> "$GITHUB_OUTPUT" + case "$COMMIT:$EVENT_NAME" in + true:push | true:schedule | true:workflow_dispatch) echo "commit=on-change" >> "$GITHUB_OUTPUT" ;; + *) echo "commit=false" >> "$GITHUB_OUTPUT" ;; + esac + + - name: Stop Docker before mounting its image disk + if: inputs.docker-images == 'true' && steps.sticky.outputs.enabled == 'true' + shell: bash + run: sudo systemctl stop docker.socket docker.service + + - name: Mount Docker image disk + if: inputs.docker-images == 'true' && steps.sticky.outputs.enabled == 'true' + uses: useblacksmith/stickydisk@b6cb58d1d31d6d32a25248fab12493d831f7bd48 # v1.7.2 + with: + key: ${{ github.repository }}-docker-images-v3-${{ runner.arch }} + path: /var/lib/docker + commit: ${{ steps.sticky.outputs.commit }} + + - name: Restart Docker on the image disk + if: inputs.docker-images == 'true' && steps.sticky.outputs.enabled == 'true' + shell: bash + run: | + sudo systemctl start docker.socket docker.service + docker info --format '{{.DockerRootDir}} {{.Driver}}' + docker image ls --format '{{.Repository}}:{{.Tag}}' | wc -l + + - name: Locate native artifact cache + id: native-artifacts + if: inputs.native-artifacts == 'true' && steps.sticky.outputs.enabled == 'true' + shell: bash + run: echo "path=${XDG_CACHE_HOME:-$HOME/.cache}/supabase-stack-artifacts" >> "$GITHUB_OUTPUT" + + - name: Mount native artifact disk + if: inputs.native-artifacts == 'true' && steps.sticky.outputs.enabled == 'true' + uses: useblacksmith/stickydisk@b6cb58d1d31d6d32a25248fab12493d831f7bd48 # v1.7.2 + with: + key: ${{ github.repository }}-stack-artifacts-v2-${{ runner.arch }} + path: ${{ steps.native-artifacts.outputs.path }} + commit: ${{ steps.sticky.outputs.commit }} + - name: Install toolchains uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 with: version: 2026.9.0 + install_args: ${{ inputs.tools }} + cache_key: ${{ inputs.tools != '' && format('mise-{0}-{1}-{2}-{3}', runner.os, runner.arch, inputs.tools, hashFiles('mise.toml', 'mise.lock', '.node-version', '.bun-version', 'package.json')) || '' }} - name: Resolve pnpm store path if: inputs.dependency-cache == 'true' id: pnpm-store shell: bash - run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + # A sticky disk mounts a block-level snapshot in seconds, while + # actions/cache downloads and untars the whole store on every job. + - name: Mount pnpm store disk + if: inputs.dependency-cache == 'true' && steps.sticky.outputs.enabled == 'true' + uses: useblacksmith/stickydisk@b6cb58d1d31d6d32a25248fab12493d831f7bd48 # v1.7.2 + with: + key: ${{ github.repository }}-pnpm-store-v3-${{ runner.arch }} + path: ${{ steps.pnpm-store.outputs.path }} + commit: ${{ steps.sticky.outputs.commit }} # Cache only the content-addressable half of the store (package files plus # the SQLite index), never the global virtual store under links/. That @@ -42,7 +156,7 @@ runs: # so restores never match the earlier whole-store archives, which still # carry a links/ tree. - name: Configure pnpm dependency cache - if: inputs.dependency-cache == 'true' + if: inputs.dependency-cache == 'true' && inputs.pnpm-store-cache == 'true' && steps.sticky.outputs.enabled != 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | @@ -91,3 +205,10 @@ runs: echo "//firewall.depthfirst.com/npm/:_authToken=${DEPENDENCY_FIREWALL_TOKEN}" } > "$npmrc" NPM_CONFIG_USERCONFIG="$npmrc" pnpm install --frozen-lockfile + + # Pruning keeps only this checkout's packages, so only the default branch + # prunes; other branches would evict each other's dependencies. + - name: Prune pnpm store disk + if: inputs.dependency-cache == 'true' && steps.sticky.outputs.commit == 'on-change' && github.ref_name == (github.event.repository.default_branch || 'develop') + shell: bash + run: pnpm store prune diff --git a/.github/workflows/fill-sticky-disks.yml b/.github/workflows/fill-sticky-disks.yml new file mode 100644 index 0000000000..5e25630251 --- /dev/null +++ b/.github/workflows/fill-sticky-disks.yml @@ -0,0 +1,88 @@ +name: Fill sticky disks + +# Test jobs mount the pnpm and Docker image caches from Blacksmith sticky +# disks but pull request runs never commit them. This workflow is the trusted +# writer: it runs on protected branches, which are the only refs Blacksmith's +# sticky-disk branch protection lets commit. +on: + push: + branches: + - develop + - next + - main + schedule: + - cron: "23 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Every ref writes the same repository-wide disks, so fills run one at a time. + group: fill-sticky-disks + cancel-in-progress: false + +jobs: + fill: + name: Fill sticky disks + # Manual dispatches from other branches would commit unreviewed caches. + if: contains(fromJSON('["refs/heads/develop", "refs/heads/next", "refs/heads/main"]'), github.ref) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" + commit-sticky-disks: "true" + docker-images: "true" + native-artifacts: "true" + + # The nightly fill clears artifacts of older service versions. + - name: Download native stack artifacts + run: | + if [ "$GITHUB_EVENT_NAME" = schedule ]; then + rm -rf "${XDG_CACHE_HOME:-$HOME/.cache}/supabase-stack-artifacts/"* + fi + pnpm --filter @supabase/stack prepare:test-artifacts + + - name: Build CLI + run: pnpm exec turbo run supabase#build + + # Starting a default project pulls every image `supabase start` uses. + - name: Pull CLI images + env: + SUPABASE_CLI_BINARY_OVERRIDE: ${{ github.workspace }}/apps/cli/dist/supabase + run: | + project="$(mktemp -d)" + cli="$GITHUB_WORKSPACE/apps/cli/dist/supabase.js" + cd "$project" + node "$cli" init + node "$cli" start + # The nightly fill drops images of older versions; the next step pulls the catalog back. + if [ "$GITHUB_EVENT_NAME" = schedule ]; then + docker image prune --all --force + fi + node "$cli" stop --no-backup + + - name: Pull stack package images + run: | + pnpm --silent --filter @supabase/stack prepare:test-artifacts --images | + xargs --max-procs 4 --max-args 1 docker pull --quiet + + # Also runs after a failed step so leftover stack containers and volumes never persist. + - name: Prune Docker before saving its image disk + if: always() + run: | + docker ps --all --quiet | xargs --no-run-if-empty docker rm --force + docker system prune --force + docker volume prune --all --force + + # The image disk only commits once nothing holds /var/lib/docker open. + - name: Stop Docker so its image disk can be saved + if: always() + run: sudo systemctl stop docker.socket docker.service diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index c37e557139..ee6a98f27c 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -33,7 +33,7 @@ jobs: test: name: Test and type-check runs-on: ubuntu-latest - # The shared setup installs the full workspace + Go toolchain via mise, which + # The shared setup installs the full workspace toolchain via mise, which # runs ~9-10 min; a 10-minute cap raced the install and got cancelled on a # cold cache. 20 gives that install headroom. (This check is heavier than it # needs to be for two scripts — slimming the setup is a possible follow-up.) diff --git a/.github/workflows/run-ci.yml b/.github/workflows/run-ci.yml index 61b40a0065..bf39a18cb6 100644 --- a/.github/workflows/run-ci.yml +++ b/.github/workflows/run-ci.yml @@ -1,9 +1,9 @@ name: run-ci -# Opt-in full develop CI for PRs that Test.yml does not already cover: stacked -# PRs (base is not develop, main, next or v*.x) and drafts. Ready PRs into -# those stay on the existing workflows so required check names and concurrency -# are unchanged. +# Opt-in full CI, Windows and macOS included, for drafts, stacked PRs, and +# ready PRs into develop or next. Ready PRs into main or v*.x already run +# everything through Test.yml. On ready PRs the Linux jobs run a second time +# alongside Test.yml's own run. # # Add the `run-ci` label to start the suite; remove it to cancel in-progress # runs via this workflow's concurrency group. Other labels do not retrigger. @@ -43,7 +43,7 @@ jobs: !startsWith(github.head_ref, 'release-notes/') && contains(github.event.pull_request.labels.*.name, 'run-ci') && (github.event.pull_request.draft || - (github.base_ref != 'develop' && github.base_ref != 'main' && github.base_ref != 'next' && + (github.base_ref != 'main' && !(startsWith(github.base_ref, 'v') && endsWith(github.base_ref, '.x')))) && ((github.event.action != 'labeled' && github.event.action != 'unlabeled') || @@ -51,5 +51,6 @@ jobs: uses: ./.github/workflows/test.yml with: force: true + full: true secrets: DF_FIREWALL_TOKEN: ${{ secrets.DF_FIREWALL_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d95ac17c2c..050fc254b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,9 @@ name: Test # # Default path: ready (non-draft) PRs targeting `develop`, `next`, `main` or a # `v*.x` maintenance branch, plus the merge queue for `develop` and `next`. -# `run-ci.yml` calls this workflow for drafts and stacked PRs. +# `run-ci.yml` calls this workflow for drafts and stacked PRs. Windows and macOS +# jobs run on PRs into develop or next only for platform-sensitive paths, and +# always in the merge queue, on PRs into main or v*.x, and under `run-ci`. on: pull_request: types: @@ -31,6 +33,10 @@ on: description: Run the suite even when the PR is a draft (used by run-ci.yml) type: boolean default: false + full: + description: Run the Windows and macOS jobs regardless of the changed paths (used by run-ci.yml) + type: boolean + default: false secrets: DF_FIREWALL_TOKEN: required: true @@ -55,6 +61,7 @@ jobs: github.event.pull_request.draft == false) name: Check code quality runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -65,7 +72,7 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - + sticky-disks: "true" - name: Check code quality run: pnpm run check:all @@ -88,6 +95,7 @@ jobs: github.event.pull_request.draft == false) name: Run unit tests runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -98,6 +106,7 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" - name: Run unit tests run: pnpm run test:unit @@ -110,6 +119,7 @@ jobs: github.event.pull_request.draft == false) name: Run integration tests (${{ matrix.name }}) runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -130,6 +140,8 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" + docker-images: "true" - name: Run integration tests run: >- @@ -147,14 +159,68 @@ jobs: src/port-reservation.e2e.test.ts --passWithNoTests=false - test-stack-ports: + # Releases /var/lib/docker so the image disk unmounts cleanly. + - name: Stop Docker + if: always() + run: sudo systemctl stop docker.socket docker.service + + platform-changes: if: | !startsWith(github.head_ref, 'release-notes/') && (github.event_name == 'merge_group' || inputs.force || github.event.pull_request.draft == false) + name: Detect platform-sensitive changes + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run: ${{ steps.detect.outputs.run }} + steps: + # The merge queue and PRs into main or v*.x run every platform; only PRs + # into develop or next defer Windows and macOS to the merge queue. + - name: Detect + id: detect + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + FULL: ${{ inputs.full }} + REPO: ${{ github.repository }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PATTERN: >- + ^(pnpm-lock\.yaml|pnpm-workspace\.yaml|package\.json|mise\.(toml|lock)|\.node-version|\.bun-version|turbo\.json|\.github/workflows/test\.yml|\.github/actions/ + |packages/stack/(package\.json|vitest[^/]*|tests/|src/(host|namespace|identity|storage|internal)/ + |src/(Artifacts|HostProcess|Ports|Network|Sweep|Owner|Proxy|HttpProxy|StackNamespace|effect)[.]|src/whole-stack\.native) + |apps/cli/src/commands/(db/reset|experimental/stack/start)/) + run: | + if [ "$EVENT" != "pull_request" ] || [ "$FULL" = "true" ] || [ "$BASE_REF" = "main" ] || [[ "$BASE_REF" == v*.x ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + pattern=$(tr -d ' \n' <<<"$PATTERN") + # The compare API lists at most 300 files; a larger diff runs every platform. + files=$(gh api "repos/$REPO/compare/$BASE_SHA...$HEAD_SHA" --jq 'if (.files | length) >= 300 then "package.json" else .files[].filename end') + matches=$(grep -E "$pattern" <<<"$files" || true) + if [ -n "$matches" ]; then + printf 'Platform-sensitive changes:\n%s\n' "$matches" + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "No platform-sensitive changes; Windows and macOS run in the merge queue. Add the run-ci label to run them here." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + test-stack-ports: + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) && + needs.platform-changes.outputs.run == 'true' + needs: platform-changes name: Run stack port tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -171,6 +237,9 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + tools: node bun pnpm + # Windows unpacks the store archive slower than it downloads the packages. + pnpm-store-cache: ${{ matrix.os != 'windows-latest' }} - name: Run stack integration tests working-directory: packages/stack @@ -211,16 +280,26 @@ jobs: !startsWith(github.head_ref, 'release-notes/') && (github.event_name == 'merge_group' || inputs.force || - github.event.pull_request.draft == false) - name: Run stack native scenarios (${{ matrix.os }}) + github.event.pull_request.draft == false) && + needs.platform-changes.outputs.run == 'true' + needs: platform-changes + name: Run stack native scenarios (${{ matrix.os }}, ${{ matrix.suite }}) runs-on: ${{ matrix.os }} - timeout-minutes: 45 + timeout-minutes: 15 strategy: fail-fast: false matrix: # Native artifacts exist for macOS and Linux only, and macOS runners have # no Docker; Linux coverage for both backends comes from test-stack-e2e. os: [blacksmith-6vcpu-macos-latest] + suite: [lifecycle, confinement, idle, isolation] + # Concurrent scenarios slow each other on macOS runners, so the + # heaviest ones run alone. + include: + - { suite: lifecycle, file: lifecycle, pattern: "^(?!.*writes stay confined)" } + - { suite: confinement, file: lifecycle, pattern: "writes stay confined" } + - { suite: idle, file: idle-parallel, pattern: "^(?!.*parallel stack isolation)" } + - { suite: isolation, file: idle-parallel, pattern: "parallel stack isolation" } steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -232,9 +311,21 @@ jobs: with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + # Keyed per suite so each job saves the artifacts it actually prepared. + - name: Cache native stack artifacts + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/supabase-stack-artifacts + key: stack-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ matrix.suite }}-${{ hashFiles('packages/stack/src/Artifacts.ts') }} + - name: Run stack native whole-stack scenarios working-directory: packages/stack - run: pnpm test:e2e:native-scenarios --passWithNoTests=false + run: >- + pnpm test:e2e:run + src/whole-stack.native.${{ matrix.file }}.e2e.test.ts + --testNamePattern='${{ matrix.pattern }}' + --sequence.concurrent + --passWithNoTests=false test-summary: if: | @@ -244,11 +335,16 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run unit and integration tests - needs: [test-unit, test-integration, test-stack-ports] + needs: [platform-changes, test-unit, test-integration, test-stack-ports] runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Verify unit and integration tests succeeded run: | + if [ "${{ needs.platform-changes.result }}" != "success" ]; then + echo "::error ::Platform change detection did not succeed: ${{ needs.platform-changes.result }}" + exit 1 + fi if [ "${{ needs.test-unit.result }}" = "failure" ] || [ "${{ needs.test-unit.result }}" = "cancelled" ] || \ [ "${{ needs.test-integration.result }}" = "failure" ] || [ "${{ needs.test-integration.result }}" = "cancelled" ]; then echo "::error ::Unit or integration tests failed: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}, stack-ports=${{ needs.test-stack-ports.result }}" @@ -268,6 +364,7 @@ jobs: github.event.pull_request.draft == false) name: Run CLI end-to-end tests (${{ matrix.name }}) runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -288,6 +385,8 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" + docker-images: "true" # The e2e harness invokes `node apps/cli/dist/supabase.js` with # `SUPABASE_CLI_BINARY_OVERRIDE` pointing at the compiled binary in @@ -299,6 +398,8 @@ jobs: - name: Run end-to-end tests if: matrix.suite == 'cli' run: pnpm exec turbo run test:e2e:run --only --filter=supabase -- --shard=${{ matrix.shard }}/${{ matrix.shards }} + env: + SUPABASE_E2E_WORKERS: "4" - name: Run cli-e2e end-to-end tests if: matrix.suite == 'cli-e2e' @@ -315,6 +416,11 @@ jobs: retention-days: 7 if-no-files-found: warn + # Releases /var/lib/docker so the image disk unmounts cleanly. + - name: Stop Docker + if: always() + run: sudo systemctl stop docker.socket docker.service + test-stack-e2e: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -323,11 +429,14 @@ jobs: github.event.pull_request.draft == false) name: Run stack end-to-end tests (${{ matrix.runtime }}, ${{ matrix.suite }}) runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 strategy: fail-fast: false matrix: runtime: [native, docker, podman] - suite: [lifecycle, idle-parallel] + # Confinement watches the whole temp tree, so it gets a job of its own + # instead of sharing one with concurrently running scenarios. + suite: [lifecycle, idle-parallel, confinement] env: SUPABASE_STACK_TEST_ENGINE: ${{ matrix.runtime == 'podman' && 'podman' || 'docker' }} steps: @@ -338,6 +447,9 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" + docker-images: ${{ matrix.runtime == 'docker' }} + native-artifacts: ${{ matrix.runtime == 'native' }} - name: Setup Podman if: matrix.runtime == 'podman' @@ -346,7 +458,9 @@ jobs: - name: Run stack end-to-end tests run: >- pnpm --filter @supabase/stack test:e2e:run - src/whole-stack.${{ matrix.runtime }}.${{ matrix.suite }}.e2e.test.ts + src/whole-stack.${{ matrix.runtime }}.${{ matrix.suite == 'idle-parallel' && 'idle-parallel' || 'lifecycle' }}.e2e.test.ts + --testNamePattern='${{ matrix.suite == 'confinement' && 'writes stay confined' || matrix.suite == 'lifecycle' && '^(?!.*writes stay confined)' || '.' }}' + --sequence.concurrent --passWithNoTests=false - name: Run public stack end-to-end tests @@ -356,6 +470,11 @@ jobs: src/public.e2e.test.ts --passWithNoTests=false + # Releases /var/lib/docker so the image disk unmounts cleanly. + - name: Stop Docker + if: always() && matrix.runtime == 'docker' + run: sudo systemctl stop docker.socket docker.service + test-stack-podman-integration: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -364,6 +483,7 @@ jobs: github.event.pull_request.draft == false) name: Run stack Podman integration tests runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 env: SUPABASE_STACK_TEST_ENGINE: podman steps: @@ -374,6 +494,7 @@ jobs: uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" - name: Setup Podman uses: ./.github/actions/setup-podman @@ -412,11 +533,23 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run end-to-end tests - needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios, test-stack-podman-integration] + needs: + [ + platform-changes, + test-e2e, + test-stack-e2e, + test-stack-native-scenarios, + test-stack-podman-integration, + ] runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Verify all shards succeeded run: | + if [ "${{ needs.platform-changes.result }}" != "success" ]; then + echo "::error ::Platform change detection did not succeed: ${{ needs.platform-changes.result }}" + exit 1 + fi if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || \ [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ] || \ [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ] || \ diff --git a/apps/cli/AGENTS.md b/apps/cli/AGENTS.md index 9ee52bc7cd..d791f8fdb3 100644 --- a/apps/cli/AGENTS.md +++ b/apps/cli/AGENTS.md @@ -152,7 +152,7 @@ the command tree), and `tsc` (a command that reads config without `withCliConfig **Exceptions** -They live in [ADR 0031](../../docs/adr/0031-config-value-precedence.md) and nowhere else. Do not add a +They live in [ADR 0032](../../docs/adr/0032-config-value-precedence.md) and nowhere else. Do not add a per-command exception without updating the ADR and the guard's exemption list together. ## Experimental feature registration diff --git a/apps/cli/docs/supabase/db/diff.md b/apps/cli/docs/supabase/db/diff.md index c46ab5499f..cdcae8eb42 100644 --- a/apps/cli/docs/supabase/db/diff.md +++ b/apps/cli/docs/supabase/db/diff.md @@ -18,8 +18,7 @@ With pg-delta, `[db.migrations] schema_paths` and declarative schema files (such With the bundled pg-delta engine, diff SQL defaults to uppercase keywords, indent 2, a maximum width of 180, trailing commas, and column/key alignment, matching its declarative export. When `-f` writes migrations, execution-aware transaction semantics are preserved as ordered per-unit files; non-transactional units carry a directive that the CLI apply path honors. Flattened review output retains the rendered SQL and preambles, but not the unit boundaries supplied to a migration runner. Configure overrides with `[experimental.pgdelta] format_options`, or set `format_options = "null"` to emit raw, unformatted statements. -While the diff command is able to capture most schema changes, there are cases where it is known to fail. Currently, this could happen if you schema contains: +Storage buckets are data, not schema, so no diff engine captures them. Under the legacy migra engine, the diff can also miss: - Changes to publication -- Changes to storage buckets - Views with `security_invoker` attributes diff --git a/apps/cli/docs/supabase/db/pull.md b/apps/cli/docs/supabase/db/pull.md index bc0f2ccae6..23d7bd636e 100644 --- a/apps/cli/docs/supabase/db/pull.md +++ b/apps/cli/docs/supabase/db/pull.md @@ -10,7 +10,12 @@ Optionally, a new row can be inserted into the migration history table to reflec If no entries exist in the migration history table, pg-delta (the default diff engine) produces the full migration from the shadow diff alone; with `--diff-engine migra`, the initial pull instead uses `pg_dump` to capture all contents of the remote schemas you have created. Otherwise, this command will only diff schema changes against the remote database, similar to running `db diff --linked`. -Pass `--declarative` to switch to the declarative pg-delta export workflow instead of writing a migration file. +Pass `--declarative` to switch to the declarative pg-delta export workflow instead of writing a migration file. The tree lives in `supabase/schemas` by default. Change the location with `experimental.pgdelta.declarative_schema_path`. + +On a project that falls back to the legacy migra engine, don't use `--declarative`. The export also points `[db.migrations].schema_paths` at the tree it writes. The next local `db diff` applies those files in lexicographic order without resolving dependencies between them, so it fails when a file sorts before an object it depends on. If you've already run it, re-enable pg-delta, or undo both changes: + +1. Delete the files the export wrote to the declarative directory (`supabase/schemas/` by default). Files that were already there are left in place. +2. Restore `[db.migrations].schema_paths` in `config.toml` to its previous value, or remove it if the export added it. pg-delta plans are execution-aware: when a plan crosses a transaction boundary — for example `ALTER TYPE ... ADD VALUE` followed by a statement that uses the new enum value, which cannot run in the same transaction — `db pull` writes one ordered migration file per plan unit instead of a single file (for example `_remote_schema_schema_changes.sql` and `_remote_schema_after_enum_values.sql`), each recorded in the migration history. The common case (a single unit) still produces exactly one `_remote_schema.sql` file. @@ -18,6 +23,8 @@ By default the emitted SQL is formatted with the same settings the declarative e pg-delta is the default diff engine: the migration-file `db pull` workflow uses pg-delta for the shadow diff step unless configured otherwise; it does not switch to declarative output. With pg-delta, `[db.migrations] schema_paths` does not affect this workflow (migra still reads it, for local targets only); use `supabase db schema declarative sync` for declarative schema files. To fall back to the legacy migra engine, set `enabled = false` under `[experimental.pgdelta]` in `config.toml`, or pass `--diff-engine migra` for a single run. +Under the pg-delta engine, the pull captures your own customizations in Supabase-managed schemas automatically. Examples are triggers on managed tables like `auth.users` and RLS policies on `storage.objects` and `storage.buckets`. Under the legacy migra engine, the initial pull's `pg_dump` skips managed schemas. The diff pass that follows appends your trigger and RLS policy changes there to the same migration. If an older CLI reports that `auth` and `storage` were excluded, capture existing customizations there with `--schema auth,storage`. + When pulling from a remote database with `--db-url`, prefer a direct connection (`db..supabase.co:5432`) over the connection pooler so pg-delta can introspect the full catalog reliably. ## Debugging empty pg-delta pulls diff --git a/apps/cli/docs/templates/examples.yaml b/apps/cli/docs/templates/examples.yaml index 8d0e70222d..f49ca5269f 100644 --- a/apps/cli/docs/templates/examples.yaml +++ b/apps/cli/docs/templates/examples.yaml @@ -236,29 +236,28 @@ supabase-db-pull: Update remote migration history table? [Y/n] Repaired migration history: [20240414044403] => applied Finished supabase db pull. - The auth and storage schemas are excluded. Run supabase db pull --schema auth,storage again to diff them. - id: local-studio name: Local studio code: supabase db pull --local response: | Connecting to local database... - Setting up initial schema.... - Creating custom roles supabase/roles.sql... + Creating shadow database... + Seeding globals from roles.sql... Applying migration 20240414044403_remote_schema.sql... + Diffing schemas... No schema changes found - The auth and storage schemas are excluded. Run supabase db pull --schema auth,storage again to diff them. - exit status 1 + The remote database is already in sync with your local migrations — nothing to pull. - id: custom-schemas name: Custom schemas - code: supabase db pull --schema auth,storage + code: supabase db pull --schema public,extensions response: | Connecting to remote database... - Setting up initial schema.... - Creating custom roles supabase/roles.sql... + Creating shadow database... + Seeding globals from roles.sql... Applying migration 20240414044403_remote_schema.sql... + Diffing schemas: public,extensions No schema changes found - Try rerunning the command with --debug to troubleshoot the error. - exit status 1 + The remote database is already in sync with your local migrations — nothing to pull. supabase-db-push: - id: basic-usage name: Basic usage diff --git a/apps/cli/package.json b/apps/cli/package.json index 2370823afb..f9c04d4228 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -36,7 +36,7 @@ "test:integration": "pnpm exec turbo run supabase#test:integration:run --", "test:integration:run": "bun --bun vitest run --project integration --coverage.reportsDirectory=coverage/integration", "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run supabase#test:e2e:run --only --", - "test:e2e:run": "bun --bun vitest run --project e2e", + "test:e2e:run": "bun --bun vitest run --project e2e --project e2e-exclusive", "test:live": "bun --bun vitest run --project live", "test:smoke": "bun run tests/smoke-test.ts", "types:check": "tsc --noEmit -p tsconfig.types.json" diff --git a/apps/cli/scripts/refresh-e2e-timings.ts b/apps/cli/scripts/refresh-e2e-timings.ts index dd5396410b..a52667501c 100644 --- a/apps/cli/scripts/refresh-e2e-timings.ts +++ b/apps/cli/scripts/refresh-e2e-timings.ts @@ -26,7 +26,7 @@ export interface MergeOptions { readonly warn?: (message: string) => void; } -const PROJECT_PREFIX = "e2e:"; +const PROJECT_PREFIXES = ["e2e:", "e2e-exclusive:"]; function major(version: string): string { return version.split(".")[0] ?? version; @@ -48,10 +48,11 @@ export function mergeTimings( ); } for (const [key, result] of cache.results) { - if (!key.startsWith(PROJECT_PREFIX)) { + const prefix = PROJECT_PREFIXES.find((candidate) => key.startsWith(candidate)); + if (prefix === undefined) { continue; } - const path = key.slice(PROJECT_PREFIX.length); + const path = key.slice(prefix.length); if (result.failed) { warn(`skipping failed ${path}`); continue; diff --git a/apps/cli/scripts/refresh-e2e-timings.unit.test.ts b/apps/cli/scripts/refresh-e2e-timings.unit.test.ts index 8571e883f7..b7c11ffae2 100644 --- a/apps/cli/scripts/refresh-e2e-timings.unit.test.ts +++ b/apps/cli/scripts/refresh-e2e-timings.unit.test.ts @@ -13,6 +13,7 @@ describe("mergeTimings", () => { [ cache( ["e2e:src/start.e2e.test.ts", { duration: 1200, failed: false }], + ["e2e-exclusive:src/health.e2e.test.ts", { duration: 900, failed: false }], ["unit:src/start.unit.test.ts", { duration: 5, failed: false }], [":src/start.integration.test.ts", { duration: 40, failed: false }], ), @@ -20,7 +21,10 @@ describe("mergeTimings", () => { { vitestVersion }, ); - expect(timings).toEqual({ "src/start.e2e.test.ts": 1200 }); + expect(timings).toEqual({ + "src/health.e2e.test.ts": 900, + "src/start.e2e.test.ts": 1200, + }); }); test("skips failed entries and reports each one", () => { diff --git a/apps/cli/src/command-internal/container-cli.ts b/apps/cli/src/command-internal/container-cli.ts index 8b054fa922..c8ccc780a1 100644 --- a/apps/cli/src/command-internal/container-cli.ts +++ b/apps/cli/src/command-internal/container-cli.ts @@ -241,11 +241,11 @@ export function runContainerCliExpectSuccess( } /** - * Like {@link containerCliExitCode}, but also returns the child's stdout — reading it instead of - * ignoring it avoids the process hanging on an unread pipe. `podmanArgs` behaves the same as on + * Like {@link containerCliExitCode}, but also returns the child's stdout and stderr — reading them + * instead of ignoring them avoids the process hanging on an unread pipe. `podmanArgs` behaves the same as on * {@link containerCliExitCode}. */ -export const containerCliExitCodeAndStdout = ( +export const containerCliOutput = ( spawner: Spawner, args: ReadonlyArray, podmanArgs?: ReadonlyArray, @@ -255,7 +255,7 @@ export const containerCliExitCodeAndStdout = ( const options = { stdin: "ignore", stdout: "pipe", - stderr: "ignore", + stderr: "pipe", } satisfies ChildProcess.CommandOptions; const handle = yield* spawnRuntime(spawner, dockerRuntime, args, options).pipe( Effect.catch(() => @@ -268,11 +268,15 @@ export const containerCliExitCodeAndStdout = ( ); // Read stdout concurrently with the exit code: Node's "exit" event can fire before a fast // process's stdio finishes draining, so awaiting the exit code first would race the stream. - const [exitCode, stdout] = yield* Effect.all( - [handle.exitCode.pipe(Effect.map(Number)), collectText(handle.stdout)], + const [exitCode, stdout, stderr] = yield* Effect.all( + [ + handle.exitCode.pipe(Effect.map(Number)), + collectText(handle.stdout), + collectText(handle.stderr), + ], { concurrency: "unbounded" }, ); - return { exitCode, stdout }; + return { exitCode, stdout, stderr }; }), ); diff --git a/apps/cli/src/command-internal/docker-remove-all.ts b/apps/cli/src/command-internal/docker-remove-all.ts index 1d5342fc85..70f92f4f5a 100644 --- a/apps/cli/src/command-internal/docker-remove-all.ts +++ b/apps/cli/src/command-internal/docker-remove-all.ts @@ -1,4 +1,4 @@ -import { Data, Effect, Result } from "effect"; +import { Data, Effect, Result, Schedule } from "effect"; import type { ChildProcessSpawner } from "effect/process/ChildProcessSpawner"; import { @@ -8,7 +8,7 @@ import { } from "../shared/telemetry/error-actionability.ts"; import { containerCliExitCode, - containerCliExitCodeAndStdout, + containerCliOutput, describeContainerCliFailure, dockerSupportsVolumePruneAllFlag, } from "./container-cli.ts"; @@ -91,6 +91,42 @@ const reportPruned = (debug: boolean, label: string, stdout: string) => process.stderr.write(`${label} [${parsePrunedNames(stdout).join(" ")}]\n`); }); +/** + * Docker runs one prune at a time per daemon and rejects concurrent ones, even with different + * label filters, so stacks stopping at the same moment wait for each other's prune. + */ +const pruneWaitingForConcurrentPrune = Effect.fn("DockerRemoveAll.prune")(function* ( + spawner: Spawner, + args: ReadonlyArray, + podmanArgs?: ReadonlyArray, +) { + let attempts = 0; + const result = yield* containerCliOutput(spawner, args, podmanArgs).pipe( + Effect.tap(() => + Effect.sync(() => { + attempts += 1; + }), + ), + Effect.repeat({ + while: (output) => + output.exitCode !== 0 && /a prune operation is already running/iu.test(output.stderr), + schedule: Schedule.spaced("250 millis").pipe(Schedule.jittered), + times: 40, + }), + ); + yield* Effect.annotateCurrentSpan({ + "docker.prune.kind": args[0] ?? "", + "docker.prune.attempts": attempts, + "docker.prune.exit_code": result.exitCode, + }); + return result; +}); + +const pruneFailureMessage = (kind: string, stderr: string) => { + const detail = stderr.trim(); + return detail.length > 0 ? `failed to prune ${kind}: ${detail}` : `failed to prune ${kind}`; +}; + /** Every failure {@link dockerRemoveAll} can produce. */ export type DockerRemoveAllError = | DockerRemoveAllListError @@ -152,7 +188,7 @@ export const dockerRemoveAll = ( // Collects stdout instead of ignoring it — reading the pipe avoids the same hang, and the // report backs the `--debug` output below. - const containerPrune = yield* containerCliExitCodeAndStdout(spawner, [ + const containerPrune = yield* pruneWaitingForConcurrentPrune(spawner, [ "container", "prune", "--force", @@ -168,7 +204,7 @@ export const dockerRemoveAll = ( ); if (containerPrune.exitCode !== 0) { return yield* new DockerRemoveAllContainerPruneError({ - message: "failed to prune containers", + message: pruneFailureMessage("containers", containerPrune.stderr), }); } yield* reportPruned(debug, "Pruned containers:", containerPrune.stdout); @@ -182,7 +218,7 @@ export const dockerRemoveAll = ( // Podman has no `--all` flag on `volume prune` at all, but already prunes every unused // volume by default, so omitting `--all` there is a lossless fallback. const dockerSupportsAll = yield* dockerSupportsVolumePruneAllFlag(spawner); - const volumePrune = yield* containerCliExitCodeAndStdout( + const volumePrune = yield* pruneWaitingForConcurrentPrune( spawner, [ "volume", @@ -202,12 +238,14 @@ export const dockerRemoveAll = ( ), ); if (volumePrune.exitCode !== 0) { - return yield* new DockerRemoveAllVolumePruneError({ message: "failed to prune volumes" }); + return yield* new DockerRemoveAllVolumePruneError({ + message: pruneFailureMessage("volumes", volumePrune.stderr), + }); } yield* reportPruned(debug, "Pruned volumes:", volumePrune.stdout); } - const networkPrune = yield* containerCliExitCodeAndStdout(spawner, [ + const networkPrune = yield* pruneWaitingForConcurrentPrune(spawner, [ "network", "prune", "--force", @@ -222,7 +260,9 @@ export const dockerRemoveAll = ( ), ); if (networkPrune.exitCode !== 0) { - return yield* new DockerRemoveAllNetworkPruneError({ message: "failed to prune networks" }); + return yield* new DockerRemoveAllNetworkPruneError({ + message: pruneFailureMessage("networks", networkPrune.stderr), + }); } // Established output text: singular "network", unlike the container/volume reports. yield* reportPruned(debug, "Pruned network:", networkPrune.stdout); diff --git a/apps/cli/src/commands/config/push/SIDE_EFFECTS.md b/apps/cli/src/commands/config/push/SIDE_EFFECTS.md index f7f8853384..6b1265ba30 100644 --- a/apps/cli/src/commands/config/push/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/config/push/SIDE_EFFECTS.md @@ -375,7 +375,7 @@ may itself contain a `.`. - **Only properties your file declares, and whose value differs from the project, are written.** Fields the API requires together ship as a group; undeclared members of that group are sent with the project's CURRENT value, read in the same run — so they do not change. Only when the read did not return a member's current value is it sent at the config schema default, and that is always disclosed (a `[group-write]` block in the confirmation output, a `forced` entry in the JSON payload, and a summary `Note:` line) — never applied silently. - **`db.ssl_enforcement`'s presence, not its decoded default, decides the gate.** `@supabase/config`'s projection recovers whether `[db.ssl_enforcement]` (and `storage.image_transformation`/`storage.s3_protocol`) were actually declared, as opposed to decoding to a schema default; an undeclared `[db.ssl_enforcement]` is treated as `disabled` — no read is needed for this any more, since row 1's single response already carries the remote value. - Optional `*pointer` sections (`db.ssl_enforcement`, `storage.image_transformation`, `storage.s3_protocol`) follow that same presence rule end to end — declared-but-absent is never confused with explicitly-disabled. -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`; matched `[remotes.*]` over base) > default. The pushed config is the declared document with these overrides applied, so a `SUPABASE_*` variable that overrides a declared key changes what is pushed. An unparsable `SUPABASE_*` value fails the diff --git a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md index c01caa7cac..aaed23e195 100644 --- a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md @@ -209,7 +209,7 @@ transaction metadata. ## Notes / Delegation -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`) > default. The whole config is decoded up front, so an invalid value fails the command. A `--linked` target also applies a matching `[remotes.*]` block. The linked-database password env is withheld when the target differs from diff --git a/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md b/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md index 85b29e5b94..cb353316bd 100644 --- a/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md @@ -83,7 +83,7 @@ shell inherits the suppressing variables and is missed. ## Notes -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`; a matched `[remotes.*]` block over the base document on `--linked`) > default. The linked-database password env is withheld when the target differs from `.temp/project-ref`: stderr gets `Not sending SUPABASE_DB_PASSWORD to : this directory is linked to . Using a temporary login role instead (needs supabase login or SUPABASE_ACCESS_TOKEN). Pass --password to use a password for .` and a temporary login role is minted. `--password` is rejected with diff --git a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md index 6024c02c57..c75c9249e7 100644 --- a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md @@ -176,7 +176,7 @@ Progress strings still go to stderr; stdout carries a single structured envelope ## Notes -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`) > default. The whole config is decoded up front, so an invalid value fails the command. A `--linked` target also applies a matching `[remotes.*]` block. The linked-database password env is withheld when the target differs from diff --git a/apps/cli/src/commands/db/push/SIDE_EFFECTS.md b/apps/cli/src/commands/db/push/SIDE_EFFECTS.md index 720dafdcad..2146375456 100644 --- a/apps/cli/src/commands/db/push/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/push/SIDE_EFFECTS.md @@ -107,7 +107,7 @@ stdout is payload-only. A single `result` object is emitted: target is a hard error rather than a silently discarded flag (deliberately stricter than `SUPABASE_PROJECT_ID`, which simply goes unused on a non-linked target). -- **Config value precedence** (ADR 0031): explicit flag > shell env > project +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`; a matched `[remotes.*]` block over the base document) > default. The whole config is decoded up front, so an invalid value fails the command before any connection. diff --git a/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md b/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md index 9a11723cdc..7d45764baf 100644 --- a/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md @@ -273,7 +273,7 @@ to those defaults (the usual outcome for an interactive terminal). standalone outside the per-file transaction batch, with the same non-atomic flush behaviour as `db push` — see `db push`'s SIDE_EFFECTS Notes (supabase/cli#5139, adopted into TS in PR supabase/cli#5671). -- **Config value precedence** (ADR 0031): explicit flag > shell env > project +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`; a matched `[remotes.*]` block over the base document) > default. The linked-database password env is withheld when the target differs from `.temp/project-ref`: stderr gets diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.e2e.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.e2e.test.ts index d630f7428f..b54627296f 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.e2e.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.e2e.test.ts @@ -1,142 +1,16 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Data, Effect, FileSystem, Path } from "effect"; +import { Effect, FileSystem, Path } from "effect"; +import { requireCliSuccess, runSupabaseEffect } from "../../../../../../tests/helpers/cli.ts"; import { - makeTempCliStackProject, - overrideStackPorts, - requireCliSuccess, - runSupabaseEffect, -} from "../../../../../../tests/helpers/cli.ts"; - -const CLI_COMMAND_TIMEOUT_MS = 60_000; -const STACK_START_TIMEOUT_MS = 280_000; -const CLEANUP_TIMEOUT_MS = 120_000; -const LIFECYCLE_MARGIN_MS = 30_000; -const CLEANUP_HOOK_TIMEOUT_MS = CLEANUP_TIMEOUT_MS + LIFECYCLE_MARGIN_MS; -const SCENARIO_COMMAND_TIMEOUT_MS = 280_000; -const BEFORE_ALL_TIMEOUT_MS = CLI_COMMAND_TIMEOUT_MS + STACK_START_TIMEOUT_MS + LIFECYCLE_MARGIN_MS; -const SCENARIO_TIMEOUT_MS = 900_000; -const TEST_TIMEOUT_MS = BEFORE_ALL_TIMEOUT_MS + SCENARIO_TIMEOUT_MS + CLEANUP_HOOK_TIMEOUT_MS; - -class SyncE2eSetupError extends Data.TaggedError("SyncE2eSetupError")<{ - readonly message: string; - readonly cause?: unknown; -}> {} - -const initialDesiredSchema = `create type public.account_state as enum ('pending', 'active'); - -create table public.disposable_note ( - id bigint generated by default as identity primary key, - body text not null -); - -create view public.auth_user_emails as -select id, email -from auth.users; -`; - -function commandFailure(result: { stdout: string; stderr: string }): string { - return `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`; -} - -const migrationFiles = (projectDir: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const migrationsDir = path.join(projectDir, "supabase", "migrations"); - return (yield* fs.exists(migrationsDir)) - ? (yield* fs.readDirectory(migrationsDir)).filter((file) => file.endsWith(".sql")).sort() - : []; - }); - -const withTempStackProject = ( - use: (project: Awaited>) => Effect.Effect, -): Effect.Effect => - Effect.acquireUseRelease( - Effect.tryPromise({ - try: () => makeTempCliStackProject("sb-pgdelta-next-e2e-"), - catch: (cause) => - new SyncE2eSetupError({ message: "temp stack project setup failed", cause }), - }), - use, - (project) => - Effect.tryPromise({ - try: () => project.cleanup(), - catch: (cause) => - new SyncE2eSetupError({ message: "temp stack project cleanup failed", cause }), - }).pipe(Effect.ignore), - ); - -const setUpProject = (projectDir: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const init = yield* runSupabaseEffect(["init"], { - cwd: projectDir, - exitTimeoutMs: CLI_COMMAND_TIMEOUT_MS, - }); - requireCliSuccess(init, "init setup"); - yield* Effect.tryPromise({ - try: () => overrideStackPorts(projectDir), - catch: (cause) => - new SyncE2eSetupError({ message: `failed to override ports in ${projectDir}`, cause }), - }); - - const configPath = path.join(projectDir, "supabase", "config.toml"); - const config = yield* fs.readFileString(configPath); - if (!config.includes("[experimental.pgdelta]\nenabled = true")) { - return yield* new SyncE2eSetupError({ - message: "init setup did not enable experimental pg-delta in config.toml", - }); - } - yield* fs.writeFileString( - configPath, - config.replace( - '# declarative_schema_path = "./schemas"', - 'declarative_schema_path = "./schemas"', - ), - ); - - const schemasDir = path.join(projectDir, "supabase", "schemas"); - yield* fs.makeDirectory(schemasDir, { recursive: true }); - yield* fs.writeFileString(path.join(schemasDir, "public.sql"), initialDesiredSchema); - const extensionsDir = path.join(schemasDir, "cluster", "extensions"); - yield* fs.makeDirectory(extensionsDir, { recursive: true }); - for (const extension of ["pg_net", "pgcrypto", "uuid-ossp"]) { - yield* fs.writeFileString( - path.join(extensionsDir, `${extension}.sql`), - `CREATE EXTENSION IF NOT EXISTS "${extension}" WITH SCHEMA "extensions";\n`, - ); - } - // Both install into a fixed schema, so no `WITH SCHEMA` clause. - for (const extension of ["pg_cron", "pgmq"]) { - yield* fs.writeFileString( - path.join(extensionsDir, `${extension}.sql`), - `CREATE EXTENSION IF NOT EXISTS "${extension}";\n`, - ); - } - - const start = yield* runSupabaseEffect( - [ - "start", - "--exclude", - "studio", - "--exclude", - "logflare", - "--exclude", - "vector", - "--exclude", - "gotrue", - "--exclude", - "realtime", - "--exclude", - "storage-api", - ], - { cwd: projectDir, exitTimeoutMs: STACK_START_TIMEOUT_MS }, - ); - requireCliSuccess(start, "start setup"); - }); + commandFailure, + migrationFiles, + scenarioCommandOptions, + setUpProject, + TEST_TIMEOUT_MS, + withTempStackProject, +} from "../../../../../../tests/helpers/declarative-sync-e2e.ts"; describe("db schema declarative sync (e2e)", () => { it.live( @@ -160,10 +34,7 @@ describe("db schema declarative sync (e2e)", () => { "initial_declarative", "--experimental", ], - { - cwd: projectDir, - exitTimeoutMs: SCENARIO_COMMAND_TIMEOUT_MS, - }, + scenarioCommandOptions(projectDir), ); expect(sync.exitCode, commandFailure(sync)).toBe(0); @@ -180,84 +51,18 @@ describe("db schema declarative sync (e2e)", () => { /CREATE\s+(?:SCHEMA|TABLE)\s+(?:IF\s+NOT\s+EXISTS\s+)?["']?(?:auth|storage|realtime)["']?/iu, ); - const reset = yield* runSupabaseEffect(["db", "reset", "--local", "--no-seed"], { - cwd: projectDir, - exitTimeoutMs: SCENARIO_COMMAND_TIMEOUT_MS, - }); + const reset = yield* runSupabaseEffect( + ["db", "reset", "--local", "--no-seed"], + scenarioCommandOptions(projectDir), + ); requireCliSuccess(reset, "db reset setup"); const converged = yield* runSupabaseEffect( ["db", "schema", "declarative", "sync", "--no-apply", "--experimental"], - { - cwd: projectDir, - exitTimeoutMs: SCENARIO_COMMAND_TIMEOUT_MS, - }, + scenarioCommandOptions(projectDir), ); expect(converged.exitCode, commandFailure(converged)).toBe(0); expect(`${converged.stdout}${converged.stderr}`).toContain("No schema changes found"); - - // Extension-managed objects on the converged tree. The stack teardown after every test - // means this continues in the same test rather than a second one. - const jobsPath = path.join(projectDir, "supabase", "schemas", "jobs.sql"); - const runSync = (name: string) => - runSupabaseEffect( - [ - "db", - "schema", - "declarative", - "sync", - "--no-apply", - "--name", - name, - "--experimental", - ], - { - cwd: projectDir, - exitTimeoutMs: SCENARIO_COMMAND_TIMEOUT_MS, - }, - ); - // A next-engine plan may span several ordered migration files; read every - // file a sync added rather than only the last one. - const syncAndReadSql = (name: string) => - Effect.gen(function* () { - const before = new Set(yield* migrationFiles(projectDir)); - const result = yield* runSync(name); - expect(result.exitCode, commandFailure(result)).toBe(0); - const added = (yield* migrationFiles(projectDir)).filter((file) => !before.has(file)); - expect(added.length, "sync did not write a migration").toBeGreaterThan(0); - return { - result, - sql: (yield* Effect.forEach(added, (file) => - fs.readFileString(path.join(projectDir, "supabase", "migrations", file)), - )).join("\n"), - }; - }); - - yield* fs.writeFileString( - jobsPath, - [ - "select cron.schedule('nightly_cleanup', '0 3 * * *', $$delete from public.disposable_note$$);", - "select pgmq.create('emails');", - "", - ].join("\n"), - ); - // pg-delta renders a job through `cron.schedule_in_database(...)`. - const added = yield* syncAndReadSql("add_jobs"); - expect(added.sql).toMatch(/cron\.schedule(?:_in_database)?\('nightly_cleanup'/); - expect(added.sql).toContain("pgmq.create('emails')"); - - // Rename the job and drop the queue — this must not be refused as a legacy export. - yield* fs.writeFileString( - jobsPath, - "select cron.schedule('weekly_cleanup', '0 3 * * 0', $$delete from public.disposable_note$$);\n", - ); - const removed = yield* syncAndReadSql("rename_job_drop_queue"); - expect(`${removed.result.stdout}${removed.result.stderr}`).not.toContain( - "legacy pg-delta export", - ); - expect(removed.sql).toContain("cron.unschedule('nightly_cleanup')"); - expect(removed.sql).toMatch(/cron\.schedule(?:_in_database)?\('weekly_cleanup'/); - expect(removed.sql).toContain("pgmq.drop_queue('emails')"); }), ).pipe(Effect.provide(BunServices.layer)), TEST_TIMEOUT_MS, diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.jobs.e2e.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.jobs.e2e.test.ts new file mode 100644 index 0000000000..59e26b4101 --- /dev/null +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.jobs.e2e.test.ts @@ -0,0 +1,89 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Path } from "effect"; + +import { runSupabaseEffect } from "../../../../../../tests/helpers/cli.ts"; +import { + commandFailure, + convergeInitialSchema, + migrationFiles, + scenarioCommandOptions, + setUpProject, + TEST_TIMEOUT_MS, + withTempStackProject, +} from "../../../../../../tests/helpers/declarative-sync-e2e.ts"; + +describe("db schema declarative sync extension jobs (e2e)", () => { + it.live( + "adds, renames, and drops extension-managed jobs and queues", + () => + withTempStackProject((project) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const projectDir = project.dir; + yield* setUpProject(projectDir); + yield* convergeInitialSchema(projectDir); + + const jobsPath = path.join(projectDir, "supabase", "schemas", "jobs.sql"); + const runSync = (name: string) => + runSupabaseEffect( + [ + "db", + "schema", + "declarative", + "sync", + "--no-apply", + "--name", + name, + "--experimental", + ], + scenarioCommandOptions(projectDir), + ); + // A next-engine plan may span several ordered migration files; read every + // file a sync added rather than only the last one. + const syncAndReadSql = (name: string) => + Effect.gen(function* () { + const before = new Set(yield* migrationFiles(projectDir)); + const result = yield* runSync(name); + expect(result.exitCode, commandFailure(result)).toBe(0); + const added = (yield* migrationFiles(projectDir)).filter((file) => !before.has(file)); + expect(added.length, "sync did not write a migration").toBeGreaterThan(0); + return { + result, + sql: (yield* Effect.forEach(added, (file) => + fs.readFileString(path.join(projectDir, "supabase", "migrations", file)), + )).join("\n"), + }; + }); + + yield* fs.writeFileString( + jobsPath, + [ + "select cron.schedule('nightly_cleanup', '0 3 * * *', $$delete from public.disposable_note$$);", + "select pgmq.create('emails');", + "", + ].join("\n"), + ); + // pg-delta renders a job through `cron.schedule_in_database(...)`. + const added = yield* syncAndReadSql("add_jobs"); + expect(added.sql).toMatch(/cron\.schedule(?:_in_database)?\('nightly_cleanup'/); + expect(added.sql).toContain("pgmq.create('emails')"); + + // Rename the job and drop the queue — this must not be refused as a legacy export. + yield* fs.writeFileString( + jobsPath, + "select cron.schedule('weekly_cleanup', '0 3 * * 0', $$delete from public.disposable_note$$);\n", + ); + const removed = yield* syncAndReadSql("rename_job_drop_queue"); + expect(`${removed.result.stdout}${removed.result.stderr}`).not.toContain( + "legacy pg-delta export", + ); + expect(removed.sql).toContain("cron.unschedule('nightly_cleanup')"); + expect(removed.sql).toMatch(/cron\.schedule(?:_in_database)?\('weekly_cleanup'/); + expect(removed.sql).toContain("pgmq.drop_queue('emails')"); + }), + ).pipe(Effect.provide(BunServices.layer)), + TEST_TIMEOUT_MS, + ); +}); diff --git a/apps/cli/src/commands/functions/deploy/SIDE_EFFECTS.md b/apps/cli/src/commands/functions/deploy/SIDE_EFFECTS.md index a28e9fe980..44549ccf7b 100644 --- a/apps/cli/src/commands/functions/deploy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/functions/deploy/SIDE_EFFECTS.md @@ -90,7 +90,7 @@ The `--output`/`-o` flag does not change deploy output. ## Notes -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`) > default. The whole config is decoded up front, so an invalid value, including an unparsable `SUPABASE_*` override, fails the command. - If no function name is provided, deploys all functions found in `supabase/functions/`. diff --git a/apps/cli/src/commands/functions/download/SIDE_EFFECTS.md b/apps/cli/src/commands/functions/download/SIDE_EFFECTS.md index c55ca3de14..f344e3ee01 100644 --- a/apps/cli/src/commands/functions/download/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/functions/download/SIDE_EFFECTS.md @@ -102,7 +102,7 @@ Same envelope as `json` above (including on the Docker-unbundle path). ## Notes -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`) > default. The whole config is decoded up front, so an invalid value, including an unparsable `SUPABASE_*` override, fails the command. - If no function name is provided, downloads all functions. diff --git a/apps/cli/src/commands/gen/types/SIDE_EFFECTS.md b/apps/cli/src/commands/gen/types/SIDE_EFFECTS.md index 1443573e1f..56387fdc9c 100644 --- a/apps/cli/src/commands/gen/types/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/gen/types/SIDE_EFFECTS.md @@ -138,7 +138,7 @@ Not applicable. shell and nested dotenv overrides, matching the CLI. The database password is the `[db].password` value (default `postgres`); `SUPABASE_DB_PASSWORD` has no effect, and the command has no `--password` flag. -- **Config value precedence** (ADR 0031): explicit flag > shell env > project `.env*` > config +- **Config value precedence** (ADR 0032): explicit flag > shell env > project `.env*` > config (`config.json` over `config.toml`; a matched `[remotes.*]` block over the base document on `--linked`/`--project-id`) > default. The whole config is decoded up front where it is loaded, so an invalid value fails the command. diff --git a/apps/cli/src/commands/migration/list/SIDE_EFFECTS.md b/apps/cli/src/commands/migration/list/SIDE_EFFECTS.md index 952dce73dd..1513e71b04 100644 --- a/apps/cli/src/commands/migration/list/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/migration/list/SIDE_EFFECTS.md @@ -63,7 +63,7 @@ Same structured `migrations` result delivered as an NDJSON `result` event. exit 1. For `--linked` the password resolves as flag > shell `SUPABASE_DB_PASSWORD` > project `.env*` > config; the env value is withheld when the target differs from `.temp/project-ref` (stderr `Not sending SUPABASE_DB_PASSWORD to : this directory is linked to . Using a temporary login role instead (needs supabase login or SUPABASE_ACCESS_TOKEN). Pass --password to use a password for .`), and a - temporary login role is minted instead (ADR 0031). + temporary login role is minted instead (ADR 0032). - `--db-url`, `--linked`, and `--local` are mutually exclusive. - **`--project-ref`** overrides ONLY the linked-ref resolution used for the connection (flag > `SUPABASE_PROJECT_ID` > `.temp/project-ref`). It never implies `--linked`: diff --git a/apps/cli/src/commands/migration/repair/SIDE_EFFECTS.md b/apps/cli/src/commands/migration/repair/SIDE_EFFECTS.md index bfc555c5de..a0e696a248 100644 --- a/apps/cli/src/commands/migration/repair/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/migration/repair/SIDE_EFFECTS.md @@ -82,7 +82,7 @@ migration history table to match local migration files?` (default **NO**). exit 1. For `--linked` the password resolves as flag > shell `SUPABASE_DB_PASSWORD` > project `.env*` > config; the env value is withheld when the target differs from `.temp/project-ref` (stderr `Not sending SUPABASE_DB_PASSWORD to : this directory is linked to . Using a temporary login role instead (needs supabase login or SUPABASE_ACCESS_TOKEN). Pass --password to use a password for .`), and a - temporary login role is minted instead (ADR 0031). + temporary login role is minted instead (ADR 0032). - **`--project-ref`** overrides ONLY the linked-ref resolution used for the connection (flag > `SUPABASE_PROJECT_ID` > `.temp/project-ref`). It never implies `--linked`: passing it with a resolved `--local`/`--db-url` target is a hard error rather diff --git a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md index a07f8b3b66..c4905f5753 100644 --- a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md @@ -148,7 +148,7 @@ code or the rest of the payload. exit 1. For `--linked` the password resolves as flag > shell `SUPABASE_DB_PASSWORD` > project `.env*` > config; the env value is withheld when the target differs from `.temp/project-ref` (stderr `Not sending SUPABASE_DB_PASSWORD to : this directory is linked to . Using a temporary login role instead (needs supabase login or SUPABASE_ACCESS_TOKEN). Pass --password to use a password for .`), and a - temporary login role is minted instead (ADR 0031). + temporary login role is minted instead (ADR 0032). - **`--project-ref`** overrides ONLY the linked-ref resolution used for the connection (flag > `SUPABASE_PROJECT_ID` > `.temp/project-ref`). It never implies `--linked`: passing it with a resolved `--local`/`--db-url` target is a hard error rather diff --git a/apps/cli/src/commands/services/SIDE_EFFECTS.md b/apps/cli/src/commands/services/SIDE_EFFECTS.md index 70ca8c048c..7a4a12a003 100644 --- a/apps/cli/src/commands/services/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/services/SIDE_EFFECTS.md @@ -83,7 +83,7 @@ NDJSON success event with the same `{ services: [...] }` payload. ## Notes -- Config values resolve as flag > shell env > project `.env*` > config (`config.json` over `config.toml`) > default (ADR 0031). An invalid config value or `SUPABASE_*` override fails the command with exit 1, naming its source. +- Config values resolve as flag > shell env > project `.env*` > config (`config.json` over `config.toml`) > default (ADR 0032). An invalid config value or `SUPABASE_*` override fails the command with exit 1, naming its source. - Backend selection follows canonical experimental-feature routing: `SUPABASE_EXPERIMENTAL_STACK=1|0` takes precedence over `experimental.stack`; unset or empty uses project config, and an invalid environment value fails. Output fields and serializers stay the same. - The legacy backend uses its baked-in service matrix and honors its existing config/version overrides. The stack backend lists services from the installed CLI artifact catalog using canonical `ghcr.io/supabase/cli/...` image names and catalog versions, reported as the upstream version without a slim revision suffix (`-rN`). Native and Docker runtimes use the same catalog versions. Since the catalog belongs to the installed CLI, an older launched CLI or a newer/mirrored stack image may differ from this inventory. The command does not inspect running containers, image pulls, service health, or live stack state. - For stack mode, PostgreSQL uses the configured major version or `SUPABASE_DB_MAJOR_VERSION` (15 or 17). An unreadable config file or an unsupported PostgreSQL major warns with the cause and falls back to default catalog versions; an invalid config value fails the command; absent config uses defaults. Legacy image pins, slim-image rewriting, and remote image overrides do not affect stack results. diff --git a/apps/cli/src/commands/start/SIDE_EFFECTS.md b/apps/cli/src/commands/start/SIDE_EFFECTS.md index c4fc2fd6b7..297c81271d 100644 --- a/apps/cli/src/commands/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/start/SIDE_EFFECTS.md @@ -6,7 +6,7 @@ uses the new [`supabase stack start` implementation](../experimental/stack/start `SUPABASE_EXPERIMENTAL_STACK=0` forces the legacy backend. See [backend selection](../../../docs/stack-commands.md). Backend routing and the legacy handler both read `supabase/config.json` when present, otherwise `supabase/config.toml`. Config values resolve as flag > shell env > project `.env*` > config > -default (ADR 0031); `start` applies no `[remotes.*]` block. Backend selection happens before command parsing. When the +default (ADR 0032); `start` applies no `[remotes.*]` block. Backend selection happens before command parsing. When the environment override is unset or empty, an unreadable, malformed, or invalid project configuration falls back to the legacy backend; an invalid environment override remains an error. diff --git a/apps/cli/src/commands/stop/stop.integration.test.ts b/apps/cli/src/commands/stop/stop.integration.test.ts index 390ccbffeb..44c2762e7d 100644 --- a/apps/cli/src/commands/stop/stop.integration.test.ts +++ b/apps/cli/src/commands/stop/stop.integration.test.ts @@ -5,6 +5,7 @@ import { Deferred, Effect, Exit, + Fiber, FileSystem, Layer, Option, @@ -14,6 +15,7 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { TestClock } from "effect/testing"; import { vi } from "vitest"; import { cliConfigValuesTestLayer } from "../../../tests/helpers/config-values-layer.ts"; @@ -618,6 +620,76 @@ describe("stop integration", () => { }).pipe(Effect.provide(BunServices.layer)), ); + const PRUNE_RUNNING = "Error response from daemon: a prune operation is already running"; + + const networkPruneRoute = (failures: number, stderr: string) => { + const route = defaultRoute(); + let remaining = failures; + return (args: ReadonlyArray): RouteResult => { + if (args[0] === "network" && args[1] === "prune" && remaining > 0) { + remaining -= 1; + return { exitCode: 1, stderr: [stderr] }; + } + return route(args); + }; + }; + + /** Each retry sleeps only after the previous attempt, so the clock steps until `stop` settles. */ + const joinAdvancingClock = (fiber: Fiber.Fiber) => + TestClock.adjust("100 millis").pipe( + Effect.repeat({ while: () => fiber.pollUnsafe() === undefined }), + Effect.andThen(Fiber.join(fiber)), + ); + + const networkPrunes = (spawned: ReadonlyArray) => + spawned.filter((s) => s.args[0] === "network" && s.args[1] === "prune"); + + it.effect("waits for another stack's prune instead of failing", () => + Effect.gen(function* () { + const { layer, child } = yield* setup({ + configuredProjectId: "demo", + route: networkPruneRoute(2, PRUNE_RUNNING), + }); + const fiber = yield* stop(flags()).pipe(Effect.provide(layer), Effect.forkChild); + yield* joinAdvancingClock(fiber); + expect(networkPrunes(child.spawned)).toHaveLength(3); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("fails once another stack's prune outlasts the retry budget", () => + Effect.gen(function* () { + const { layer, child } = yield* setup({ + configuredProjectId: "demo", + route: networkPruneRoute(Number.POSITIVE_INFINITY, PRUNE_RUNNING), + }); + const fiber = yield* stop(flags()).pipe(Effect.provide(layer), Effect.exit, Effect.forkChild); + const exit = yield* joinAdvancingClock(fiber); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Cause.pretty(exit.cause)).toContain("StopNetworkPruneError"); + expect(Cause.pretty(exit.cause)).toContain("a prune operation is already running"); + } + expect(networkPrunes(child.spawned)).toHaveLength(41); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("fails on the first unrelated prune error", () => + Effect.gen(function* () { + const { layer, child } = yield* setup({ + configuredProjectId: "demo", + route: networkPruneRoute(1, "Error response from daemon: permission denied"), + }); + const exit = yield* Effect.exit(stop(flags()).pipe(Effect.provide(layer))); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Cause.pretty(exit.cause)).toContain( + "failed to prune networks: Error response from daemon: permission denied", + ); + } + expect(networkPrunes(child.spawned)).toHaveLength(1); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.effect("deletes data volumes with --no-backup", () => Effect.gen(function* () { const { layer, child } = yield* setup({ diff --git a/apps/cli/src/shared/services/Dockerfile b/apps/cli/src/shared/services/Dockerfile index 8ddea5a6ac..563c96ccd7 100644 --- a/apps/cli/src/shared/services/Dockerfile +++ b/apps/cli/src/shared/services/Dockerfile @@ -35,7 +35,7 @@ FROM supabase/gotrue:v2.197.0 AS gotrue FROM supabase/realtime:v2.143.4 AS realtime -FROM supabase/storage-api:v1.82.0 AS storage +FROM supabase/storage-api:v1.82.1 AS storage FROM supabase/logflare:1.53.0 AS logflare # One-shot job images (not started as long-running containers); differ's alias diff --git a/apps/cli/src/shared/telemetry/error-actionability.ts b/apps/cli/src/shared/telemetry/error-actionability.ts index 46bd328e98..c865d8c508 100644 --- a/apps/cli/src/shared/telemetry/error-actionability.ts +++ b/apps/cli/src/shared/telemetry/error-actionability.ts @@ -1,4 +1,5 @@ import { Cause, Option, Predicate } from "effect"; +import type { PlatformError } from "effect"; import type { CliError as EffectCliError } from "effect/cli"; import type { StackError, StackFailureKind } from "@supabase/stack/effect"; @@ -1167,10 +1168,55 @@ const defectSystemErrorCodes = new Set([ "EXDEV", ]); +// Data descriptors only, so reading a defect never runs a getter. +const ownValue = (record: object, key: string): unknown => + Object.getOwnPropertyDescriptor(record, key)?.value; + +const platformErrorReasons: ReadonlySet = new Set([ + "AlreadyExists", + "BadResource", + "Busy", + "InvalidData", + "NotFound", + "PermissionDenied", + "TimedOut", + "UnexpectedEof", + "Unknown", + "WouldBlock", + "WriteZero", +]); + +// The Effect platform services, and the CLI's stdin reader, that raise PlatformError for local I/O. +const platformErrorModules: ReadonlySet = new Set([ + "ChildProcess", + "Crypto", + "FileSystem", + "Path", + "Stdin", + "Stdio", + "Terminal", +]); + +/** + * Names a PlatformError's closed reason and the platform module and method that raised it. The + * method is an identifier those modules pass as a literal, never a caller value. + */ +function platformErrorOperation(error: Error): string | undefined { + const reason = ownValue(error, "reason"); + if (!isErrorRecord(reason)) return undefined; + const tag = ownValue(reason, "_tag"); + const module = ownValue(reason, "module"); + const method = ownValue(reason, "method"); + if (typeof tag !== "string" || !platformErrorReasons.has(tag)) return undefined; + if (typeof module !== "string" || !platformErrorModules.has(module)) return undefined; + const name = typeof method === "string" ? safeIdentifier(method) : undefined; + return name === undefined ? undefined : `${tag}:${module}.${name}`; +} + /** * Names an unclassified defect by its class's prototype tag and a known system error code, or by its type - * for a non-error value. Instance fields other than an errno code are never read, so only - * source-owned identifiers reach telemetry. + * for a non-error value. A PlatformError adds its closed reason and known raising operation. No + * other instance field is read, so only source-owned identifiers reach telemetry. */ function defectIdentity(defect: unknown): string { try { @@ -1178,8 +1224,11 @@ function defectIdentity(defect: unknown): string { if (!isErrorRecord(error)) return typeof error; if (!(error instanceof Error)) return "object"; const base = readStableTaggedPrototypeName(error, true) ?? "Error"; - // A data descriptor, so reading the code never runs a getter. - const code = Object.getOwnPropertyDescriptor(error, "code")?.value; + if (base === "PlatformError") { + const operation = platformErrorOperation(error); + if (operation !== undefined) return `${base}:${operation}`; + } + const code = ownValue(error, "code"); return typeof code === "string" && defectSystemErrorCodes.has(code) ? `${base}:${code}` : base; } catch { return "Unreadable"; diff --git a/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts b/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts index 6c9fa5b4eb..246b9baaa0 100644 --- a/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts +++ b/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts @@ -1,4 +1,4 @@ -import { Cause, Data } from "effect"; +import { Cause, Data, PlatformError } from "effect"; import { CliError } from "effect/cli"; import { describe, expect, it } from "vitest"; import { SupabaseApiInputError, markSupabaseApiInputErrorAsUserInput } from "@supabase/api/effect"; @@ -297,7 +297,17 @@ describe("classifyCliCauseActionability", () => { }); }); - it("names an unclassified defect by its source-owned tag and errno code, never instance data", () => { + const platformDefectWith = (field: "_tag" | "module" | "method", value: string) => { + const reason = new PlatformError.SystemError({ + _tag: "Busy", + module: "FileSystem", + method: "readFile", + }); + Object.defineProperty(reason, field, { value }); + return new PlatformError.PlatformError(reason); + }; + + it("names an unclassified defect by its source-owned tag, errno code, or platform operation, never instance data", () => { const secret = "privateToken"; class NamespaceDefect extends Data.TaggedError("Namespace.NamespaceError")<{ readonly message: string; @@ -323,6 +333,27 @@ describe("classifyCliCauseActionability", () => { "error:Defect:Error", ], [secret, "error:Defect:string"], + [ + new PlatformError.PlatformError( + new PlatformError.SystemError({ + _tag: "Busy", + module: "FileSystem", + method: "readFile", + pathOrDescriptor: secret, + description: secret, + }), + ), + "error:Defect:PlatformError:Busy:FileSystem.readFile", + ], + [ + new PlatformError.PlatformError( + new PlatformError.SystemError({ _tag: "WouldBlock", module: "Stdin", method: "read" }), + ), + "error:Defect:PlatformError:WouldBlock:Stdin.read", + ], + [platformDefectWith("_tag", secret), "error:Defect:PlatformError"], + [platformDefectWith("module", secret), "error:Defect:PlatformError"], + [platformDefectWith("method", "a b"), "error:Defect:PlatformError"], ]; for (const [defect, fingerprint] of cases) { diff --git a/apps/cli/tests/e2e-sequencer.ts b/apps/cli/tests/e2e-sequencer.ts index 576b9e16bd..0ce50c1f43 100644 --- a/apps/cli/tests/e2e-sequencer.ts +++ b/apps/cli/tests/e2e-sequencer.ts @@ -4,6 +4,7 @@ import { BaseSequencer, type TestSpecification } from "vitest/node"; /** Bun `--timings` file of measured e2e file durations, refreshed from CI artifacts. */ const TIMINGS_FILE = "tests/e2e-timings.json"; +const E2E_PROJECTS = new Set(["e2e", "e2e-exclusive"]); /** Duration assumed for every file while no timing has been measured yet. */ export const DEFAULT_DURATION_MS = 30_000; @@ -92,13 +93,13 @@ export function readTimings(root: string): Readonly> | un } /** - * Shards the e2e project by measured duration. Other projects, and the e2e project while + * Shards the e2e projects by measured duration. Other projects, and the e2e projects while * the timings file is absent, keep Vitest's hash split. */ export class E2eSequencer extends BaseSequencer { override async shard(files: TestSpecification[]): Promise { const { root, shard } = this.ctx.config; - const timings = files.every((spec) => spec.project.name === "e2e") + const timings = files.every((spec) => E2E_PROJECTS.has(spec.project.name)) ? readTimings(root) : undefined; if (shard === undefined || timings === undefined) { diff --git a/apps/cli/tests/e2e-timings.json b/apps/cli/tests/e2e-timings.json index 0594a390ef..620d433cef 100644 --- a/apps/cli/tests/e2e-timings.json +++ b/apps/cli/tests/e2e-timings.json @@ -17,7 +17,8 @@ "src/commands/db/push/push.e2e.test.ts": 543, "src/commands/db/reset/reset.e2e.test.ts": 229, "src/commands/db/reset/reset.stack.e2e.test.ts": 9406, - "src/commands/db/schema/declarative/sync/sync.e2e.test.ts": 134676, + "src/commands/db/schema/declarative/sync/sync.e2e.test.ts": 57895, + "src/commands/db/schema/declarative/sync/sync.jobs.e2e.test.ts": 66192, "src/commands/db/start/start.e2e.test.ts": 17188, "src/commands/db/start/start.roles.e2e.test.ts": 13597, "src/commands/encryption/encryption.e2e.test.ts": 412, diff --git a/apps/cli/tests/helpers/cli.ts b/apps/cli/tests/helpers/cli.ts index 9b2d6b9931..e76a2e37df 100644 --- a/apps/cli/tests/helpers/cli.ts +++ b/apps/cli/tests/helpers/cli.ts @@ -173,7 +173,23 @@ export const withTempHome = ( }), ); -function pickFreePort(): Promise { +/** + * Concurrent e2e workers each keep only ports in their own slot, so a port one worker picked and + * released before Docker binds it is never handed to another worker. Slots skip the low bit + * because Linux hands out mostly odd ports for `listen(0)`. + */ +const e2eWorkers = Number(process.env.SUPABASE_E2E_WORKERS || "1"); +const e2eWorkerSlot = Number(process.env.VITEST_POOL_ID || "1") % e2eWorkers; + +async function pickFreePort(): Promise { + for (let attempt = 0; attempt < 1000; attempt++) { + const port = await pickReleasedPort(); + if (Math.floor(port / 2) % e2eWorkers === e2eWorkerSlot) return port; + } + throw new Error(`No free port for e2e worker slot ${e2eWorkerSlot} of ${e2eWorkers}`); +} + +function pickReleasedPort(): Promise { return new Promise((resolve, reject) => { const server = createServer(); server.listen(0, "127.0.0.1", () => { @@ -276,6 +292,14 @@ export async function makeTempCliStackProject( return stackProject; } +/** Concurrent e2e files each run a stack, and Docker names its containers after the project id. */ +const projectIdFor = (dir: string) => + `e2e-${path + .basename(dir) + .toLowerCase() + .replace(/[^a-z0-9]/g, "") + .slice(-8)}`; + export async function makeTempStackProject(prefix = "supabase-stack-e2e-") { const project = await makeTempProject(prefix); const ports = { @@ -304,7 +328,7 @@ export async function makeTempStackProject(prefix = "supabase-stack-e2e-") { await writeFile( path.join(supabaseDir, "config.toml"), [ - 'project_id = "e2e"', + `project_id = "${projectIdFor(project.dir)}"`, "", "[api]", `port = ${ports.apiPort}`, diff --git a/apps/cli/tests/helpers/declarative-sync-e2e.ts b/apps/cli/tests/helpers/declarative-sync-e2e.ts new file mode 100644 index 0000000000..bc09358ff2 --- /dev/null +++ b/apps/cli/tests/helpers/declarative-sync-e2e.ts @@ -0,0 +1,181 @@ +import { Data, Effect, FileSystem, Path } from "effect"; + +import { + makeTempCliStackProject, + overrideStackPorts, + requireCliSuccess, + runSupabaseEffect, +} from "./cli.ts"; + +const CLI_COMMAND_TIMEOUT_MS = 60_000; +const STACK_START_TIMEOUT_MS = 280_000; +const CLEANUP_TIMEOUT_MS = 120_000; +const LIFECYCLE_MARGIN_MS = 30_000; +const CLEANUP_HOOK_TIMEOUT_MS = CLEANUP_TIMEOUT_MS + LIFECYCLE_MARGIN_MS; +const SCENARIO_COMMAND_TIMEOUT_MS = 280_000; +const BEFORE_ALL_TIMEOUT_MS = CLI_COMMAND_TIMEOUT_MS + STACK_START_TIMEOUT_MS + LIFECYCLE_MARGIN_MS; +const SCENARIO_TIMEOUT_MS = 900_000; +export const TEST_TIMEOUT_MS = + BEFORE_ALL_TIMEOUT_MS + SCENARIO_TIMEOUT_MS + CLEANUP_HOOK_TIMEOUT_MS; + +class SyncE2eSetupError extends Data.TaggedError("SyncE2eSetupError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const initialDesiredSchema = `create type public.account_state as enum ('pending', 'active'); + +create table public.disposable_note ( + id bigint generated by default as identity primary key, + body text not null +); + +create view public.auth_user_emails as +select id, email +from auth.users; +`; + +export function commandFailure(result: { stdout: string; stderr: string }): string { + return `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`; +} + +export const migrationFiles = (projectDir: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const migrationsDir = path.join(projectDir, "supabase", "migrations"); + return (yield* fs.exists(migrationsDir)) + ? (yield* fs.readDirectory(migrationsDir)).filter((file) => file.endsWith(".sql")).sort() + : []; + }); + +export const withTempStackProject = ( + use: (project: Awaited>) => Effect.Effect, +): Effect.Effect => + Effect.acquireUseRelease( + Effect.tryPromise({ + try: () => makeTempCliStackProject("sb-pgdelta-next-e2e-"), + catch: (cause) => + new SyncE2eSetupError({ message: "temp stack project setup failed", cause }), + }), + use, + (project) => + Effect.tryPromise({ + try: () => project.cleanup(), + catch: (cause) => + new SyncE2eSetupError({ message: "temp stack project cleanup failed", cause }), + }).pipe(Effect.ignore), + ); + +/** Scenario commands share one home with the shipped shadow cache on, so only the first one provisions a cold shadow. */ +export const scenarioCommandOptions = (projectDir: string) => ({ + cwd: projectDir, + exitTimeoutMs: SCENARIO_COMMAND_TIMEOUT_MS, + home: `${projectDir}/.supabase-home`, + env: { SUPABASE_SHADOW_CACHE: undefined }, +}); + +export const setUpProject = (projectDir: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(path.join(projectDir, ".supabase-home")); + const init = yield* runSupabaseEffect(["init"], { + cwd: projectDir, + exitTimeoutMs: CLI_COMMAND_TIMEOUT_MS, + }); + requireCliSuccess(init, "init setup"); + yield* Effect.tryPromise({ + try: () => overrideStackPorts(projectDir), + catch: (cause) => + new SyncE2eSetupError({ message: `failed to override ports in ${projectDir}`, cause }), + }); + + const configPath = path.join(projectDir, "supabase", "config.toml"); + const config = yield* fs.readFileString(configPath); + if (!config.includes("[experimental.pgdelta]\nenabled = true")) { + return yield* new SyncE2eSetupError({ + message: "init setup did not enable experimental pg-delta in config.toml", + }); + } + yield* fs.writeFileString( + configPath, + config.replace( + '# declarative_schema_path = "./schemas"', + 'declarative_schema_path = "./schemas"', + ), + ); + + const schemasDir = path.join(projectDir, "supabase", "schemas"); + yield* fs.makeDirectory(schemasDir, { recursive: true }); + yield* fs.writeFileString(path.join(schemasDir, "public.sql"), initialDesiredSchema); + const extensionsDir = path.join(schemasDir, "cluster", "extensions"); + yield* fs.makeDirectory(extensionsDir, { recursive: true }); + for (const extension of ["pg_net", "pgcrypto", "uuid-ossp"]) { + yield* fs.writeFileString( + path.join(extensionsDir, `${extension}.sql`), + `CREATE EXTENSION IF NOT EXISTS "${extension}" WITH SCHEMA "extensions";\n`, + ); + } + // Both install into a fixed schema, so no `WITH SCHEMA` clause. + for (const extension of ["pg_cron", "pgmq"]) { + yield* fs.writeFileString( + path.join(extensionsDir, `${extension}.sql`), + `CREATE EXTENSION IF NOT EXISTS "${extension}";\n`, + ); + } + + const start = yield* runSupabaseEffect( + [ + "start", + "--exclude", + "studio", + "--exclude", + "logflare", + "--exclude", + "vector", + "--exclude", + "gotrue", + "--exclude", + "realtime", + "--exclude", + "storage-api", + ], + { cwd: projectDir, exitTimeoutMs: STACK_START_TIMEOUT_MS }, + ); + requireCliSuccess(start, "start setup"); + }); + +/** Writes the initial declarative migration, resets onto it, and requires a converged plan. */ +export const convergeInitialSchema = (projectDir: string) => + Effect.gen(function* () { + const sync = yield* runSupabaseEffect( + [ + "db", + "schema", + "declarative", + "sync", + "--no-apply", + "--name", + "initial_declarative", + "--experimental", + ], + scenarioCommandOptions(projectDir), + ); + requireCliSuccess(sync, "initial declarative sync"); + const reset = yield* runSupabaseEffect( + ["db", "reset", "--local", "--no-seed"], + scenarioCommandOptions(projectDir), + ); + requireCliSuccess(reset, "db reset setup"); + const converged = yield* runSupabaseEffect( + ["db", "schema", "declarative", "sync", "--no-apply", "--experimental"], + scenarioCommandOptions(projectDir), + ); + requireCliSuccess(converged, "converged declarative sync"); + if (!`${converged.stdout}${converged.stderr}`.includes("No schema changes found")) { + return yield* new SyncE2eSetupError({ + message: `initial schema did not converge:\n${commandFailure(converged)}`, + }); + } + }); diff --git a/apps/cli/vitest.config.ts b/apps/cli/vitest.config.ts index 1c20786369..531aefc51c 100644 --- a/apps/cli/vitest.config.ts +++ b/apps/cli/vitest.config.ts @@ -1,8 +1,32 @@ import { readFileSync } from "node:fs"; import { defaultClientConditions, defaultServerConditions } from "vite"; -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; import { E2eSequencer } from "./tests/e2e-sequencer.ts"; +/** E2e files that run concurrently; each stack project picks free ports and its own project id. */ +const e2eWorkers = Number(process.env.SUPABASE_E2E_WORKERS || "1"); +if (!Number.isSafeInteger(e2eWorkers) || e2eWorkers < 1) { + throw new Error( + `SUPABASE_E2E_WORKERS must be a positive integer, got "${process.env.SUPABASE_E2E_WORKERS}"`, + ); +} + +/** + * E2e files that run alone: `start.health` tags a broken image under the tag every stack's + * Mailpit resolves to, and the rest provision pg-delta shadow databases on host ports the CLI + * picks and releases before Docker binds them, which a concurrent stack could take. + */ +const E2E_EXCLUSIVE_FILES = [ + "src/commands/start/start.health.e2e.test.ts", + "src/command-internal/db-bootstrap/shadow-cache.e2e.test.ts", + "src/commands/db/diff/diff.e2e.test.ts", + "src/commands/db/diff/diff.declarative.e2e.test.ts", + "src/commands/db/pull/pull.e2e.test.ts", + "src/commands/db/schema/declarative/sync/sync.e2e.test.ts", + "src/commands/db/schema/declarative/sync/sync.jobs.e2e.test.ts", + "src/commands/db/start/start.roles.e2e.test.ts", +]; + function dockerfileTextPlugin() { return { name: "dockerfile-text-loader", @@ -75,8 +99,23 @@ export default defineConfig({ test: { name: "e2e", include: ["**/*.e2e.test.ts"], + exclude: [...configDefaults.exclude, ...E2E_EXCLUSIVE_FILES], + fileParallelism: e2eWorkers > 1, + maxWorkers: e2eWorkers, + setupFiles: ["tests/e2e-setup.ts"], + globalSetup: ["../../packages/stack/tests/docker-volume-run.ts"], + testTimeout: 120_000, + hookTimeout: 120_000, + }, + }, + { + test: { + // Runs after the e2e project, one file at a time. + name: "e2e-exclusive", + include: E2E_EXCLUSIVE_FILES, fileParallelism: false, maxWorkers: 1, + sequence: { groupOrder: 2 }, setupFiles: ["tests/e2e-setup.ts"], globalSetup: ["../../packages/stack/tests/docker-volume-run.ts"], testTimeout: 120_000, diff --git a/docs/adr/0030-go-cli-removal.md b/docs/adr/0030-go-cli-removal.md deleted file mode 100644 index 45bcf3cbf8..0000000000 --- a/docs/adr/0030-go-cli-removal.md +++ /dev/null @@ -1,132 +0,0 @@ -# 0030. Go CLI Removal - -**Status**: accepted -**Date**: 2026-09-14 - -## Problem Statement - -ADR 0016 described a residual Go delegation surface in `src/legacy/`: a handful of commands still -proxying to the Go binary (`GoProxy`) while the rest of the CLI had already been natively ported to -TypeScript. The remaining deprecated surfaces were removed rather than ported (CLI-2432); each now -exits 1 with replacement guidance: - -- `db remote changes` → `db diff --linked` -- `gen keys` → `projects api-keys` -- `db diff --use-pg-schema` → the default engine or `--use-pg-delta` -- `functions download --legacy-bundle` → `--use-api` -- `db branch create|delete|list|switch` has no local replacement (hosted preview branches via - `supabase branches` are a different feature) - -No command in the TypeScript CLI spawns `supabase-go` anymore. - -That leaves `apps/cli-go/` — roughly 300 tracked files across three Go modules (the root CLI -module, the `fsevents` helper module, and the separately-tagged `pkg` module) — serving nothing. -It still carries its own Go toolchain pin, CI (test/coverage/lint/CodeQL), Dependabot ecosystem, -release packaging (a second binary per platform, signed and archived alongside the TS binary), and -documentation, all to build and ship a binary nothing calls. - -## Decision - -Delete `apps/cli-go/` entirely. The CLI ships as a single compiled binary per platform -(`packages/cli-/bin/supabase`), built by `apps/cli/scripts/build.ts` with no Go -compilation step. - -The one file the Go tree owned that TypeScript still needed — the service-image Dockerfile -manifest parsed by `dockerfileServiceImages` — was already relocated into the TS tree -(`apps/cli/src/shared/services/Dockerfile`) by a prep commit on this branch before the tree was -deleted, so no runtime behavior depends on `apps/cli-go/` by the time it's removed. - -The tree's own `pkg` module (`github.com/supabase/cli/pkg`, a separately-tagged, externally -published Go module) already had a structurally broken publish path before this ADR, not merely -one this ADR chooses not to continue. At `pkg/v1.2.3` (2026-04-16, `ae7642d508`) the module lived -at repo-root `pkg/`, matching Go's subdirectory-module convention that a `pkg/vX.Y.Z` tag resolves -by finding `pkg/go.mod` at the repository root. `cad095718` ("chore(monorepo): move CLI sources -under apps/cli-go/", 2026-05-06) relocated it to `apps/cli-go/pkg/` while keeping the same module -path, so from that point any new `pkg/vX` tag would have been unresolvable — the module path no -longer matched where `go.mod` actually lived. `pkg/v1.2.3` is therefore the last resolvable -version, and was already roughly five months stale relative to `apps/cli-go/pkg`'s actual content -before this deletion. Deleting the tree makes that unresolvability permanent rather than causing -it. The durability guarantee for existing importers rests on two things: `refs/tags/pkg/v*` is -protected against deletion, creation, and force-push by a repository ruleset ("Protect pkg/v* -release tags", id 23327496), and on top of that the Go module proxy (`proxy.golang.org`, the -default `GOPROXY`) caches published module zips independently of this repository's current tree, -so an importer already pinned to a version keeps resolving it through the proxy even without -direct access to the tag. - -## Consequences - -### Positive - -- The repo ships zero Go: no toolchain pin, no CI surface, no packaging step, no documentation - drift between two implementations of the same CLI. -- Release archives, install scripts, and the Homebrew formula carry one binary instead of two, - removing an entire class of "sidecar present but stale/missing" failure mode. -- `cli-go-ci.yml`'s test/coverage/lint/codegen jobs, the per-PR Go module dependency cache in the - shared setup action, and Dependabot's `gomod` ecosystem entry are all gone — but these jobs were - path-filtered to `apps/cli-go/**` and so ran only on PRs that touched the Go tree already; most - ordinary PRs never paid this cost. CodeQL's shape changes rather than shrinking uniformly: its - `go` language matrix entry is gone entirely, including from the `merge_group` trigger it used to - run on, while the remaining `javascript-typescript` entry now runs on every `pull_request` push - — coverage it effectively never had before, since `merge_group` was CodeQL's only unfiltered - trigger — bounded by a new concurrency group so pushes to the same PR no longer queue up - uncancelled analysis runs. - -### Negative - -- No further `pkg` module releases. External importers pinned to existing tags are unaffected, but - anyone needing content newer than `pkg/v1.2.3` has no upgrade path within this module. Two - external importers are known from a code-search during planning: `supabase/terraform-provider-supabase` - (same org, pins `pkg v1.2.3`) and `turbot/steampipe-plugin-supabase` (third-party); both import - only `pkg/api`. Because `pkg/api` is pure `oapi-codegen` output, the concrete unblock for both is - to run `oapi-codegen` in the consuming repo against the Management API OpenAPI document — - the same one `packages/api` already consumes — and drop the dependency on - `github.com/supabase/cli/pkg` entirely. One caveat: the deleted `apps/cli-go/api/README.md` - documented the Go snapshot as generated from **staging** (`https://api.supabase.green/api/v1-yaml`), - while `packages/api` generates from **production** (`https://api.supabase.com`, see - `packages/api/scripts/download-openapi.ts`), so a consumer following this unblock gets a - different — production, not staging — surface than `pkg/api` had, which is worth knowing now - that the source doc explaining that distinction is gone. Beyond these two, this repository has - no visibility into other importers beyond the module proxy's own download counts, which aren't - queried here. -- Go-specific CI ends entirely: Coveralls coverage upload (Go-side) had no TypeScript-side - equivalent and simply stops with this PR — there is no CI-enforced coverage threshold for the - TypeScript CLI today. This is a known gap, not something this PR fixes; it is a candidate for a - follow-up if coverage enforcement is wanted. -- golangci-lint and the `go` CodeQL analysis both end, along with any lint/security coverage they - provided that had no TypeScript-side equivalent (e.g. Go-specific vulnerability classes CodeQL's - `javascript-typescript` matrix entry doesn't check for). -- The four `*.go-payload.ts` output-encoding specs (which drive `-o yaml`/`-o toml` byte-compatible - output) become frozen, hand-maintained contracts with no live Go source left to diff against for - drift. A prior commit on this branch already retargeted their drift test at the OpenAPI schema - instead of Go source, so this is a note about the ongoing nature of that contract, not new work. - `7b469f5b3` (2026-08-12, the CLI-1970 pin) is the last commit with the full pre-shrink Go source - tree intact and remains the closest available provenance reference for that contract's origin, - but it does not reflect `apps/cli-go/pkg/api`'s final state: that package kept regenerating via - the API-sync workflow for roughly another month after that pin, until this PR's tree deletion. - -## Alternatives Considered - -1. **Keep `pkg/` alone as a standalone Go module in this repo.** Rejected — this reintroduces the - entire Go toolchain, CI, and Dependabot surface this PR removes, to serve at most a couple of - external consumers of a single module. -2. **Extract `pkg/` to its own separate repository before deleting it here.** Not chosen for this - PR. Because the publish path was already structurally broken since the May 2026 monorepo move - (see Decision), this would be a restart from the last resolvable state (`pkg/v1.2.3`), not a - continuation of an active release cadence. The module shipped 14 packages (`api`, `cast`, - `config`, `diff`, `fetcher`, `function`, `migration`, `parser`, `pgtest`, `pgxv5`, `queue`, - `storage`, `vault`, plus the root module); "internal" audience is only evidenced for the two - known importers named above, both of which use only `pkg/api` — not for the module as a whole. - -## Related Decisions - -- Supersedes [ADR 0016](0016-legacy-port-completion-and-go-cli-authority-scope.md), whose - residual authority scope no longer applies now that `apps/cli-go/` is gone. -- [ADR 0011](0011-cli-release-and-distribution-strategy.md) and - [ADR 0014](0014-macos-code-signing-and-notarization.md) remain historically accurate records of - decisions made while the Go CLI still existed; they are not updated by this ADR. -- Linear: CLI-2432. - -## See Also - -- [`apps/cli/docs/binary-distribution.md`](../../apps/cli/docs/binary-distribution.md) -- [`apps/cli/docs/release-process.md`](../../apps/cli/docs/release-process.md) diff --git a/docs/adr/0031-config-value-precedence.md b/docs/adr/0032-config-value-precedence.md similarity index 99% rename from docs/adr/0031-config-value-precedence.md rename to docs/adr/0032-config-value-precedence.md index 95c2cd3068..1914fac638 100644 --- a/docs/adr/0031-config-value-precedence.md +++ b/docs/adr/0032-config-value-precedence.md @@ -1,4 +1,4 @@ -# 0031. Config value precedence +# 0032. Config value precedence **Status**: proposed **Date**: 2026-10-08 diff --git a/docs/adr/README.md b/docs/adr/README.md index bbac0486b2..532f634d5f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -73,6 +73,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0029 | [Native Postgres Privilege Step-Down](0029-native-postgres-privilege-step-down.md) | proposed | | 0030 | [Hotfix Releases](0030-hotfix-releases.md) | accepted | | 0031 | [Go CLI Removal](0031-go-cli-removal.md) | accepted | +| 0032 | [Config Value Precedence](0032-config-value-precedence.md) | proposed | ## Template @@ -118,4 +119,3 @@ Why did we choose this approach? ``` > **Note**: Additional sections (Implementation Notes, Open Questions, Verification Checklist) may be added as needed. -> | 0031 | [Config Value Precedence](0031-config-value-precedence.md) | proposed | diff --git a/packages/config/docs/cli-config-loading.md b/packages/config/docs/cli-config-loading.md index d842344919..403cbf1906 100644 --- a/packages/config/docs/cli-config-loading.md +++ b/packages/config/docs/cli-config-loading.md @@ -362,7 +362,7 @@ distinct from the `CliConfig` document. ## CLI value precedence The CLI resolves every config value, whether from `CliConfig` or from a `SUPABASE_*` variable, with -one order (ADR 0031): +one order (ADR 0032): 1. an explicit flag 2. shell environment diff --git a/packages/stack/package.json b/packages/stack/package.json index a8ba06429f..95313fb0f9 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -18,6 +18,7 @@ }, "scripts": { "generate": "bun run scripts/generate-functions-bootstrap.ts", + "prepare:test-artifacts": "bun run scripts/prepare-test-artifacts.ts", "test": "pnpm run test:unit && pnpm run test:integration", "test:unit": "pnpm exec turbo run @supabase/stack#test:unit:run --", "test:unit:run": "bun --bun vitest run --project unit", diff --git a/packages/stack/scripts/prepare-test-artifacts.ts b/packages/stack/scripts/prepare-test-artifacts.ts new file mode 100644 index 0000000000..fd182c98f4 --- /dev/null +++ b/packages/stack/scripts/prepare-test-artifacts.ts @@ -0,0 +1,26 @@ +import { NodeHttpClient, NodeRuntime, NodeServices } from "@effect/platform-node"; +import { Console, Effect, Layer } from "effect"; +import { artifactServiceKinds, catalogPins, prepareNativeArtifact } from "../src/Artifacts.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; + +/** + * Downloads every default native artifact into the shared test cache, as CI fills its disk. + * With `--images`, prints every catalog container image instead so CI can pull them. + */ +const prepareNative = Effect.forEach( + [...artifactServiceKinds(), "vector" as const], + (service) => + prepareNativeArtifact({ service }, testArtifactCacheRoot).pipe( + Effect.tap((artifact) => Console.log(`${service} ${artifact.version}`)), + ), + { concurrency: "unbounded", discard: true }, +).pipe(Effect.withSpan("Stack.prepareNativeArtifacts")); + +const printImages = Effect.forEach(catalogPins(), ({ pin }) => Console.log(pin.image), { + discard: true, +}); + +(process.argv.includes("--images") ? printImages : prepareNative).pipe( + Effect.provide(Layer.mergeAll(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + NodeRuntime.runMain, +); diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 184195ea71..3ad3e92387 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -267,23 +267,23 @@ const definitions: Readonly> = { storage: definition( "storage", { - upstreamVersion: "v1.82.0", + upstreamVersion: "v1.82.1", revision: 0, image: - "ghcr.io/supabase/cli/storage:v1.82.0-r0@sha256:c71adb01633ce0147162f53fd5d4fa861fcf2707586707f6d88acfaa8db2eeb4", - upstreamImage: "supabase/storage-api:v1.82.0", + "ghcr.io/supabase/cli/storage:v1.82.1-r0@sha256:68c0f36557430219eb318df22ee817915992f2d5c976971e32574c3fe429be3c", + upstreamImage: "supabase/storage-api:v1.82.1", natives: { "darwin-arm64": { - archive: "e225cd6ce14cd8ee1ede02c8e3b14e366ae2342920d904b6bd30727cbc1fd6ab", - manifest: "b2e126721482284b60cb32d82b530e17e9603cbe95cbfcc35dcb7fcedad0696d", + archive: "69a3308f1633db400bc718ee363aaa3d0d8ae846f52af4ea3a61d4cf04064e69", + manifest: "37ac0f369085e88952fde5c7493e2291d9e1034ade6c763509900e4cafa7d978", }, "linux-amd64": { - archive: "de14d47658694a3d084d26bd12c0d2ec8ff30a7abbc1dcdcbc7fa1b098010360", - manifest: "5de5ef546a2f579b6c2dd025eaf9052013849218db80449b87feda23b8d8db59", + archive: "f8716410f5997cf2a89cd3fe13565fe21b6bb239ef0be9818be385c0b397afba", + manifest: "ca82ac18ccccd91ec3370d65497c7f17c938f8601d7982a5fb48893ad5c1b159", }, "linux-arm64": { - archive: "3e436461a05a18579da27b3734df0c367195731046c54044a0f81e442dccec39", - manifest: "a9eed25be05723dc34a2f7240cf8127c3d3c85fbfb82fae903f470122debf2fa", + archive: "45e623ac2913dc0aa2e11c3b58ea437227c51bc8e55a5184a2736fb1c13ab49e", + manifest: "024d11f58d90d54ebe62f55b88f75d5b0e541f1612211f5bf6a351472c6cb466", }, }, }, diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index d0cb69bdfa..71ea4f4d3a 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -598,12 +598,18 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => ).pipe(Effect.provide(NodeServices.layer)), ); +/** Ports Windows runners listen on: RDP, WinRM, Delivery Optimization and Windows services. */ +const windowsHeldPorts = new Set([3389, 5040, 5357, 5985, 5986, 7680]); + /** * A pinned port below the stack's native and auto port ranges and every OS ephemeral range, so * neither another test's auto allocation nor an outbound socket can already hold it. It stays * under 9000, where other files pin fixed ports. */ -const pinnedPort = () => 1024 + (Math.abs(Hash.string(randomUUID())) % (9_000 - 1024)); +const pinnedPort = (): number => { + const port = 1024 + (Math.abs(Hash.string(randomUUID())) % (9_000 - 1024)); + return windowsHeldPorts.has(port) ? pinnedPort() : port; +}; it.live("addresses a pinned endpoint from its configured port without a registry row", () => Effect.scoped(