test: split SDK suites into a mocked unit tier and an opt-in e2e tier

.changeset/plenty-pugs-organize.md

@@ -0,0 +1,7 @@
+---
+'@e2b/python-sdk': patch
+'@e2b/cli': patch
+'e2b': patch
+---
+
+Split the test suites into a fully mocked default tier and an opt-in `E2B_E2E=1` end-to-end tier (tests only, no runtime changes)

.github/workflows/cli_tests.yml

@@ -7,6 +7,11 @@ on:
         required: false
         type: string
         default: ''
+      e2e:
+        description: 'Run the e2e tier (drives the CLI against real sandboxes) instead of the mocked unit tier'
+        required: false
+        type: boolean
+        default: false
     secrets:
       E2B_API_KEY:
         required: true
@@ -64,9 +69,20 @@ jobs:
         run: pnpm build
         working-directory: ./packages/cli
 
+      # The default tier: fully mocked, no sandboxes, no credentials needed.
       - name: Run tests
+        if: ${{ !inputs.e2e }}
         run: pnpm test
         working-directory: ./packages/cli
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
           E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
+
+      # The opt-in tier: drives the built CLI against real sandboxes.
+      - name: Run e2e tests
+        if: ${{ inputs.e2e }}
+        run: pnpm test:e2e
+        working-directory: ./packages/cli
+        env:
+          E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+          E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}

.github/workflows/js_sdk_tests.yml

@@ -12,6 +12,11 @@ on:
         required: false
         type: boolean
         default: false
+      e2e:
+        description: 'Run the e2e tier (provisions sandboxes and builds templates) instead of the mocked unit tier'
+        required: false
+        type: boolean
+        default: false
     secrets:
       E2B_API_KEY:
         required: true
@@ -83,62 +88,74 @@ jobs:
           pnpm install --frozen-lockfile
 
       # Only the Node runtime runs the vitest `browser` project, which drives
-      # Chromium through Playwright.
+      # Chromium through Playwright — and that project is e2e (it provisions a
+      # sandbox from a browser bundle).
       - name: Get Playwright version
-        if: matrix.runtime == 'node'
+        if: matrix.runtime == 'node' && inputs.e2e
         id: playwright-version
         run: echo "version=$(node -p "require('playwright/package.json').version")" >> "$GITHUB_OUTPUT"
 
       - name: Cache Playwright browsers
-        if: matrix.runtime == 'node'
+        if: matrix.runtime == 'node' && inputs.e2e
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ${{ matrix.os == 'windows-latest' && '~/AppData/Local/ms-playwright' || '~/.cache/ms-playwright' }}
           key: playwright-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
 
       - name: Install Playwright Chromium
-        if: matrix.runtime == 'node'
+        if: matrix.runtime == 'node' && inputs.e2e
         run: pnpm run playwright:install
 
       # The unit bundle test and the Cloudflare deploy config fail in CI when
       # the build output is missing.
       - name: Test build
         run: pnpm build
 
+      # The default tier: fully mocked, no sandboxes, no credentials needed.
       - name: Run Node tests
-        if: matrix.runtime == 'node'
+        if: matrix.runtime == 'node' && !inputs.e2e
         run: pnpm test
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
           E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
 
+      # The opt-in tier: real sandboxes, envd round-trips and template builds.
+      - name: Run Node e2e tests
+        if: matrix.runtime == 'node' && inputs.e2e
+        run: |
+          pnpm test:e2e
+          pnpm test:browser
+        env:
+          E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+          E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
+
       - name: Install Bun
-        if: matrix.runtime == 'bun'
+        if: matrix.runtime == 'bun' && !inputs.e2e
         uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
 
       - name: Run test suite under Bun
-        if: matrix.runtime == 'bun'
+        if: matrix.runtime == 'bun' && !inputs.e2e
         run: pnpm test:bun
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
           E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
 
       - name: Install Deno
-        if: matrix.runtime == 'deno'
+        if: matrix.runtime == 'deno' && !inputs.e2e
         uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5
         with:
           deno-version: v${{ env.TOOL_VERSION_DENO }}
 
       - name: Run test suite under Deno
-        if: matrix.runtime == 'deno'
+        if: matrix.runtime == 'deno' && !inputs.e2e
         run: pnpm test:deno
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
           E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
 
       # Full unit + connectionConfig suite inside workerd (vitest-pool-workers).
       - name: Run test suite under Cloudflare workerd
-        if: matrix.runtime == 'cloudflare'
+        if: matrix.runtime == 'cloudflare' && !inputs.e2e
         run: pnpm test:cf
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
@@ -148,7 +165,7 @@ jobs:
       # global setup (wrangler deploy --temporary, no Cloudflare credentials
       # needed) and deletes the worker in teardown.
       - name: Run Cloudflare Workers deploy tests
-        if: matrix.runtime == 'cloudflare-deploy'
+        if: matrix.runtime == 'cloudflare-deploy' && !inputs.e2e
         run: pnpm test:cf:deploy
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}

.github/workflows/python_sdk_tests.yml

@@ -7,6 +7,11 @@ on:
         required: false
         type: string
         default: ''
+      e2e:
+        description: 'Run the e2e tier (provisions sandboxes and builds templates) instead of the mocked unit tier'
+        required: false
+        type: boolean
+        default: false
     secrets:
       E2B_API_KEY:
         required: true
@@ -50,8 +55,18 @@ jobs:
       - name: Test build
         run: uv build
 
+      # The default tier: fully mocked, no sandboxes, no credentials needed.
       - name: Run tests
+        if: ${{ !inputs.e2e }}
         run: uv run pytest --verbose --numprocesses=4
         env:
           E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
           E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}
+
+      # The opt-in tier: real sandboxes, envd round-trips and template builds.
+      - name: Run e2e tests
+        if: ${{ inputs.e2e }}
+        run: uv run pytest -m e2e --verbose --numprocesses=4
+        env:
+          E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+          E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }}

.github/workflows/sdk_e2e_tests.yml

@@ -0,0 +1,56 @@
+name: SDK E2E Tests
+
+# The opt-in tier. These jobs provision sandboxes, talk to envd and build
+# templates against live infrastructure, so they are not part of the required
+# PR checks (see sdk_tests.yml, which runs the fully mocked default tier).
+#
+# Run them manually from the Actions tab, or by adding the `e2e` label to a PR.
+on:
+  workflow_dispatch:
+    inputs:
+      staging:
+        description: 'Run against staging instead of production'
+        required: false
+        type: boolean
+        default: false
+  pull_request:
+    branches:
+      - main
+    types: [opened, synchronize, reopened, labeled]
+
+permissions:
+  contents: read
+
+jobs:
+  js-e2e:
+    name: E2E / JS SDK
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'e2e') }}
+    uses: ./.github/workflows/js_sdk_tests.yml
+    with:
+      e2e: true
+      # The e2e tier only runs under Node — the Bun/Deno/Cloudflare legs cover
+      # the mocked tier in the default workflow.
+      node-only: true
+      E2B_DOMAIN: ${{ inputs.staging && vars.E2B_DOMAIN_STAGING || '' }}
+    secrets:
+      E2B_API_KEY: ${{ inputs.staging && secrets.E2B_API_KEY_STAGING || secrets.E2B_API_KEY }}
+
+  python-e2e:
+    name: E2E / Python SDK
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'e2e') }}
+    uses: ./.github/workflows/python_sdk_tests.yml
+    with:
+      e2e: true
+      E2B_DOMAIN: ${{ inputs.staging && vars.E2B_DOMAIN_STAGING || '' }}
+    secrets:
+      E2B_API_KEY: ${{ inputs.staging && secrets.E2B_API_KEY_STAGING || secrets.E2B_API_KEY }}
+
+  cli-e2e:
+    name: E2E / CLI
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'e2e') }}
+    uses: ./.github/workflows/cli_tests.yml
+    with:
+      e2e: true
+      E2B_DOMAIN: ${{ inputs.staging && vars.E2B_DOMAIN_STAGING || '' }}
+    secrets:
+      E2B_API_KEY: ${{ inputs.staging && secrets.E2B_API_KEY_STAGING || secrets.E2B_API_KEY }}

.github/workflows/sdk_tests.yml

@@ -1,5 +1,8 @@
 name: SDK Tests
 
+# The default tier: fully mocked, deterministic, no sandboxes and no template
+# builds. The behavioral tests that need live infrastructure live in the opt-in
+# sdk_e2e_tests.yml workflow (`e2e` PR label or manual dispatch).
 on:
   pull_request:
     branches:
@@ -76,6 +79,9 @@ jobs:
     secrets:
       E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
 
+  # The staging legs re-run the same mocked tier against the staging domain, so
+  # they only catch environment-specific breakage now; backend compatibility is
+  # verified by the e2e workflow (`workflow_dispatch` with `staging: true`).
   js-tests-staging:
     name: Staging / JS SDK Tests
     needs: changes

CONTRIBUTING.md

@@ -1,2 +1,22 @@
 # Contributing
 If you want to contribute, open a PR, issue, or start a discussion on our [Discord](https://discord.gg/dSBY3ms2Qr).
+
+## Tests
+
+Every package splits its tests into two tiers:
+
+- **unit (default)** — fully mocked, deterministic, no sandboxes and no
+  credentials.
+- **e2e (opt-in)** — real sandboxes, envd round-trips and template builds;
+  requires `E2B_E2E=1` and an API key.
+
+| Package | Unit | E2E |
+| --- | --- | --- |
+| `packages/js-sdk` | `pnpm test` | `pnpm test:e2e`, `pnpm test:browser` |
+| `packages/python-sdk` | `uv run pytest` | `uv run pytest -m e2e` |
+| `packages/cli` | `pnpm test` | `pnpm test:e2e` |
+
+Details, and where a new test belongs, are in each package's
+`tests/README.md`. On CI the `SDK Tests` workflow runs the unit tier for every
+PR; the e2e tier runs in the opt-in `SDK E2E Tests` workflow (add the `e2e`
+label to a PR or dispatch it manually).

packages/cli/package.json

@@ -43,6 +43,7 @@
     "format": "prettier --write src",
     "test:interactive": "pnpm build && ./dist/index.js",
     "test": "vitest run",
+    "test:e2e": "vitest run --config vitest.e2e.config.mts",
     "test:watch": "vitest watch",
     "test:coverage": "vitest run --coverage",
     "check-deps": "knip"

packages/cli/tests/README.md

@@ -0,0 +1,35 @@
+# CLI tests
+
+The suite has two tiers.
+
+## Unit tier (default)
+
+```bash
+pnpm test
+```
+
+Fully mocked (`vi.mock` over `e2b` and the CLI's API modules) or driving the
+built CLI against local input only — deterministic, no sandboxes, no
+credentials. It asserts on argument parsing, validation, output formatting and
+the calls the CLI makes into the SDK.
+
+## E2E tier (opt-in)
+
+```bash
+E2B_API_KEY=e2b_... pnpm test:e2e
+```
+
+Tests that drive the built CLI against a real sandbox (`exec` piping,
+`backend_integration`). They need `E2B_E2E=1` (set by the script above) plus
+credentials, from `E2B_API_KEY` or `~/.e2b/config.json`; without both they are
+skipped.
+
+Use `e2eTest` (or `skipE2E` in a `beforeAll`) from [`setup.ts`](./setup.ts),
+which also resolves the shared `e2eApiKey`/`e2eDomain`. `E2B_DEBUG` is a
+separate axis and disables the e2e tier because it points the CLI at a local
+stack.
+
+## CI
+
+`SDK Tests` runs the unit tier on every PR. The e2e tier runs in the opt-in
+`SDK E2E Tests` workflow — add the `e2e` label to a PR or dispatch it manually.