Add critical user flow test gate

.github/workflows/ci.yml

@@ -44,7 +44,21 @@ jobs:
         run: pnpm build
 
       - name: Install Playwright browsers
-        run: pnpm exec playwright install --with-deps chromium
+        run: pnpm exec playwright install --with-deps chromium webkit
+
+      - name: Run critical user flows
+        run: pnpm exec playwright test --config playwright.critical.config.ts
 
       - name: Run tests
         run: pnpm test
+
+      - name: Upload critical Playwright failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-critical-${{ github.run_attempt }}
+          path: |
+            test-results/critical
+            playwright-report/critical
+          if-no-files-found: ignore
+          retention-days: 7

package.json

@@ -12,6 +12,7 @@
     "test": "pnpm build && vitest run && pnpm exec playwright test",
     "test:smoke": "vitest run",
     "test:e2e": "pnpm exec playwright test",
+    "test:critical": "pnpm build && pnpm exec playwright test --config playwright.critical.config.ts",
     "test:docs-verify": "vitest run --config vitest.docs-verify.config.ts",
     "verify:images": "node scripts/verify-resource-images.mjs",
     "optimize:images": "bash scripts/optimize-images.sh",

playwright.critical.config.ts

@@ -0,0 +1,59 @@
+import { defineConfig, devices } from "@playwright/test";
+
+const PORT = 4173;
+const isCI = Boolean(process.env.CI);
+
+export default defineConfig({
+  testDir: "tests/critical",
+  outputDir: "test-results/critical",
+  fullyParallel: false,
+  forbidOnly: isCI,
+  retries: isCI ? 2 : 0,
+  workers: 1,
+  timeout: 45_000,
+  reporter: isCI
+    ? [
+        ["list"],
+        [
+          "html",
+          {
+            open: "never",
+            outputFolder: "playwright-report/critical",
+          },
+        ],
+      ]
+    : "list",
+  use: {
+    baseURL: `http://localhost:${PORT}`,
+    screenshot: "only-on-failure",
+    trace: "retain-on-failure",
+    video: "retain-on-failure",
+  },
+  projects: [
+    {
+      name: "desktop-chromium",
+      use: {
+        ...devices["Desktop Chrome"],
+        permissions: ["clipboard-read", "clipboard-write"],
+      },
+    },
+    {
+      name: "emulated-pixel-chromium",
+      use: { ...devices["Pixel 7"] },
+    },
+    {
+      name: "desktop-webkit",
+      use: { ...devices["Desktop Safari"] },
+    },
+    {
+      name: "emulated-iphone-webkit",
+      use: { ...devices["iPhone 15"] },
+    },
+  ],
+  webServer: {
+    command: `pnpm exec next start -p ${PORT}`,
+    port: PORT,
+    reuseExistingServer: false,
+    timeout: 60_000,
+  },
+});

tests/critical/README.md

@@ -0,0 +1,47 @@
+# Critical user flow tests
+
+This suite is the release gate for the smallest set of DevHub journeys whose
+failure prevents a developer or coding agent from getting useful content. It
+runs against a local production build. It does not poll the production site,
+send alerts, or run on a schedule.
+
+## Test plan
+
+| ID           | Priority | User flow                                               | Required outcome                                                                                                        |
+| ------------ | -------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| TC-WEB-001   | P0       | Home → Copy agent prompt                                | The real bootstrap API returns a complete prompt and Chromium copies it through the browser Clipboard API.              |
+| TC-WEB-002   | P0       | Global search → Databricks CLI docs → Copy Markdown     | Cmd/Ctrl-K on desktop or the mobile-menu trigger opens search, navigates to the right doc, and exports useful Markdown. |
+| TC-WEB-003   | P0       | Templates → search → SaaS Tracker → agent handoff       | The selected template copies a complete prompt and generates a valid Replit URL.                                        |
+| TC-WEB-004   | P1       | Template copy → simulated clipboard error → retry       | The failure is visible and a second attempt succeeds without reloading.                                                 |
+| TC-AGENT-001 | P0       | Agent requests bootstrap, llms.txt, raw docs, and tools | Machine-readable entry points respond, and the MCP server advertises and executes its required tools.                   |
+
+The four browser journeys run in desktop Chromium, Playwright's emulated Pixel
+7 Chromium profile, desktop WebKit, and Playwright's emulated iPhone 15 WebKit
+profile. Device profiles emulate browser settings such as viewport, user agent,
+touch input, and device scale factor; they are not physical devices or installed
+mobile Safari/Chrome. The browser-independent API contract runs once in desktop
+Chromium.
+
+Successful copy flows use Chromium's browser Clipboard API with explicit
+permissions in the desktop Chromium project. Other browser profiles use a
+deterministic clipboard implementation because headless clipboard permission
+support differs by engine. The clipboard failure and retry flow always uses a
+controlled failure so the error state is reproducible.
+
+## Commands
+
+```bash
+# Production build plus only the release-gating journeys
+pnpm test:critical
+
+# Unit tests and the broad Chromium regression suite
+pnpm test
+
+# The same two release gates CI runs
+pnpm test:critical && pnpm test
+```
+
+On CI, a failed critical journey keeps its screenshot, trace, and video as a
+short-lived workflow artifact. The broader Playwright and Vitest suites remain
+the regression layer; tests should only move into this folder when a failure
+blocks the core developer outcome described above.

tests/critical/critical-user-flow-contracts.ts

@@ -0,0 +1,36 @@
+export const criticalUserFlowContracts = {
+  bootstrapPrompt: {
+    id: "TC-WEB-001",
+    priority: "P0",
+    route: "/",
+    minimumPromptLength: 500,
+    requiredPromptFragments: ["# About DevHub", "Databricks CLI", "llms.txt"],
+  },
+  docsSearchAndExport: {
+    id: "TC-WEB-002",
+    priority: "P0",
+    route: "/docs/tools/databricks-cli",
+    query: "Databricks CLI",
+    minimumMarkdownLength: 500,
+    requiredMarkdownFragments: ["# Databricks CLI"],
+  },
+  templateDiscoveryAndHandoff: {
+    id: "TC-WEB-003",
+    priority: "P0",
+    route: "/templates/saas-tracker",
+    query: "SaaS Subscription Tracker",
+    title: "SaaS Subscription Tracker",
+    minimumPromptLength: 1_000,
+    requiredPromptFragments: ["# About DevHub", "# SaaS Subscription Tracker"],
+  },
+  copyRecovery: {
+    id: "TC-WEB-004",
+    priority: "P1",
+    route: "/templates/saas-tracker",
+  },
+  agentEntryPoints: {
+    id: "TC-AGENT-001",
+    priority: "P0",
+    mcpTools: ["get_doc_resource", "list_docs_resources"],
+  },
+} as const;