From d6e9920fe4fce879086474cbf61c0eaf625ad224 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 14 Oct 2026 14:50:06 -0700 Subject: [PATCH 1/2] fix(docs): preserve negotiated response cache headers --- .github/workflows/docs-negotiation.yml | 60 ++++++++++++++++ apps/docs/package.json | 1 + apps/docs/scripts/test-negotiation-e2e.ts | 87 +++++++++++++++++++++++ bun.lock | 1 + package.json | 3 +- patches/README.md | 36 ++++++++++ patches/next@16.4.0.patch | 75 +++++++++++++++++++ 7 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs-negotiation.yml create mode 100644 apps/docs/scripts/test-negotiation-e2e.ts create mode 100644 patches/next@16.4.0.patch diff --git a/.github/workflows/docs-negotiation.yml b/.github/workflows/docs-negotiation.yml new file mode 100644 index 00000000000..09bc78c76fd --- /dev/null +++ b/.github/workflows/docs-negotiation.yml @@ -0,0 +1,60 @@ +name: docs content negotiation + +on: + pull_request: + paths: + - 'apps/docs/**' + - 'patches/next@*.patch' + - 'package.json' + - 'bun.lock' + - '.github/workflows/docs-negotiation.yml' + push: + branches: [staging] + paths: + - 'apps/docs/**' + - 'patches/next@*.patch' + - 'package.json' + - 'bun.lock' + - '.github/workflows/docs-negotiation.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-negotiation-${{ github.ref }} + cancel-in-progress: true + +jobs: + http: + name: docs HTTP + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'linux-x64-8-core' }} + timeout-minutes: 15 + env: + DATABASE_URL: postgresql://docs_test:docs_test@127.0.0.1:1/docs_test + DOCS_E2E_BASE_URL: http://127.0.0.1:3001 + DOCS_NEGOTIATION_REPORT_PATH: ${{ github.workspace }}/test-results/docs-negotiation.json + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - name: Setup workspace + uses: ./.github/actions/setup + with: + provider: ${{ vars.CI_PROVIDER }} + - name: Build production docs + run: bun run --cwd apps/docs build + - name: Verify HTTP content negotiation + run: | + mkdir -p test-results + bun run --cwd apps/docs start --port 3001 >test-results/docs-server.log 2>&1 & + DOCS_SERVER_PID=$! + trap 'kill "$DOCS_SERVER_PID" 2>/dev/null || true' EXIT + bun run --cwd apps/docs test:negotiation:e2e + - name: Upload HTTP regression report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: docs-negotiation-report + path: test-results/ + if-no-files-found: warn + retention-days: 14 diff --git a/apps/docs/package.json b/apps/docs/package.json index eb0b47b77e5..c28227be6ac 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -11,6 +11,7 @@ "start": "next start", "postinstall": "fumadocs-mdx", "test": "vitest run", + "test:negotiation:e2e": "bun scripts/test-negotiation-e2e.ts", "type-check": "fumadocs-mdx && tsc --noEmit", "lint": "biome check --write --unsafe .", "lint:check": "biome check .", diff --git a/apps/docs/scripts/test-negotiation-e2e.ts b/apps/docs/scripts/test-negotiation-e2e.ts new file mode 100644 index 00000000000..609ef3744e1 --- /dev/null +++ b/apps/docs/scripts/test-negotiation-e2e.ts @@ -0,0 +1,87 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' + +const logger = createLogger('DocsNegotiationE2E') +const origin = process.env.DOCS_E2E_BASE_URL +const reportPath = process.env.DOCS_NEGOTIATION_REPORT_PATH +if (!origin || !reportPath) { + throw new Error('DOCS_E2E_BASE_URL and DOCS_NEGOTIATION_REPORT_PATH are required') +} +const url = new URL('/introduction', origin) +if (!['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) { + throw new Error('The documentation server must use a loopback origin') +} + +interface CheckResult { + name: string + status: 'passed' | 'failed' + durationMs: number + contentType?: string + vary?: string + error?: string +} + +const cases = [ + { name: 'HTML advertises its representation and router cache dimensions', accept: 'text/html' }, + { name: 'Markdown negotiates at the same URL', accept: 'text/markdown' }, + { name: 'HTML remains available after a Markdown request', accept: 'text/html' }, +] as const + +for (let attempt = 0; attempt < 20; attempt += 1) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(1000) }) + await response.body?.cancel() + if (response.ok) break + } catch {} + await sleep(250) +} + +const checks: CheckResult[] = [] +for (const entry of cases) { + const started = performance.now() + const result: CheckResult = { name: entry.name, status: 'failed', durationMs: 0 } + try { + const response = await fetch(url, { + headers: { Accept: entry.accept }, + signal: AbortSignal.timeout(10000), + }) + await response.body?.cancel() + result.contentType = response.headers.get('Content-Type') ?? '' + result.vary = response.headers.get('Vary') ?? '' + const fields = result.vary.split(',').map((field) => field.trim().toLowerCase()) + if (!response.ok) throw new Error(`Unexpected HTTP status ${response.status}`) + if (!result.contentType.startsWith(entry.accept)) { + throw new Error(`Expected ${entry.accept}; received ${result.contentType}`) + } + if (!fields.includes('accept')) throw new Error('The response lost Vary: Accept') + if (entry.accept === 'text/html') { + for (const field of [ + 'rsc', + 'next-router-state-tree', + 'next-router-prefetch', + 'next-router-segment-prefetch', + ]) { + if (!fields.includes(field)) throw new Error(`The response lost Vary: ${field}`) + } + } + if (fields.length !== new Set(fields).size) throw new Error('Vary contains duplicate fields') + result.status = 'passed' + } catch (error) { + result.error = getErrorMessage(error) + } + result.durationMs = Math.round(performance.now() - started) + checks.push(result) +} + +await mkdir(dirname(reportPath), { recursive: true }) +await writeFile(reportPath, `${JSON.stringify({ checks }, null, 2)}\n`) +const failures = checks.filter((check) => check.status === 'failed') +if (failures.length) { + logger.error('Documentation negotiation failed', { failures }) + process.exitCode = 1 +} else { + logger.info('Documentation negotiation passed', { checks: checks.length }) +} diff --git a/bun.lock b/bun.lock index 7d81a32fcad..7f0e555445e 100644 --- a/bun.lock +++ b/bun.lock @@ -842,6 +842,7 @@ "echarts@6.1.0": "patches/echarts@6.1.0.patch", "@better-auth/oauth-provider@1.6.27": "patches/@better-auth%2Foauth-provider@1.6.27.patch", "drizzle-kit@0.31.10": "patches/drizzle-kit@0.31.10.patch", + "next@16.4.0": "patches/next@16.4.0.patch", "oracledb@7.0.1": "patches/oracledb@7.0.1.patch", "postgres@3.4.9": "patches/postgres@3.4.9.patch", }, diff --git a/package.json b/package.json index e23838fb81c..720bcf8ceea 100644 --- a/package.json +++ b/package.json @@ -212,6 +212,7 @@ "drizzle-kit@0.31.10": "patches/drizzle-kit@0.31.10.patch", "postgres@3.4.9": "patches/postgres@3.4.9.patch", "oracledb@7.0.1": "patches/oracledb@7.0.1.patch", - "echarts@6.1.0": "patches/echarts@6.1.0.patch" + "echarts@6.1.0": "patches/echarts@6.1.0.patch", + "next@16.4.0": "patches/next@16.4.0.patch" } } diff --git a/patches/README.md b/patches/README.md index 929ce944a2a..6ebc804a5d6 100644 --- a/patches/README.md +++ b/patches/README.md @@ -89,3 +89,39 @@ rich-text mode, preserving the chart document's untrusted-markup boundary. `apps/sim/lib/charts/tooltip.test.ts` exercises actual ECharts tooltip rendering, font overrides, generated tokens, and box sizing. Remove this patch when an upstream release provides equivalent font handling and these tests pass. + +# Next.js application Vary preservation + +`next@16.4.0` replaces application `Vary` values when the Node App Page handler +sets its router cache dimensions. The docs proxy's `Vary: Accept` therefore +vanishes from the final HTML response, allowing downstream caches to confuse +HTML with negotiated Markdown. + +The pinned patch applies the merging semantics of [upstream PR #98104](https://github.com/vercel/next.js/pull/98104) +at Next's native Node outgoing-header merger, which is used before the bundled +App Page handler. Both published CommonJS and ESM entry points preserve existing +Vary fields and add later fields without case-sensitive duplicates. Wildcards +remain intact, and native header validation still rejects invalid values. + +Vary writes become cumulative across Node server responses in both the docs and +Sim workspaces; explicit header removal, cookie handling, other headers, router +selection, cache lifetimes, and authentication are unchanged. Edge SSR and direct +handlers outside Next's Node server are outside this patch's scope. + +The `docs content negotiation` CI workflow builds and serves the actual docs, +then runs `apps/docs/scripts/test-negotiation-e2e.ts`. Its HTTP checks verify +HTML → Markdown → HTML at one URL, `Vary: Accept` on both representations, and +preservation of Next's router dimensions. It uploads a JSON report and server +log. The HTML checks fail with the unpatched dependency even when the proxy +already sets `Vary: Accept`. + +To repeat against a running production docs server: + +```bash +DOCS_E2E_BASE_URL=http://127.0.0.1:3001 \ +DOCS_NEGOTIATION_REPORT_PATH=/tmp/docs-negotiation.json \ +bun run --cwd apps/docs test:negotiation:e2e +``` + +Remove the patch when upgrading to a release with equivalent Vary preservation, +and retain the HTTP regression check. diff --git a/patches/next@16.4.0.patch b/patches/next@16.4.0.patch new file mode 100644 index 00000000000..9c4f122d5ce --- /dev/null +++ b/patches/next@16.4.0.patch @@ -0,0 +1,75 @@ +diff --git a/dist/esm/server/lib/patch-set-header.js b/dist/esm/server/lib/patch-set-header.js +index f2236f7ae222afcb8554267ed295ada59dceee6a..3be146fc8fa48196254d93a8a21c2ceac2596b09 100644 +--- a/dist/esm/server/lib/patch-set-header.js ++++ b/dist/esm/server/lib/patch-set-header.js +@@ -1,3 +1,4 @@ ++import { validateHeaderValue } from 'node:http'; + import { getRequestMeta } from '../request-meta'; + const PATCHED_SET_HEADER = Symbol('next.patchSetHeaderWithCookieSupport'); + /** +@@ -20,6 +21,26 @@ const PATCHED_SET_HEADER = Symbol('next.patchSetHeaderWithCookieSupport'); + if ('headersSent' in res && res.headersSent) { + return res; + } ++ if (name.toLowerCase() === 'vary' && (typeof value === 'string' || Array.isArray(value))) { ++ validateHeaderValue(name, value); ++ const fields = []; ++ const seen = new Set(); ++ const collect = (candidate)=>{ ++ const values = Array.isArray(candidate) ? candidate : typeof candidate === 'string' ? [candidate] : []; ++ for (const entry of values){ ++ for (const field of entry.split(',')){ ++ const trimmed = field.trim(); ++ const normalized = trimmed.toLowerCase(); ++ if (!trimmed || seen.has(normalized)) continue; ++ seen.add(normalized); ++ fields.push(trimmed); ++ } ++ } ++ }; ++ collect(res.getHeader(name)); ++ collect(value); ++ value = seen.has('*') ? '*' : fields.join(', '); ++ } + if (name.toLowerCase() === 'set-cookie') { + const middlewareValue = getRequestMeta(req, 'middlewareCookie'); + if (!middlewareValue || !Array.isArray(value) || !value.every((item, idx)=>item === middlewareValue[idx])) { +diff --git a/dist/server/lib/patch-set-header.js b/dist/server/lib/patch-set-header.js +index 16cdc4703f3e9957a16c68c962cfaf5ee5d04561..a1c3c7061bc917284e01ed9947cd914818cfd708 100644 +--- a/dist/server/lib/patch-set-header.js ++++ b/dist/server/lib/patch-set-header.js +@@ -8,6 +8,7 @@ Object.defineProperty(exports, "patchSetHeaderWithCookieSupport", { + return patchSetHeaderWithCookieSupport; + } + }); ++const _nodehttp = require("node:http"); + const _requestmeta = require("../request-meta"); + const PATCHED_SET_HEADER = Symbol('next.patchSetHeaderWithCookieSupport'); + function patchSetHeaderWithCookieSupport(req, res) { +@@ -24,6 +25,26 @@ function patchSetHeaderWithCookieSupport(req, res) { + if ('headersSent' in res && res.headersSent) { + return res; + } ++ if (name.toLowerCase() === 'vary' && (typeof value === 'string' || Array.isArray(value))) { ++ (0, _nodehttp.validateHeaderValue)(name, value); ++ const fields = []; ++ const seen = new Set(); ++ const collect = (candidate)=>{ ++ const values = Array.isArray(candidate) ? candidate : typeof candidate === 'string' ? [candidate] : []; ++ for (const entry of values){ ++ for (const field of entry.split(',')){ ++ const trimmed = field.trim(); ++ const normalized = trimmed.toLowerCase(); ++ if (!trimmed || seen.has(normalized)) continue; ++ seen.add(normalized); ++ fields.push(trimmed); ++ } ++ } ++ }; ++ collect(res.getHeader(name)); ++ collect(value); ++ value = seen.has('*') ? '*' : fields.join(', '); ++ } + if (name.toLowerCase() === 'set-cookie') { + const middlewareValue = (0, _requestmeta.getRequestMeta)(req, 'middlewareCookie'); + if (!middlewareValue || !Array.isArray(value) || !value.every((item, idx)=>item === middlewareValue[idx])) { From 88f5ac09a4d02e975b3af4f57e7eabd9b0ceccd3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 14 Oct 2026 15:15:48 -0700 Subject: [PATCH 2/2] fix(docs): retain native header regression checks --- .github/workflows/docs-negotiation.yml | 3 + apps/docs/package.json | 1 + apps/docs/scripts/test-header-merge-e2e.ts | 206 +++++++++++++++++++++ patches/README.md | 6 +- 4 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 apps/docs/scripts/test-header-merge-e2e.ts diff --git a/.github/workflows/docs-negotiation.yml b/.github/workflows/docs-negotiation.yml index 09bc78c76fd..87f82e99180 100644 --- a/.github/workflows/docs-negotiation.yml +++ b/.github/workflows/docs-negotiation.yml @@ -34,6 +34,7 @@ jobs: DATABASE_URL: postgresql://docs_test:docs_test@127.0.0.1:1/docs_test DOCS_E2E_BASE_URL: http://127.0.0.1:3001 DOCS_NEGOTIATION_REPORT_PATH: ${{ github.workspace }}/test-results/docs-negotiation.json + DOCS_HEADER_REPORT_PATH: ${{ github.workspace }}/test-results/docs-headers.json steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -43,6 +44,8 @@ jobs: provider: ${{ vars.CI_PROVIDER }} - name: Build production docs run: bun run --cwd apps/docs build + - name: Verify native header edge cases + run: bun run --cwd apps/docs test:headers:e2e - name: Verify HTTP content negotiation run: | mkdir -p test-results diff --git a/apps/docs/package.json b/apps/docs/package.json index c28227be6ac..87d61d75d13 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -12,6 +12,7 @@ "postinstall": "fumadocs-mdx", "test": "vitest run", "test:negotiation:e2e": "bun scripts/test-negotiation-e2e.ts", + "test:headers:e2e": "bun scripts/test-header-merge-e2e.ts", "type-check": "fumadocs-mdx && tsc --noEmit", "lint": "biome check --write --unsafe .", "lint:check": "biome check .", diff --git a/apps/docs/scripts/test-header-merge-e2e.ts b/apps/docs/scripts/test-header-merge-e2e.ts new file mode 100644 index 00000000000..b2c2f37be93 --- /dev/null +++ b/apps/docs/scripts/test-header-merge-e2e.ts @@ -0,0 +1,206 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { createRequire } from 'node:module' +import { dirname } from 'node:path' +import { pathToFileURL } from 'node:url' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { toRecord } from '@sim/utils/object' +import { patchSetHeaderWithCookieSupport } from 'next/dist/server/lib/patch-set-header' +import { addRequestMeta } from 'next/dist/server/request-meta' + +const logger = createLogger('DocsHeaderMergeE2E') +const reportPath = process.env.DOCS_HEADER_REPORT_PATH +if (!reportPath) throw new Error('DOCS_HEADER_REPORT_PATH is required') + +interface HeaderCase { + name: string + header: string + before?: string | string[] + value: string | string[] + expected?: string[] + rejected?: boolean + middlewareCookies?: string[] +} + +const cases: HeaderCase[] = [ + { + name: 'existing Vary survives a later write', + header: 'Vary', + before: 'Origin', + value: 'RSC', + expected: ['origin', 'rsc'], + }, + { + name: 'array fields survive a later write', + header: 'Vary', + before: ['Accept', 'rSc'], + value: ['RSC', 'Next-Url'], + expected: ['accept', 'rsc', 'next-url'], + }, + { + name: 'mixed casing does not duplicate fields', + header: 'Vary', + before: 'Accept, rSc', + value: 'RSC, aCcEpT, Next-Url, RSC', + expected: ['accept', 'rsc', 'next-url'], + }, + { + name: 'wildcard remains the only field', + header: 'Vary', + before: '*', + value: 'RSC', + expected: ['*'], + }, + { + name: 'invalid bytes cannot be trimmed away', + header: 'Vary', + value: 'RSC\r\n', + rejected: true, + }, + { + name: 'wildcard cannot hide invalid array bytes', + header: 'Vary', + value: ['*', 'bad\r\nheader'], + rejected: true, + }, + { + name: 'other headers retain replacement behavior', + header: 'X-Probe', + before: 'old', + value: 'new', + expected: ['new'], + }, + { + name: 'middleware cookies remain present', + header: 'Set-Cookie', + value: ['probe_b=2'], + middlewareCookies: ['probe_a=1'], + expected: ['probe_a=1', 'probe_b=2'], + }, +] + +const require = createRequire(import.meta.url) +const esmPath = require.resolve('next/dist/esm/server/lib/patch-set-header.js') +const imported: unknown = await import(pathToFileURL(esmPath).href) +const esmPatch = toRecord(imported).patchSetHeaderWithCookieSupport +if (typeof esmPatch !== 'function') throw new Error('The ESM header merger is unavailable') + +const variants: Array<{ + name: string + apply: (req: IncomingMessage, res: ServerResponse) => void +}> = [ + { name: 'CommonJS', apply: patchSetHeaderWithCookieSupport }, + { + name: 'ESM', + apply: (req, res) => { + esmPatch(req, res) + }, + }, +] +const checks: Array<{ + name: string + status: 'passed' | 'failed' + durationMs: number + error?: string +}> = [] + +const suiteStarted = performance.now() +try { + for (const variant of variants) { + const server = createServer((req, res) => { + const entry = cases[Number(req.url?.slice(1))] + if (!entry) { + res.writeHead(404).end() + return + } + try { + variant.apply(req, res) + if (entry.middlewareCookies) + addRequestMeta(req, 'middlewareCookie', entry.middlewareCookies) + if (entry.before) res.setHeader(entry.header, entry.before) + } catch (error) { + res.statusCode = 500 + res.end(JSON.stringify({ error: getErrorMessage(error) })) + return + } + try { + res.setHeader(entry.header, entry.value) + res.end(JSON.stringify({ accepted: true })) + } catch (error) { + res.statusCode = 400 + res.end(JSON.stringify({ rejected: true, code: toRecord(error).code })) + } + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + try { + const address = server.address() + if (!address || typeof address === 'string') + throw new Error('No loopback listener is available') + for (const [index, entry] of cases.entries()) { + const started = performance.now() + let error: string | undefined + try { + const response = await fetch(`http://127.0.0.1:${address.port}/${index}`, { + signal: AbortSignal.timeout(5000), + }) + const payload: unknown = await response.json() + if (entry.rejected) { + const body = toRecord(payload) + if ( + response.status !== 400 || + body.rejected !== true || + body.code !== 'ERR_INVALID_CHAR' + ) + throw new Error('Invalid header bytes were accepted') + } else { + const fields = + entry.header === 'Set-Cookie' + ? response.headers.getSetCookie() + : (response.headers.get(entry.header) ?? '') + .split(',') + .map((field) => field.trim().toLowerCase()) + const expected = [...(entry.expected ?? [])].sort() + if ( + response.status !== 200 || + new Set(fields).size !== fields.length || + JSON.stringify([...fields].sort()) !== JSON.stringify(expected) + ) + throw new Error(`Unexpected response fields: ${JSON.stringify(fields)}`) + } + } catch (caught) { + error = getErrorMessage(caught) + } + checks.push({ + name: `${variant.name}: ${entry.name}`, + status: error ? 'failed' : 'passed', + durationMs: Math.round(performance.now() - started), + ...(error ? { error } : {}), + }) + } + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ) + } + } +} catch (error) { + checks.push({ + name: 'Header suite lifecycle', + status: 'failed', + durationMs: Math.round(performance.now() - suiteStarted), + error: getErrorMessage(error), + }) +} finally { + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, `${JSON.stringify({ checks }, null, 2)}\n`) +} + +const failures = checks.filter((check) => check.status === 'failed') +if (failures.length) { + logger.error('Native header regression checks failed', { failures }) + process.exitCode = 1 +} else logger.info('Native header regression checks passed', { checks: checks.length }) diff --git a/patches/README.md b/patches/README.md index 6ebc804a5d6..253a8bfbd15 100644 --- a/patches/README.md +++ b/patches/README.md @@ -113,7 +113,11 @@ then runs `apps/docs/scripts/test-negotiation-e2e.ts`. Its HTTP checks verify HTML → Markdown → HTML at one URL, `Vary: Accept` on both representations, and preservation of Next's router dimensions. It uploads a JSON report and server log. The HTML checks fail with the unpatched dependency even when the proxy -already sets `Vary: Accept`. +already sets `Vary: Accept`. `test:headers:e2e` exercises the actual CommonJS and +ESM mergers over loopback HTTP for string/array fields, mixed casing, wildcards, +invalid values, other-header replacement, and middleware cookies. Its checks +fail independently when the merge or validation guard is removed. CI uploads +both reports. To repeat against a running production docs server: