Some content
+ + diff --git a/dev-packages/browser-integration-tests/suites/tracing/metrics/web-vitals-cls-without-array-at/test.ts b/dev-packages/browser-integration-tests/suites/tracing/metrics/web-vitals-cls-without-array-at/test.ts new file mode 100644 index 000000000000..68be3f8f7063 --- /dev/null +++ b/dev-packages/browser-integration-tests/suites/tracing/metrics/web-vitals-cls-without-array-at/test.ts @@ -0,0 +1,34 @@ +import { expect } from '@playwright/test'; +import { sentryTest } from '../../../../utils/fixtures'; +import { hidePage, shouldSkipTracingTest } from '../../../../utils/helpers'; +import { getSpanOp, waitForStreamedSpan } from '../../../../utils/spanUtils'; + +sentryTest.beforeEach(async ({ browserName, page }) => { + if (shouldSkipTracingTest() || browserName !== 'chromium') { + sentryTest.skip(); + } + + await page.setViewportSize({ width: 800, height: 1200 }); +}); + +// Chromium 80-91 is within our supported range and reports layout shifts, but has no `Array.prototype.at`, +// which web-vitals' CLS code relies on. +sentryTest('does not throw on layout shifts without Array.prototype.at', async ({ getLocalTestUrl, page }) => { + const pageErrors: string[] = []; + page.on('pageerror', error => pageErrors.push(error.message)); + + await page.addInitScript(() => { + // @ts-expect-error - emulating Chromium < 92 + delete Array.prototype.at; + }); + + const url = await getLocalTestUrl({ testDir: __dirname }); + const pageloadSpanPromise = waitForStreamedSpan(page, span => getSpanOp(span) === 'pageload'); + + await page.goto(url); + await page.evaluate(() => new Promise(resolve => window.addEventListener('cls-done', resolve))); + await hidePage(page); + await pageloadSpanPromise; + + expect(pageErrors).toEqual([]); +}); diff --git a/dev-packages/browser-min-version-tests/.gitignore b/dev-packages/browser-min-version-tests/.gitignore new file mode 100644 index 000000000000..36e1248d6bd9 --- /dev/null +++ b/dev-packages/browser-min-version-tests/.gitignore @@ -0,0 +1 @@ +.chromium/ diff --git a/dev-packages/browser-min-version-tests/install-deps.sh b/dev-packages/browser-min-version-tests/install-deps.sh new file mode 100755 index 000000000000..8316317a7d05 --- /dev/null +++ b/dev-packages/browser-min-version-tests/install-deps.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Old Chromium links against libXss, which current runner images don't ship. Installing from the image's +# package lists is fast; only refresh them when they are too stale to resolve the package. +set -eu + +if ! sudo apt-get install -y --no-install-recommends libxss1; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends libxss1 +fi diff --git a/dev-packages/browser-min-version-tests/package.json b/dev-packages/browser-min-version-tests/package.json new file mode 100644 index 000000000000..172040f165b6 --- /dev/null +++ b/dev-packages/browser-min-version-tests/package.json @@ -0,0 +1,13 @@ +{ + "name": "@sentry-internal/browser-min-version-tests", + "version": "11.12.0", + "license": "MIT", + "private": true, + "type": "module", + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0 <23.0.0 || >=23.2.0" + }, + "devDependencies": { + "@sentry-internal/test-utils": "11.12.0" + } +} diff --git a/dev-packages/browser-min-version-tests/page.html b/dev-packages/browser-min-version-tests/page.html new file mode 100644 index 000000000000..c918731ca689 --- /dev/null +++ b/dev-packages/browser-min-version-tests/page.html @@ -0,0 +1,29 @@ + + + + + + + + +Shifting
+ + + + diff --git a/dev-packages/browser-min-version-tests/run.mjs b/dev-packages/browser-min-version-tests/run.mjs new file mode 100644 index 000000000000..64d3b686010c --- /dev/null +++ b/dev-packages/browser-min-version-tests/run.mjs @@ -0,0 +1,275 @@ +// Loads the CDN bundles in the oldest Chromium we support and checks that the web vitals it can +// measure are sent, and that nothing throws. Playwright only drives the Chromium it ships with, so +// this talks to the browser over the DevTools protocol directly. Run it after `pnpm build:bundle` in +// `packages/browser`. +import { execFileSync, spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { CDPClient, createBasicSentryServer, getSpanOp } from '@sentry-internal/test-utils'; + +// Chromium 80.0.3987.0, the minimum Chrome version in our supported browsers docs. +const CHROMIUM_REVISION = '722234'; +const CHROMIUM_ZIP_SHA256 = '2a7f1d1cf4fd933f4090739667f86b52187431a5a22e5f33a4a2be158d4574dc'; +const DIR = path.dirname(fileURLToPath(import.meta.url)); +const CHROMIUM_DIR = path.join(DIR, '.chromium', CHROMIUM_REVISION); +const CHROME = path.join(CHROMIUM_DIR, 'chrome-linux', 'chrome'); +const BUNDLES_DIR = path.join(DIR, '../../packages/browser/build/bundles'); +const BUNDLES = [ + 'bundle.tracing.min.js', + // The debug build logs errors the SDK catches internally, which would otherwise go unnoticed. + 'bundle.tracing.replay.feedback.logs.metrics.debug.min.js', +]; +const PAGE = readFileSync(path.join(DIR, 'page.html'), 'utf8'); + +// What Chromium 80 can report. CLS is skipped there because web-vitals' CLS code needs +// `Array.prototype.at` (Chromium 92), and INP needs `PerformanceEventTiming.interactionId` (Chromium 96). +const EXPECTED_PAGELOAD_VITALS = ['fcp', 'fp', 'ttfb']; +const UNEXPECTED_SPAN_OPS = ['ui.webvital.cls', 'ui.interaction.click']; + +async function ensureChromium() { + if (existsSync(CHROME)) { + return; + } + const url = `https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/${CHROMIUM_REVISION}/chrome-linux.zip`; + const response = await fetch(url, { signal: AbortSignal.timeout(120_000) }); + if (!response.ok) { + throw new Error(`Downloading Chromium failed: ${response.status} ${url}`); + } + const data = Buffer.from(await response.arrayBuffer()); + const sha256 = createHash('sha256').update(data).digest('hex'); + if (sha256 !== CHROMIUM_ZIP_SHA256) { + throw new Error(`Chromium download has SHA-256 ${sha256}, expected ${CHROMIUM_ZIP_SHA256}`); + } + mkdirSync(CHROMIUM_DIR, { recursive: true }); + const zip = `${CHROMIUM_DIR}.zip`; + writeFileSync(zip, data); + execFileSync('unzip', ['-q', zip, '-d', CHROMIUM_DIR]); + rmSync(zip); +} + +// The page and the DSN are on different ports, so envelopes are sent cross-origin like they are to sentry.io. +function startPageServer(bundlePath, sentryPort) { + const page = PAGE.replace('__DSN__', `http://public@localhost:${sentryPort}/1337`); + const server = createServer((req, res) => { + if (req.url === '/bundle.js') { + res.setHeader('content-type', 'text/javascript'); + res.end(readFileSync(bundlePath)); + return; + } + res.setHeader('content-type', 'text/html'); + res.end(page); + }); + return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve(server))); +} + +function launchChrome(userDataDir) { + const chrome = spawn(CHROME, [ + '--headless', + '--no-sandbox', + '--disable-gpu', + '--remote-debugging-port=0', + '--window-size=800,1200', + `--user-data-dir=${userDataDir}`, + 'about:blank', + ]); + const port = new Promise((resolve, reject) => { + let stderr = ''; + chrome.stderr.on('data', chunk => { + stderr += chunk; + const match = stderr.match(/DevTools listening on ws:\/\/127\.0\.0\.1:(\d+)\//); + if (match) { + resolve(match[1]); + } + }); + chrome.on('exit', code => reject(new Error(`Chromium exited with ${code}:\n${stderr}`))); + }); + return { chrome, port }; +} + +async function connect(port) { + // The DevTools server can come up before the initial page target exists. + let page; + for (let attempt = 0; !page && attempt < 50; attempt++) { + const targets = await (await fetch(`http://127.0.0.1:${port}/json/list`)).json(); + page = targets.find(target => target.type === 'page'); + if (!page) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + if (!page) { + throw new Error('Chromium has no page target'); + } + const cdp = new CDPClient({ url: page.webSocketDebuggerUrl, retries: 1 }); + await cdp.connect(); + return cdp; +} + +function waitForEvent(cdp, method) { + return new Promise(resolve => { + const off = cdp.on(method, () => { + off(); + resolve(); + }); + }); +} + +async function evaluate(cdp, expression) { + const { result, exceptionDetails } = await cdp.send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (exceptionDetails) { + throw new Error(`Evaluating failed: ${exceptionDetails.exception?.description ?? exceptionDetails.text}`); + } + return result.value; +} + +function collectItems(spans, events) { + return ([, items]) => { + for (const [header, payload] of items) { + if (header.type === 'span') { + spans.push(...payload.items); + } else if (header.type === 'event') { + events.push(payload); + } + } + }; +} + +const isPageload = span => span.is_segment && getSpanOp(span) === 'pageload'; + +async function runBundle(bundle) { + const spans = []; + const events = []; + const errors = []; + const [sentryPort, closeSentryServer] = await createBasicSentryServer(collectItems(spans, events)); + const server = await startPageServer(path.join(BUNDLES_DIR, bundle), sentryPort); + const userDataDir = mkdtempSync(path.join(tmpdir(), 'chromium-profile-')); + const { chrome, port } = launchChrome(userDataDir); + let cdp; + + try { + cdp = await connect(await port); + cdp.on('Runtime.exceptionThrown', ({ exceptionDetails }) => { + errors.push(exceptionDetails.exception?.description ?? exceptionDetails.text); + }); + cdp.on('Runtime.consoleAPICalled', ({ type, args }) => { + if (type === 'error') { + errors.push(args.map(arg => arg.value ?? arg.description).join(' ')); + } + }); + cdp.on('Log.entryAdded', ({ entry }) => { + if (entry.level === 'error') { + errors.push(entry.text); + } + }); + await Promise.all([cdp.send('Runtime.enable'), cdp.send('Log.enable'), cdp.send('Page.enable')]); + + const loaded = waitForEvent(cdp, 'Page.loadEventFired'); + await cdp.send('Page.navigate', { url: `http://127.0.0.1:${server.address().port}/` }); + await loaded; + + // Layout shifts, then a click: shifts within 500ms after an input don't count towards CLS. + await evaluate( + cdp, + `(async () => { + const element = document.getElementById('shift'); + for (let i = 1; i <= 10; i++) { + element.style.top = 200 + i * 40 + 'px'; + await new Promise(resolve => setTimeout(resolve, 50)); + } + })()`, + ); + const { x, y } = await evaluate( + cdp, + `(() => { + const rect = document.getElementById('button').getBoundingClientRect(); + return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }; + })()`, + ); + for (const type of ['mousePressed', 'mouseReleased']) { + await cdp.send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 }); + } + await new Promise(resolve => setTimeout(resolve, 300)); + + // Without these, the checks that CLS and INP are not sent would pass without testing anything. + const { layoutShifts, clicked } = await evaluate( + cdp, + `new Promise(resolve => { + const done = layoutShifts => resolve({ layoutShifts, clicked: document.getElementById('button').dataset.clicked === 'true' }); + new PerformanceObserver(list => done(list.getEntries().length)).observe({ type: 'layout-shift', buffered: true }); + setTimeout(() => done(0), 500); + })`, + ); + if (!layoutShifts || !clicked) { + throw new Error(`Page did not shift (${layoutShifts} layout shifts) or was not clicked (${clicked})`); + } + + // Hiding the page is what makes the SDK finalize LCP and CLS. + await evaluate( + cdp, + `(() => { + Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => 'hidden' }); + document.dispatchEvent(new Event('visibilitychange')); + })()`, + ); + // The pageload span, which carries FCP, FP and TTFB, ends once the page has been idle for a second. + const deadline = Date.now() + 10_000; + while (!spans.some(isPageload) && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } finally { + await cdp?.close(); + chrome.kill(); + server.close(); + closeSentryServer(); + rmSync(userDataDir, { recursive: true, force: true }); + } + + const pageload = spans.find(isPageload); + const ops = new Set(spans.map(getSpanOp)); + const failures = [...new Set(errors)].map(error => `page error: ${error}`); + + for (const event of events) { + failures.push(`error event sent: ${event.exception?.values?.[0]?.value ?? event.message}`); + } + if (!pageload) { + failures.push('no pageload span sent'); + } else { + if (!pageload.attributes['user_agent.original']?.value.includes('Chrome/80.')) { + failures.push(`not running in Chrome 80: ${pageload.attributes['user_agent.original']?.value}`); + } + for (const vital of EXPECTED_PAGELOAD_VITALS) { + if (typeof pageload.attributes[`browser.web_vital.${vital}.value`]?.value !== 'number') { + failures.push(`pageload span has no ${vital.toUpperCase()}`); + } + } + } + const lcp = spans.find(span => getSpanOp(span) === 'ui.webvital.lcp'); + if (typeof lcp?.attributes['browser.web_vital.lcp.value']?.value !== 'number') { + failures.push('no LCP span sent'); + } + for (const unexpected of UNEXPECTED_SPAN_OPS) { + if (ops.has(unexpected)) { + failures.push(`unexpected ${unexpected} span sent`); + } + } + + return failures; +} + +await ensureChromium(); + +const results = await Promise.all(BUNDLES.map(runBundle)); +results.forEach((failures, index) => { + // eslint-disable-next-line no-console + console.log( + `${failures.length ? '✘' : '✓'} ${BUNDLES[index]}${failures.map(failure => `\n ${failure}`).join('')}`, + ); +}); +process.exitCode = results.some(failures => failures.length) ? 1 : 0; diff --git a/dev-packages/test-utils/src/cdp-client.ts b/dev-packages/test-utils/src/cdp-client.ts index 491d7bfa77d9..7d0aaead7474 100644 --- a/dev-packages/test-utils/src/cdp-client.ts +++ b/dev-packages/test-utils/src/cdp-client.ts @@ -135,18 +135,24 @@ export class CDPClient { this.#log('Sending:', method, params || ''); return new Promise((resolve, reject) => { - this.#pendingRequests.set(id, { - resolve: value => resolve(value as T), - reject, - }); - this.#ws!.send(message); - - setTimeout(() => { + // Cleared on response, so a finished request doesn't keep the process alive until the timeout. + const timeoutId = setTimeout(() => { if (this.#pendingRequests.has(id)) { this.#pendingRequests.delete(id); reject(new Error(`CDP request ${method} timed out after ${timeout}ms`)); } }, timeout); + this.#pendingRequests.set(id, { + resolve: value => { + clearTimeout(timeoutId); + resolve(value as T); + }, + reject: error => { + clearTimeout(timeoutId); + reject(error); + }, + }); + this.#ws!.send(message); }); } diff --git a/dev-packages/test-utils/src/server.ts b/dev-packages/test-utils/src/server.ts index 3fef94be89a0..3e9da4320a81 100644 --- a/dev-packages/test-utils/src/server.ts +++ b/dev-packages/test-utils/src/server.ts @@ -22,6 +22,8 @@ export function createBasicSentryServer(onEnvelope: (env: Envelope) => void): Pr console.error(e); } + // Browsers send envelopes cross-origin, as they do to sentry.io, and reject the response without this. + res.set('Access-Control-Allow-Origin', '*'); res.status(200).send(); }); diff --git a/package.json b/package.json index 7ea21a7e2067..2d7ebb25303f 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "packages/wasm", "packages/webpack-plugin", "dev-packages/browser-integration-tests", + "dev-packages/browser-min-version-tests", "dev-packages/bundler-plugin-integration-tests", "dev-packages/e2e-tests", "dev-packages/node-integration-tests", diff --git a/packages/browser-utils/src/instrumentation/performanceObserver.ts b/packages/browser-utils/src/instrumentation/performanceObserver.ts index 4424a5f3ef87..c8a312e40af8 100644 --- a/packages/browser-utils/src/instrumentation/performanceObserver.ts +++ b/packages/browser-utils/src/instrumentation/performanceObserver.ts @@ -322,6 +322,13 @@ function unlessBfcacheDisabled(callback: (metric: Metric) => void): (metric: Met } function instrumentCls(): StopListening { + // web-vitals' CLS code calls `Array.prototype.at`, which Chromium only ships from 92. Chromium 80-91 + // still emits layout shifts, so web-vitals would throw an uncaught TypeError on every one of them. + // TODO(v12): Remove this check if the minimum supported Chromium version is raised to 92 or higher. + if (!('at' in Array.prototype)) { + return; + } + return onCLS( unlessBfcacheDisabled(metric => { triggerHandlers('cls', { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a3f4846f21cd..33991319be79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -319,6 +319,12 @@ importers: specifier: ^5.112.2 version: 5.112.2(clean-css@5.2.2)(cssnano@7.1.9(postcss@8.5.32))(csso@5.0.5)(esbuild@0.28.1)(html-minifier-terser@6.1.0)(lightningcss@1.33.0)(postcss@8.5.32)(sharp@0.35.5(@types/node@20.19.43))(svgo@4.1.0)(uglify-js@3.13.3) + dev-packages/browser-min-version-tests: + devDependencies: + '@sentry-internal/test-utils': + specifier: 11.12.0 + version: link:../test-utils + dev-packages/bun-integration-tests: devDependencies: '@sentry-internal/node-integration-tests': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cb3f595a481f..902a3950f327 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -49,6 +49,7 @@ packages: - 'packages/wasm' - 'packages/webpack-plugin' - 'dev-packages/browser-integration-tests' + - 'dev-packages/browser-min-version-tests' - 'dev-packages/bundler-plugin-integration-tests' - 'dev-packages/e2e-tests' - 'dev-packages/node-integration-tests'