From f7eb9cada8280b0ceb4756e5c4108d8e32d95879 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 19 Mar 2026 14:08:34 +0000 Subject: [PATCH] chore: extract large snapshots in the trace as resources --- .../src/server/trace/recorder/snapshotter.ts | 13 +++- .../src/server/trace/recorder/tracing.ts | 2 +- .../src/tools/trace/traceCli.ts | 11 +-- .../src/utils/isomorphic/lruCache.ts | 4 +- .../isomorphic/trace/snapshotRenderer.ts | 69 ++++++++++++------- .../utils/isomorphic/trace/snapshotServer.ts | 4 +- .../utils/isomorphic/trace/snapshotStorage.ts | 11 ++- .../src/utils/isomorphic/trace/traceLoader.ts | 2 +- .../utils/isomorphic/trace/traceModernizer.ts | 7 +- packages/playwright/src/worker/testTracing.ts | 2 +- packages/trace-viewer/src/sw/main.ts | 2 +- packages/trace/src/snapshot.ts | 5 +- packages/trace/src/trace.ts | 2 +- tests/library/tracing.spec.ts | 12 +++- 14 files changed, 95 insertions(+), 51 deletions(-) diff --git a/packages/playwright-core/src/server/trace/recorder/snapshotter.ts b/packages/playwright-core/src/server/trace/recorder/snapshotter.ts index 6b0c558507891..c81a15df79f5d 100644 --- a/packages/playwright-core/src/server/trace/recorder/snapshotter.ts +++ b/packages/playwright-core/src/server/trace/recorder/snapshotter.ts @@ -131,15 +131,24 @@ export class Snapshotter { frameId: frame.guid, frameUrl: data.url, doctype: data.doctype, - html: data.html, viewport: data.viewport, timestamp: monotonicTime(), wallTime: data.wallTime, collectionTime: data.collectionTime, - resourceOverrides: [], isMainFrame: page.mainFrame() === frame }; + const htmlJson = JSON.stringify(data.html); + if (htmlJson.length < 200) { + snapshot.html = data.html; + } else { + const buffer = Buffer.from(htmlJson); + const sha1 = calculateSha1(buffer) + '.json'; + snapshot.sha1 = sha1; + this._delegate.onSnapshotterBlob({ sha1, buffer }); + } for (const { url, content, contentType } of data.resourceOverrides) { + if (!snapshot.resourceOverrides) + snapshot.resourceOverrides = []; if (typeof content === 'string') { const buffer = Buffer.from(content); const sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(contentType) || 'dat'); diff --git a/packages/playwright-core/src/server/trace/recorder/tracing.ts b/packages/playwright-core/src/server/trace/recorder/tracing.ts index 93d9ce5817fe6..32d82519d415f 100644 --- a/packages/playwright-core/src/server/trace/recorder/tracing.ts +++ b/packages/playwright-core/src/server/trace/recorder/tracing.ts @@ -52,7 +52,7 @@ import type { Progress } from '@protocol/progress'; import type * as types from '../../types'; import type { ScreencastListener } from '../../screencast'; -const version: trace.VERSION = 8; +const version: trace.VERSION = 9; export type TracerOptions = { name?: string; diff --git a/packages/playwright-core/src/tools/trace/traceCli.ts b/packages/playwright-core/src/tools/trace/traceCli.ts index e08111364937e..a07fc6f5ec4f8 100644 --- a/packages/playwright-core/src/tools/trace/traceCli.ts +++ b/packages/playwright-core/src/tools/trace/traceCli.ts @@ -690,7 +690,7 @@ export async function traceSnapshot(traceFile: string, actionId: string, options const snapshotKey = `${snapshotName}@${callId}`; - const rendered = renderer.render(); + const rendered = await renderer.render(); const defaultName = `snapshot-${actionId}-${snapshotName}.html`; if (options.serve) { @@ -704,10 +704,11 @@ export async function traceSnapshot(traceFile: string, actionId: string, options const url = new URL('http://localhost' + request.url!); const searchParams = url.searchParams; searchParams.set('name', snapshotKey); - const snapshotResponse = snapshotServer.serveSnapshot(pageId, searchParams, '/snapshot'); - response.statusCode = snapshotResponse.status; - snapshotResponse.headers.forEach((value, key) => response.setHeader(key, value)); - snapshotResponse.text().then(text => response.end(text)); + snapshotServer.serveSnapshot(pageId, searchParams, '/snapshot').then(snapshotResponse => { + response.statusCode = snapshotResponse.status; + snapshotResponse.headers.forEach((value, key) => response.setHeader(key, value)); + snapshotResponse.text().then(text => response.end(text)); + }); return true; }); diff --git a/packages/playwright-core/src/utils/isomorphic/lruCache.ts b/packages/playwright-core/src/utils/isomorphic/lruCache.ts index 2d5eae8542497..d4c4b1a4594f8 100644 --- a/packages/playwright-core/src/utils/isomorphic/lruCache.ts +++ b/packages/playwright-core/src/utils/isomorphic/lruCache.ts @@ -25,7 +25,7 @@ export class LRUCache { this._size = 0; } - getOrCompute(key: K, compute: () => { value: V, size: number }): V { + async getOrCompute(key: K, compute: () => Promise<{ value: V, size: number }>): Promise { if (this._map.has(key)) { const result = this._map.get(key)!; // reinserting makes this the least recently used entry @@ -34,7 +34,7 @@ export class LRUCache { return result.value; } - const result = compute(); + const result = await compute(); while (this._map.size && this._size + result.size > this._maxSize) { const [firstKey, firstValue] = this._map.entries().next().value!; diff --git a/packages/playwright-core/src/utils/isomorphic/trace/snapshotRenderer.ts b/packages/playwright-core/src/utils/isomorphic/trace/snapshotRenderer.ts index 6dbf28e9fcd19..5460ef5e3976f 100644 --- a/packages/playwright-core/src/utils/isomorphic/trace/snapshotRenderer.ts +++ b/packages/playwright-core/src/utils/isomorphic/trace/snapshotRenderer.ts @@ -46,9 +46,11 @@ export class SnapshotRenderer { private _snapshot: FrameSnapshot; private _callId: string; private _screencastFrames: PageEntry['screencastFrames']; + private _readSha1: (sha1: string) => Promise; - constructor(htmlCache: LRUCache, resources: ResourceSnapshot[], snapshots: FrameSnapshot[], screencastFrames: PageEntry['screencastFrames'], index: number) { + constructor(htmlCache: LRUCache, readSha1: (sha1: string) => Promise, resources: ResourceSnapshot[], snapshots: FrameSnapshot[], screencastFrames: PageEntry['screencastFrames'], index: number) { this._htmlCache = htmlCache; + this._readSha1 = readSha1; this._resources = resources; this._snapshots = snapshots; this._index = index; @@ -74,9 +76,9 @@ export class SnapshotRenderer { return closestFrame?.sha1; } - render(): RenderedFrameSnapshot { + async render(): Promise { const result: string[] = []; - const visit = (n: NodeSnapshot, snapshotIndex: number, parentTag: string | undefined, parentAttrs: [string, string][] | undefined) => { + const visit = async (n: NodeSnapshot, snapshotIndex: number, parentTag: string | undefined, parentAttrs: [string, string][] | undefined) => { // Text node. if (typeof n === 'string') { // Best-effort Electron support: rewrite custom protocol in url() links in stylesheets. @@ -92,7 +94,12 @@ export class SnapshotRenderer { // Node reference. const referenceIndex = snapshotIndex - n[0][0]; if (referenceIndex >= 0 && referenceIndex <= snapshotIndex) { - const nodes = snapshotNodes(this._snapshots[referenceIndex]); + const refSnapshot = this._snapshots[referenceIndex]; + let nodes: NodeSnapshot[] = (refSnapshot as any)._nodes; + if (!nodes) { + nodes = snapshotNodes(await this._ensureHtml(refSnapshot)); + (refSnapshot as any)._nodes = nodes; + } const nodeIndex = n[0][1]; if (nodeIndex >= 0 && nodeIndex < nodes.length) return visit(nodes[nodeIndex], referenceIndex, parentTag, parentAttrs); @@ -134,7 +141,7 @@ export class SnapshotRenderer { } result.push('>'); for (const child of children) - visit(child, snapshotIndex, nodeName, attrs); + await visit(child, snapshotIndex, nodeName, attrs); if (!autoClosing.has(nodeName)) result.push(''); return; @@ -145,8 +152,8 @@ export class SnapshotRenderer { }; const snapshot = this._snapshot; - const html = this._htmlCache.getOrCompute(this, () => { - visit(snapshot.html, this._index, undefined, undefined); + const html = await this._htmlCache.getOrCompute(this, async () => { + await visit(await this._ensureHtml(snapshot), this._index, undefined, undefined); const prefix = snapshot.doctype ? `` : ''; const html = prefix + [ // Hide the document in order to prevent flickering. We will unhide once script has processed shadow. @@ -159,6 +166,19 @@ export class SnapshotRenderer { return { html, pageId: snapshot.pageId, frameId: snapshot.frameId, index: this._index }; } + private async _ensureHtml(snapshot: FrameSnapshot): Promise { + if (!snapshot.html) { + try { + const blob = await this._readSha1(snapshot.sha1!); + const text = await blob!.text(); + snapshot.html = JSON.parse(text); + } catch { + snapshot.html = ['html']; + } + } + return snapshot.html!; + } + resourceByUrl(url: string, method: string): ResourceSnapshot | undefined { const snapshot = this._snapshot; let sameFrameResource: ResourceSnapshot | undefined; @@ -193,12 +213,12 @@ export class SnapshotRenderer { let result = sameFrameResource ?? otherFrameResource; if (result && method.toUpperCase() === 'GET') { // Patch override if necessary. - let override = snapshot.resourceOverrides.find(o => o.url === url); + let override = snapshot.resourceOverrides?.find(o => o.url === url); if (override?.ref) { // "ref" means use the same content as "ref" snapshots ago. const index = this._index - override.ref; if (index >= 0 && index < this._snapshots.length) - override = this._snapshots[index].resourceOverrides.find(o => o.url === url); + override = this._snapshots[index].resourceOverrides?.find(o => o.url === url); } if (override?.sha1) { result = { @@ -220,23 +240,20 @@ export class SnapshotRenderer { const autoClosing = new Set(['AREA', 'BASE', 'BR', 'COL', 'COMMAND', 'EMBED', 'HR', 'IMG', 'INPUT', 'KEYGEN', 'LINK', 'MENUITEM', 'META', 'PARAM', 'SOURCE', 'TRACK', 'WBR']); -function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] { - if (!(snapshot as any)._nodes) { - const nodes: NodeSnapshot[] = []; - const visit = (n: NodeSnapshot) => { - if (typeof n === 'string') { - nodes.push(n); - } else if (isNodeNameAttributesChildNodesSnapshot(n)) { - const [,, ...children] = n; - for (const child of children) - visit(child); - nodes.push(n); - } - }; - visit(snapshot.html); - (snapshot as any)._nodes = nodes; - } - return (snapshot as any)._nodes; +function snapshotNodes(html: NodeSnapshot): NodeSnapshot[] { + const nodes: NodeSnapshot[] = []; + const visit = (n: NodeSnapshot) => { + if (typeof n === 'string') { + nodes.push(n); + } else if (isNodeNameAttributesChildNodesSnapshot(n)) { + const [,, ...children] = n; + for (const child of children) + visit(child); + nodes.push(n); + } + }; + visit(html); + return nodes; } type ViewportSize = { width: number, height: number }; diff --git a/packages/playwright-core/src/utils/isomorphic/trace/snapshotServer.ts b/packages/playwright-core/src/utils/isomorphic/trace/snapshotServer.ts index e2703a0060fc3..5a62088a4e459 100644 --- a/packages/playwright-core/src/utils/isomorphic/trace/snapshotServer.ts +++ b/packages/playwright-core/src/utils/isomorphic/trace/snapshotServer.ts @@ -29,12 +29,12 @@ export class SnapshotServer { this._resourceLoader = resourceLoader; } - serveSnapshot(pageOrFrameId: string, searchParams: URLSearchParams, snapshotUrl: string): Response { + async serveSnapshot(pageOrFrameId: string, searchParams: URLSearchParams, snapshotUrl: string): Promise { const snapshot = this._snapshot(pageOrFrameId, searchParams); if (!snapshot) return new Response(null, { status: 404 }); - const renderedSnapshot = snapshot.render(); + const renderedSnapshot = await snapshot.render(); this._snapshotIds.set(snapshotUrl, snapshot); return new Response(renderedSnapshot.html, { status: 200, headers: { 'Content-Type': 'text/html; charset=utf-8' } }); } diff --git a/packages/playwright-core/src/utils/isomorphic/trace/snapshotStorage.ts b/packages/playwright-core/src/utils/isomorphic/trace/snapshotStorage.ts index 28eb3eedae7a6..85a1836f92209 100644 --- a/packages/playwright-core/src/utils/isomorphic/trace/snapshotStorage.ts +++ b/packages/playwright-core/src/utils/isomorphic/trace/snapshotStorage.ts @@ -29,6 +29,11 @@ export class SnapshotStorage { private _cache = new LRUCache(100_000_000); // 100MB per each trace private _contextToResources = new Map(); private _resourceUrlsWithOverrides = new Set(); + private _readSha1: (sha1: string) => Promise; + + constructor(readSha1: (sha1: string) => Promise) { + this._readSha1 = readSha1; + } addResource(contextId: string, resource: ResourceSnapshot): void { resource.request.url = rewriteURLForCustomProtocol(resource.request.url); @@ -36,7 +41,7 @@ export class SnapshotStorage { } addFrameSnapshot(contextId: string, snapshot: FrameSnapshot, screencastFrames: PageEntry['screencastFrames']) { - for (const override of snapshot.resourceOverrides) + for (const override of snapshot.resourceOverrides ?? []) override.url = rewriteURLForCustomProtocol(override.url); let frameSnapshots = this._frameSnapshots.get(snapshot.frameId); if (!frameSnapshots) { @@ -50,7 +55,7 @@ export class SnapshotStorage { } frameSnapshots.raw.push(snapshot); const resources = this._ensureResourcesForContext(contextId); - const renderer = new SnapshotRenderer(this._cache, resources, frameSnapshots.raw, screencastFrames, frameSnapshots.raw.length - 1); + const renderer = new SnapshotRenderer(this._cache, this._readSha1, resources, frameSnapshots.raw, screencastFrames, frameSnapshots.raw.length - 1); frameSnapshots.renderers.push(renderer); return renderer; } @@ -72,7 +77,7 @@ export class SnapshotStorage { // while serving snapshots with different override values. for (const frameSnapshots of this._frameSnapshots.values()) { for (const snapshot of frameSnapshots.raw) { - for (const override of snapshot.resourceOverrides) + for (const override of snapshot.resourceOverrides ?? []) this._resourceUrlsWithOverrides.add(override.url); } } diff --git a/packages/playwright-core/src/utils/isomorphic/trace/traceLoader.ts b/packages/playwright-core/src/utils/isomorphic/trace/traceLoader.ts index 5c9a1e3e22465..38543b2634125 100644 --- a/packages/playwright-core/src/utils/isomorphic/trace/traceLoader.ts +++ b/packages/playwright-core/src/utils/isomorphic/trace/traceLoader.ts @@ -53,7 +53,7 @@ export class TraceLoader { if (!ordinals.length) throw new Error('Cannot find .trace file'); - this._snapshotStorage = new SnapshotStorage(); + this._snapshotStorage = new SnapshotStorage(sha1 => this._backend.readBlob('resources/' + sha1)); // 3 * ordinals progress increments below. const total = ordinals.length * 3; diff --git a/packages/playwright-core/src/utils/isomorphic/trace/traceModernizer.ts b/packages/playwright-core/src/utils/isomorphic/trace/traceModernizer.ts index 20e703145f553..6a6cc2b5262cf 100644 --- a/packages/playwright-core/src/utils/isomorphic/trace/traceModernizer.ts +++ b/packages/playwright-core/src/utils/isomorphic/trace/traceModernizer.ts @@ -33,7 +33,8 @@ export class TraceVersionError extends Error { // 6 => 10/2023 ~1.40 // 7 => 05/2024 ~1.45 -const latestVersion: trace.VERSION = 8; +// 8 => 03/2026 ~1.58 +const latestVersion: trace.VERSION = 9; export class TraceModernizer { private _contextEntry: ContextEntry; @@ -438,4 +439,8 @@ export class TraceModernizer { } return result; } + + _modernize_8_to_9(events: traceV8.TraceEvent[]): trace.TraceEvent[] { + return events as trace.TraceEvent[]; + } } diff --git a/packages/playwright/src/worker/testTracing.ts b/packages/playwright/src/worker/testTracing.ts index 27811874506e0..a0327c008c441 100644 --- a/packages/playwright/src/worker/testTracing.ts +++ b/packages/playwright/src/worker/testTracing.ts @@ -31,7 +31,7 @@ import type EventEmitter from 'events'; export type Attachment = TestInfo['attachments'][0]; export const testTraceEntryName = 'test.trace'; -const version: trace.VERSION = 8; +const version: trace.VERSION = 9; let traceOrdinal = 0; type TraceFixtureValue = PlaywrightWorkerOptions['trace'] | undefined; diff --git a/packages/trace-viewer/src/sw/main.ts b/packages/trace-viewer/src/sw/main.ts index 02cb68aeaff0d..8cdf13b786cdc 100644 --- a/packages/trace-viewer/src/sw/main.ts +++ b/packages/trace-viewer/src/sw/main.ts @@ -165,7 +165,7 @@ async function doFetch(event: FetchEvent): Promise { if (errorResponse) return errorResponse; const pageOrFrameId = relativePath.substring('/snapshot/'.length); - const response = loadedTrace!.snapshotServer.serveSnapshot(pageOrFrameId, url.searchParams, url.href); + const response = await loadedTrace!.snapshotServer.serveSnapshot(pageOrFrameId, url.searchParams, url.href); if (isDeployedAsHttps) response.headers.set('Content-Security-Policy', 'upgrade-insecure-requests'); return response; diff --git a/packages/trace/src/snapshot.ts b/packages/trace/src/snapshot.ts index 43d002f616860..baee30eebf0f1 100644 --- a/packages/trace/src/snapshot.ts +++ b/packages/trace/src/snapshot.ts @@ -47,8 +47,9 @@ export type FrameSnapshot = { wallTime?: number, collectionTime: number, doctype?: string, - html: NodeSnapshot, - resourceOverrides: ResourceOverride[], + html?: NodeSnapshot, + sha1?: string, + resourceOverrides?: ResourceOverride[], viewport: { width: number, height: number }, isMainFrame: boolean, }; diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts index d11ac4dc91045..2d063bb85f19e 100644 --- a/packages/trace/src/trace.ts +++ b/packages/trace/src/trace.ts @@ -21,7 +21,7 @@ import type { Point, SerializedError, StackFrame } from '@protocol/channels'; export type Size = { width: number, height: number }; // Make sure you add _modernize_N_to_N1(event: any) to traceModernizer.ts. -export type VERSION = 8; +export type VERSION = 9; export type BrowserContextEventOptions = { baseURL?: string, diff --git a/tests/library/tracing.spec.ts b/tests/library/tracing.spec.ts index 7dc0771616a57..134ece4dce8c1 100644 --- a/tests/library/tracing.spec.ts +++ b/tests/library/tracing.spec.ts @@ -232,7 +232,7 @@ test('should respect tracesDir and name', async ({ browserType, server, mode }, function resourceNames(resources: Map) { return [...resources.keys()].map(file => { return file.replace(/^resources\/.*\.(html|css)$/, 'resources/XXX.$1'); - }).sort(); + }).sort().filter(name => !name.endsWith('.json')); } { @@ -587,8 +587,14 @@ test('should ignore iframes in head', async ({ context, page, server }, testInfo const trace = await parseTraceRaw(testInfo.outputPath('trace.zip')); expect(trace.actions).toEqual(['Click']); - expect(trace.events.find(e => e.type === 'frame-snapshot')).toBeTruthy(); - expect(trace.events.find(e => e.type === 'frame-snapshot' && JSON.stringify(e.snapshot.html).includes('IFRAME'))).toBeFalsy(); + const events = trace.events.filter(e => e.type === 'frame-snapshot'); + expect(events.length).toBeGreaterThan(0); + for (const event of events) { + let html = event.snapshot.html ? JSON.stringify(event.snapshot.html) : undefined; + if (!html) + html = trace.resources.get('resources/' + event.snapshot.sha1).toString(); + expect(html).not.toContain('IFRAME'); + } }); test('should hide internal stack frames', async ({ context, page }, testInfo) => {