diff --git a/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/Description.mdx b/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/Description.mdx index 59997c51..0b9f052a 100644 --- a/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/Description.mdx +++ b/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/Description.mdx @@ -1,4 +1,4 @@ -These CSR browser performance tests use Lighthouse to measure paint timings and a controlled navigation interaction. +These CSR browser performance tests use Lighthouse to measure paint timings and a controlled navigation interaction. The interaction is also scored as [work to commit](/methodology/#interaction-work-to-commit) from the same trace. Methodology: [Client Side Rendered Tests (Lighthouse)](/methodology/#client-side-rendered-tests-lighthouse). diff --git a/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/LighthouseFrameworkClientRenderedVersionCharts.astro b/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/LighthouseFrameworkClientRenderedVersionCharts.astro index 3c85d2a5..c32c605d 100644 --- a/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/LighthouseFrameworkClientRenderedVersionCharts.astro +++ b/packages/docs/src/benchmarks/lighthouse-client-side-rendered-tests/LighthouseFrameworkClientRenderedVersionCharts.astro @@ -4,7 +4,9 @@ import { prepareLighthouseClientSideRenderedTestsFcpMsHistory, prepareLighthouseClientSideRenderedTestsFirstPaintMsHistory, prepareLighthouseClientSideRenderedTestsInteractionHistory, + prepareClientSideRenderedInteractionWorkHistory, } from '../../lib/prepare-chart' +import { INTERACTION_WORK } from '../../lib/interaction-phases' import VersionLineChart from '../../components/VersionLineChart.astro' import InteractionHistoryChart from '../shared/InteractionHistoryChart.astro' @@ -16,6 +18,8 @@ const { versions } = Astro.props const interactionData = prepareLighthouseClientSideRenderedTestsInteractionHistory(versions) +const interactionWorkData = + prepareClientSideRenderedInteractionWorkHistory(versions) --- +{ + interactionWorkData.length > 0 ? ( + + ) : null +} { interactionData.length > 0 ? ( +{ + interactionWorkData.length > 0 ? ( + + ) : null +} { interactionData.length > 0 ? ( ({ +const { title, data, metric = LIGHTHOUSE_INTERACTION } = Astro.props +const series = metric.parts.map(({ key, label, color }) => ({ key, label, color, @@ -37,7 +41,7 @@ const chart1Table = prepareInteractionBreakdownTable(chart1) defaultValue="chart" > - + diff --git a/packages/docs/src/benchmarks/shared/InteractionChartCaption.astro b/packages/docs/src/benchmarks/shared/InteractionChartCaption.astro index a3d891bb..1253f6a0 100644 --- a/packages/docs/src/benchmarks/shared/InteractionChartCaption.astro +++ b/packages/docs/src/benchmarks/shared/InteractionChartCaption.astro @@ -1,22 +1,23 @@ --- -import { INTERACTION_PHASES } from '../../lib/interaction-phases' +import { + LIGHTHOUSE_INTERACTION, + type InteractionMetric, +} from '../../lib/interaction-phases' interface Props { title?: string + metric?: InteractionMetric } -const { title } = Astro.props +const { title, metric = LIGHTHOUSE_INTERACTION } = Astro.props --- {title && {title}} - - Total latency is split into the three consecutive phases reported by - Lighthouse. - + {metric.summary} { - INTERACTION_PHASES.map(({ label, description }) => ( + metric.parts.map(({ label, description }) => ( The {label.toLowerCase()}, {description} @@ -25,7 +26,7 @@ const { title } = Astro.props { - INTERACTION_PHASES.map(({ label, color }) => ( + metric.parts.map(({ label, color }) => ( {label} diff --git a/packages/docs/src/benchmarks/shared/InteractionHistoryChart.astro b/packages/docs/src/benchmarks/shared/InteractionHistoryChart.astro index 4bcbe36d..750891d4 100644 --- a/packages/docs/src/benchmarks/shared/InteractionHistoryChart.astro +++ b/packages/docs/src/benchmarks/shared/InteractionHistoryChart.astro @@ -1,33 +1,42 @@ --- import StackedAreaChart from '../../components/StackedAreaChart.astro' -import { INTERACTION_PHASES } from '../../lib/interaction-phases' -import type { InteractionHistoryDatum } from '../../lib/types' +import { + LIGHTHOUSE_INTERACTION, + type InteractionMetric, +} from '../../lib/interaction-phases' +import type { + InteractionHistoryDatum, + InteractionWorkHistoryDatum, +} from '../../lib/types' import InteractionChartCaption from './InteractionChartCaption.astro' interface Props { title: string - data: InteractionHistoryDatum[] + data: (InteractionHistoryDatum | InteractionWorkHistoryDatum)[] + metric?: InteractionMetric } -const { title, data } = Astro.props +const { title, data, metric = LIGHTHOUSE_INTERACTION } = Astro.props +const valueOf = (datum: Props['data'][number], key: string) => + (datum as unknown as Record)[key]! --- ({ name: datum.name, - total: datum.interactionLatency, + total: valueOf(datum, metric.totalKey), values: Object.fromEntries( - INTERACTION_PHASES.map(({ key }) => [key, datum[key]]), + metric.parts.map(({ key }) => [key, valueOf(datum, key)]), ), }))} - series={INTERACTION_PHASES.map(({ key, label, color }) => ({ + series={metric.parts.map(({ key, label, color }) => ({ key, label, color, }))} valueFormat="ms" - yAxisLabel="Interaction latency (ms)" + yAxisLabel={metric.yAxisLabel} > - + diff --git a/packages/docs/src/benchmarks/shared/LighthouseRenderedInteractionCharts.astro b/packages/docs/src/benchmarks/shared/LighthouseRenderedInteractionCharts.astro index d82d0acc..b009a1e0 100644 --- a/packages/docs/src/benchmarks/shared/LighthouseRenderedInteractionCharts.astro +++ b/packages/docs/src/benchmarks/shared/LighthouseRenderedInteractionCharts.astro @@ -1,5 +1,9 @@ --- -import { getRenderedInteractionBreakdownData } from '../../lib/prepare-chart' +import { + getRenderedInteractionBreakdownData, + getRenderedInteractionWorkBreakdownData, +} from '../../lib/prepare-chart' +import { INTERACTION_WORK } from '../../lib/interaction-phases' import InteractionBreakdownChart from './InteractionBreakdownChart.astro' interface Props { @@ -7,12 +11,31 @@ interface Props { } const { testsKey } = Astro.props -const data = getRenderedInteractionBreakdownData(testsKey).sort( - (a, b) => - a.interactionLatency - b.interactionLatency || a.name.localeCompare(b.name), +const work = getRenderedInteractionWorkBreakdownData(testsKey) +const workToCommit = new Map( + work.map((datum) => [datum.name, datum.workToCommit]), ) +const byWorkToCommit = ( + a: { name: string; interactionLatency?: number }, + b: { name: string; interactionLatency?: number }, +) => + (workToCommit.get(a.name) ?? Infinity) - + (workToCommit.get(b.name) ?? Infinity) || + (a.interactionLatency ?? 0) - (b.interactionLatency ?? 0) || + a.name.localeCompare(b.name) +const data = getRenderedInteractionBreakdownData(testsKey).sort(byWorkToCommit) +work.sort(byWorkToCommit) --- +{ + work.length > 0 ? ( + + ) : null +} { data.length > 0 ? ( key), +] as const + +function hasInteractionWork( + timings: InteractionWorkTimings | undefined, +): timings is Required { + return ( + timings != null && + INTERACTION_WORK_KEYS.every((key) => finiteValue(timings[key]) != null) + ) +} + +function interactionWorkHistory( + records: readonly ChartMeasurement[], + testsKey: 'clientSideRendered' | 'serverSideRendered', +): InteractionWorkHistoryDatum[] { + return records.flatMap((record) => { + const timings = record[testsKey]?.interactionWork + if (!hasInteractionWork(timings)) return [] + + const version = record.frameworkVersion ?? 'Unknown' + return { + id: `${record.package ?? 'unknown'}:${version}`, + name: `v${version}`, + workToCommit: timings.workToCommit, + inputDelay: timings.inputDelay, + processingDuration: timings.processingDuration, + renderingWork: timings.renderingWork, + } + }) +} + +export function prepareClientSideRenderedInteractionWorkHistory( + records: readonly ChartMeasurement[], +): InteractionWorkHistoryDatum[] { + return interactionWorkHistory(records, 'clientSideRendered') +} + +export function prepareServerSideRenderedInteractionWorkHistory( + records: readonly ChartMeasurement[], +): InteractionWorkHistoryDatum[] { + return interactionWorkHistory(records, 'serverSideRendered') +} + export function prepareLighthouseClientSideRenderedTestsFcpMsMonthlyComparison( history: readonly ChartMeasurement[], latest: readonly ChartMeasurement[], diff --git a/packages/docs/src/lib/interaction-history.test.ts b/packages/docs/src/lib/interaction-history.test.ts index 116fbefd..ea449a0e 100644 --- a/packages/docs/src/lib/interaction-history.test.ts +++ b/packages/docs/src/lib/interaction-history.test.ts @@ -1,8 +1,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { + prepareClientSideRenderedInteractionWorkHistory, prepareLighthouseClientSideRenderedTestsInteractionHistory, prepareLighthouseServerSideRenderedTestsInteractionHistory, + prepareServerSideRenderedInteractionWorkHistory, } from './chart-measurements.ts' import { formatLighthouse } from './result-content.ts' @@ -119,3 +121,129 @@ test('history is empty when no version has interaction data', () => { [], ) }) + +const work = ( + inputDelay: number, + processingDuration: number, + renderingWork: number, +) => ({ + inputDelay, + processingDuration, + renderingWork, + workToCommit: inputDelay + processingDuration + renderingWork, +}) +const scalars = >(values: T) => + Object.fromEntries( + Object.entries(values).map(([key, value]) => [ + key, + { value, unit: 'ms' as const }, + ]), + ) as { [K in keyof T]: { value: number; unit: 'ms' } } + +function lighthouseResult(runs: ReturnType[] | number) { + const count = typeof runs === 'number' ? runs : runs.length + return { + name: 'csrLighthouse' as const, + project: 'app' as const, + iterations: { + runs: count, + iterations: Array.from({ length: count }, (_, index) => ({ + firstPaint: { value: 100, unit: 'ms' as const }, + firstContentfulPaint: { value: 100, unit: 'ms' as const }, + interaction: scalars(timings(1, 2, 20 - index)), + ...(typeof runs === 'number' + ? {} + : { interactionWork: scalars(runs[index]!) }), + })), + }, + firstPaint: metric(100), + firstContentfulPaint: metric(100), + interaction: { + interactionLatency: metric(20), + inputDelay: metric(1), + processingDuration: metric(2), + presentationDelay: metric(17), + }, + ...(typeof runs === 'number' + ? {} + : { + interactionWork: { + workToCommit: metric(5), + inputDelay: metric(1), + processingDuration: metric(2), + renderingWork: metric(2), + }, + }), + } +} + +test('rendered tests take interaction work parts from the run with the median work to commit', () => { + const formatted = formatLighthouse( + lighthouseResult([ + work(1, 9, 1), + work(1, 2, 1), + work(1, 3, 2), + work(2, 8, 4), + work(1, 2, 4), + ]), + ) + + assert.deepEqual( + { + workToCommit: formatted.interactionWork?.workToCommit, + inputDelay: formatted.interactionWork?.inputDelay, + processingDuration: formatted.interactionWork?.processingDuration, + renderingWork: formatted.interactionWork?.renderingWork, + }, + work(1, 2, 4), + ) + assert.equal(formatted.interactionTests.interactionLatency, 21) +}) + +test('rendered tests recorded without interaction work keep the Lighthouse breakdown only', () => { + const formatted = formatLighthouse(lighthouseResult(3)) + + assert.equal(formatted.interactionWork, undefined) + assert.equal(formatted.interactionTests.interactionLatency, 22) + + const partial = lighthouseResult([work(1, 2, 1), work(1, 2, 2)]) + delete (partial.iterations.iterations[1] as { interactionWork?: unknown }) + .interactionWork + assert.equal(formatLighthouse(partial).interactionWork, undefined) +}) + +test('work to commit history skips versions without interaction work', () => { + assert.deepEqual( + prepareClientSideRenderedInteractionWorkHistory([ + { + frameworkVersion: '1.0.0', + clientSideRendered: { interactionTests: timings(1, 2, 3) }, + }, + { + frameworkVersion: '1.5.0', + clientSideRendered: { + interactionTests: timings(1, 2, 3), + interactionWork: { ...work(1, 2, 1), renderingWork: Number.NaN }, + }, + }, + { + frameworkVersion: '2.0.0', + clientSideRendered: { + interactionTests: timings(1, 2, 3), + interactionWork: work(1, 2, 1), + }, + }, + ]), + [{ id: 'unknown:2.0.0', name: 'v2.0.0', ...work(1, 2, 1) }], + ) + assert.equal( + prepareServerSideRenderedInteractionWorkHistory([ + { + frameworkVersion: '2.0.0', + clientSideRendered: { interactionWork: work(1, 1, 1) }, + serverSideRendered: { interactionWork: work(4, 5, 6) }, + }, + ])[0]?.workToCommit, + 15, + ) +}) diff --git a/packages/docs/src/lib/interaction-phases.ts b/packages/docs/src/lib/interaction-phases.ts index db0875d5..e4c2b41f 100644 --- a/packages/docs/src/lib/interaction-phases.ts +++ b/packages/docs/src/lib/interaction-phases.ts @@ -1,32 +1,78 @@ import { stackGrays } from './chart-styles.ts' -import type { InteractionPhaseKey } from './types' +import type { InteractionPhaseKey, InteractionWorkPartKey } from './types' -/** Lighthouse INP phases in the order they occur, which is also the stack order bottom to top. */ -export const INTERACTION_PHASES: readonly { - key: InteractionPhaseKey +interface InteractionPart { + key: K label: string description: string color: string -}[] = [ - { - key: 'inputDelay', - label: 'Input delay', - description: - 'which starts when the user initiates an interaction with the page, and ends when the event callbacks for the interaction begin to run.', - color: stackGrays[0], - }, - { - key: 'processingDuration', - label: 'Processing duration', - description: - 'which consists of the time it takes for event callbacks to run to completion.', - color: stackGrays[1], - }, - { - key: 'presentationDelay', - label: 'Presentation delay', - description: - 'which is the time it takes for the browser to present the next frame which contains the visual result of the interaction.', - color: stackGrays[2], - }, -] +} + +const inputDelay = { + key: 'inputDelay', + label: 'Input delay', + description: + 'which starts when the user initiates an interaction with the page, and ends when the event callbacks for the interaction begin to run.', + color: stackGrays[0], +} as const + +const processingDuration = { + key: 'processingDuration', + label: 'Processing duration', + description: + 'which consists of the time it takes for event callbacks to run to completion.', + color: stackGrays[1], +} as const + +/** Lighthouse INP phases in the order they occur, which is also the stack order bottom to top. */ +export const INTERACTION_PHASES: readonly InteractionPart[] = + [ + inputDelay, + processingDuration, + { + key: 'presentationDelay', + label: 'Presentation delay', + description: + 'which is the time it takes for the browser to present the next frame which contains the visual result of the interaction.', + color: stackGrays[2], + }, + ] + +/** Parts of interaction work to commit, in stack order bottom to top. */ +export const INTERACTION_WORK_PARTS: readonly InteractionPart[] = + [ + inputDelay, + processingDuration, + { + key: 'renderingWork', + label: 'Rendering work', + description: + "which is the time the main thread is busy after the event callbacks, until the interaction's frame commits. It includes framework updates, style, layout, and paint, but not idle time waiting for the frame.", + color: stackGrays[2], + }, + ] + +export interface InteractionMetric { + totalKey: 'interactionLatency' | 'workToCommit' + summary: string + yAxisLabel: string + parts: readonly InteractionPart< + InteractionPhaseKey | InteractionWorkPartKey + >[] +} + +export const LIGHTHOUSE_INTERACTION: InteractionMetric = { + totalKey: 'interactionLatency', + summary: + 'Total latency is split into the three consecutive phases reported by Lighthouse.', + yAxisLabel: 'Interaction latency (ms)', + parts: INTERACTION_PHASES, +} + +export const INTERACTION_WORK: InteractionMetric = { + totalKey: 'workToCommit', + summary: + 'Work to commit is the sum of three parts, measured from the Chrome trace of the interaction. Idle time waiting for the next frame, and work after the frame is committed, such as raster and presentation, are not included.', + yAxisLabel: 'Work to commit (ms)', + parts: INTERACTION_WORK_PARTS, +} diff --git a/packages/docs/src/lib/prepare-chart.ts b/packages/docs/src/lib/prepare-chart.ts index 1a16a30f..9d1a5d8f 100644 --- a/packages/docs/src/lib/prepare-chart.ts +++ b/packages/docs/src/lib/prepare-chart.ts @@ -1,4 +1,7 @@ -import { INTERACTION_PHASES } from './interaction-phases' +import { + INTERACTION_PHASES, + INTERACTION_WORK_PARTS, +} from './interaction-phases' import { stackGrays } from './chart-styles' import { prepareInitialLoadResources } from './initial-load' import { getCollection } from 'astro:content' @@ -100,6 +103,23 @@ export function getRenderedInteractionBreakdownData( }) } +export function getRenderedInteractionWorkBreakdownData( + testsKey: RenderedTestsKey, +) { + return ssrRequestThroughputStats.flatMap((framework) => { + const work = framework[testsKey]?.interactionWork + if (work == null) return [] + + return { + name: framework.name, + workToCommit: work.workToCommit, + values: Object.fromEntries( + INTERACTION_WORK_PARTS.map(({ key }) => [key, work[key]]), + ), + } + }) +} + export type CWV = 'lcp' | 'cls' | 'fcp' | 'ttfb' | 'inp' export function getCWVStatsChartData(cwv: CWV, device: Device) { diff --git a/packages/docs/src/lib/result-content.ts b/packages/docs/src/lib/result-content.ts index bca60cd2..45223760 100644 --- a/packages/docs/src/lib/result-content.ts +++ b/packages/docs/src/lib/result-content.ts @@ -27,6 +27,25 @@ function interactionValues( } } +function formatInteractionWork( + value: NonNullable, +) { + const runs = value.iterations.iterations.flatMap((run) => + run.interactionWork ? [run.interactionWork] : [], + ) + if (!value.interactionWork || runs.length !== value.iterations.runs) return + const medianRun = runs.sort( + (a, b) => a.workToCommit.value - b.workToCommit.value, + )[Math.floor(runs.length / 2)]! + return { + ...value.interactionWork, + workToCommit: medianRun.workToCommit.value, + inputDelay: medianRun.inputDelay.value, + processingDuration: medianRun.processingDuration.value, + renderingWork: medianRun.renderingWork.value, + } +} + export function formatThroughput( value: NonNullable, ) { @@ -95,6 +114,7 @@ export function formatLighthouse( ...value.interaction, ...interactionValues(medianRun.interaction), }, + interactionWork: formatInteractionWork(value), } } diff --git a/packages/docs/src/lib/types.ts b/packages/docs/src/lib/types.ts index 453c6eaf..31e21cdb 100644 --- a/packages/docs/src/lib/types.ts +++ b/packages/docs/src/lib/types.ts @@ -93,6 +93,22 @@ export type InteractionTimings = Partial< Record > +export type InteractionWorkPartKey = + 'inputDelay' | 'processingDuration' | 'renderingWork' + +export type InteractionWorkTimings = Partial< + Record +> + +/** One framework version's interaction work to commit split into its parts. */ +export interface InteractionWorkHistoryDatum extends Record< + InteractionWorkPartKey | 'workToCommit', + number +> { + id: string + name: string +} + /** One framework version's interaction latency split into its phases. */ export interface InteractionHistoryDatum extends Record< InteractionPhaseKey | 'interactionLatency', @@ -133,6 +149,7 @@ export interface ChartMeasurement { firstPaint?: number firstContentfulPaint?: number interactionTests?: InteractionTimings + interactionWork?: InteractionWorkTimings } coldBuild?: { medianMs?: number @@ -145,6 +162,7 @@ export interface ChartMeasurement { firstPaint?: number firstContentfulPaint?: number interactionTests?: InteractionTimings + interactionWork?: InteractionWorkTimings } ssrRequestThroughput?: { requestThroughput?: number diff --git a/packages/results/src/benchmark-schemas.ts b/packages/results/src/benchmark-schemas.ts index 0f436302..54451b95 100644 --- a/packages/results/src/benchmark-schemas.ts +++ b/packages/results/src/benchmark-schemas.ts @@ -177,6 +177,24 @@ export const LighthouseInteractionTimingSummary = z.object({ presentationDelay: createIterationSummary('ms'), }) +/** + * Interaction scored from the trace: input delay and processing duration, plus renderer main-thread busy time from + * the end of processing until the interaction's frame commits. Absent from results recorded before it was measured. + */ +export const InteractionWorkIteration = z.object({ + workToCommit: createValueWithUnit('ms', z.number().positive()), + inputDelay: createValueWithUnit('ms', z.number().nonnegative()), + processingDuration: createValueWithUnit('ms', z.number().nonnegative()), + renderingWork: createValueWithUnit('ms', z.number().nonnegative()), +}) + +export const InteractionWorkSummary = z.object({ + workToCommit: createIterationSummary('ms'), + inputDelay: createIterationSummary('ms'), + processingDuration: createIterationSummary('ms'), + renderingWork: createIterationSummary('ms'), +}) + export const CSRLighthouseBenchmark = BaseBenchmark.extend({ project: z.literal('app'), name: z.literal('csrLighthouse'), @@ -185,11 +203,13 @@ export const CSRLighthouseBenchmark = BaseBenchmark.extend({ firstPaint: createValueWithUnit('ms', z.number().positive()), firstContentfulPaint: createValueWithUnit('ms', z.number().positive()), interaction: LighthouseInteractionTimingIteration, + interactionWork: InteractionWorkIteration.optional(), }), ), firstPaint: createIterationSummary('ms'), firstContentfulPaint: createIterationSummary('ms'), interaction: LighthouseInteractionTimingSummary, + interactionWork: InteractionWorkSummary.optional(), }) export const SSRLighthouseBenchmark = BaseBenchmark.extend({ @@ -200,11 +220,13 @@ export const SSRLighthouseBenchmark = BaseBenchmark.extend({ firstPaint: createValueWithUnit('ms', z.number().positive()), firstContentfulPaint: createValueWithUnit('ms', z.number().positive()), interaction: LighthouseInteractionTimingIteration, + interactionWork: InteractionWorkIteration.optional(), }), ), firstPaint: createIterationSummary('ms'), firstContentfulPaint: createIterationSummary('ms'), interaction: LighthouseInteractionTimingSummary, + interactionWork: InteractionWorkSummary.optional(), }) export const InitialLoadResourceType = z.enum(InitialLoadResourceOptions) diff --git a/packages/stats-generator/src/benchmark-artifacts.ts b/packages/stats-generator/src/benchmark-artifacts.ts index feda009b..bd6eafa7 100644 --- a/packages/stats-generator/src/benchmark-artifacts.ts +++ b/packages/stats-generator/src/benchmark-artifacts.ts @@ -98,6 +98,28 @@ const routerLoad = SSRLoadRouterLinksBenchmark.refine( loadArtifactError, ) +function hasInteractionWork(value: { + interactionWork?: unknown + iterations: { iterations: { interactionWork?: unknown }[] } +}) { + return ( + value.interactionWork !== undefined && + value.iterations.iterations.every( + (run) => run.interactionWork !== undefined, + ) + ) +} + +const interactionWorkError = 'Expected interaction work for every run' +const csrLighthouse = CSRLighthouseBenchmark.refine( + hasInteractionWork, + interactionWorkError, +) +const ssrLighthouse = SSRLighthouseBenchmark.refine( + hasInteractionWork, + interactionWorkError, +) + export const benchmarkArtifacts = [ [ 'install', @@ -223,13 +245,13 @@ export const benchmarkArtifacts = [ 'clientSideRendered', 'client-side-rendered', 'ci-stats.json', - browser.extend({ csrLighthouse: CSRLighthouseBenchmark }), + browser.extend({ csrLighthouse }), ], [ 'serverSideRendered', 'server-side-rendered', 'ci-stats.json', - browser.extend({ ssrLighthouse: SSRLighthouseBenchmark }), + browser.extend({ ssrLighthouse }), ], [ 'initialLoad', diff --git a/packages/stats-generator/src/client-rendering.test.ts b/packages/stats-generator/src/client-rendering.test.ts index 9203703b..44dddff6 100644 --- a/packages/stats-generator/src/client-rendering.test.ts +++ b/packages/stats-generator/src/client-rendering.test.ts @@ -2,6 +2,7 @@ import { test } from 'node:test' import assert from 'node:assert/strict' import { assembleClientRendering } from './client-rendering.ts' import { CSRLighthouseBenchmark } from '@framework-performance/results/benchmark-schemas' +import { benchmarkArtifacts } from './benchmark-artifacts.ts' test('assembles client rendering from exact raw browser measurements', () => { const iterations = [1, 3].map((value) => ({ @@ -13,6 +14,12 @@ test('assembles client rendering from exact raw browser measurements', () => { processingDuration: value, presentationDelay: 0, }, + interactionWork: { + workToCommit: value + 1, + inputDelay: 0, + processingDuration: value, + renderingWork: 1, + }, })) const result = assembleClientRendering(iterations) assert.deepEqual( @@ -23,6 +30,19 @@ test('assembles client rendering from exact raw browser measurements', () => { ], ) assert.equal(result.firstPaint.mean, 2) + assert.deepEqual(result.interactionWork?.workToCommit, { + unit: 'ms', + mean: 3, + median: 3, + min: 2, + max: 4, + }) + assert.deepEqual(result.iterations.iterations[1].interactionWork, { + workToCommit: { value: 4, unit: 'ms' }, + inputDelay: { value: 0, unit: 'ms' }, + processingDuration: { value: 3, unit: 'ms' }, + renderingWork: { value: 1, unit: 'ms' }, + }) assert.equal( CSRLighthouseBenchmark.safeParse({ ...result, @@ -38,3 +58,65 @@ test('assembles client rendering from exact raw browser measurements', () => { false, ) }) + +test('accepts results without interaction work but requires it in new artifacts', () => { + const result = assembleClientRendering( + [1, 3].map((value) => ({ + firstPaint: value, + firstContentfulPaint: value, + interactionTests: { + interactionLatency: value, + inputDelay: 0, + processingDuration: value, + presentationDelay: 0, + }, + interactionWork: { + workToCommit: value, + inputDelay: 0, + processingDuration: value, + renderingWork: 0, + }, + })), + ) + const { interactionWork: _interactionWork, ...stored } = result + const legacy = { + ...stored, + iterations: { + ...result.iterations, + iterations: result.iterations.iterations.map( + ({ interactionWork: _work, ...run }) => run, + ), + }, + } + assert.equal(CSRLighthouseBenchmark.safeParse(legacy).success, true) + + const artifact = benchmarkArtifacts.find( + ([name]) => name === 'clientSideRendered', + )![3] + const parse = (value: unknown) => + artifact.safeParse({ + frameworkVersion: '1.0.0', + browserVersion: 'Chrome', + csrLighthouse: value, + }).success + assert.equal(parse(result), true) + assert.equal(parse(legacy), false) + assert.equal(parse(stored), false) + assert.equal( + parse({ + ...result, + iterations: { + ...result.iterations, + iterations: [ + result.iterations.iterations[0], + legacy.iterations.iterations[1], + ], + }, + }), + false, + ) + assert.equal( + parse({ ...legacy, interactionWork: result.interactionWork }), + false, + ) +}) diff --git a/packages/stats-generator/src/client-rendering.ts b/packages/stats-generator/src/client-rendering.ts index 058b42c4..038998ff 100644 --- a/packages/stats-generator/src/client-rendering.ts +++ b/packages/stats-generator/src/client-rendering.ts @@ -35,5 +35,18 @@ export function assembleClientRendering(iterations: RenderedTestSample[]) { ]), ), }, + interactionWork: Object.fromEntries( + ( + [ + 'workToCommit', + 'inputDelay', + 'processingDuration', + 'renderingWork', + ] as const + ).map((key) => [ + key, + summarizeMetric(iterations.map((value) => value.interactionWork[key])), + ]), + ), }) } diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index b1981e3b..0021f6ee 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -10,6 +10,7 @@ import { getNavigationInteractionTimingFromLighthouse, prepareInteractionTraceForLighthouse, } from '../interaction-timing.ts' +import { getInteractionWorkFromTrace } from '../interaction-work.ts' import type { ClientSideRenderedBenchmarkResult, ClientSideRenderedRunResult, @@ -112,6 +113,10 @@ async function runOnce( const interactionArtifacts = flow.createArtifactsJson().gatherSteps[1]?.artifacts + // Scored before the Lighthouse normalization below changes Event Timing durations. + const interactionWork = getInteractionWorkFromTrace( + interactionArtifacts?.Trace, + ) if (fullDocumentNavigation) { prepareInteractionTraceForLighthouse(interactionArtifacts?.Trace) } @@ -137,7 +142,7 @@ async function runOnce( ) await page.close() - return { firstPaint, firstContentfulPaint, interaction } + return { firstPaint, firstContentfulPaint, interaction, interactionWork } } finally { await browser.close() } @@ -192,10 +197,20 @@ export async function runBenchmark( ) } + const interactionWork = results + .map((r) => r.interactionWork) + .filter((value) => value !== null) + if (interactionWork.length !== results.length) { + throw new Error( + `Interaction work could not be measured in ${results.length - interactionWork.length} of ${results.length} runs`, + ) + } + const samples = results.map((_, index) => ({ firstPaint: fp[index], firstContentfulPaint: firstContentfulPaint[index], interactionTests: interactions[index], + interactionWork: interactionWork[index], })) return { name: packageName, diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index 28ea7e75..644ae7b8 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -1,11 +1,13 @@ import type { z } from 'zod' import type { CSRLighthouseBenchmark } from '@framework-performance/results/benchmark-schemas' import type { InteractionTiming } from '../types.ts' +import type { InteractionWork } from '../interaction-work.ts' export interface ClientSideRenderedRunResult { firstPaint: number | null firstContentfulPaint: number | null interaction: InteractionTiming | null + interactionWork: InteractionWork | null } export interface ClientSideRenderedBenchmarkResult { diff --git a/packages/stats-generator/src/interaction-work.test.ts b/packages/stats-generator/src/interaction-work.test.ts new file mode 100644 index 00000000..1a3ae586 --- /dev/null +++ b/packages/stats-generator/src/interaction-work.test.ts @@ -0,0 +1,260 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { getInteractionWorkFromTrace } from './interaction-work.ts' + +const PID = 1 +const MAIN_TID = 10 +const COMPOSITOR_TID = 11 +const OTHER_PID = 2 +const TIME_ORIGIN = 2_000_000 + +const at = (ms: number) => TIME_ORIGIN + ms * 1000 + +function metadata() { + return [ + threadName(PID, MAIN_TID, 'CrRendererMain'), + threadName(PID, COMPOSITOR_TID, 'Compositor'), + threadName(OTHER_PID, MAIN_TID, 'CrRendererMain'), + ] +} + +function threadName(pid: number, tid: number, name: string) { + return { name: 'thread_name', ph: 'M', pid, tid, ts: 0, args: { name } } +} + +function eventTiming(data: { + interactionId?: number + timeStamp: number + duration: number + processingStart: number + processingEnd: number + commitFinishTime?: number +}) { + return { + name: 'EventTiming', + ph: 'b', + pid: PID, + tid: MAIN_TID, + ts: at(data.timeStamp), + args: { data: { interactionId: 7, ...data } }, + } +} + +function task( + start: number, + end: number, + { name = 'RunTask', pid = PID, tid = MAIN_TID } = {}, +) { + return { name, ph: 'X', pid, tid, ts: at(start), dur: (end - start) * 1000 } +} + +function click({ + commitFinishTime = 1006 as number | null, + tasks = [] as ReturnType[], +} = {}) { + return { + traceEvents: [ + ...metadata(), + eventTiming({ + timeStamp: 1000, + duration: 16, + processingStart: 1000.5, + processingEnd: 1001, + }), + eventTiming({ + timeStamp: 1000.2, + duration: 15.8, + processingStart: 1001.2, + processingEnd: 1003, + ...(commitFinishTime === null ? {} : { commitFinishTime }), + }), + task(999.8, 1003.2), + task(1003.5, 1005, { tid: COMPOSITOR_TID }), + task(1003.5, 1005, { pid: OTHER_PID }), + ...tasks, + task(1008, 1010), + ], + } +} + +test('excludes idle time before the frame and work after commit', () => { + assert.deepEqual( + getInteractionWorkFromTrace(click({ tasks: [task(1004.5, 1006.5)] })), + { + workToCommit: 4.7, + inputDelay: 0.5, + processingDuration: 2.5, + renderingWork: 1.7, + }, + ) +}) + +test('counts all main-thread time up to commit when there is no idle time', () => { + assert.deepEqual( + getInteractionWorkFromTrace( + click({ tasks: [task(1003.2, 1005), task(1005, 1006.5)] }), + ), + { + workToCommit: 6, + inputDelay: 0.5, + processingDuration: 2.5, + renderingWork: 3, + }, + ) +}) + +test('counts overlapping tasks once', () => { + assert.equal( + getInteractionWorkFromTrace( + click({ + tasks: [task(1005.2, 1006.2), task(1003.5, 1005.5), task(1004, 1005)], + }), + )?.renderingWork, + 2.7, + ) +}) + +test('returns null without commitFinishTime', () => { + assert.equal( + getInteractionWorkFromTrace( + click({ + commitFinishTime: null, + tasks: [task(1004.5, 1006.5), task(1005.5, 1005.8, { name: 'Commit' })], + }), + ), + null, + ) +}) + +test('returns null when commitFinishTime does not follow processing', () => { + assert.equal( + getInteractionWorkFromTrace( + click({ commitFinishTime: 1002, tasks: [task(1004.5, 1006.5)] }), + ), + null, + ) + assert.equal( + getInteractionWorkFromTrace({ + traceEvents: [ + ...metadata(), + eventTiming({ + timeStamp: 1000, + duration: 16, + processingStart: 1000.5, + processingEnd: 1001, + commitFinishTime: 1002, + }), + eventTiming({ + timeStamp: 1000.2, + duration: 15.8, + processingStart: 1001.2, + processingEnd: 1003, + }), + task(999.8, 1006.5), + ], + }), + null, + ) +}) + +test('caps the commit boundary at the end of the Event Timing duration', () => { + assert.equal( + getInteractionWorkFromTrace( + click({ commitFinishTime: 1030, tasks: [task(1004, 1020)] }), + )?.renderingWork, + 12.2, + ) +}) + +test('scores an interaction that ends with processing without commit data', () => { + assert.deepEqual( + getInteractionWorkFromTrace({ + traceEvents: [ + ...metadata(), + eventTiming({ + timeStamp: 1000.287, + duration: 0.754, + processingStart: 1000.791, + processingEnd: 1001.041, + }), + task(999.8, 1003.2), + ], + }), + { + workToCommit: 0.754, + inputDelay: 0.504, + processingDuration: 0.25, + renderingWork: 0, + }, + ) +}) + +test('returns null without an interaction or a renderer main thread', () => { + assert.equal(getInteractionWorkFromTrace(undefined), null) + assert.equal(getInteractionWorkFromTrace({ traceEvents: [] }), null) + assert.equal( + getInteractionWorkFromTrace({ + traceEvents: [ + ...metadata(), + eventTiming({ + interactionId: 0, + timeStamp: 1000, + duration: 16, + processingStart: 1000.5, + processingEnd: 1003, + commitFinishTime: 1006, + }), + task(999.8, 1003.2), + ], + }), + null, + ) + assert.equal( + getInteractionWorkFromTrace({ + traceEvents: click().traceEvents.filter( + (event) => event.name !== 'thread_name', + ), + }), + null, + ) + assert.equal( + getInteractionWorkFromTrace({ + traceEvents: [ + ...metadata(), + eventTiming({ + timeStamp: 1000, + duration: 0, + processingStart: 1000, + processingEnd: 1000, + }), + task(999.8, 1003.2), + ], + }), + null, + ) +}) + +test('scores the interaction with the longest Event Timing duration', () => { + const trace = click({ tasks: [task(1004.5, 1006.5)] }) + trace.traceEvents.unshift( + eventTiming({ + interactionId: 3, + timeStamp: 900, + duration: 8, + processingStart: 901, + processingEnd: 905, + commitFinishTime: 906, + }), + ) + trace.traceEvents.push( + eventTiming({ + interactionId: 9, + timeStamp: 1100, + duration: 4, + processingStart: 1101, + processingEnd: 1102, + commitFinishTime: 1103, + }), + ) + assert.equal(getInteractionWorkFromTrace(trace)?.workToCommit, 4.7) +}) diff --git a/packages/stats-generator/src/interaction-work.ts b/packages/stats-generator/src/interaction-work.ts new file mode 100644 index 00000000..60088dab --- /dev/null +++ b/packages/stats-generator/src/interaction-work.ts @@ -0,0 +1,220 @@ +export interface InteractionWork { + /** Input delay plus processing duration plus rendering work, in ms. */ + workToCommit: number + /** Input timestamp to the start of the first event handler, in ms. */ + inputDelay: number + /** Start of the first event handler to the end of the last, in ms. */ + processingDuration: number + /** Renderer main-thread busy time from the end of processing until the interaction's frame commits, in ms. */ + renderingWork: number +} + +interface EventTimingData { + interactionId?: number + timeStamp?: number + duration?: number + processingStart?: number + processingEnd?: number + commitFinishTime?: number +} + +interface TraceEvent { + name?: string + ph?: string + pid?: number + tid?: number + ts?: number + dur?: number + args?: { name?: string; data?: EventTimingData } +} + +type CompleteEvent = TraceEvent & { ts: number; dur: number } + +/** Trace-clock timestamps in microseconds. */ +interface Interaction { + pid: number + start: number + end: number + processingStart: number + processingEnd: number + commitFinish?: number +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) +} + +function maxDefined(a: number | undefined, b: number | undefined) { + return a === undefined ? b : b === undefined ? a : Math.max(a, b) +} + +/** + * Event Timing fields are milliseconds relative to the document's time origin, with microsecond precision. + * The start event's `ts` is the trace-clock time of its `timeStamp`. + */ +function toInteraction(event: TraceEvent): Interaction | undefined { + const data = event.args?.data + if ( + !isFiniteNumber(event.pid) || + !isFiniteNumber(event.ts) || + !isFiniteNumber(data?.timeStamp) || + !isFiniteNumber(data.duration) || + !isFiniteNumber(data.processingStart) || + !isFiniteNumber(data.processingEnd) + ) + return undefined + + const { ts } = event + const { timeStamp } = data + const toTraceTime = (ms: number) => Math.round(ts + (ms - timeStamp) * 1000) + return { + pid: event.pid, + start: ts, + end: toTraceTime(timeStamp + data.duration), + processingStart: toTraceTime(data.processingStart), + processingEnd: toTraceTime(data.processingEnd), + commitFinish: isFiniteNumber(data.commitFinishTime) + ? toTraceTime(data.commitFinishTime) + : undefined, + } +} + +/** Merges the Event Timing entries of each interaction and returns the one with the longest duration. */ +function findLongestInteraction( + events: readonly TraceEvent[], +): Interaction | undefined { + const interactions = new Map() + for (const event of events) { + const interactionId = event.args?.data?.interactionId + if (event.name !== 'EventTiming' || event.ph !== 'b' || !interactionId) + continue + const current = toInteraction(event) + if (!current) continue + + const key = `${current.pid}:${interactionId}` + const existing = interactions.get(key) + interactions.set( + key, + existing + ? { + pid: current.pid, + start: Math.min(existing.start, current.start), + end: Math.max(existing.end, current.end), + processingStart: Math.min( + existing.processingStart, + current.processingStart, + ), + processingEnd: Math.max( + existing.processingEnd, + current.processingEnd, + ), + commitFinish: maxDefined( + existing.commitFinish, + current.commitFinish, + ), + } + : current, + ) + } + + let longest: Interaction | undefined + for (const interaction of interactions.values()) { + if ( + !longest || + interaction.end - interaction.start > longest.end - longest.start + ) + longest = interaction + } + return longest +} + +function getMainThreadTasks(events: readonly TraceEvent[], pid: number) { + const threads = new Set( + events + .filter( + (event) => + event.ph === 'M' && + event.name === 'thread_name' && + event.args?.name === 'CrRendererMain' && + event.pid === pid && + isFiniteNumber(event.tid), + ) + .map((event) => event.tid), + ) + if (threads.size === 0) return undefined + return events.filter( + (event): event is CompleteEvent => + event.ph === 'X' && + event.name === 'RunTask' && + event.pid === pid && + threads.has(event.tid) && + isFiniteNumber(event.ts) && + isFiniteNumber(event.dur), + ) +} + +/** Total length of the union of the tasks' intervals, clipped to [from, to]. */ +function getBusyTime( + tasks: readonly CompleteEvent[], + from: number, + to: number, +): number { + const intervals = tasks + .map(({ ts, dur }) => [Math.max(ts, from), Math.min(ts + dur, to)]) + .filter(([start, end]) => end > start) + .sort((a, b) => a[0] - b[0]) + + let busy = 0 + let current: number[] | undefined + for (const interval of intervals) { + if (current && interval[0] <= current[1]) { + current[1] = Math.max(current[1], interval[1]) + continue + } + if (current) busy += current[1] - current[0] + current = interval + } + if (current) busy += current[1] - current[0] + return busy +} + +/** + * Scores the longest interaction in a Chrome trace as input delay plus processing duration, plus the renderer main + * thread's busy time from the end of processing until `commitFinishTime`, capped at the end of the Event Timing + * duration. Returns null when the trace has no interaction or renderer main thread, or when the duration extends past + * processing without a later `commitFinishTime`. + */ +export function getInteractionWorkFromTrace( + trace: unknown, +): InteractionWork | null { + const events = + (trace as { traceEvents?: readonly TraceEvent[] } | undefined) + ?.traceEvents ?? [] + const interaction = findLongestInteraction(events) + if (!interaction) return null + const tasks = getMainThreadTasks(events, interaction.pid) + const { start, end, processingStart, processingEnd, commitFinish } = + interaction + const commitBoundary = + end <= processingEnd + ? end + : commitFinish === undefined || commitFinish <= processingEnd + ? undefined + : Math.min(commitFinish, end) + if (!tasks || commitBoundary === undefined) return null + + const inputDelay = Math.round(processingStart - start) + const processingDuration = Math.round(processingEnd - processingStart) + const renderingWork = Math.round( + getBusyTime(tasks, processingEnd, commitBoundary), + ) + const workToCommit = inputDelay + processingDuration + renderingWork + if (inputDelay < 0 || processingDuration < 0 || workToCommit <= 0) return null + + return { + workToCommit: workToCommit / 1000, + inputDelay: inputDelay / 1000, + processingDuration: processingDuration / 1000, + renderingWork: renderingWork / 1000, + } +} diff --git a/packages/stats-generator/src/rendered-sample-metrics.ts b/packages/stats-generator/src/rendered-sample-metrics.ts index 45fdbf87..7fe86080 100644 --- a/packages/stats-generator/src/rendered-sample-metrics.ts +++ b/packages/stats-generator/src/rendered-sample-metrics.ts @@ -2,6 +2,7 @@ import type { RenderedTestSample } from './types.ts' export function renderedSampleMetrics(sample: RenderedTestSample) { const timing = sample.interactionTests + const work = sample.interactionWork return { firstPaint: { value: sample.firstPaint, unit: 'ms' }, firstContentfulPaint: { value: sample.firstContentfulPaint, unit: 'ms' }, @@ -11,5 +12,11 @@ export function renderedSampleMetrics(sample: RenderedTestSample) { processingDuration: { value: timing.processingDuration, unit: 'ms' }, presentationDelay: { value: timing.presentationDelay, unit: 'ms' }, }, + interactionWork: { + workToCommit: { value: work.workToCommit, unit: 'ms' }, + inputDelay: { value: work.inputDelay, unit: 'ms' }, + processingDuration: { value: work.processingDuration, unit: 'ms' }, + renderingWork: { value: work.renderingWork, unit: 'ms' }, + }, } } diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index d1540974..76646a13 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -46,6 +46,9 @@ async function main() { console.info(` Input delay: ${interaction.inputDelay.mean}ms`) console.info(` Processing: ${interaction.processingDuration.mean}ms`) console.info(` Presentation: ${interaction.presentationDelay.mean}ms`) + const work = result.clientSideRendered.interactionWork + console.info(` Work to commit: ${work?.workToCommit.mean}ms`) + console.info(` Rendering work: ${work?.renderingWork.mean}ms`) } main().catch((error) => { diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index fd0a5a89..cdb30d13 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -46,6 +46,9 @@ async function main() { console.info(` Input delay: ${interaction.inputDelay.mean}ms`) console.info(` Processing: ${interaction.processingDuration.mean}ms`) console.info(` Presentation: ${interaction.presentationDelay.mean}ms`) + const work = result.serverSideRendered.interactionWork + console.info(` Work to commit: ${work?.workToCommit.mean}ms`) + console.info(` Rendering work: ${work?.renderingWork.mean}ms`) } main().catch((error) => { diff --git a/packages/stats-generator/src/server-rendering.test.ts b/packages/stats-generator/src/server-rendering.test.ts index aa2c7c1f..a19ac3ae 100644 --- a/packages/stats-generator/src/server-rendering.test.ts +++ b/packages/stats-generator/src/server-rendering.test.ts @@ -2,6 +2,7 @@ import { test } from 'node:test' import assert from 'node:assert/strict' import { assembleServerRendering } from './server-rendering.ts' import { SSRLighthouseBenchmark } from '@framework-performance/results/benchmark-schemas' +import { benchmarkArtifacts } from './benchmark-artifacts.ts' test('assembles server rendering from exact raw browser measurements', () => { const iterations = [1, 3].map((value) => ({ @@ -13,6 +14,12 @@ test('assembles server rendering from exact raw browser measurements', () => { processingDuration: value, presentationDelay: 0, }, + interactionWork: { + workToCommit: value + 1, + inputDelay: 0, + processingDuration: value, + renderingWork: 1, + }, })) const result = assembleServerRendering(iterations) assert.deepEqual( @@ -23,6 +30,19 @@ test('assembles server rendering from exact raw browser measurements', () => { ], ) assert.equal(result.firstPaint.mean, 2) + assert.deepEqual(result.interactionWork?.workToCommit, { + unit: 'ms', + mean: 3, + median: 3, + min: 2, + max: 4, + }) + assert.deepEqual(result.iterations.iterations[1].interactionWork, { + workToCommit: { value: 4, unit: 'ms' }, + inputDelay: { value: 0, unit: 'ms' }, + processingDuration: { value: 3, unit: 'ms' }, + renderingWork: { value: 1, unit: 'ms' }, + }) assert.equal( SSRLighthouseBenchmark.safeParse({ ...result, @@ -38,3 +58,65 @@ test('assembles server rendering from exact raw browser measurements', () => { false, ) }) + +test('accepts results without interaction work but requires it in new artifacts', () => { + const result = assembleServerRendering( + [1, 3].map((value) => ({ + firstPaint: value, + firstContentfulPaint: value, + interactionTests: { + interactionLatency: value, + inputDelay: 0, + processingDuration: value, + presentationDelay: 0, + }, + interactionWork: { + workToCommit: value, + inputDelay: 0, + processingDuration: value, + renderingWork: 0, + }, + })), + ) + const { interactionWork: _interactionWork, ...stored } = result + const legacy = { + ...stored, + iterations: { + ...result.iterations, + iterations: result.iterations.iterations.map( + ({ interactionWork: _work, ...run }) => run, + ), + }, + } + assert.equal(SSRLighthouseBenchmark.safeParse(legacy).success, true) + + const artifact = benchmarkArtifacts.find( + ([name]) => name === 'serverSideRendered', + )![3] + const parse = (value: unknown) => + artifact.safeParse({ + frameworkVersion: '1.0.0', + browserVersion: 'Chrome', + ssrLighthouse: value, + }).success + assert.equal(parse(result), true) + assert.equal(parse(legacy), false) + assert.equal(parse(stored), false) + assert.equal( + parse({ + ...result, + iterations: { + ...result.iterations, + iterations: [ + result.iterations.iterations[0], + legacy.iterations.iterations[1], + ], + }, + }), + false, + ) + assert.equal( + parse({ ...legacy, interactionWork: result.interactionWork }), + false, + ) +}) diff --git a/packages/stats-generator/src/server-rendering.ts b/packages/stats-generator/src/server-rendering.ts index 7596b900..d1ca0a0a 100644 --- a/packages/stats-generator/src/server-rendering.ts +++ b/packages/stats-generator/src/server-rendering.ts @@ -35,5 +35,18 @@ export function assembleServerRendering(iterations: RenderedTestSample[]) { ]), ), }, + interactionWork: Object.fromEntries( + ( + [ + 'workToCommit', + 'inputDelay', + 'processingDuration', + 'renderingWork', + ] as const + ).map((key) => [ + key, + summarizeMetric(iterations.map((value) => value.interactionWork[key])), + ]), + ), }) } diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index b6dfc832..e5ac1c73 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -8,6 +8,7 @@ import { getNavigationInteractionTimingFromLighthouse, prepareInteractionTraceForLighthouse, } from '../interaction-timing.ts' +import { getInteractionWorkFromTrace } from '../interaction-work.ts' import type { ServerSideRenderedBenchmarkResult, ServerSideRenderedRunResult, @@ -86,6 +87,10 @@ async function runOnce( const interactionArtifacts = flow.createArtifactsJson().gatherSteps[1]?.artifacts + // Scored before the Lighthouse normalization below changes Event Timing durations. + const interactionWork = getInteractionWorkFromTrace( + interactionArtifacts?.Trace, + ) if (fullDocumentNavigation) { prepareInteractionTraceForLighthouse(interactionArtifacts?.Trace) } @@ -111,7 +116,7 @@ async function runOnce( ) await page.close() - return { firstPaint, firstContentfulPaint, interaction } + return { firstPaint, firstContentfulPaint, interaction, interactionWork } } finally { await browser.close() } @@ -168,10 +173,20 @@ export async function runBenchmark( ) } + const interactionWork = results + .map((r) => r.interactionWork) + .filter((value) => value !== null) + if (interactionWork.length !== results.length) { + throw new Error( + `Interaction work could not be measured in ${results.length - interactionWork.length} of ${results.length} runs`, + ) + } + const samples = results.map((_, index) => ({ firstPaint: fp[index], firstContentfulPaint: firstContentfulPaint[index], interactionTests: interactions[index], + interactionWork: interactionWork[index], })) return { name: packageName, diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 1e28590b..9e949e61 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -1,10 +1,12 @@ import type { z } from 'zod' import type { SSRLighthouseBenchmark } from '@framework-performance/results/benchmark-schemas' import type { InteractionTiming } from '../types.ts' +import type { InteractionWork } from '../interaction-work.ts' export interface ServerSideRenderedRunResult { firstPaint: number | null firstContentfulPaint: number | null interaction: InteractionTiming | null + interactionWork: InteractionWork | null } export interface ServerSideRenderedBenchmarkResult { diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 446c9c36..61e5b4e9 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -1,3 +1,4 @@ +import type { InteractionWork } from './interaction-work.ts' export type { FrameworkConfig, TestConfig, @@ -14,6 +15,7 @@ export interface RenderedTestSample { firstPaint: number firstContentfulPaint: number interactionTests: InteractionTiming + interactionWork: InteractionWork } export interface TimeStat {
{title}
- Total latency is split into the three consecutive phases reported by - Lighthouse. -
{metric.summary}