From 6a472130a67362a844281a075ca96797e351879d Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 2 Jun 2026 06:41:15 +0200 Subject: [PATCH 1/2] Add opt-in sourcemap upload failure metrics Add metrics for Error Tracking sourcemap upload retries and final upload failures. These make intake instability observable without parsing CI logs. Emit the metrics only when the existing metrics plugin is enabled, preserving the default behavior for customers who use Error Tracking without build metrics. Batch metric emission after each upload phase and keep metric publishing non-fatal so sourcemap uploads are not affected by telemetry failures. Include tags for bundler, plugin version, service, site, status code, error type, and CI job context when available. --- .../plugins/error-tracking/src/index.test.ts | 27 ++++ packages/plugins/error-tracking/src/index.ts | 3 + .../error-tracking/src/sourcemaps/index.ts | 1 + .../src/sourcemaps/sender.test.ts | 101 +++++++++++- .../error-tracking/src/sourcemaps/sender.ts | 28 +++- .../src/sourcemaps/upload-metrics.ts | 150 ++++++++++++++++++ 6 files changed, 303 insertions(+), 7 deletions(-) create mode 100644 packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts diff --git a/packages/plugins/error-tracking/src/index.test.ts b/packages/plugins/error-tracking/src/index.test.ts index 5573868d4..5f11f2726 100644 --- a/packages/plugins/error-tracking/src/index.test.ts +++ b/packages/plugins/error-tracking/src/index.test.ts @@ -32,6 +32,33 @@ describe('Error Tracking Plugin', () => { expect(uploadSourcemapsMock).toHaveBeenCalledTimes(BUNDLERS.length); }); + test('Should not send sourcemap upload metrics unless metrics are enabled.', async () => { + await runBundlers({ + enableGit: false, + errorTracking: { + sourcemaps: getSourcemapsConfiguration(), + }, + }); + + expect(uploadSourcemapsMock.mock.calls[0][1]).toMatchObject({ + sendMetrics: false, + }); + }); + + test('Should send sourcemap upload metrics when metrics are enabled.', async () => { + await runBundlers({ + enableGit: false, + errorTracking: { + sourcemaps: getSourcemapsConfiguration(), + }, + metrics: {}, + }); + + expect(uploadSourcemapsMock.mock.calls[0][1]).toMatchObject({ + sendMetrics: true, + }); + }); + test('Should not process the sourcemaps with no options.', async () => { await runBundlers({ enableGit: false, diff --git a/packages/plugins/error-tracking/src/index.ts b/packages/plugins/error-tracking/src/index.ts index 5d8d353b3..8b44a755f 100644 --- a/packages/plugins/error-tracking/src/index.ts +++ b/packages/plugins/error-tracking/src/index.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { resolveEnable } from '@dd/core/helpers/options'; import { shouldGetGitInfo } from '@dd/core/helpers/plugins'; import type { BuildReport, GetPlugins, RepositoryData } from '@dd/core/types'; @@ -22,6 +23,7 @@ export const getPlugins: GetPlugins = ({ options, context }) => { const timeOptions = log.time('validate options'); const validatedOptions = validateOptions(options, log); timeOptions.end(); + const sendSourcemapUploadMetrics = resolveEnable(options, 'metrics', log); let gitInfo: RepositoryData | undefined; let buildReport: BuildReport | undefined; @@ -42,6 +44,7 @@ export const getPlugins: GetPlugins = ({ options, context }) => { git: gitInfo, outDir: context.bundler.outDir, outputs: buildReport?.outputs || [], + sendMetrics: sendSourcemapUploadMetrics, site: context.auth.site, version: context.version, }, diff --git a/packages/plugins/error-tracking/src/sourcemaps/index.ts b/packages/plugins/error-tracking/src/sourcemaps/index.ts index c2105d76f..c72329c1b 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/index.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/index.ts @@ -36,6 +36,7 @@ export const uploadSourcemaps = async ( bundlerName: context.bundlerName, git: context.git, outDir: context.outDir, + sendMetrics: context.sendMetrics, site: context.site, version: context.version, }, diff --git a/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts b/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts index 56ad02553..9cccd7491 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts @@ -11,6 +11,7 @@ import { SOURCEMAPS_API_SUBDOMAIN, SOURCEMAPS_API_PATH, } from '@dd/error-tracking-plugin/sourcemaps/sender'; +import { SOURCEMAP_UPLOAD_METRIC_PREFIX } from '@dd/error-tracking-plugin/sourcemaps/upload-metrics'; import { getContextMock, mockLogFn, @@ -165,6 +166,8 @@ describe('Error Tracking Plugin Sourcemaps', () => { describe('upload', () => { beforeEach(() => { + doRequestMock.mockReset(); + // Add some fixtures. addFixtureFiles({ '/path/to/minified.min.js': 'Some JS File with some content.', @@ -173,7 +176,7 @@ describe('Error Tracking Plugin Sourcemaps', () => { }); test('Should not throw', async () => { - doRequestMock.mockImplementation(jest.fn()); + doRequestMock.mockResolvedValue(undefined); const payloads = [getPayloadMock()]; @@ -190,7 +193,9 @@ describe('Error Tracking Plugin Sourcemaps', () => { }); test('Should alert in case of errors', async () => { - doRequestMock.mockRejectedValue(new Error('Fake Error')); + doRequestMock + .mockRejectedValueOnce(new Error('Fake Error')) + .mockResolvedValueOnce(undefined); const payloads = [getPayloadMock()]; const { warnings, errors } = await upload( @@ -209,10 +214,13 @@ describe('Error Tracking Plugin Sourcemaps', () => { error: new Error('Fake Error'), }); expect(warnings).toHaveLength(0); + expect(doRequestMock).toHaveBeenCalledTimes(1); }); test('Should throw in case of errors with bailOnError', async () => { - doRequestMock.mockRejectedValue(new Error('Fake Error')); + doRequestMock + .mockRejectedValueOnce(new Error('Fake Error')) + .mockResolvedValueOnce(undefined); const payloads = [getPayloadMock()]; await expect( @@ -224,5 +232,92 @@ describe('Error Tracking Plugin Sourcemaps', () => { ), ).rejects.toThrow('Fake Error'); }); + + test('Should send retry metrics for temporary upload failures', async () => { + const retryError = new Error('HTTP 408 Request Timeout\nstream timeout'); + doRequestMock.mockImplementation(async (opts) => { + opts.onRetry?.(retryError, 1); + }); + + const payloads = [getPayloadMock()]; + const { warnings, errors } = await upload( + payloads, + getSourcemapsConfiguration(), + { ...uploadContextMock, sendMetrics: true }, + mockLogger, + ); + + expect(warnings).toHaveLength(1); + expect(errors).toHaveLength(0); + expect(doRequestMock).toHaveBeenCalledTimes(2); + const metricsRequest = doRequestMock.mock.calls[1][0]; + expect(metricsRequest).toMatchObject({ + method: 'POST', + url: `https://api.${uploadContextMock.site}/api/v1/series?api_key=${uploadContextMock.apiKey}`, + getData: expect.any(Function), + }); + const metricData = await metricsRequest.getData!(); + + expect(JSON.parse(metricData.data as string)).toMatchObject({ + series: [ + { + metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.retry`, + type: 'count', + points: [[expect.any(Number), 1]], + tags: expect.arrayContaining([ + `bundler:${uploadContextMock.bundlerName}`, + `plugin_version:${uploadContextMock.version}`, + 'service:error-tracking-build-plugin-sourcemaps', + `site:${uploadContextMock.site}`, + 'attempt:1', + 'status_code:408', + 'error_type:http_408', + ]), + }, + ], + }); + }); + + test('Should send final failure metrics for exhausted upload retries', async () => { + doRequestMock + .mockRejectedValueOnce(new Error('HTTP 408 Request Timeout\nstream timeout')) + .mockResolvedValueOnce(undefined); + + const payloads = [getPayloadMock()]; + const { warnings, errors } = await upload( + payloads, + getSourcemapsConfiguration(), + { ...uploadContextMock, sendMetrics: true }, + mockLogger, + ); + + expect(warnings).toHaveLength(0); + expect(errors).toHaveLength(1); + const metricsRequest = doRequestMock.mock.calls[1][0]; + expect(metricsRequest).toMatchObject({ + method: 'POST', + url: `https://api.${uploadContextMock.site}/api/v1/series?api_key=${uploadContextMock.apiKey}`, + getData: expect.any(Function), + }); + const metricData = await metricsRequest.getData!(); + + expect(JSON.parse(metricData.data as string)).toMatchObject({ + series: [ + { + metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.failure`, + type: 'count', + points: [[expect.any(Number), 1]], + tags: expect.arrayContaining([ + `bundler:${uploadContextMock.bundlerName}`, + `plugin_version:${uploadContextMock.version}`, + 'service:error-tracking-build-plugin-sourcemaps', + `site:${uploadContextMock.site}`, + 'status_code:408', + 'error_type:http_408', + ]), + }, + ], + }); + }); }); }); diff --git a/packages/plugins/error-tracking/src/sourcemaps/sender.ts b/packages/plugins/error-tracking/src/sourcemaps/sender.ts index 51e88b8f7..39e143aa5 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/sender.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/sender.ts @@ -4,8 +4,13 @@ import { getDDEnvValue } from '@dd/core/helpers/env'; import { getFile } from '@dd/core/helpers/fs'; -import { createRequestData, type RequestData } from '@dd/core/helpers/request'; -import { doRequest, getOriginHeaders, NB_RETRIES } from '@dd/core/helpers/request'; +import { + createRequestData, + doRequest, + getOriginHeaders, + NB_RETRIES, + type RequestData, +} from '@dd/core/helpers/request'; import { formatDuration, prettyObject } from '@dd/core/helpers/strings'; import type { Logger, RepositoryData } from '@dd/core/types'; import chalk from 'chalk'; @@ -15,6 +20,12 @@ import type { SourcemapsOptionsWithDefaults, Sourcemap } from '../types'; import type { Metadata, MultipartFileValue, Payload } from './payload'; import { getPayload } from './payload'; +import { + createSourcemapUploadMetrics, + recordSourcemapUploadFailure, + recordSourcemapUploadRetry, + sendSourcemapUploadMetrics, +} from './upload-metrics'; const green = chalk.green.bold; const yellow = chalk.yellow.bold; @@ -59,6 +70,7 @@ export const getData = export type UploadContext = { apiKey?: string; bundlerName: string; + sendMetrics?: boolean; site: string; version: string; outDir: string; @@ -93,6 +105,7 @@ export const upload = async ( plugin: 'sourcemaps', version: context.version, }); + const uploadMetrics = createSourcemapUploadMetrics(options, context); // Show a pretty summary of the configuration. const configurationString = prettyObject({ @@ -130,6 +143,7 @@ export const upload = async ( getData: getData(payload, defaultHeaders), // On retry we store the error as a warning. onRetry: (error: Error, attempt: number) => { + recordSourcemapUploadRetry(uploadMetrics, error, attempt); const warningMessage = `Failed to upload ${yellow(metadata.sourcemap)} | ${yellow(metadata.file)}:\n ${error.message}\nRetrying ${attempt}/${NB_RETRIES}`; // This will be logged at the end of the process. warnings.push(warningMessage); @@ -137,6 +151,7 @@ export const upload = async ( }, }); } catch (e: any) { + recordSourcemapUploadFailure(uploadMetrics, e); errors.push({ metadata, error: e }); // Depending on the configuration we throw or not. if (options.bailOnError === true) { @@ -149,8 +164,13 @@ export const upload = async ( queueTimer.end(); log.debug(`Queued ${green(payloads.length.toString())} uploads.`); - await Promise.all(addPromises); - await queue.onIdle(); + try { + await Promise.all(addPromises); + await queue.onIdle(); + } finally { + await sendSourcemapUploadMetrics(uploadMetrics, context, log); + } + return { warnings, errors }; }; diff --git a/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts b/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts new file mode 100644 index 000000000..abda07dff --- /dev/null +++ b/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts @@ -0,0 +1,150 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { doRequest } from '@dd/core/helpers/request'; +import type { Logger, Metric } from '@dd/core/types'; + +import type { SourcemapsOptionsWithDefaults } from '../types'; + +import type { UploadContext } from './sender'; + +export const SOURCEMAP_UPLOAD_METRIC_PREFIX = 'datadog.build_plugins.sourcemaps.upload'; +const METRICS_API_PATH = 'api/v1/series'; + +type UploadMetricName = 'retry' | 'failure'; + +type UploadMetric = { + name: UploadMetricName; + value: number; + tags: string[]; +}; + +export type SourcemapUploadMetrics = { + metrics: Map; + baseTags: string[]; +}; + +const getMetricKey = (name: UploadMetricName, tags: string[]) => `${name}|${tags.join('|')}`; + +const getStatusCodeTag = (error: Error): string => { + const match = error.message.match(/HTTP (\d{3})/); + return match ? `status_code:${match[1]}` : 'status_code:unknown'; +}; + +const normalizeTagValue = (value: string): string => + value + .trim() + .toLowerCase() + .replace(/[^a-z0-9_:./-]+/g, '_') + .replace(/^_+|_+$/g, '') || 'unknown'; + +const getErrorTypeTag = (error: Error): string => { + const statusCodeTag = getStatusCodeTag(error); + if (statusCodeTag !== 'status_code:unknown') { + return `error_type:http_${statusCodeTag.replace('status_code:', '')}`; + } + + return `error_type:${normalizeTagValue(error.name || 'unknown')}`; +}; + +const getBaseMetricTags = (options: SourcemapsOptionsWithDefaults, context: UploadContext) => [ + `bundler:${context.bundlerName}`, + `plugin_version:${context.version}`, + `service:${options.service}`, + `site:${context.site}`, + ...(process.env.CI_JOB_NAME ? [`jobname:${normalizeTagValue(process.env.CI_JOB_NAME)}`] : []), + ...(process.env.BRANCH_TYPE + ? [`branchtype:${normalizeTagValue(process.env.BRANCH_TYPE)}`] + : []), +]; + +export const createSourcemapUploadMetrics = ( + options: SourcemapsOptionsWithDefaults, + context: UploadContext, +): SourcemapUploadMetrics => ({ + metrics: new Map(), + baseTags: getBaseMetricTags(options, context), +}); + +const incrementUploadMetric = ( + uploadMetrics: SourcemapUploadMetrics, + name: UploadMetricName, + tags: string[], +) => { + const metricKey = getMetricKey(name, tags); + const currentMetric = uploadMetrics.metrics.get(metricKey); + if (currentMetric) { + currentMetric.value++; + return; + } + + uploadMetrics.metrics.set(metricKey, { name, value: 1, tags }); +}; + +export const recordSourcemapUploadRetry = ( + uploadMetrics: SourcemapUploadMetrics, + error: Error, + attempt: number, +) => { + incrementUploadMetric(uploadMetrics, 'retry', [ + ...uploadMetrics.baseTags, + `attempt:${attempt}`, + getStatusCodeTag(error), + getErrorTypeTag(error), + ]); +}; + +export const recordSourcemapUploadFailure = ( + uploadMetrics: SourcemapUploadMetrics, + error: Error, +) => { + incrementUploadMetric(uploadMetrics, 'failure', [ + ...uploadMetrics.baseTags, + getStatusCodeTag(error), + getErrorTypeTag(error), + ]); +}; + +const buildUploadMetricSeries = (uploadMetrics: SourcemapUploadMetrics): Metric[] => { + const timestamp = Math.floor(Date.now() / 1000); + return Array.from(uploadMetrics.metrics.values()).map((metric) => ({ + metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.${metric.name}`, + type: 'count', + points: [[timestamp, metric.value]], + tags: metric.tags, + })); +}; + +export const sendSourcemapUploadMetrics = async ( + uploadMetrics: SourcemapUploadMetrics, + context: UploadContext, + log: Logger, +) => { + if (!context.sendMetrics) { + return; + } + + if (!uploadMetrics.metrics.size) { + return; + } + + if (!context.apiKey) { + log.debug(`Won't send sourcemap upload metrics to Datadog: missing API Key.`); + return; + } + + const series = buildUploadMetricSeries(uploadMetrics); + const url = `https://api.${context.site}/${METRICS_API_PATH}?api_key=${context.apiKey}`; + try { + await doRequest({ + method: 'POST', + url, + getData: () => ({ + data: JSON.stringify({ series }), + }), + }); + } catch (error) { + log.debug(`Error sending sourcemap upload metrics: ${error}`); + } +}; From d1d3c15ce4bca5a31ed3b07e4cfafb161d6a1238 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 3 Jun 2026 14:06:11 +0200 Subject: [PATCH 2/2] Route upload failure metrics through metrics plugin Replace the sourcemap-specific metrics sender with a shared internal metrics collector exposed on the global context. This lets error-tracking add upload retry and failure metrics while the metrics plugin still owns filtering, prefixing, hooks, and sending. Flush any collected metrics at true end so late sourcemap uploads keep the old awaited send behavior. --- packages/core/src/types.ts | 2 + packages/factory/src/helpers/context.ts | 4 + packages/factory/src/index.ts | 1 + packages/plugins/error-tracking/src/index.ts | 1 + .../error-tracking/src/sourcemaps/index.ts | 1 + .../src/sourcemaps/sender.test.ts | 83 +++++++---------- .../error-tracking/src/sourcemaps/sender.ts | 9 +- .../src/sourcemaps/upload-metrics.ts | 31 ++----- packages/plugins/metrics/src/collector.ts | 11 +++ .../plugins/metrics/src/common/filters.ts | 3 + packages/plugins/metrics/src/index.test.ts | 90 ++++++++++++++++++- packages/plugins/metrics/src/index.ts | 48 +++++++++- packages/tests/src/_jest/helpers/mocks.ts | 2 + packages/tools/src/helpers.ts | 1 + 14 files changed, 205 insertions(+), 82 deletions(-) create mode 100644 packages/plugins/metrics/src/collector.ts diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 1e6c45234..d466e5e81 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -179,6 +179,7 @@ export type TriggerHook = ( ...args: Parameters> ) => R; export type GlobalContext = { + addMetric: (metric: Metric) => void; asyncHook: TriggerHook>; auth: AuthOptionsWithDefaults; build: BuildReport; @@ -319,6 +320,7 @@ export type GlobalData = { export type GlobalStores = { errors: string[]; logs: Log[]; + metrics: Set; queue: Promise[]; timings: Timer[]; warnings: string[]; diff --git a/packages/factory/src/helpers/context.ts b/packages/factory/src/helpers/context.ts index 2df0aed7c..b0fdcdbde 100644 --- a/packages/factory/src/helpers/context.ts +++ b/packages/factory/src/helpers/context.ts @@ -34,6 +34,10 @@ export const getContext = ({ bundler: data.bundler, }; const context: GlobalContext = { + // This will be updated in the metrics plugin on initialization. + addMetric: () => { + throw new Error('AddMetric function called before it was initialized.'); + }, auth: options.auth, pluginNames: [], bundler: { diff --git a/packages/factory/src/index.ts b/packages/factory/src/index.ts index cd3ba8c2b..775800e2b 100644 --- a/packages/factory/src/index.ts +++ b/packages/factory/src/index.ts @@ -119,6 +119,7 @@ export const buildPluginFactory = ({ const stores: GlobalStores = { errors: [], logs: [], + metrics: new Set(), queue: [], timings: [], warnings: [], diff --git a/packages/plugins/error-tracking/src/index.ts b/packages/plugins/error-tracking/src/index.ts index 8b44a755f..12c51f3d7 100644 --- a/packages/plugins/error-tracking/src/index.ts +++ b/packages/plugins/error-tracking/src/index.ts @@ -42,6 +42,7 @@ export const getPlugins: GetPlugins = ({ options, context }) => { apiKey: context.auth.apiKey, bundlerName: context.bundler.name, git: gitInfo, + addMetric: context.addMetric, outDir: context.bundler.outDir, outputs: buildReport?.outputs || [], sendMetrics: sendSourcemapUploadMetrics, diff --git a/packages/plugins/error-tracking/src/sourcemaps/index.ts b/packages/plugins/error-tracking/src/sourcemaps/index.ts index c72329c1b..d8c0e2a77 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/index.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/index.ts @@ -32,6 +32,7 @@ export const uploadSourcemaps = async ( sourcemaps, options.sourcemaps, { + addMetric: context.addMetric, apiKey: context.apiKey, bundlerName: context.bundlerName, git: context.git, diff --git a/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts b/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts index 9cccd7491..4f1f68bac 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/sender.test.ts @@ -43,6 +43,7 @@ const doRequestMock = jest.mocked(doRequest); const contextMock = getContextMock(); const uploadContextMock = { + addMetric: contextMock.addMetric, apiKey: contextMock.auth.apiKey, bundlerName: contextMock.bundler.name, site: contextMock.auth.site, @@ -167,6 +168,7 @@ describe('Error Tracking Plugin Sourcemaps', () => { describe('upload', () => { beforeEach(() => { doRequestMock.mockReset(); + jest.mocked(contextMock.addMetric).mockReset(); // Add some fixtures. addFixtureFiles({ @@ -233,7 +235,7 @@ describe('Error Tracking Plugin Sourcemaps', () => { ).rejects.toThrow('Fake Error'); }); - test('Should send retry metrics for temporary upload failures', async () => { + test('Should add retry metrics for temporary upload failures', async () => { const retryError = new Error('HTTP 408 Request Timeout\nstream timeout'); doRequestMock.mockImplementation(async (opts) => { opts.onRetry?.(retryError, 1); @@ -249,36 +251,24 @@ describe('Error Tracking Plugin Sourcemaps', () => { expect(warnings).toHaveLength(1); expect(errors).toHaveLength(0); - expect(doRequestMock).toHaveBeenCalledTimes(2); - const metricsRequest = doRequestMock.mock.calls[1][0]; - expect(metricsRequest).toMatchObject({ - method: 'POST', - url: `https://api.${uploadContextMock.site}/api/v1/series?api_key=${uploadContextMock.apiKey}`, - getData: expect.any(Function), - }); - const metricData = await metricsRequest.getData!(); - - expect(JSON.parse(metricData.data as string)).toMatchObject({ - series: [ - { - metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.retry`, - type: 'count', - points: [[expect.any(Number), 1]], - tags: expect.arrayContaining([ - `bundler:${uploadContextMock.bundlerName}`, - `plugin_version:${uploadContextMock.version}`, - 'service:error-tracking-build-plugin-sourcemaps', - `site:${uploadContextMock.site}`, - 'attempt:1', - 'status_code:408', - 'error_type:http_408', - ]), - }, - ], + expect(doRequestMock).toHaveBeenCalledTimes(1); + expect(uploadContextMock.addMetric).toHaveBeenCalledWith({ + metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.retry`, + type: 'count', + points: [[expect.any(Number), 1]], + tags: expect.arrayContaining([ + `bundler:${uploadContextMock.bundlerName}`, + `plugin_version:${uploadContextMock.version}`, + 'service:error-tracking-build-plugin-sourcemaps', + `site:${uploadContextMock.site}`, + 'attempt:1', + 'status_code:408', + 'error_type:http_408', + ]), }); }); - test('Should send final failure metrics for exhausted upload retries', async () => { + test('Should add final failure metrics for exhausted upload retries', async () => { doRequestMock .mockRejectedValueOnce(new Error('HTTP 408 Request Timeout\nstream timeout')) .mockResolvedValueOnce(undefined); @@ -293,30 +283,19 @@ describe('Error Tracking Plugin Sourcemaps', () => { expect(warnings).toHaveLength(0); expect(errors).toHaveLength(1); - const metricsRequest = doRequestMock.mock.calls[1][0]; - expect(metricsRequest).toMatchObject({ - method: 'POST', - url: `https://api.${uploadContextMock.site}/api/v1/series?api_key=${uploadContextMock.apiKey}`, - getData: expect.any(Function), - }); - const metricData = await metricsRequest.getData!(); - - expect(JSON.parse(metricData.data as string)).toMatchObject({ - series: [ - { - metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.failure`, - type: 'count', - points: [[expect.any(Number), 1]], - tags: expect.arrayContaining([ - `bundler:${uploadContextMock.bundlerName}`, - `plugin_version:${uploadContextMock.version}`, - 'service:error-tracking-build-plugin-sourcemaps', - `site:${uploadContextMock.site}`, - 'status_code:408', - 'error_type:http_408', - ]), - }, - ], + expect(doRequestMock).toHaveBeenCalledTimes(1); + expect(uploadContextMock.addMetric).toHaveBeenCalledWith({ + metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.failure`, + type: 'count', + points: [[expect.any(Number), 1]], + tags: expect.arrayContaining([ + `bundler:${uploadContextMock.bundlerName}`, + `plugin_version:${uploadContextMock.version}`, + 'service:error-tracking-build-plugin-sourcemaps', + `site:${uploadContextMock.site}`, + 'status_code:408', + 'error_type:http_408', + ]), }); }); }); diff --git a/packages/plugins/error-tracking/src/sourcemaps/sender.ts b/packages/plugins/error-tracking/src/sourcemaps/sender.ts index 39e143aa5..503ad55c2 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/sender.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/sender.ts @@ -12,7 +12,7 @@ import { type RequestData, } from '@dd/core/helpers/request'; import { formatDuration, prettyObject } from '@dd/core/helpers/strings'; -import type { Logger, RepositoryData } from '@dd/core/types'; +import type { Logger, Metric, RepositoryData } from '@dd/core/types'; import chalk from 'chalk'; import PQueue from 'p-queue'; @@ -21,10 +21,10 @@ import type { SourcemapsOptionsWithDefaults, Sourcemap } from '../types'; import type { Metadata, MultipartFileValue, Payload } from './payload'; import { getPayload } from './payload'; import { + addSourcemapUploadMetrics, createSourcemapUploadMetrics, recordSourcemapUploadFailure, recordSourcemapUploadRetry, - sendSourcemapUploadMetrics, } from './upload-metrics'; const green = chalk.green.bold; @@ -68,6 +68,7 @@ export const getData = }; export type UploadContext = { + addMetric: (metric: Metric) => void; apiKey?: string; bundlerName: string; sendMetrics?: boolean; @@ -168,7 +169,7 @@ export const upload = async ( await Promise.all(addPromises); await queue.onIdle(); } finally { - await sendSourcemapUploadMetrics(uploadMetrics, context, log); + addSourcemapUploadMetrics(uploadMetrics, context); } return { warnings, errors }; @@ -230,6 +231,8 @@ export const sendSourcemaps = async ( version: context.version, outDir: context.outDir, site: context.site, + sendMetrics: context.sendMetrics, + addMetric: context.addMetric, }, log, ); diff --git a/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts b/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts index abda07dff..6d4249d84 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/upload-metrics.ts @@ -2,15 +2,13 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { doRequest } from '@dd/core/helpers/request'; -import type { Logger, Metric } from '@dd/core/types'; +import type { Metric } from '@dd/core/types'; import type { SourcemapsOptionsWithDefaults } from '../types'; import type { UploadContext } from './sender'; -export const SOURCEMAP_UPLOAD_METRIC_PREFIX = 'datadog.build_plugins.sourcemaps.upload'; -const METRICS_API_PATH = 'api/v1/series'; +export const SOURCEMAP_UPLOAD_METRIC_PREFIX = 'sourcemaps.upload'; type UploadMetricName = 'retry' | 'failure'; @@ -106,7 +104,7 @@ export const recordSourcemapUploadFailure = ( ]); }; -const buildUploadMetricSeries = (uploadMetrics: SourcemapUploadMetrics): Metric[] => { +export const getSourcemapUploadMetrics = (uploadMetrics: SourcemapUploadMetrics): Metric[] => { const timestamp = Math.floor(Date.now() / 1000); return Array.from(uploadMetrics.metrics.values()).map((metric) => ({ metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.${metric.name}`, @@ -116,10 +114,9 @@ const buildUploadMetricSeries = (uploadMetrics: SourcemapUploadMetrics): Metric[ })); }; -export const sendSourcemapUploadMetrics = async ( +export const addSourcemapUploadMetrics = ( uploadMetrics: SourcemapUploadMetrics, context: UploadContext, - log: Logger, ) => { if (!context.sendMetrics) { return; @@ -129,22 +126,8 @@ export const sendSourcemapUploadMetrics = async ( return; } - if (!context.apiKey) { - log.debug(`Won't send sourcemap upload metrics to Datadog: missing API Key.`); - return; - } - - const series = buildUploadMetricSeries(uploadMetrics); - const url = `https://api.${context.site}/${METRICS_API_PATH}?api_key=${context.apiKey}`; - try { - await doRequest({ - method: 'POST', - url, - getData: () => ({ - data: JSON.stringify({ series }), - }), - }); - } catch (error) { - log.debug(`Error sending sourcemap upload metrics: ${error}`); + const metrics = getSourcemapUploadMetrics(uploadMetrics); + for (const metric of metrics) { + context.addMetric(metric); } }; diff --git a/packages/plugins/metrics/src/collector.ts b/packages/plugins/metrics/src/collector.ts new file mode 100644 index 000000000..f29269f5e --- /dev/null +++ b/packages/plugins/metrics/src/collector.ts @@ -0,0 +1,11 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { GlobalContext, GlobalStores } from '@dd/core/types'; + +export const initializeMetricsCollector = (context: GlobalContext, stores: GlobalStores) => { + context.addMetric = (metric) => { + stores.metrics.add(metric); + }; +}; diff --git a/packages/plugins/metrics/src/common/filters.ts b/packages/plugins/metrics/src/common/filters.ts index 8a204a7c6..b6686df8b 100644 --- a/packages/plugins/metrics/src/common/filters.ts +++ b/packages/plugins/metrics/src/common/filters.ts @@ -25,6 +25,9 @@ const filterMetricsOnThreshold = (metric: Metric): Metric | null => { count: 10, duration: 1000, }; + if (/^sourcemaps\.upload\.(failure|retry)$/.test(metric.metric)) { + thresholds.count = 0; + } // Allow count for smaller results. if (/(entries|loaders|warnings|errors)\.count$/.test(metric.metric)) { thresholds.count = 0; diff --git a/packages/plugins/metrics/src/index.test.ts b/packages/plugins/metrics/src/index.test.ts index 8b580f42d..64a88837e 100644 --- a/packages/plugins/metrics/src/index.test.ts +++ b/packages/plugins/metrics/src/index.test.ts @@ -5,7 +5,11 @@ import { DEFAULT_SITE } from '@dd/core/constants'; import type { Options, Metric } from '@dd/core/types'; import { getPlugins } from '@dd/metrics-plugin'; -import { getGetPluginsArg, hardProjectEntries } from '@dd/tests/_jest/helpers/mocks'; +import { + getGetPluginsArg, + getMockBuildReport, + hardProjectEntries, +} from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; import type { Bundler } from '@dd/tests/_jest/helpers/types'; import nock from 'nock'; @@ -97,6 +101,90 @@ describe('Metrics Universal Plugin', () => { test('Should initialize the plugin', async () => { expect(getPlugins(getGetPluginsArg({ metrics: {} })).length).toBeGreaterThan(0); }); + + test('Should include collected metrics in the metrics package', async () => { + const arg = getGetPluginsArg({ + metrics: { + filters: [], + }, + }); + arg.stores.metrics.add({ + metric: 'sourcemaps.upload.failure', + type: 'count', + points: [[123, 1]], + tags: ['status_code:408'], + }); + + const plugins = getPlugins(arg); + const universalPlugin = plugins.find( + (plugin) => plugin.name === 'datadog-universal-metrics-plugin', + ); + + if (typeof universalPlugin?.buildReport === 'function') { + const buildReport = getMockBuildReport(); + await universalPlugin.buildReport(buildReport); + } + + expect(arg.context.asyncHook).toHaveBeenCalledWith('metrics', expect.any(Set)); + const metricsArg = jest.mocked(arg.context.asyncHook).mock.calls[0][1]; + if (!(metricsArg instanceof Set)) { + throw new Error('Expected the metrics hook payload to be a Set.'); + } + const metrics = Array.from(metricsArg); + expect(metrics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + metric: 'build.esbuild.sourcemaps.upload.failure', + points: [[123, 1]], + tags: ['status_code:408'], + toSend: true, + type: 'count', + }), + ]), + ); + }); + + test('Should flush collected metrics at the end of the build', async () => { + const arg = getGetPluginsArg({ + metrics: { + filters: [], + }, + }); + arg.stores.metrics.add({ + metric: 'sourcemaps.upload.retry', + type: 'count', + points: [[123, 1]], + tags: ['status_code:408'], + }); + + const plugins = getPlugins(arg); + const universalPlugin = plugins.find( + (plugin) => plugin.name === 'datadog-universal-metrics-plugin', + ); + + if (typeof universalPlugin?.asyncTrueEnd === 'function') { + await universalPlugin.asyncTrueEnd(); + } + + expect(arg.context.asyncHook).toHaveBeenCalledWith('metrics', expect.any(Set)); + const metricsArg = jest.mocked(arg.context.asyncHook).mock.calls[0][1]; + if (!(metricsArg instanceof Set)) { + throw new Error('Expected the metrics hook payload to be a Set.'); + } + const metrics = Array.from(metricsArg); + expect(metrics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + metric: 'build.esbuild.sourcemaps.upload.retry', + points: [[123, 1]], + tags: ['status_code:408'], + toSend: true, + type: 'count', + }), + ]), + ); + expect(arg.stores.metrics.size).toBe(0); + }); }); describe('With enableTracing', () => { diff --git a/packages/plugins/metrics/src/index.ts b/packages/plugins/metrics/src/index.ts index 209ae79c5..e6048541d 100644 --- a/packages/plugins/metrics/src/index.ts +++ b/packages/plugins/metrics/src/index.ts @@ -4,6 +4,7 @@ import type { BuildReport, GetPlugins, Metric, PluginOptions, TimingsReport } from '@dd/core/types'; +import { initializeMetricsCollector } from './collector'; import { getUniversalMetrics, getPluginMetrics, getLoaderMetrics } from './common/aggregator'; import { defaultFilters } from './common/filters'; import { getMetricsToSend, getTimestamp, validateOptions } from './common/helpers'; @@ -26,9 +27,10 @@ export type types = { MetricsOptions: MetricsOptions; }; -export const getPlugins: GetPlugins = ({ options, context }) => { +export const getPlugins: GetPlugins = ({ options, context, stores }) => { const log = context.getLogger(PLUGIN_NAME); let realBuildEnd: number = 0; + initializeMetricsCollector(context, stores); const validatedOptions = validateOptions(options, context.bundler.name); const plugins: PluginOptions[] = []; @@ -51,6 +53,39 @@ export const getPlugins: GetPlugins = ({ options, context }) => { let timingsReport: TimingsReport | undefined; let buildReport: BuildReport; + const getMetricsToSendFromCollectedMetrics = () => { + const metrics = new Set(stores.metrics); + stores.metrics.clear(); + + return getMetricsToSend( + metrics, + validatedOptions.timestamp, + validatedOptions.filters, + validatedOptions.tags, + validatedOptions.prefix, + ); + }; + + const sendCollectedMetrics = async () => { + if (!stores.metrics.size) { + return; + } + + const timeMetrics = log.time(`aggregating collected metrics`); + const metricsToSend = getMetricsToSendFromCollectedMetrics(); + + await context.asyncHook('metrics', metricsToSend); + timeMetrics.end(); + + const timeSend = log.time('sending collected metrics to Datadog'); + await sendMetrics( + metricsToSend, + { apiKey: context.auth.apiKey, site: context.auth.site }, + log, + ); + timeSend.end(); + }; + const computeMetrics = async () => { context.build.end = Date.now(); context.build.duration = context.build.end - context.build.start!; @@ -69,7 +104,13 @@ export const getPlugins: GetPlugins = ({ options, context }) => { const loaderMetrics = getLoaderMetrics(timingsReport?.loaders, timestamp); timeLoader.end(); - const allMetrics = new Set([...universalMetrics, ...pluginMetrics, ...loaderMetrics]); + const allMetrics = new Set([ + ...universalMetrics, + ...pluginMetrics, + ...loaderMetrics, + ...stores.metrics, + ]); + stores.metrics.clear(); const metricsToSend = getMetricsToSend( allMetrics, @@ -129,6 +170,9 @@ export const getPlugins: GetPlugins = ({ options, context }) => { await computeMetrics(); } }, + async asyncTrueEnd() { + await sendCollectedMetrics(); + }, }; if (validatedOptions.enableTracing) { diff --git a/packages/tests/src/_jest/helpers/mocks.ts b/packages/tests/src/_jest/helpers/mocks.ts index 5524afd68..decb1924f 100644 --- a/packages/tests/src/_jest/helpers/mocks.ts +++ b/packages/tests/src/_jest/helpers/mocks.ts @@ -89,6 +89,7 @@ export const getMockStores = (overrides: Partial = {}): GlobalStor logs: [], errors: [], warnings: [], + metrics: new Set(), queue: [], timings: [], ...overrides, @@ -227,6 +228,7 @@ export const getContextMock = (overrides: Partial = {}): GlobalCo env: 'test', getLogger: jest.fn(() => getMockLogger()), asyncHook: jest.fn(), + addMetric: jest.fn(), hook: jest.fn(), inject: jest.fn(), pluginNames: [], diff --git a/packages/tools/src/helpers.ts b/packages/tools/src/helpers.ts index fa6c7bcde..4d6382875 100644 --- a/packages/tools/src/helpers.ts +++ b/packages/tools/src/helpers.ts @@ -199,6 +199,7 @@ export const getSupportedBundlers = (getPlugins: GetPlugins) => { errors: [], warnings: [], logs: [], + metrics: new Set(), timings: [], queue: [], };