Add sourcemap upload failure metrics

packages/core/src/types.ts

@@ -179,6 +179,7 @@ export type TriggerHook<R> = <K extends keyof CustomHooks>(
     ...args: Parameters<NonNullable<CustomHooks[K]>>
 ) => R;
 export type GlobalContext = {
+    addMetric: (metric: Metric) => void;
     asyncHook: TriggerHook<Promise<void[]>>;
     auth: AuthOptionsWithDefaults;
     build: BuildReport;
@@ -319,6 +320,7 @@ export type GlobalData = {
 export type GlobalStores = {
     errors: string[];
     logs: Log[];
+    metrics: Set<Metric>;
     queue: Promise<any>[];
     timings: Timer[];
     warnings: string[];

packages/factory/src/helpers/context.ts

@@ -34,6 +34,10 @@ export const getContext = ({
         bundler: data.bundler,
     };
     const context: GlobalContext = {
+        // This will be updated in the metrics plugin on initialization.
+        addMetric: () => {
+            throw new Error('AddMetric function called before it was initialized.');
+        },
         auth: options.auth,
         pluginNames: [],
         bundler: {

packages/factory/src/index.ts

@@ -119,6 +119,7 @@ export const buildPluginFactory = ({
         const stores: GlobalStores = {
             errors: [],
             logs: [],
+            metrics: new Set(),
             queue: [],
             timings: [],
             warnings: [],

packages/plugins/error-tracking/src/index.test.ts

@@ -32,6 +32,33 @@ describe('Error Tracking Plugin', () => {
         expect(uploadSourcemapsMock).toHaveBeenCalledTimes(BUNDLERS.length);
     });
 
+    test('Should not send sourcemap upload metrics unless metrics are enabled.', async () => {
+        await runBundlers({
+            enableGit: false,
+            errorTracking: {
+                sourcemaps: getSourcemapsConfiguration(),
+            },
+        });
+
+        expect(uploadSourcemapsMock.mock.calls[0][1]).toMatchObject({
+            sendMetrics: false,
+        });
+    });
+
+    test('Should send sourcemap upload metrics when metrics are enabled.', async () => {
+        await runBundlers({
+            enableGit: false,
+            errorTracking: {
+                sourcemaps: getSourcemapsConfiguration(),
+            },
+            metrics: {},
+        });
+
+        expect(uploadSourcemapsMock.mock.calls[0][1]).toMatchObject({
+            sendMetrics: true,
+        });
+    });
+
     test('Should not process the sourcemaps with no options.', async () => {
         await runBundlers({
             enableGit: false,

packages/plugins/error-tracking/src/index.ts

@@ -2,6 +2,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
+import { resolveEnable } from '@dd/core/helpers/options';
 import { shouldGetGitInfo } from '@dd/core/helpers/plugins';
 import type { BuildReport, GetPlugins, RepositoryData } from '@dd/core/types';
 
@@ -22,6 +23,7 @@ export const getPlugins: GetPlugins = ({ options, context }) => {
     const timeOptions = log.time('validate options');
     const validatedOptions = validateOptions(options, log);
     timeOptions.end();
+    const sendSourcemapUploadMetrics = resolveEnable(options, 'metrics', log);
 
     let gitInfo: RepositoryData | undefined;
     let buildReport: BuildReport | undefined;
@@ -40,8 +42,10 @@ export const getPlugins: GetPlugins = ({ options, context }) => {
                 apiKey: context.auth.apiKey,
                 bundlerName: context.bundler.name,
                 git: gitInfo,
+                addMetric: context.addMetric,
                 outDir: context.bundler.outDir,
                 outputs: buildReport?.outputs || [],
+                sendMetrics: sendSourcemapUploadMetrics,
                 site: context.auth.site,
                 version: context.version,
             },

packages/plugins/error-tracking/src/sourcemaps/index.ts

@@ -32,10 +32,12 @@ export const uploadSourcemaps = async (
         sourcemaps,
         options.sourcemaps,
         {
+            addMetric: context.addMetric,
             apiKey: context.apiKey,
             bundlerName: context.bundlerName,
             git: context.git,
             outDir: context.outDir,
+            sendMetrics: context.sendMetrics,
             site: context.site,
             version: context.version,
         },

packages/plugins/error-tracking/src/sourcemaps/sender.test.ts

@@ -11,6 +11,7 @@ import {
     SOURCEMAPS_API_SUBDOMAIN,
     SOURCEMAPS_API_PATH,
 } from '@dd/error-tracking-plugin/sourcemaps/sender';
+import { SOURCEMAP_UPLOAD_METRIC_PREFIX } from '@dd/error-tracking-plugin/sourcemaps/upload-metrics';
 import {
     getContextMock,
     mockLogFn,
@@ -42,6 +43,7 @@ const doRequestMock = jest.mocked(doRequest);
 
 const contextMock = getContextMock();
 const uploadContextMock = {
+    addMetric: contextMock.addMetric,
     apiKey: contextMock.auth.apiKey,
     bundlerName: contextMock.bundler.name,
     site: contextMock.auth.site,
@@ -165,6 +167,9 @@ describe('Error Tracking Plugin Sourcemaps', () => {
 
     describe('upload', () => {
         beforeEach(() => {
+            doRequestMock.mockReset();
+            jest.mocked(contextMock.addMetric).mockReset();
+
             // Add some fixtures.
             addFixtureFiles({
                 '/path/to/minified.min.js': 'Some JS File with some content.',
@@ -173,7 +178,7 @@ describe('Error Tracking Plugin Sourcemaps', () => {
         });
 
         test('Should not throw', async () => {
-            doRequestMock.mockImplementation(jest.fn());
+            doRequestMock.mockResolvedValue(undefined);
 
             const payloads = [getPayloadMock()];
 
@@ -190,7 +195,9 @@ describe('Error Tracking Plugin Sourcemaps', () => {
         });
 
         test('Should alert in case of errors', async () => {
-            doRequestMock.mockRejectedValue(new Error('Fake Error'));
+            doRequestMock
+                .mockRejectedValueOnce(new Error('Fake Error'))
+                .mockResolvedValueOnce(undefined);
 
             const payloads = [getPayloadMock()];
             const { warnings, errors } = await upload(
@@ -209,10 +216,13 @@ describe('Error Tracking Plugin Sourcemaps', () => {
                 error: new Error('Fake Error'),
             });
             expect(warnings).toHaveLength(0);
+            expect(doRequestMock).toHaveBeenCalledTimes(1);
         });
 
         test('Should throw in case of errors with bailOnError', async () => {
-            doRequestMock.mockRejectedValue(new Error('Fake Error'));
+            doRequestMock
+                .mockRejectedValueOnce(new Error('Fake Error'))
+                .mockResolvedValueOnce(undefined);
 
             const payloads = [getPayloadMock()];
             await expect(
@@ -224,5 +234,69 @@ describe('Error Tracking Plugin Sourcemaps', () => {
                 ),
             ).rejects.toThrow('Fake Error');
         });
+
+        test('Should add retry metrics for temporary upload failures', async () => {
+            const retryError = new Error('HTTP 408 Request Timeout\nstream timeout');
+            doRequestMock.mockImplementation(async (opts) => {
+                opts.onRetry?.(retryError, 1);
+            });
+
+            const payloads = [getPayloadMock()];
+            const { warnings, errors } = await upload(
+                payloads,
+                getSourcemapsConfiguration(),
+                { ...uploadContextMock, sendMetrics: true },
+                mockLogger,
+            );
+
+            expect(warnings).toHaveLength(1);
+            expect(errors).toHaveLength(0);
+            expect(doRequestMock).toHaveBeenCalledTimes(1);
+            expect(uploadContextMock.addMetric).toHaveBeenCalledWith({
+                metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.retry`,
+                type: 'count',
+                points: [[expect.any(Number), 1]],
+                tags: expect.arrayContaining([
+                    `bundler:${uploadContextMock.bundlerName}`,
+                    `plugin_version:${uploadContextMock.version}`,
+                    'service:error-tracking-build-plugin-sourcemaps',
+                    `site:${uploadContextMock.site}`,
+                    'attempt:1',
+                    'status_code:408',
+                    'error_type:http_408',
+                ]),
+            });
+        });
+
+        test('Should add final failure metrics for exhausted upload retries', async () => {
+            doRequestMock
+                .mockRejectedValueOnce(new Error('HTTP 408 Request Timeout\nstream timeout'))
+                .mockResolvedValueOnce(undefined);
+
+            const payloads = [getPayloadMock()];
+            const { warnings, errors } = await upload(
+                payloads,
+                getSourcemapsConfiguration(),
+                { ...uploadContextMock, sendMetrics: true },
+                mockLogger,
+            );
+
+            expect(warnings).toHaveLength(0);
+            expect(errors).toHaveLength(1);
+            expect(doRequestMock).toHaveBeenCalledTimes(1);
+            expect(uploadContextMock.addMetric).toHaveBeenCalledWith({
+                metric: `${SOURCEMAP_UPLOAD_METRIC_PREFIX}.failure`,
+                type: 'count',
+                points: [[expect.any(Number), 1]],
+                tags: expect.arrayContaining([
+                    `bundler:${uploadContextMock.bundlerName}`,
+                    `plugin_version:${uploadContextMock.version}`,
+                    'service:error-tracking-build-plugin-sourcemaps',
+                    `site:${uploadContextMock.site}`,
+                    'status_code:408',
+                    'error_type:http_408',
+                ]),
+            });
+        });
     });
 });

packages/plugins/error-tracking/src/sourcemaps/sender.ts

@@ -4,17 +4,28 @@
 
 import { getDDEnvValue } from '@dd/core/helpers/env';
 import { getFile } from '@dd/core/helpers/fs';
-import { createRequestData, type RequestData } from '@dd/core/helpers/request';
-import { doRequest, getOriginHeaders, NB_RETRIES } from '@dd/core/helpers/request';
+import {
+    createRequestData,
+    doRequest,
+    getOriginHeaders,
+    NB_RETRIES,
+    type RequestData,
+} from '@dd/core/helpers/request';
 import { formatDuration, prettyObject } from '@dd/core/helpers/strings';
-import type { Logger, RepositoryData } from '@dd/core/types';
+import type { Logger, Metric, RepositoryData } from '@dd/core/types';
 import chalk from 'chalk';
 import PQueue from 'p-queue';
 
 import type { SourcemapsOptionsWithDefaults, Sourcemap } from '../types';
 
 import type { Metadata, MultipartFileValue, Payload } from './payload';
 import { getPayload } from './payload';
+import {
+    addSourcemapUploadMetrics,
+    createSourcemapUploadMetrics,
+    recordSourcemapUploadFailure,
+    recordSourcemapUploadRetry,
+} from './upload-metrics';
 
 const green = chalk.green.bold;
 const yellow = chalk.yellow.bold;
@@ -57,8 +68,10 @@ export const getData =
     };
 
 export type UploadContext = {
+    addMetric: (metric: Metric) => void;
     apiKey?: string;
     bundlerName: string;
+    sendMetrics?: boolean;
     site: string;
     version: string;
     outDir: string;
@@ -93,6 +106,7 @@ export const upload = async (
         plugin: 'sourcemaps',
         version: context.version,
     });
+    const uploadMetrics = createSourcemapUploadMetrics(options, context);
 
     // Show a pretty summary of the configuration.
     const configurationString = prettyObject({
@@ -130,13 +144,15 @@ export const upload = async (
                         getData: getData(payload, defaultHeaders),
                         // On retry we store the error as a warning.
                         onRetry: (error: Error, attempt: number) => {
+                            recordSourcemapUploadRetry(uploadMetrics, error, attempt);
                             const warningMessage = `Failed to upload ${yellow(metadata.sourcemap)} | ${yellow(metadata.file)}:\n  ${error.message}\nRetrying ${attempt}/${NB_RETRIES}`;
                             // This will be logged at the end of the process.
                             warnings.push(warningMessage);
                             log.debug(warningMessage);
                         },
                     });
                 } catch (e: any) {
+                    recordSourcemapUploadFailure(uploadMetrics, e);
                     errors.push({ metadata, error: e });
                     // Depending on the configuration we throw or not.
                     if (options.bailOnError === true) {
@@ -149,8 +165,13 @@ export const upload = async (
     queueTimer.end();
     log.debug(`Queued ${green(payloads.length.toString())} uploads.`);
 
-    await Promise.all(addPromises);
-    await queue.onIdle();
+    try {
+        await Promise.all(addPromises);
+        await queue.onIdle();
+    } finally {
+        addSourcemapUploadMetrics(uploadMetrics, context);
+    }
+
     return { warnings, errors };
 };
 
@@ -210,6 +231,8 @@ export const sendSourcemaps = async (
             version: context.version,
             outDir: context.outDir,
             site: context.site,
+            sendMetrics: context.sendMetrics,
+            addMetric: context.addMetric,
         },
         log,
     );