Ja/jr/auto instrumentation in server

Production.dockerfile

@@ -28,4 +28,4 @@ ENV NODE_ENV=production
 # Expose the port that the application listens on
 EXPOSE 9000
 
-CMD ["node", "/app/server.js"]
+CMD ["node", "--require", "/app/instrumentation.js", "/app/server.js"]

dotcom-rendering/package.json

@@ -46,6 +46,10 @@
 		"@guardian/source-development-kitchen": "29.1.0",
 		"@guardian/support-dotcom-components": "11.0.0",
 		"@guardian/tsconfig": "catalog:",
+		"@opentelemetry/api": "1.9.1",
+		"@opentelemetry/exporter-trace-otlp-proto": "0.219.0",
+		"@opentelemetry/instrumentation-http": "0.219.0",
+		"@opentelemetry/sdk-node": "0.219.0",
 		"@playwright/test": "1.60.0",
 		"@sentry/browser": "10.65.0",
 		"@storybook/addon-a11y": "10.5.7",

dotcom-rendering/src/server/instrumentation.ts

@@ -0,0 +1,54 @@
+import { diag, DiagLogLevel } from '@opentelemetry/api';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+// The OTel packages are pinned to 0.219: 0.220+ breaks webpack production builds until the fix
+// in webpack/webpack#21623 ships. See open-telemetry/opentelemetry-js#6981.
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { logger } from './lib/logging';
+
+// Route OTel's internal diagnostics through DCR's logger so that export failures appear in CloudWatch rather than
+// being silently ignored.
+//
+// This is important because the OTLP exporter will silently fail if it can't reach the endpoint, which can happen if
+// the (nonessential) endpoint is down.
+diag.setLogger(
+	{
+		error: (message, ...args) => logger.error(message, ...args),
+		warn: (message, ...args) => logger.warn(message, ...args),
+		info: (message, ...args) => logger.info(message, ...args),
+		debug: (message, ...args) => logger.debug(message, ...args),
+		verbose: (message, ...args) => logger.debug(message, ...args),
+	},
+	DiagLogLevel.WARN, // WARN and above only — ERROR covers export failures
+);
+
+// This code is mostly boilerplate taken from the Node SDK README
+//
+// Note: we can't use auto instrumentation as it doesn't work natively with webpack,
+// so we call the only module that works which is HttpInstrumentation
+//
+// @see https://opentelemetry.io/docs/languages/js/getting-started/nodejs/
+// @see https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-sdk-node/README.md
+const sdk = new NodeSDK({
+	traceExporter: new OTLPTraceExporter(),
+	instrumentations: [
+		new HttpInstrumentation({
+			// The load balancer polls this every few seconds
+			ignoreIncomingRequestHook: (request) =>
+				request.url === '/_healthcheck',
+			// Rendering makes no outbound calls, so all we'd trace is AWS SDK background chatter
+			ignoreOutgoingRequestHook: () => true,
+		}),
+	],
+});
+
+sdk.start();
+
+process.on('SIGTERM', () => {
+	sdk.shutdown()
+		.then(
+			() => console.log('SDK shut down successfully'),
+			(err) => console.log('Error shutting down SDK', err),
+		)
+		.finally(() => process.exit(0));
+});

dotcom-rendering/src/server/server.prod.ts

@@ -1,5 +1,11 @@
+import { trace } from '@opentelemetry/api';
 import compression from 'compression';
-import type { ErrorRequestHandler, Request, Response } from 'express';
+import type {
+	ErrorRequestHandler,
+	Request,
+	RequestHandler,
+	Response,
+} from 'express';
 import express from 'express';
 import { NotRenderableInDCR } from '../lib/errors/not-renderable-in-dcr';
 import { handleAllEditorialNewslettersPage } from './handler.allEditorialNewslettersPage.web';
@@ -32,12 +38,30 @@ import { logger } from './lib/logging';
 import { requestLoggerMiddleware } from './lib/logging-middleware';
 import { recordError } from './lib/logging-store';
 
+// Spans are hand-made because @opentelemetry/instrumentation-express hooks
+// `require`, and webpack has inlined express into the server bundle.
+// There is a community plugin for webpack that enables this auto-instrumentation
+// but we'd prefer to avoid using it.
+// @see https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-instrumentation/README.md#limitations
+// @see https://opentelemetry.io/docs/languages/js/instrumentation/#acquiring-a-tracer
+const tracer = trace.getTracer('dotcom-rendering');
+
 export const prodServer = (): void => {
 	logger.info('dotcom-rendering is GO.');
 
 	const app = express();
 
-	app.use(express.json({ limit: '50mb' }));
+	// Starts a span just before calling express.json, and ends it on its callback
+	const expressJson = express.json({ limit: '50mb' });
+	const expressJsonWrapper: RequestHandler = (req, res, next) => {
+		const span = tracer.startSpan('express.json');
+		expressJson(req, res, (error) => {
+			span.end();
+			next(error);
+		});
+	};
+
+	app.use(expressJsonWrapper);
 	app.use(requestLoggerMiddleware);
 	app.use(compression());
 	app.use(responseHeaderMiddleware);
@@ -53,6 +77,17 @@ export const prodServer = (): void => {
 		app.use('/assets', express.static(__dirname));
 	}
 
+	// Starts a span until the http connection is closed
+	// Because it's the last middleware before the request handler,
+	// the span will effectively "wrap" the request handler
+	const requestHandlerWrapper: RequestHandler = (req, res, next) => {
+		const span = tracer.startSpan('request handler');
+		res.on('close', () => span.end());
+		next();
+	};
+
+	app.use(requestHandlerWrapper);
+
 	app.post('/Article', handleArticle);
 	app.post('/Interactive', handleInteractive);
 	app.post('/Blocks', handleBlocks);

dotcom-rendering/webpack/webpack.config.server.js

@@ -27,6 +27,7 @@ const swcLoader = [
 module.exports = {
 	entry: {
 		server: './src/server/server.ts',
+		instrumentation: './src/server/instrumentation.ts',
 	},
 	output: {
 		filename: `[name].js`,