From 802e6134211856f0d289eb3c80f1f2087306264c Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 28 Jul 2026 09:51:28 +0100 Subject: [PATCH 1/6] fix(webapp,redis-worker): stop logging raw metadata, alert payloads, and job items Debug logging around run metadata writes previously included the full, unfiltered metadata object on every flush. It now logs key/operation counts and byte sizes instead, and skips the log entirely when there is nothing buffered. The webapp logger's redaction list also now covers `metadata` and `seedMetadata` as a backstop. Alert webhook delivery failures no longer log the outgoing request body or the (useless, non-serializable) HMAC signature; they log the response status, the webhook URL host, and the relevant ids instead. The redis-worker's failure, retry, and dead-letter logs no longer include the raw job item. The item is still retrievable by id when needed, and the default worker logger now filters the `item` key as a backstop. --- apps/webapp/app/services/logger.server.ts | 2 +- .../metadata/updateMetadata.server.ts | 17 ++++++---- .../v3/services/alerts/deliverAlert.server.ts | 34 +++++++++++++------ packages/redis-worker/src/worker.ts | 10 +++--- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/apps/webapp/app/services/logger.server.ts b/apps/webapp/app/services/logger.server.ts index 15b248f0d94..b03802afefd 100644 --- a/apps/webapp/app/services/logger.server.ts +++ b/apps/webapp/app/services/logger.server.ts @@ -50,7 +50,7 @@ function flattenArgs(args: Array | undefined>) { export const logger = new Logger( "webapp", (process.env.APP_LOG_LEVEL ?? "info") as LogLevel, - ["examples", "output", "connectionString", "payload"], + ["examples", "output", "connectionString", "payload", "metadata", "seedMetadata"], sensitiveDataReplacer, () => { const fields = currentFieldsStore.getStore(); diff --git a/apps/webapp/app/services/metadata/updateMetadata.server.ts b/apps/webapp/app/services/metadata/updateMetadata.server.ts index c44dacf65c5..a7449be387d 100644 --- a/apps/webapp/app/services/metadata/updateMetadata.server.ts +++ b/apps/webapp/app/services/metadata/updateMetadata.server.ts @@ -91,9 +91,14 @@ export class UpdateMetadataService { this._bufferedOperations.clear(); yield* Effect.sync(() => { - if (this.flushLoggingEnabled) { + if (this.flushLoggingEnabled && currentOperations.size > 0) { + const operationCount = Array.from(currentOperations.values()).reduce( + (sum, ops) => sum + ops.length, + 0 + ); this.logger.debug(`[UpdateMetadataService] Flushing operations`, { - operations: Object.fromEntries(currentOperations), + runCount: currentOperations.size, + operationCount, }); } }); @@ -520,9 +525,9 @@ export class UpdateMetadataService { if (this.flushLoggingEnabled) { this.logger.debug(`[updateRunMetadataWithOperations] Updated metadata for run`, { - metadata: applyResults.newMetadata, - operations: operations, runId, + metadataKeyCount: Object.keys(applyResults.newMetadata).length, + operationCount: operations.length, }); } @@ -567,8 +572,8 @@ export class UpdateMetadataService { ) { if (this.flushLoggingEnabled) { this.logger.debug(`[updateRunMetadataDirectly] Updating metadata directly for run`, { - metadata: metadataPacket.data, runId, + metadataSizeBytes: metadataPacket.data?.length ?? 0, }); } @@ -607,7 +612,7 @@ export class UpdateMetadataService { if (this.flushLoggingEnabled) { this.logger.debug(`[ingestRunOperations] Ingesting operations for run`, { runId, - bufferedOperations, + operationCount: bufferedOperations.length, }); } diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index 0905f7c768f..b24ec825b74 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -455,7 +455,7 @@ export class DeliverAlertService extends BaseService { error, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id, runId: alert.taskRun.friendlyId }); break; } case "v2": { @@ -516,7 +516,7 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id, runId: alert.taskRun.friendlyId }); break; } @@ -577,7 +577,7 @@ export class DeliverAlertService extends BaseService { vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl), }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); break; } case "v2": { @@ -616,7 +616,7 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); break; } @@ -671,7 +671,7 @@ export class DeliverAlertService extends BaseService { vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl), }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); break; } case "v2": { @@ -716,7 +716,7 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data); + await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); break; } @@ -1017,7 +1017,11 @@ export class DeliverAlertService extends BaseService { } } - async #deliverWebhook(payload: T, webhook: ProjectAlertWebhookProperties) { + async #deliverWebhook( + payload: T, + webhook: ProjectAlertWebhookProperties, + context: { webhookId: string; runId?: string } + ) { const rawPayload = JSON.stringify(payload); const hashPayload = Buffer.from(rawPayload, "utf-8"); @@ -1046,12 +1050,14 @@ export class DeliverAlertService extends BaseService { }); if (!response.ok) { + // Never log the request/response body here: it is customer-controlled alert + // content and may include stack traces or other application data. logger.info("[DeliverAlert] Failed to send alert webhook", { status: response.status, statusText: response.statusText, - url: webhook.url, - body: payload, - signature, + urlHost: safeUrlHost(webhook.url), + webhookId: context.webhookId, + runId: context.runId, }); throw new Error(`Failed to send alert webhook to ${webhook.url}`); @@ -1435,3 +1441,11 @@ function isWebAPIHTTPError(error: unknown): error is WebAPIHTTPError { function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError { return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError; } + +function safeUrlHost(url: string): string { + try { + return new URL(url).host; + } catch { + return "unknown"; + } +} diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts index 58988b81896..b9240acaa63 100644 --- a/packages/redis-worker/src/worker.ts +++ b/packages/redis-worker/src/worker.ts @@ -140,7 +140,7 @@ class Worker { > = new Map(); constructor(private options: WorkerOptions) { - this.logger = options.logger ?? new Logger("Worker", "debug"); + this.logger = options.logger ?? new Logger("Worker", "debug", ["item"]); this.tracer = options.tracer ?? trace.getTracer(options.name); this.meter = options.meter ?? metrics.getMeter(options.name); @@ -608,7 +608,8 @@ class Worker { this.logger.error("Unhandled error in processItem:", { error: err, workerId, - item, + id: queueItem.id, + job: queueItem.job, }); } ); @@ -933,11 +934,12 @@ class Worker { const errorLogLevel = error && typeof error === "object" && "logLevel" in error ? error.logLevel : undefined; + // Never include the raw item/payload here: it is job data that may be + // customer-controlled. It is retrievable via `getJob(id)` if needed for triage. const logAttributes = { name: this.options.name, id, job, - item, visibilityTimeoutMs, error, errorMessage, @@ -994,7 +996,6 @@ class Worker { name: this.options.name, id, job, - item, retryDate, retryDelay, visibilityTimeoutMs, @@ -1015,7 +1016,6 @@ class Worker { name: this.options.name, id, job, - item, visibilityTimeoutMs, error: requeueError, } From cdacca56845976548b31badd8ecfff00b2e8404b Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 28 Jul 2026 10:36:38 +0100 Subject: [PATCH 2/6] fix(webapp): sanitize metadata and webhook error logs --- .../metadata/updateMetadata.server.ts | 2 +- .../v3/services/alerts/deliverAlert.server.ts | 28 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/services/metadata/updateMetadata.server.ts b/apps/webapp/app/services/metadata/updateMetadata.server.ts index a7449be387d..9bf60798282 100644 --- a/apps/webapp/app/services/metadata/updateMetadata.server.ts +++ b/apps/webapp/app/services/metadata/updateMetadata.server.ts @@ -573,7 +573,7 @@ export class UpdateMetadataService { if (this.flushLoggingEnabled) { this.logger.debug(`[updateRunMetadataDirectly] Updating metadata directly for run`, { runId, - metadataSizeBytes: metadataPacket.data?.length ?? 0, + metadataSizeBytes: Buffer.byteLength(metadataPacket.data ?? "", "utf8"), }); } diff --git a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts index b24ec825b74..070be65f7ad 100644 --- a/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts +++ b/apps/webapp/app/v3/services/alerts/deliverAlert.server.ts @@ -455,7 +455,10 @@ export class DeliverAlertService extends BaseService { error, }; - await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id, runId: alert.taskRun.friendlyId }); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + runId: alert.taskRun.friendlyId, + }); break; } case "v2": { @@ -516,7 +519,10 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id, runId: alert.taskRun.friendlyId }); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + runId: alert.taskRun.friendlyId, + }); break; } @@ -577,7 +583,9 @@ export class DeliverAlertService extends BaseService { vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl), }; - await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } case "v2": { @@ -616,7 +624,9 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } @@ -671,7 +681,9 @@ export class DeliverAlertService extends BaseService { vercel: this.#buildWebhookVercelObject(deploymentMeta.vercelDeploymentUrl), }; - await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } case "v2": { @@ -716,7 +728,9 @@ export class DeliverAlertService extends BaseService { }, }; - await this.#deliverWebhook(payload, webhookProperties.data, { webhookId: alert.channel.id }); + await this.#deliverWebhook(payload, webhookProperties.data, { + webhookId: alert.channel.id, + }); break; } @@ -1060,7 +1074,7 @@ export class DeliverAlertService extends BaseService { runId: context.runId, }); - throw new Error(`Failed to send alert webhook to ${webhook.url}`); + throw new Error(`Failed to send alert webhook to ${safeUrlHost(webhook.url)}`); } } From 747eea42f952fd5ea023a8717a95f91494cd5144 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 28 Jul 2026 15:28:42 +0100 Subject: [PATCH 3/6] chore: add redis worker changeset --- .changeset/redact-worker-logs.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/redact-worker-logs.md diff --git a/.changeset/redact-worker-logs.md b/.changeset/redact-worker-logs.md new file mode 100644 index 00000000000..e7e6e150f1e --- /dev/null +++ b/.changeset/redact-worker-logs.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/redis-worker": patch +--- + +Prevent sensitive job data from being included in worker failure logs. From c9e570a78eca68d5df8880c26f3f829cca156a73 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 28 Jul 2026 16:10:38 +0100 Subject: [PATCH 4/6] fix(webapp): reuse validated metadata byte length --- .../services/metadata/updateMetadata.server.ts | 15 +++++++++++---- apps/webapp/app/utils/packets.ts | 10 +++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/services/metadata/updateMetadata.server.ts b/apps/webapp/app/services/metadata/updateMetadata.server.ts index 9bf60798282..6c7de249a76 100644 --- a/apps/webapp/app/services/metadata/updateMetadata.server.ts +++ b/apps/webapp/app/services/metadata/updateMetadata.server.ts @@ -6,7 +6,11 @@ import type { import { applyMetadataOperations, parsePacket } from "@trigger.dev/core/v3"; import type { PrismaClientOrTransaction } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; -import { handleMetadataPacket, MetadataTooLargeError } from "~/utils/packets"; +import { + handleMetadataPacket, + handleMetadataPacketWithByteLength, + MetadataTooLargeError, +} from "~/utils/packets"; import { ServiceValidationError } from "~/v3/services/common.server"; import { Effect, Schedule, Duration, Fiber } from "effect"; import { type RuntimeFiber } from "effect/Fiber"; @@ -554,16 +558,19 @@ export class UpdateMetadataService { body: UpdateMetadataRequestBody, existingMetadata: IOPacket ): Promise<{ metadata: Record | undefined; updatedAtMs?: number }> { - const metadataPacket = handleMetadataPacket( + const metadataPacketWithByteLength = handleMetadataPacketWithByteLength( body.metadata, "application/json", this.maximumSize ); - if (!metadataPacket) { + if (!metadataPacketWithByteLength) { return { metadata: {} }; } + const { packet: metadataPacket, byteLength: metadataSizeBytes } = + metadataPacketWithByteLength; + let updatedAtMs: number | undefined; if ( @@ -573,7 +580,7 @@ export class UpdateMetadataService { if (this.flushLoggingEnabled) { this.logger.debug(`[updateRunMetadataDirectly] Updating metadata directly for run`, { runId, - metadataSizeBytes: Buffer.byteLength(metadataPacket.data ?? "", "utf8"), + metadataSizeBytes, }); } diff --git a/apps/webapp/app/utils/packets.ts b/apps/webapp/app/utils/packets.ts index 7a522d6f7af..27217c7f2ae 100644 --- a/apps/webapp/app/utils/packets.ts +++ b/apps/webapp/app/utils/packets.ts @@ -13,6 +13,14 @@ export function handleMetadataPacket( metadataType: string, maximumSize: number ): IOPacket | undefined { + return handleMetadataPacketWithByteLength(metadata, metadataType, maximumSize)?.packet; +} + +export function handleMetadataPacketWithByteLength( + metadata: any, + metadataType: string, + maximumSize: number +): { packet: IOPacket; byteLength: number } | undefined { let metadataPacket: IOPacket | undefined = undefined; if (typeof metadata === "string") { @@ -33,5 +41,5 @@ export function handleMetadataPacket( throw new MetadataTooLargeError(`Metadata exceeds maximum size of ${maximumSize} bytes`); } - return metadataPacket; + return { packet: metadataPacket, byteLength }; } From 865294782f35c92eec8ee5e8d140f40a825eeb20 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 28 Jul 2026 16:21:25 +0100 Subject: [PATCH 5/6] format --- apps/webapp/app/services/metadata/updateMetadata.server.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/webapp/app/services/metadata/updateMetadata.server.ts b/apps/webapp/app/services/metadata/updateMetadata.server.ts index 6c7de249a76..07dc236c976 100644 --- a/apps/webapp/app/services/metadata/updateMetadata.server.ts +++ b/apps/webapp/app/services/metadata/updateMetadata.server.ts @@ -568,8 +568,7 @@ export class UpdateMetadataService { return { metadata: {} }; } - const { packet: metadataPacket, byteLength: metadataSizeBytes } = - metadataPacketWithByteLength; + const { packet: metadataPacket, byteLength: metadataSizeBytes } = metadataPacketWithByteLength; let updatedAtMs: number | undefined; From bcec492e502cac81ce9074d373f61f7d6ac97237 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Wed, 29 Jul 2026 17:38:32 +0100 Subject: [PATCH 6/6] Delete .changeset/redact-worker-logs.md --- .changeset/redact-worker-logs.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/redact-worker-logs.md diff --git a/.changeset/redact-worker-logs.md b/.changeset/redact-worker-logs.md deleted file mode 100644 index e7e6e150f1e..00000000000 --- a/.changeset/redact-worker-logs.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/redis-worker": patch ---- - -Prevent sensitive job data from being included in worker failure logs.