From b2007a9585eb80ec36d070c400aebf8994bf7830 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 20:40:56 +0900 Subject: [PATCH 01/53] Observe ordinary outbound send starts Copy observer registrations and await start callbacks for ordinary outbound sends, isolating observer errors from delivery. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 11 + packages/fedify/src/federation/federation.ts | 18 ++ packages/fedify/src/federation/middleware.ts | 90 ++++--- packages/fedify/src/federation/mod.ts | 4 + .../src/federation/outbound-observation.ts | 33 +++ .../src/federation/outbound-report.test.ts | 248 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 39 +++ packages/testing/src/mock.test.ts | 6 + packages/testing/src/mock.ts | 5 + 9 files changed, 416 insertions(+), 38 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-observation.ts create mode 100644 packages/fedify/src/federation/outbound-report.test.ts create mode 100644 packages/fedify/src/federation/outbound-report.ts diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 4b660ba0d..6f3f8f7f8 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -1,3 +1,4 @@ +import type { OutboundObservers } from "./outbound-report.ts"; import type { InboxRequestFinishedHandler } from "./inbox-report.ts"; import { assertPath, @@ -183,6 +184,9 @@ export class FederationBuilderImpl inboxListeners?: ActivityListenerSet>; outboxListeners?: ActivityListenerSet>; inboxRequestFinishedHandler?: InboxRequestFinishedHandler; + outboundObservers: OutboundObservers = globalThis.Object.freeze( + {}, + ); inboxErrorHandler?: InboxErrorHandler; outboxListenerErrorHandler?: OutboxListenerErrorHandler; @@ -288,6 +292,7 @@ export class FederationBuilderImpl f.inboxListeners = this.inboxListeners?.clone(); f.outboxListeners = this.outboxListeners?.clone(); f.inboxRequestFinishedHandler = this.inboxRequestFinishedHandler; + f.setOutboundObservers(this.outboundObservers); f.inboxErrorHandler = this.inboxErrorHandler; f.outboxListenerErrorHandler = this.outboxListenerErrorHandler; f.outboxAuthorizePredicate = this.outboxAuthorizePredicate; @@ -658,6 +663,12 @@ export class FederationBuilderImpl return setters; } + setOutboundObservers(observers: OutboundObservers): void { + this.outboundObservers = globalThis.Object.freeze({ + onBroadcast: observers.onBroadcast, + }); + } + setNodeInfoDispatcher( path: string, dispatcher: NodeInfoDispatcher, diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 286d9c830..34651ae75 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -1,3 +1,4 @@ +import type { OutboundObservers } from "./outbound-report.ts"; import type { InboxListenerOptions } from "./activity-listener.ts"; import type { InboxRequestFinishedHandler } from "./inbox-report.ts"; import type { @@ -82,6 +83,23 @@ export interface FederationStartQueueOptions { * @since 1.6.0 */ export interface Federatable extends TaskRegistry { + /** + * Registers observers for outgoing activity processing. + * + * Each call replaces all previously registered outbound observers. + * Pass an empty object to clear them. Fedify copies the callbacks; + * later changes to the supplied object do not affect registration. + * Available on builders and directly created federations. Building copies + * the callbacks, and an active send keeps the callback selected at entry. + * Currently only ordinary {@link Context.sendActivity} starts are observed; + * outbox-context overrides, forwarding, workers, and preparation completion + * are not observed yet. + * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. + * @param observers The optional callbacks to register. + * @since 2.5.0 + */ + setOutboundObservers(observers: OutboundObservers): void; + /** * Registers a NodeInfo dispatcher. * @param path The URI path pattern for the NodeInfo dispatcher. The syntax diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 5889daa7a..e32324b5d 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1,3 +1,4 @@ +import { observeOutboundBroadcastStart } from "./outbound-observation.ts"; import { InboxObservation } from "./inbox-observation.ts"; import { type Path, RouterError } from "@fedify/uri-template"; import type { @@ -5646,44 +5647,57 @@ export class ContextImpl implements Context { activity: Activity, options: SendActivityOptionsForCollection = {}, ): Promise { - const tracer = this.tracerProvider.getTracer( - metadata.name, - metadata.version, - ); - return tracer.startActiveSpan( - this.federation.outboxQueue == null || options.immediate - ? "activitypub.outbox" - : "activitypub.fanout", - { - kind: this.federation.outboxQueue == null || options.immediate - ? SpanKind.CLIENT - : SpanKind.PRODUCER, - attributes: { - "activitypub.activity.type": getTypeId(activity).href, - "activitypub.activity.to": activity.toIds.map((to) => to.href), - "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), - "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href), - "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href), - }, - }, - async (span) => { - try { - if (activity.id != null) { - span.setAttribute("activitypub.activity.id", activity.id.href); - } - await this.sendActivityInternal( - sender, - recipients, - activity, - options, - span, - ); - } catch (e) { - span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); - throw e; - } finally { - span.end(); - } + return observeOutboundBroadcastStart( + this, + this.federation.outboundObservers.onBroadcast, + () => { + const tracer = this.tracerProvider.getTracer( + metadata.name, + metadata.version, + ); + return tracer.startActiveSpan( + this.federation.outboxQueue == null || options.immediate + ? "activitypub.outbox" + : "activitypub.fanout", + { + kind: this.federation.outboxQueue == null || options.immediate + ? SpanKind.CLIENT + : SpanKind.PRODUCER, + attributes: { + "activitypub.activity.type": getTypeId(activity).href, + "activitypub.activity.to": activity.toIds.map((to) => to.href), + "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), + "activitypub.activity.bto": activity.btoIds.map((bto) => + bto.href + ), + "activitypub.activity.bcc": activity.bccIds.map((bcc) => + bcc.href + ), + }, + }, + async (span) => { + try { + if (activity.id != null) { + span.setAttribute("activitypub.activity.id", activity.id.href); + } + await this.sendActivityInternal( + sender, + recipients, + activity, + options, + span, + ); + } catch (e) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: String(e), + }); + throw e; + } finally { + span.end(); + } + }, + ); }, ); } diff --git a/packages/fedify/src/federation/mod.ts b/packages/fedify/src/federation/mod.ts index 79ecf68a3..6d4f2c29d 100644 --- a/packages/fedify/src/federation/mod.ts +++ b/packages/fedify/src/federation/mod.ts @@ -22,6 +22,10 @@ export { type IsActorAuthenticatedOptions, } from "./inbox-authentication.ts"; export * from "./inbox-report.ts"; +export type { + OutboundBroadcastEvent, + OutboundObservers, +} from "./outbound-report.ts"; export * from "./kv.ts"; export { createFederation, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts new file mode 100644 index 000000000..0b239f693 --- /dev/null +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -0,0 +1,33 @@ +import { getLogger } from "@logtape/logtape"; +import type { Context } from "./context.ts"; +import type { OutboundObservers } from "./outbound-report.ts"; + +/** Observe entry without changing the unobserved send's execution path. */ +export function observeOutboundBroadcastStart( + context: Context, + observer: OutboundObservers["onBroadcast"], + send: () => Promise, +): Promise { + if (observer == null) return send(); + return notify().then(send); + + async function notify(): Promise { + const event = Object.freeze({ + broadcastId: crypto.randomUUID(), + operation: "send" as const, + stage: "started" as const, + }); + try { + await observer!(context, event); + } catch (error) { + try { + getLogger(["fedify", "federation", "outbox"]).error( + "An unexpected error occurred in outbound broadcast observer:\n{error}", + { error }, + ); + } catch { + // A logging sink must not replace the send's result either. + } + } + } +} diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts new file mode 100644 index 000000000..404724a36 --- /dev/null +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -0,0 +1,248 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Person } from "@fedify/vocab"; +import { configure, reset } from "@logtape/logtape"; +import fetchMock from "fetch-mock"; +import { + deepStrictEqual, + notStrictEqual, + ok, + rejects, + strictEqual, +} from "node:assert/strict"; +import { createFederationBuilder } from "./builder.ts"; +import { MemoryKvStore } from "./kv.ts"; +import { createFederation } from "./middleware.ts"; +import type { OutboundBroadcastEvent, OutboundObservers } from "./mod.ts"; +import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; + +const activity = new Create({ + id: new URL("https://example.com/create"), + actor: new URL("https://example.com/alice"), +}); +const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; +const options = { + kv: new MemoryKvStore(), + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, +}; + +test("outbound observers report fresh immutable starts without tracing or extra loading", async (t) => { + const { AlwaysOffSampler, BasicTracerProvider } = await import( + "@opentelemetry/sdk-trace-base" + ); + const provider = new BasicTracerProvider({ sampler: new AlwaysOffSampler() }); + try { + for (const tracerProvider of [undefined, provider]) { + await t.step( + tracerProvider == null ? "no provider" : "sampling disabled", + async () => { + let loads = 0; + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + tracerProvider, + documentLoaderFactory: () => () => { + loads++; + throw new Error("unexpected observation lookup"); + }, + contextLoaderFactory: () => () => { + loads++; + throw new Error("unexpected observation lookup"); + }, + }); + const ctx = federation.createContext(new URL("https://example.com"), { + label: "app", + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(context, event) { + strictEqual(context, ctx); + strictEqual(context.data.label, "app"); + strictEqual(event.stage, "started"); + strictEqual(event.operation, "send"); + ok(Object.isFrozen(event)); + events.push(event); + }, + }); + await ctx.sendActivity(sender, [], activity); + await ctx.sendActivity(sender, [], activity); + strictEqual(events.length, 2); + ok(events[0].broadcastId.length > 0); + notStrictEqual(events[0].broadcastId, events[1].broadcastId); + strictEqual(loads, 0); + }, + ); + } + } finally { + await provider.shutdown(); + } +}); + +test("outbound observers await entry before preparation and retain the selected callback", async () => { + const federation = createFederation(options); + let preparations = 0; + const failure = Symbol("original preparation failure"); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .mapHandle(() => { + preparations++; + throw failure; + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + let entered!: () => void; + const entry = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const calls: string[] = []; + federation.setOutboundObservers({ + async onBroadcast() { + calls.push("original"); + entered(); + await gate; + throw new Error("asynchronous observer failure"); + }, + }); + const pending = rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + await entry; + strictEqual(preparations, 0); + federation.setOutboundObservers({ + onBroadcast() { + calls.push("replacement"); + }, + }); + release(); + await pending; + await rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + deepStrictEqual(calls, ["original", "replacement"]); + strictEqual(preparations, 2); +}); + +test("outbound observers isolate callback and logging errors from success and original failures", async () => { + let logCalls = 0; + await configure({ + reset: true, + sinks: { + broken() { + logCalls++; + throw new Error("sink failed"); + }, + }, + loggers: [ + { + category: ["fedify", "federation", "outbox"], + sinks: ["broken"], + lowestLevel: "error", + }, + { category: ["logtape", "meta"], sinks: [] }, + ], + }); + try { + const federation = createFederation(options); + const failure = { original: true }; + let calls = 0; + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .mapHandle(() => { + throw failure; + }); + federation.setOutboundObservers({ + onBroadcast() { + calls++; + throw "observer failed"; + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await ctx.sendActivity(sender, [], activity); + await rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + strictEqual(calls, 2); + strictEqual(logCalls, 2); + } finally { + await reset(); + } +}); + +test("outbound observers copy registration and isolate builder instances", async () => { + const builder = createFederationBuilder(); + const calls: string[] = []; + const registration = { + onBroadcast() { + calls.push("original"); + }, + } satisfies OutboundObservers; + builder.setOutboundObservers(registration); + registration.onBroadcast = () => { + calls.push("mutated"); + }; + const first = await builder.build(options); + const second = await builder.build(options); + builder.setOutboundObservers({ + onBroadcast() { + calls.push("new builder"); + }, + }); + const third = await builder.build(options); + first.setOutboundObservers({ + onBroadcast() { + calls.push("first only"); + }, + }); + for (const federation of [first, second, third]) { + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, [], activity); + } + deepStrictEqual(calls, ["first only", "original", "new builder"]); + second.setOutboundObservers({}); + await second.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, [], activity); + strictEqual(calls.length, 3); +}); + +test("outbound observers do not veto an actual immediate delivery", async () => { + const order: string[] = []; + fetchMock.mockGlobal().post("https://receiver.example/inbox", (call) => { + order.push("fetch"); + strictEqual(JSON.parse(String(call.options.body)).id, activity.id!.href); + return 202; + }); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onBroadcast() { + order.push("started"); + throw new Error("recording failed"); + }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + sender, + new Person({ + id: new URL("https://receiver.example/actor"), + inbox: new URL("https://receiver.example/inbox"), + }), + activity, + { immediate: true }, + ); + deepStrictEqual(order, ["started", "fetch"]); + } finally { + fetchMock.hardReset(); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts new file mode 100644 index 000000000..1fc97a4fd --- /dev/null +++ b/packages/fedify/src/federation/outbound-report.ts @@ -0,0 +1,39 @@ +import type { Context } from "./context.ts"; + +/** + * The start of an ordinary {@link Context.sendActivity} invocation. + * + * This reports entry before preparation, not queue acceptance or delivery. + * @since 2.5.0 + */ +export interface OutboundBroadcastEvent { + /** Fresh opaque identity for this invocation, independent of tracing. */ + readonly broadcastId: string; + /** This increment observes ordinary sends of a new activity. */ + readonly operation: "send"; + /** Preparation has not started yet. */ + readonly stage: "started"; +} + +/** + * Optional observers of outgoing activity processing. + * + * Currently only ordinary {@link Context.sendActivity} starts are observed. + * Callbacks are awaited without a default timeout. Their errors, including + * errors logging observer failures, do not replace the send's result. A + * callback that never settles delays the send indefinitely. Detached recording + * must handle its own rejection and shutdown; no persistence is guaranteed. + * @template TContextData Application context data. + * @since 2.5.0 + */ +export interface OutboundObservers { + /** + * Observes an invocation before sender/recipient resolution or signing. + * The existing context is passed without loading data for observation. + * Returning or throwing cannot veto sending. + */ + readonly onBroadcast?: ( + context: Context, + event: OutboundBroadcastEvent, + ) => void | Promise; +} diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index 799d451a0..5cc118ff8 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -207,6 +207,12 @@ test("reset clears sent activities", async () => { deepStrictEqual(mockFederation.sentActivities.length, 0); }); +test("MockFederation accepts outbound observer registration", () => { + const federation = createFederation(); + federation.setOutboundObservers({ onBroadcast: () => {} }); + federation.setOutboundObservers({}); +}); + test("MockFederation accepts inbox request observation registration", () => { const federation = createFederation(); const setters = federation.setInboxListeners("/users/{identifier}/inbox"); diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index fbcbe71c1..09f218346 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -7,6 +7,7 @@ import type { FederationStartQueueOptions, GetObjectOptions, Message, + OutboundObservers, ParseUriOptions, ParseUriResult, PortableRequest, @@ -592,6 +593,10 @@ class MockFederation implements Federation { return setters; } + setOutboundObservers(_observers: OutboundObservers): void { + // Registration compatibility; mock sends do not run transport observers. + } + setInboxListeners(inboxPath: any, sharedInboxPath?: string): any { this.inboxPath = inboxPath; this.sharedInboxPath = sharedInboxPath; From b6caa065a7c1bbafcdfc5528e0d11ca57fe0dccf Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 21:17:35 +0900 Subject: [PATCH 02/53] Observe outbound preparation results Report readiness, skips, and failures with one broadcast identity. Retain the prepared result after send or enqueue failure. Documentation and changelog updates are deferred until section B closes. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/federation.ts | 4 +- packages/fedify/src/federation/middleware.ts | 63 ++- packages/fedify/src/federation/mod.ts | 5 +- .../src/federation/outbound-observation.ts | 120 ++++- .../src/federation/outbound-report.test.ts | 428 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 94 +++- 6 files changed, 650 insertions(+), 64 deletions(-) diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 34651ae75..2a8b15681 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -91,8 +91,8 @@ export interface Federatable extends TaskRegistry { * later changes to the supplied object do not affect registration. * Available on builders and directly created federations. Building copies * the callbacks, and an active send keeps the callback selected at entry. - * Currently only ordinary {@link Context.sendActivity} starts are observed; - * outbox-context overrides, forwarding, workers, and preparation completion + * Currently only ordinary {@link Context.sendActivity} entry and preparation + * results are observed; outbox-context overrides, forwarding, and workers * are not observed yet. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index e32324b5d..00965acd8 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1,4 +1,7 @@ -import { observeOutboundBroadcastStart } from "./outbound-observation.ts"; +import { + observeOutboundBroadcast, + type OutboundBroadcastObservation, +} from "./outbound-observation.ts"; import { InboxObservation } from "./inbox-observation.ts"; import { type Path, RouterError } from "@fedify/uri-template"; import type { @@ -3180,7 +3183,14 @@ export class FederationImpl options: SendActivityInternalOptions, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); - const { immediate, collectionSync, orderingKey, context: ctx } = options; + const { + immediate, + collectionSync, + orderingKey, + context: ctx, + observation, + } = options; + observation?.setPhase("validation"); if (activity.id == null) { throw new TypeError("The activity to send must have an id."); } @@ -3191,6 +3201,7 @@ export class FederationImpl } else if (keys.length < 1) { throw new TypeError("The keys must not be empty."); } + observation?.setPhase("serialization"); const contextLoader = this.contextLoaderFactory( this.#getLoaderOptions(ctx.origin), ); @@ -3200,6 +3211,7 @@ export class FederationImpl // nor any key other than the DID's can do for a portable actor's // activity; its Object Integrity Proof does that instead: const ldSignable = !hasPortableActor(activity); + observation?.setPhase("key-resolution"); for (const { keyId, privateKey } of keys) { validateCryptoKey(privateKey, "private"); if ( @@ -3212,6 +3224,7 @@ export class FederationImpl } // If Object Integrity Proofs were already created before fanout (e.g., in // sendActivityInternal()), skip signing to avoid duplicates. + observation?.setPhase("signing"); const signed = await signOutgoingActivity( activity, keys.map(({ keyId, privateKey }) => ({ @@ -3226,6 +3239,7 @@ export class FederationImpl ); activity = signed.activity; const { hasProof, proofCreated } = signed; + observation?.setPhase("serialization"); let jsonLd = !proofCreated && options.activityJsonLd != null ? options.activityJsonLd : await activity.toJsonLd({ @@ -3241,6 +3255,7 @@ export class FederationImpl preserveNestedSecuredDocuments: true, }); } + observation?.setPhase("validation"); assertSupportedCompoundProofShape(jsonLd, activityId); if (rsaKey == null && !ldSignable) { logger.debug( @@ -3264,6 +3279,7 @@ export class FederationImpl ); } else { try { + observation?.setPhase("signing"); jsonLd = await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, { contextLoader, tracerProvider: this.tracerProvider, @@ -3292,6 +3308,7 @@ export class FederationImpl }, ); } + observation?.setPhase("request-preparation"); if (immediate || this.outboxQueue == null) { if (immediate) { logger.debug( @@ -3307,8 +3324,10 @@ export class FederationImpl } const promises: Promise[] = []; for (const inbox in inboxes) { - promises.push( - sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, { + const sending = sendActivityToInbox( + inbox, + inboxes[inbox].gatewayInboxes, + { allowPrivateAddress: this.allowPrivateAddress, keys, activity: jsonLd, @@ -3330,9 +3349,16 @@ export class FederationImpl ), meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, - }), + }, ); + if (observation != null) { + // The preparation callback may await while an earlier send rejects. + // Keep its original rejection for Promise.all without an unhandled one. + sending.catch(() => {}); + } + promises.push(sending); } + if (observation != null) await observation.prepared("delivery"); await Promise.all(promises); return; } @@ -3340,12 +3366,14 @@ export class FederationImpl "Enqueuing activity {activityId} to send later.", { activityId: activity.id!.href, activity: jsonLd }, ); + observation?.setPhase("key-resolution"); const keyJwkPairs: SenderKeyJwkPair[] = []; for (const { keyId, privateKey } of keys) { const privateKeyJwk = await exportJwk(privateKey); keyJwkPairs.push({ keyId: keyId.href, privateKey: privateKeyJwk }); } if (!this.manuallyStartQueue) this._startQueueInternal(ctx.data); + observation?.setPhase("request-preparation"); const carrier: Record = {}; propagation.inject(context.active(), carrier); const messages: { message: OutboxMessage; orderingKey?: string }[] = []; @@ -3380,6 +3408,7 @@ export class FederationImpl }; messages.push({ message, orderingKey: messageOrderingKey }); } + if (observation != null) await observation.prepared("delivery"); const { outboxQueue } = this; // enqueueMany does not support per-message orderingKey, so fall back to // individual enqueues whenever orderingKey is specified or the backend @@ -5647,10 +5676,10 @@ export class ContextImpl implements Context { activity: Activity, options: SendActivityOptionsForCollection = {}, ): Promise { - return observeOutboundBroadcastStart( + return observeOutboundBroadcast( this, this.federation.outboundObservers.onBroadcast, - () => { + (observation) => { const tracer = this.tracerProvider.getTracer( metadata.name, metadata.version, @@ -5686,6 +5715,7 @@ export class ContextImpl implements Context { activity, options, span, + observation, ); } catch (e) { span.setStatus({ @@ -5712,8 +5742,10 @@ export class ContextImpl implements Context { activity: Activity, options: SendActivityOptionsForCollection, span: Span, + observation?: OutboundBroadcastObservation, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); + observation?.setPhase("sender-resolution"); let keys: SenderKeyPair[]; let identifier: string | null = null; let actorKeyPairs: ActorKeyPair[] | null = null; @@ -5740,6 +5772,7 @@ export class ContextImpl implements Context { } } span.setAttribute("fedify.actor.identifier", identifier); + observation?.setPhase("key-resolution"); if (this.federation.actorCallbacks?.keyPairsDispatcher == null) { throw new Error("No actor key pairs dispatcher registered."); } @@ -5754,6 +5787,7 @@ export class ContextImpl implements Context { privateKey: kp.privateKey, })); } else if (Array.isArray(sender)) { + observation?.setPhase("key-resolution"); if (sender.length < 1) { throw new Error("The sender's key pairs are empty."); } @@ -5764,9 +5798,11 @@ export class ContextImpl implements Context { if (keys.length < 1) { throw new TypeError("The sender's keys must not be empty."); } + observation?.setPhase("key-resolution"); for (const { privateKey } of keys) { validateCryptoKey(privateKey, "private"); } + observation?.setPhase("recipient-resolution"); let expandedRecipients: Recipient[]; let collectionSync: string | undefined; if (Array.isArray(recipients)) { @@ -5798,16 +5834,19 @@ export class ContextImpl implements Context { } const opts: SendActivityInternalOptions = { context: this, + observation, orderingKey: options.orderingKey, collectionSync, immediate: options.immediate, normalizeExistingProofs: options.normalizeExistingProofs, }; span.setAttribute("activitypub.inboxes", expandedRecipients.length); + observation?.setPhase("transformation"); for (const activityTransformer of this.federation.activityTransformers) { activity = activityTransformer(activity, this); } span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? ""); + observation?.setPhase("validation"); if (activity.actorId == null) { logger.error( "Activity {activityId} to send does not have an actor.", @@ -5834,6 +5873,7 @@ export class ContextImpl implements Context { // does not accept. This mirrors the guard `FederationImpl.sendActivity()` // applies before signing. Actor key pairs keep appending outside that // profile, as they always have. + observation?.setPhase("signing"); const { activity: signedActivity, proofCreated } = await signOutgoingActivity( activity, @@ -5855,16 +5895,19 @@ export class ContextImpl implements Context { }, ); activity = signedActivity; + observation?.setPhase("destination-planning"); const inboxes = extractInboxes({ recipients: expandedRecipients, preferSharedInbox: options.preferSharedInbox, excludeBaseUris: options.excludeBaseUris, }); + observation?.planned(globalThis.Object.keys(inboxes).length); if (globalThis.Object.keys(inboxes).length < 1) { logger.debug("No inboxes found for activity {activityId}.", { activityId: activity.id?.href, activity, }); + if (observation != null) await observation.skipped(); return false; } logger.debug("Sending activity {activityId} to inboxes:\n{inboxes}", { @@ -5872,6 +5915,7 @@ export class ContextImpl implements Context { activityId: activity.id?.href, activity, }); + observation?.setPhase("serialization"); const activityJsonLd = await activity.toJsonLd({ format: "compact", contextLoader: this.contextLoader, @@ -5904,13 +5948,16 @@ export class ContextImpl implements Context { } // Reject before anything is enqueued, so the caller learns about it // instead of the fanout worker. + observation?.setPhase("validation"); assertSupportedCompoundProofShape(activityJsonLd, activity.id?.href); + observation?.setPhase("key-resolution"); const keyJwkPairs = await Promise.all( keys.map(async ({ keyId, privateKey }) => ({ keyId: keyId.href, privateKey: await exportJwk(privateKey), })), ); + observation?.setPhase("request-preparation"); const carrier: Record = {}; propagation.inject(context.active(), carrier); const message: FanoutMessage = { @@ -5938,6 +5985,7 @@ export class ContextImpl implements Context { if (!this.federation.manuallyStartQueue) { this.federation._startQueueInternal(this.data); } + if (observation != null) await observation.prepared("fanout"); await this.federation.fanoutQueue.enqueue( message, { orderingKey: options.orderingKey }, @@ -7225,6 +7273,7 @@ export class OutboxContextImpl extends ContextImpl } interface SendActivityInternalOptions { + readonly observation?: OutboundBroadcastObservation; readonly immediate?: boolean; readonly collectionSync?: string; readonly orderingKey?: string; diff --git a/packages/fedify/src/federation/mod.ts b/packages/fedify/src/federation/mod.ts index 6d4f2c29d..dac2763b5 100644 --- a/packages/fedify/src/federation/mod.ts +++ b/packages/fedify/src/federation/mod.ts @@ -22,10 +22,7 @@ export { type IsActorAuthenticatedOptions, } from "./inbox-authentication.ts"; export * from "./inbox-report.ts"; -export type { - OutboundBroadcastEvent, - OutboundObservers, -} from "./outbound-report.ts"; +export * from "./outbound-report.ts"; export * from "./kv.ts"; export { createFederation, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 0b239f693..a6acc9d45 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,24 +1,98 @@ import { getLogger } from "@logtape/logtape"; import type { Context } from "./context.ts"; -import type { OutboundObservers } from "./outbound-report.ts"; +import type { + OutboundBroadcastEvent, + OutboundObservers, + OutboundPreparationResult, +} from "./outbound-report.ts"; -/** Observe entry without changing the unobserved send's execution path. */ -export function observeOutboundBroadcastStart( - context: Context, - observer: OutboundObservers["onBroadcast"], - send: () => Promise, -): Promise { - if (observer == null) return send(); - return notify().then(send); - - async function notify(): Promise { - const event = Object.freeze({ - broadcastId: crypto.randomUUID(), - operation: "send" as const, - stage: "started" as const, - }); +type PreparationFailure = Extract< + OutboundPreparationResult, + { type: "failed" } +>; + +/** State owned by one ordinary send invocation, never serialized into queues. */ +export class OutboundBroadcastObservation { + readonly #broadcastId = crypto.randomUUID(); + #finished = false; + #phase: PreparationFailure["failure"]["phase"] = "unknown"; + #planning: PreparationFailure["planning"] = "not-started"; + #destinationCount = 0; + + constructor( + readonly context: Context, + readonly observer: NonNullable< + OutboundObservers["onBroadcast"] + >, + ) {} + + setPhase(phase: PreparationFailure["failure"]["phase"]): void { + this.#phase = phase; + if (phase === "destination-planning") this.#planning = "incomplete"; + } + + planned(destinationCount: number): void { + this.#planning = "complete"; + this.#destinationCount = destinationCount; + } + + prepared(preparedFor: "fanout" | "delivery"): Promise { + return this.#finish(Object.freeze({ + type: "prepared", + preparedFor, + destinationCount: this.#destinationCount, + })); + } + + skipped(): Promise { + return this.#finish(Object.freeze({ + type: "skipped", + reason: "no-destinations", + })); + } + + async run( + send: ( + observation?: OutboundBroadcastObservation, + ) => Promise, + ): Promise { + await this.#notify(Object.freeze({ + broadcastId: this.#broadcastId, + operation: "send", + stage: "started", + })); + try { + await send(this); + } catch (error) { + const failure: PreparationFailure["failure"] = { + kind: this.#phase === "unknown" ? "unknown" : "local", + phase: this.#phase, + error, + }; + await this.#finish(Object.freeze({ + type: "failed", + failure: Object.freeze(failure), + planning: this.#planning, + plannedDestinationCount: this.#destinationCount, + })); + throw error; + } + } + + async #finish(result: OutboundPreparationResult): Promise { + if (this.#finished) return; + this.#finished = true; + await this.#notify(Object.freeze({ + broadcastId: this.#broadcastId, + operation: "send", + stage: "preparation-finished", + result, + })); + } + + async #notify(event: OutboundBroadcastEvent): Promise { try { - await observer!(context, event); + await this.observer(this.context, event); } catch (error) { try { getLogger(["fedify", "federation", "outbox"]).error( @@ -31,3 +105,15 @@ export function observeOutboundBroadcastStart( } } } + +/** Observe preparation without changing the unobserved send's execution path. */ +export function observeOutboundBroadcast( + context: Context, + observer: OutboundObservers["onBroadcast"], + send: ( + observation?: OutboundBroadcastObservation, + ) => Promise, +): Promise { + if (observer == null) return send(); + return new OutboundBroadcastObservation(context, observer).run(send); +} diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 404724a36..c06613279 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -1,6 +1,11 @@ -import { mockDocumentLoader, test } from "@fedify/fixture"; +import { + createTestTracerProvider, + mockDocumentLoader, + test, +} from "@fedify/fixture"; import { Create, Person } from "@fedify/vocab"; import { configure, reset } from "@logtape/logtape"; +import { SpanStatusCode } from "@opentelemetry/api"; import fetchMock from "fetch-mock"; import { deepStrictEqual, @@ -12,7 +17,12 @@ import { import { createFederationBuilder } from "./builder.ts"; import { MemoryKvStore } from "./kv.ts"; import { createFederation } from "./middleware.ts"; -import type { OutboundBroadcastEvent, OutboundObservers } from "./mod.ts"; +import type { + Message, + MessageQueue, + OutboundBroadcastEvent, + OutboundObservers, +} from "./mod.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; const activity = new Create({ @@ -57,7 +67,6 @@ test("outbound observers report fresh immutable starts without tracing or extra onBroadcast(context, event) { strictEqual(context, ctx); strictEqual(context.data.label, "app"); - strictEqual(event.stage, "started"); strictEqual(event.operation, "send"); ok(Object.isFrozen(event)); events.push(event); @@ -65,9 +74,16 @@ test("outbound observers report fresh immutable starts without tracing or extra }); await ctx.sendActivity(sender, [], activity); await ctx.sendActivity(sender, [], activity); - strictEqual(events.length, 2); + deepStrictEqual(preparation(events.slice(0, 2)), { + type: "skipped", + reason: "no-destinations", + }); + deepStrictEqual(preparation(events.slice(2)), { + type: "skipped", + reason: "no-destinations", + }); ok(events[0].broadcastId.length > 0); - notStrictEqual(events[0].broadcastId, events[1].broadcastId); + notStrictEqual(events[0].broadcastId, events[2].broadcastId); strictEqual(loads, 0); }, ); @@ -100,8 +116,8 @@ test("outbound observers await entry before preparation and retain the selected }); const calls: string[] = []; federation.setOutboundObservers({ - async onBroadcast() { - calls.push("original"); + async onBroadcast(_ctx, event) { + calls.push(`original:${event.stage}`); entered(); await gate; throw new Error("asynchronous observer failure"); @@ -114,8 +130,8 @@ test("outbound observers await entry before preparation and retain the selected await entry; strictEqual(preparations, 0); federation.setOutboundObservers({ - onBroadcast() { - calls.push("replacement"); + onBroadcast(_ctx, event) { + calls.push(`replacement:${event.stage}`); }, }); release(); @@ -124,7 +140,12 @@ test("outbound observers await entry before preparation and retain the selected ctx.sendActivity({ username: "alice" }, [], activity), (error: unknown) => error === failure, ); - deepStrictEqual(calls, ["original", "replacement"]); + deepStrictEqual(calls, [ + "original:started", + "original:preparation-finished", + "replacement:started", + "replacement:preparation-finished", + ]); strictEqual(preparations, 2); }); @@ -170,8 +191,8 @@ test("outbound observers isolate callback and logging errors from success and or ctx.sendActivity({ username: "alice" }, [], activity), (error: unknown) => error === failure, ); - strictEqual(calls, 2); - strictEqual(logCalls, 2); + strictEqual(calls, 4); + strictEqual(logCalls, 4); } finally { await reset(); } @@ -206,11 +227,18 @@ test("outbound observers copy registration and isolate builder instances", async await federation.createContext(new URL("https://example.com"), undefined) .sendActivity(sender, [], activity); } - deepStrictEqual(calls, ["first only", "original", "new builder"]); + deepStrictEqual(calls, [ + "first only", + "first only", + "original", + "original", + "new builder", + "new builder", + ]); second.setOutboundObservers({}); await second.createContext(new URL("https://example.com"), undefined) .sendActivity(sender, [], activity); - strictEqual(calls.length, 3); + strictEqual(calls.length, 6); }); test("outbound observers do not veto an actual immediate delivery", async () => { @@ -226,8 +254,8 @@ test("outbound observers do not veto an actual immediate delivery", async () => allowPrivateAddress: true, }); federation.setOutboundObservers({ - onBroadcast() { - order.push("started"); + onBroadcast(_ctx, event) { + order.push(event.stage); throw new Error("recording failed"); }, }); @@ -241,8 +269,374 @@ test("outbound observers do not veto an actual immediate delivery", async () => activity, { immediate: true }, ); - deepStrictEqual(order, ["started", "fetch"]); + strictEqual(order[0], "started"); + strictEqual(order.filter((entry) => entry === "fetch").length, 1); + strictEqual( + order.filter((entry) => entry === "preparation-finished").length, + 1, + ); } finally { fetchMock.hardReset(); } }); + +function preparation(events: OutboundBroadcastEvent[]) { + deepStrictEqual(events.map((event) => event.stage), [ + "started", + "preparation-finished", + ]); + strictEqual(events[0].broadcastId, events[1].broadcastId); + ok(events.every((event) => Object.isFrozen(event))); + const event = events[1]; + ok(event.stage === "preparation-finished"); + ok(Object.isFrozen(event.result)); + return event.result; +} + +const recipient = new Person({ + id: new URL("https://receiver.example/actor"), + inbox: new URL("https://receiver.example/inbox"), +}); + +test("outbound preparation preserves failure provenance and plan completeness", async (t) => { + for ( + const phase of [ + "sender-resolution", + "transformation", + "destination-planning", + "serialization", + ] as const + ) { + await t.step(phase, async () => { + const failure = Symbol(phase); + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + ...(phase === "transformation" + ? { + activityTransformers: [() => { + throw failure; + }], + } + : { activityTransformers: [] }), + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + let sentActivity = activity; + let target = recipient; + if (phase === "serialization") { + sentActivity = activity.clone({}); + Object.defineProperty(sentActivity, "toJsonLd", { + value: () => Promise.reject(failure), + }); + } + if (phase === "destination-planning") { + target = recipient.clone({}); + Object.defineProperty(target, "inboxId", { + get() { + throw failure; + }, + }); + } + if (phase === "sender-resolution") { + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .mapHandle(() => { + throw failure; + }); + } + await rejects( + federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + phase === "sender-resolution" ? { username: "alice" } : sender, + target, + sentActivity, + ), + (error: unknown) => error === failure, + ); + deepStrictEqual(preparation(events), { + type: "failed", + failure: { kind: "local", phase, error: failure }, + planning: phase === "serialization" + ? "complete" + : phase === "destination-planning" + ? "incomplete" + : "not-started", + plannedDestinationCount: phase === "serialization" ? 1 : 0, + }); + }); + } +}); + +test("outbound preparation precedes queue acceptance and is not rewritten by enqueue failure", async (t) => { + for (const role of ["outbox", "fanout"] as const) { + for (const rejected of [false, true]) { + await t.step( + `${role}: ${rejected ? "enqueue rejects" : "enqueue accepts"}`, + async () => { + const failure = Symbol("enqueue failure"); + const events: OutboundBroadcastEvent[] = []; + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + if (rejected) return Promise.reject(failure); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { [role]: queue }, + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await gate; + } + }, + }); + const sending = federation.createContext( + new URL("https://example.com"), + undefined, + ) + .sendActivity(sender, [recipient, recipient], activity, { + fanout: role === "fanout" ? "force" : "skip", + }); + const settled = rejected + ? rejects(sending, (error: unknown) => error === failure) + : sending; + await prepared; + strictEqual(messages.length, 0); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: role === "fanout" ? "fanout" : "delivery", + destinationCount: 1, + }); + release(); + await settled; + strictEqual(messages.length, 1); + strictEqual(messages[0].type, role); + strictEqual(events.length, 2); + ok(!("observation" in messages[0])); + }, + ); + } + } +}); + +test("outbound preparation finishes before immediate sends settle without reporting later failures twice", async (t) => { + for (const status of [202, 503]) { + await t.step(`HTTP ${status}`, async () => { + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let releaseResponse!: () => void; + const responseGate = new Promise((resolve) => { + releaseResponse = resolve; + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let releaseObserver!: () => void; + const observerGate = new Promise((resolve) => { + releaseObserver = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + const [tracerProvider, spanExporter] = createTestTracerProvider(); + let sendEnded!: () => void; + const concreteSendEnded = new Promise((resolve) => { + sendEnded = resolve; + }); + const exportSpans = spanExporter.export.bind(spanExporter); + spanExporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (spans.some((span) => span.name === "activitypub.send_activity")) { + sendEnded(); + } + }; + fetchMock.mockGlobal().post( + "https://receiver.example/inbox", + async () => { + fetched(); + await responseGate; + return { status, body: "receiver response" }; + }, + ); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + }); + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await observerGate; + } + }, + }); + let settled = false; + const sending = federation.createContext( + new URL("https://example.com"), + undefined, + ) + .sendActivity(sender, recipient, activity, { immediate: true }) + .finally(() => { + settled = true; + }); + const result = status === 202 + ? sending + : rejects(sending, (error: unknown) => { + return error instanceof Error && "statusCode" in error && + error.statusCode === status; + }); + await Promise.all([prepared, fetching]); + strictEqual(settled, false); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + releaseResponse(); + // Wait for the actual concrete send to finish while the preparation + // observer remains blocked, rather than assuming a timer is long enough. + await concreteSendEnded; + const span = spanExporter.getSpan("activitypub.send_activity"); + ok(span); + if (status === 503) strictEqual(span.status.code, SpanStatusCode.ERROR); + // Only now yield a turn so an unhandled rejection would be detected + // before releasing the observer and joining the send promises. + await new Promise((resolve) => setTimeout(resolve, 0)); + strictEqual(settled, false); + releaseObserver(); + await result; + strictEqual(events.length, 2); + } finally { + releaseResponse(); + releaseObserver(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound preparation preserves earlier handoffs when later destination setup fails", async (t) => { + for (const immediate of [true, false]) { + await t.step( + immediate + ? "immediate handoff survives" + : "queued messages are not handed off yet", + async () => { + const events: OutboundBroadcastEvent[] = []; + const messages: Message[] = []; + const firstInbox = "https://receiver.example/first-inbox"; + const broken = recipient.clone({ + inbox: new URL("https://receiver.example/second-inbox"), + }); + // Fault injection: planning can retain this value, but collection-sync + // header construction cannot interpret it as a URL. + Object.defineProperty(broken, "id", { + value: { href: "invalid actor URL" }, + }); + let firstFetched!: () => void; + const fetching = new Promise((resolve) => { + firstFetched = resolve; + }); + fetchMock.mockGlobal().post(firstInbox, () => { + firstFetched(); + return new Response(null, { status: 202 }); + }); + try { + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + ...(immediate ? {} : { queue: { outbox: queue } }), + }); + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .setKeyPairsDispatcher( + () => [{ + privateKey: rsaPrivateKey2, + publicKey: rsaPublicKey2.publicKey!, + }], + ); + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + () => ({ + items: [recipient.clone({ inbox: new URL(firstInbox) }), broken], + }), + ); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + let originalError: unknown; + await rejects( + federation.createContext(new URL("https://example.com"), undefined) + .sendActivity({ identifier: "alice" }, "followers", activity, { + immediate, + syncCollection: true, + }), + (error: unknown) => { + originalError = error; + return error instanceof TypeError; + }, + ); + deepStrictEqual(preparation(events), { + type: "failed", + failure: { + kind: "local", + phase: "request-preparation", + error: originalError, + }, + planning: "complete", + plannedDestinationCount: 2, + }); + strictEqual(messages.length, 0); + if (immediate) { + // Prove the earlier send still reaches fetch after producer failure. + await fetching; + } + strictEqual(events.length, 2); + } finally { + fetchMock.hardReset(); + } + }, + ); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 1fc97a4fd..0163d53d9 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -1,36 +1,96 @@ import type { Context } from "./context.ts"; /** - * The start of an ordinary {@link Context.sendActivity} invocation. + * The current producer's preparation result, not delivery completion. + * @since 2.5.0 + */ +export type OutboundPreparationResult = + | { + /** The producer finished preparing its continuation. */ + readonly type: "prepared"; + /** Fanout-ready work still needs worker-side delivery preparation. */ + readonly preparedFor: "fanout" | "delivery"; + /** Number of deduplicated destinations, not actors or successful sends. */ + readonly destinationCount: number; + } + | { + /** A deliberate no-send decision. */ + readonly type: "skipped"; + /** The completed destination plan contained no obligations. */ + readonly reason: "no-destinations"; + } + | { + /** Preparation failed before the producer reached readiness. */ + readonly type: "failed"; + /** The operation that failed and its original thrown value. */ + readonly failure: { + /** Local processing failure, or unavailable operation provenance. */ + readonly kind: "local" | "unknown"; + /** The observed operation, not a diagnosis from the error message. */ + readonly phase: + | "sender-resolution" + | "key-resolution" + | "recipient-resolution" + | "transformation" + | "validation" + | "signing" + | "destination-planning" + | "serialization" + | "request-preparation" + | "unknown"; + /** Original thrown value, including non-Error values. */ + readonly error: unknown; + }; + /** Whether a complete destination plan was established before failure. */ + readonly planning: "not-started" | "incomplete" | "complete"; + /** Confirmed planned obligations so far, not an audience estimate. */ + readonly plannedDestinationCount: number; + }; + +/** + * An ordinary {@link Context.sendActivity} invocation's producer lifecycle. * - * This reports entry before preparation, not queue acceptance or delivery. + * Preparation results do not report queue acceptance or delivery completion. * @since 2.5.0 */ -export interface OutboundBroadcastEvent { - /** Fresh opaque identity for this invocation, independent of tracing. */ - readonly broadcastId: string; - /** This increment observes ordinary sends of a new activity. */ - readonly operation: "send"; - /** Preparation has not started yet. */ - readonly stage: "started"; -} +export type OutboundBroadcastEvent = + & { + /** Fresh opaque identity shared by this invocation's producer events. */ + readonly broadcastId: string; + /** The outgoing operation being observed. */ + readonly operation: "send"; + } + & ( + | { + /** Entry before sender/recipient resolution or signing. */ + readonly stage: "started"; + } + | { + /** The current producer's preparation settled. */ + readonly stage: "preparation-finished"; + /** Readiness, a deliberate skip, or the original preparation failure. */ + readonly result: OutboundPreparationResult; + } + ); /** * Optional observers of outgoing activity processing. * - * Currently only ordinary {@link Context.sendActivity} starts are observed. - * Callbacks are awaited without a default timeout. Their errors, including - * errors logging observer failures, do not replace the send's result. A - * callback that never settles delays the send indefinitely. Detached recording - * must handle its own rejection and shutdown; no persistence is guaranteed. + * Currently only ordinary {@link Context.sendActivity} producer events are + * observed. Callbacks are awaited without a default timeout. Their errors, + * including errors logging observer failures, do not replace the send's result. + * A callback that never settles delays execution indefinitely. Detached + * recording must handle its own rejection and shutdown; no persistence is + * guaranteed. * @template TContextData Application context data. * @since 2.5.0 */ export interface OutboundObservers { /** - * Observes an invocation before sender/recipient resolution or signing. + * Observes invocation entry and the current producer's preparation result. * The existing context is passed without loading data for observation. - * Returning or throwing cannot veto sending. + * Returning or throwing cannot veto sending. Earlier immediate sends may + * already be running when preparation finishes; their outcomes are separate. */ readonly onBroadcast?: ( context: Context, From 35ec8e9327b671bb8d1ce4c1d3641da6b228ed5c Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 21:50:25 +0900 Subject: [PATCH 03/53] Document outbound observer preparation Describe the current observer events, guarantees, and limitations. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- changes.d/fedify/outbound-observers.md | 12 ++++ claude-plugin/skills/fedify/SKILL.md | 16 +++++ docs/manual/send.md | 65 +++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 2 +- 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 changes.d/fedify/outbound-observers.md diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md new file mode 100644 index 000000000..fa882c5b8 --- /dev/null +++ b/changes.d/fedify/outbound-observers.md @@ -0,0 +1,12 @@ +--- +links: + '#1316': https://github.com/fedify-dev/fedify/issues/1316 +--- + - Added `setOutboundObservers()` on federation builders and instances to + observe ordinary `Context.sendActivity()` starts and preparation results + with application context and independent broadcast IDs, without requiring + OpenTelemetry. Register `onBroadcast` to record preparation separately from + queue acceptance and delivery success. Callbacks are awaited and their + errors do not change the send's result. Outbox-context overrides, + forwarding, replication, workers, and HTTP exchanges are not yet observed. + [[#1316] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index c6385e05b..448af7ffd 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -244,6 +244,22 @@ deliveries have no retry. > those must be distinct activities. Use a fresh UUID or counter in the > fragment. +Use `federation.setOutboundObservers({ onBroadcast(ctx, event) { ... } })` +(also available on builders) to observe ordinary `Context.sendActivity()` entry +and preparation results. This replaces the whole observer set; `{}` clears it. +Registration and builds copy callbacks, and active sends retain their selected +callback. The callback receives existing context/data without extra loading. +Events share an independent `broadcastId` and work without OpenTelemetry. +Narrow `event.stage` before reading `event.result`, then narrow `result.type` +(`prepared`, `skipped`, or `failed`). Preparation readiness is separate from +queue acceptance and delivery success. Callbacks are awaited without a timeout; +errors are isolated, but a nonsettling callback delays execution indefinitely. +Detached recording must handle its own rejection, shutdown, and persistence. +Outbox-context overrides, forwarding, portable replication, workers, attempts, +HTTP exchanges, and response capture are not observed yet. Mock federations +accept registration without executing observers. Incoming C2S outbox listeners +use the separate `setOutboxListeners()` API. + See . diff --git a/docs/manual/send.md b/docs/manual/send.md index 1f442a27d..c6948839f 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -59,6 +59,71 @@ async function sendFollow( > the *Context* section. +Observing outgoing preparation +------------------------------ + +*This API is available since Fedify 2.5.0.* + +Use `Federatable.setOutboundObservers()` on a federation builder or a directly +created federation to observe ordinary `~Context.sendActivity()` invocations: + +~~~~ typescript twoslash +import type { Federation, OutboundBroadcastEvent } from "@fedify/fedify"; +declare const federation: Federation<{ requestId: string }>; +declare function recordBroadcast( + requestId: string, + event: OutboundBroadcastEvent, +): Promise; +// ---cut-before--- +federation.setOutboundObservers({ + async onBroadcast(ctx, event) { + await recordBroadcast(ctx.data.requestId, event); + if (event.stage === "preparation-finished" && event.result.type === "failed") { + console.error(event.result.failure.phase, event.result.failure.error); + } + }, +}); +~~~~ + +Here, `recordBroadcast()` is your application's recording function. The callback +receives the existing context and application data without loading additional +data for observation. Each invocation has a fresh `broadcastId`, +shared by its `started` and `preparation-finished` events. These IDs work +without OpenTelemetry, including when no tracer provider is configured or +sampling is disabled. + +`started` precedes sender and recipient resolution and signing. The +`preparation-finished` event contains an `OutboundPreparationResult`: + + - `prepared` means the current producer reached readiness for fanout or + delivery. `destinationCount` counts deduplicated destinations, not + successful deliveries. Fanout workers still need to prepare deliveries. + - `skipped` with reason `no-destinations` means there was nothing to send. + - `failed` reports the preparation phase, the original thrown value (which + need not be an `Error`), and the extent of destination planning. + +Preparation readiness does not confirm queue acceptance or successful delivery. +Immediate sends can already be running when preparation finishes, and a later +send or enqueue failure does not replace the preparation result. + +Each registration call replaces all previous outbound observers; pass `{}` to +clear them. Fedify copies the supplied callbacks, and building a federation +copies the builder's registration. An active invocation keeps the callback +selected at entry even if registration changes while it is running. + +Callbacks are awaited without a default timeout. Their return values and errors +cannot veto sending, and callback or error-logging failures do not replace the +send's original result. Slow callbacks delay processing; a callback that never +settles delays it indefinitely. If you detach recording work, handle its +rejections and shutdown yourself; Fedify does not guarantee its persistence. + +Currently this API covers ordinary sends only. Sends through an +`OutboxContext` override, forwarding, automatic portable replication, workers, +delivery attempts, HTTP exchanges, and response bodies are not observed yet. +Incoming C2S `setOutboxListeners()` registration is separate. Mock federations +accept outbound observer registration but do not execute these observers. + + Specifying a sender ------------------- diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 0163d53d9..5648965df 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -55,7 +55,7 @@ export type OutboundPreparationResult = */ export type OutboundBroadcastEvent = & { - /** Fresh opaque identity shared by this invocation's producer events. */ + /** Fresh opaque identity shared by producer events, independent of tracing. */ readonly broadcastId: string; /** The outgoing operation being observed. */ readonly operation: "send"; From 43d60d7ffa8309f308a9a56934aa90dd092fdf54 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 22:26:49 +0900 Subject: [PATCH 04/53] Observe outbox-context send preparation Report broadcast preparation for sends from outbox contexts while preserving delivery state and existing send behavior. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/federation.ts | 6 +- packages/fedify/src/federation/middleware.ts | 92 ++--- .../src/federation/outbound-observation.ts | 2 +- .../src/federation/outbound-report.test.ts | 313 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 10 +- 5 files changed, 375 insertions(+), 48 deletions(-) diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 2a8b15681..9f8b0d28e 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -91,9 +91,9 @@ export interface Federatable extends TaskRegistry { * later changes to the supplied object do not affect registration. * Available on builders and directly created federations. Building copies * the callbacks, and an active send keeps the callback selected at entry. - * Currently only ordinary {@link Context.sendActivity} entry and preparation - * results are observed; outbox-context overrides, forwarding, and workers - * are not observed yet. + * Currently {@link Context.sendActivity} entry and preparation results are + * observed, including the outbox-context override. Forwarding and + * workers are not observed yet. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 00965acd8..4e3b69c16 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -7178,45 +7178,59 @@ export class OutboxContextImpl extends ContextImpl activity: Activity, options: SendActivityOptionsForCollection = {}, ): Promise { - const tracer = this.tracerProvider.getTracer( - metadata.name, - metadata.version, - ); - return tracer.startActiveSpan( - this.federation.outboxQueue == null || options.immediate - ? "activitypub.outbox" - : "activitypub.fanout", - { - kind: this.federation.outboxQueue == null || options.immediate - ? SpanKind.CLIENT - : SpanKind.PRODUCER, - attributes: { - "activitypub.activity.type": getTypeId(activity).href, - "activitypub.activity.to": activity.toIds.map((to) => to.href), - "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), - "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href), - "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href), - }, - }, - async (span) => { - try { - if (activity.id != null) { - span.setAttribute("activitypub.activity.id", activity.id.href); - } - const delivered = await this.sendActivityInternal( - sender, - recipients, - activity, - options, - span, - ); - if (delivered) this.#deliveryState.delivered = true; - } catch (e) { - span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); - throw e; - } finally { - span.end(); - } + return observeOutboundBroadcast( + this, + this.federation.outboundObservers.onBroadcast, + (observation) => { + const tracer = this.tracerProvider.getTracer( + metadata.name, + metadata.version, + ); + return tracer.startActiveSpan( + this.federation.outboxQueue == null || options.immediate + ? "activitypub.outbox" + : "activitypub.fanout", + { + kind: this.federation.outboxQueue == null || options.immediate + ? SpanKind.CLIENT + : SpanKind.PRODUCER, + attributes: { + "activitypub.activity.type": getTypeId(activity).href, + "activitypub.activity.to": activity.toIds.map((to) => to.href), + "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), + "activitypub.activity.bto": activity.btoIds.map((bto) => + bto.href + ), + "activitypub.activity.bcc": activity.bccIds.map((bcc) => + bcc.href + ), + }, + }, + async (span) => { + try { + if (activity.id != null) { + span.setAttribute("activitypub.activity.id", activity.id.href); + } + const delivered = await this.sendActivityInternal( + sender, + recipients, + activity, + options, + span, + observation, + ); + if (delivered) this.#deliveryState.delivered = true; + } catch (e) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: String(e), + }); + throw e; + } finally { + span.end(); + } + }, + ); }, ); } diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index a6acc9d45..37cad9633 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -11,7 +11,7 @@ type PreparationFailure = Extract< { type: "failed" } >; -/** State owned by one ordinary send invocation, never serialized into queues. */ +/** State owned by one send invocation, never serialized into queues. */ export class OutboundBroadcastObservation { readonly #broadcastId = crypto.randomUUID(); #finished = false; diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index c06613279..31f66b45e 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -16,7 +16,7 @@ import { } from "node:assert/strict"; import { createFederationBuilder } from "./builder.ts"; import { MemoryKvStore } from "./kv.ts"; -import { createFederation } from "./middleware.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; import type { Message, MessageQueue, @@ -640,3 +640,314 @@ test("outbound preparation preserves earlier handoffs when later destination set ); } }); + +test("outbox-context observers await starts and preserve context, registration, and failures", async () => { + const federation = createFederation<{ label: string }>(options); + const failure = Symbol("sender resolution failed"); + let resolutions = 0; + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .mapHandle(() => { + resolutions++; + throw failure; + }); + const base = federation.createContext(new URL("https://example.com"), { + label: "app", + }); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + let entered!: () => void; + const entry = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + let replacementCalls = 0; + federation.setOutboundObservers({ + async onBroadcast(context, event) { + contexts.push(context); + events.push(event); + if (event.stage === "started") { + entered(); + await gate; + } + throw new Error("recorder failed"); + }, + }); + const sending = rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + await entry; + strictEqual(resolutions, 0); + strictEqual(ctx.hasDeliveredActivity(), false); + federation.setOutboundObservers({ + onBroadcast() { + replacementCalls++; + }, + }); + release(); + await sending; + strictEqual(resolutions, 1); + strictEqual(replacementCalls, 0); + strictEqual(contexts.length, 2); + for (const context of contexts) strictEqual(context, ctx); + strictEqual(ctx.data.label, "app"); + deepStrictEqual(preparation(events), { + type: "failed", + failure: { kind: "local", phase: "sender-resolution", error: failure }, + planning: "not-started", + plannedDestinationCount: 0, + }); + strictEqual(ctx.hasDeliveredActivity(), false); + federation.setOutboundObservers({}); + await rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + strictEqual(resolutions, 2); + strictEqual(events.length, 2); + strictEqual(ctx.hasDeliveredActivity(), false); +}); + +test("outbox-context observers preserve empty sends without extra loading", async () => { + let loads = 0; + const federation = createFederation({ + kv: new MemoryKvStore(), + documentLoaderFactory: () => () => { + loads++; + throw new Error("unexpected lookup"); + }, + contextLoaderFactory: () => () => { + loads++; + throw new Error("unexpected lookup"); + }, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + await ctx.sendActivity(sender, [], activity); + deepStrictEqual(preparation(events), { + type: "skipped", + reason: "no-destinations", + }); + strictEqual(ctx.hasDeliveredActivity(), false); + federation.setOutboundObservers({}); + await ctx.sendActivity(sender, [], activity); + strictEqual(ctx.hasDeliveredActivity(), false); + strictEqual(events.length, 2); + strictEqual(loads, 0); +}); + +test("outbox-context observers report queue readiness before acceptance and preserve delivery state", async (t) => { + for (const role of ["outbox", "fanout"] as const) { + for (const rejected of [false, true]) { + await t.step( + `${role}: ${rejected ? "rejected" : "accepted"}`, + async () => { + const failure = Symbol("enqueue failed"); + const events: OutboundBroadcastEvent[] = []; + const messages: Message[] = []; + const checkpoints: { + context: unknown; + delivered: boolean; + enqueued: number; + }[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { [role]: queue }, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const clone = ctx.clone(undefined); + federation.setOutboundObservers({ + onBroadcast(context, event) { + events.push(event); + checkpoints.push({ + context, + delivered: ctx.hasDeliveredActivity(), + enqueued: messages.length, + }); + // Callback failure must not change acceptance or delivery state. + throw new Error("recorder failed"); + }, + }); + const sending = ctx.sendActivity( + sender, + [recipient, recipient], + activity, + { + fanout: role === "fanout" ? "force" : "skip", + }, + ); + if (rejected) { + await rejects(sending, (error: unknown) => error === failure); + } else await sending; + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: role === "fanout" ? "fanout" : "delivery", + destinationCount: 1, + }); + strictEqual(checkpoints.length, 2); + for (const checkpoint of checkpoints) { + strictEqual(checkpoint.context, ctx); + strictEqual(checkpoint.delivered, false); + strictEqual(checkpoint.enqueued, 0); + } + strictEqual(messages.length, 1); + strictEqual(messages[0].type, role); + strictEqual(ctx.hasDeliveredActivity(), !rejected); + strictEqual(clone.hasDeliveredActivity(), !rejected); + // Clearing observers keeps successful sends and the shared state working. + if (!rejected) { + federation.setOutboundObservers({}); + await ctx.sendActivity(sender, recipient, activity, { + fanout: role === "fanout" ? "force" : "skip", + }); + await ctx.sendActivity(sender, [], activity); + strictEqual(messages.length, 2); + strictEqual(events.length, 2); + strictEqual(ctx.hasDeliveredActivity(), true); + strictEqual(clone.hasDeliveredActivity(), true); + } + }, + ); + } + } +}); + +test("outbox-context observers report immediate readiness before transport settles", async (t) => { + for (const status of [202, 503]) { + await t.step(`HTTP ${status}`, async () => { + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + const [tracerProvider, spanExporter] = createTestTracerProvider(); + fetchMock.mockGlobal().post( + "https://receiver.example/inbox", + async () => { + fetched(); + await gate; + return new Response(null, { status }); + }, + ); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const clone = ctx.clone(undefined); + federation.setOutboundObservers({ + onBroadcast(context, event) { + contexts.push(context); + events.push(event); + if (event.stage === "preparation-finished") ready(); + }, + }); + let settled = false; + const sending = ctx.sendActivity(sender, recipient, activity, { + immediate: true, + }) + .finally(() => { + settled = true; + }); + const result = status === 202 + ? sending + : rejects(sending, (error: unknown) => { + return error instanceof Error && "statusCode" in error && + error.statusCode === status; + }); + await Promise.all([fetching, prepared]); + strictEqual(settled, false); + strictEqual(ctx.hasDeliveredActivity(), false); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + release(); + await result; + strictEqual(events.length, 2); + for (const context of contexts) strictEqual(context, ctx); + strictEqual(ctx.hasDeliveredActivity(), status === 202); + strictEqual(clone.hasDeliveredActivity(), status === 202); + const spans = spanExporter.getSpans("activitypub.outbox"); + strictEqual(spans.length, 1); + strictEqual( + spans[0].attributes["activitypub.activity.id"], + activity.id!.href, + ); + if (status === 503) { + strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + } + } finally { + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 5648965df..0ee08c84e 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -48,7 +48,8 @@ export type OutboundPreparationResult = }; /** - * An ordinary {@link Context.sendActivity} invocation's producer lifecycle. + * A {@link Context.sendActivity} invocation's producer lifecycle, including + * the outbox-context override. * * Preparation results do not report queue acceptance or delivery completion. * @since 2.5.0 @@ -76,9 +77,10 @@ export type OutboundBroadcastEvent = /** * Optional observers of outgoing activity processing. * - * Currently only ordinary {@link Context.sendActivity} producer events are - * observed. Callbacks are awaited without a default timeout. Their errors, - * including errors logging observer failures, do not replace the send's result. + * Currently {@link Context.sendActivity} producer events are observed, including + * the outbox-context override. Callbacks are awaited without a default timeout. + * Their errors, including errors logging observer failures, do not replace the + * send's result. * A callback that never settles delays execution indefinitely. Detached * recording must handle its own rejection and shutdown; no persistence is * guaranteed. From 964dbb7d97bc64fe449273f70e6a6e1a2a6a5e7c Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 22:35:33 +0900 Subject: [PATCH 05/53] Observe original-payload forwarding Report forwarding preparation and skipped unsigned activities to outbound observers. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/federation.ts | 6 +- packages/fedify/src/federation/middleware.ts | 105 ++++-- .../src/federation/outbound-observation.ts | 33 +- .../src/federation/outbound-report.test.ts | 353 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 13 +- 5 files changed, 450 insertions(+), 60 deletions(-) diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 9f8b0d28e..9a03631c3 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -91,9 +91,9 @@ export interface Federatable extends TaskRegistry { * later changes to the supplied object do not affect registration. * Available on builders and directly created federations. Building copies * the callbacks, and an active send keeps the callback selected at entry. - * Currently {@link Context.sendActivity} entry and preparation results are - * observed, including the outbox-context override. Forwarding and - * workers are not observed yet. + * Currently send and original-payload forwarding entry and preparation results + * are observed, including outbox-context operations. Automatic portable + * replication and workers are not observed yet. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 4e3b69c16..4d617eb76 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -6791,39 +6791,47 @@ function forwardActivity( recipients: Recipient | Recipient[] | "followers", options?: ForwardActivityOptions, ): Promise { - const tracer = ctx.tracerProvider.getTracer( - metadata.name, - metadata.version, - ); - return tracer.startActiveSpan( - ctx.federation.outboxQueue == null || options?.immediate - ? `activitypub.${loggerCategory}` - : "activitypub.fanout", - { - kind: ctx.federation.outboxQueue == null || options?.immediate - ? SpanKind.CLIENT - : SpanKind.PRODUCER, - attributes: { "activitypub.activity.type": ctx.activityType }, - }, - async (span) => { - try { - if (ctx.activityId != null) { - span.setAttribute("activitypub.activity.id", ctx.activityId); - } - return await forwardActivityInternal( - ctx, - loggerCategory, - forwarder, - recipients, - options, - ); - } catch (e) { - span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); - throw e; - } finally { - span.end(); - } + return observeOutboundBroadcast( + ctx, + ctx.federation.outboundObservers.onBroadcast, + (observation) => { + const tracer = ctx.tracerProvider.getTracer( + metadata.name, + metadata.version, + ); + return tracer.startActiveSpan( + ctx.federation.outboxQueue == null || options?.immediate + ? `activitypub.${loggerCategory}` + : "activitypub.fanout", + { + kind: ctx.federation.outboxQueue == null || options?.immediate + ? SpanKind.CLIENT + : SpanKind.PRODUCER, + attributes: { "activitypub.activity.type": ctx.activityType }, + }, + async (span) => { + try { + if (ctx.activityId != null) { + span.setAttribute("activitypub.activity.id", ctx.activityId); + } + return await forwardActivityInternal( + ctx, + loggerCategory, + forwarder, + recipients, + options, + observation, + ); + } catch (e) { + span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); + throw e; + } finally { + span.end(); + } + }, + ); }, + "forward", ); } @@ -6837,8 +6845,10 @@ async function forwardActivityInternal( | { username: string }, recipients: Recipient | Recipient[] | "followers", options?: ForwardActivityOptions, + observation?: OutboundBroadcastObservation, ): Promise { const logger = getLogger(["fedify", "federation", loggerCategory]); + observation?.setPhase("sender-resolution"); let keys: SenderKeyPair[]; let identifier: string | null = null; if ( @@ -6865,6 +6875,7 @@ async function forwardActivityInternal( identifier = mapped; } } + observation?.setPhase("key-resolution"); const actorKeyPairs = await ctx.getActorKeyPairs(identifier); if (actorKeyPairs.length < 1) { throw new Error( @@ -6876,6 +6887,7 @@ async function forwardActivityInternal( privateKey: kp.privateKey, })); } else if (Array.isArray(forwarder)) { + observation?.setPhase("key-resolution"); if (forwarder.length < 1) { throw new Error("The forwarder's key pairs are empty."); } @@ -6883,10 +6895,14 @@ async function forwardActivityInternal( } else { keys = [forwarder]; } + observation?.setPhase("validation"); if (!hasSignatureLike(ctx.activity)) { const hasProof = hasProofLike(ctx.activity); if (!hasProof) { - if (options?.skipIfUnsigned) return false; + if (options?.skipIfUnsigned) { + if (observation != null) await observation.skipped("unsigned"); + return false; + } logger.warn( "The activity {activityId} is not signed; even if it is " + "forwarded to other servers as is, it may not be accepted by " + @@ -6899,6 +6915,7 @@ async function forwardActivityInternal( ); } } + observation?.setPhase("recipient-resolution"); if (recipients === "followers") { if (identifier == null) { throw new Error( @@ -6912,17 +6929,20 @@ async function forwardActivityInternal( } recipients = followers; } + observation?.setPhase("destination-planning"); const inboxes = extractInboxes({ recipients: Array.isArray(recipients) ? recipients : [recipients], preferSharedInbox: options?.preferSharedInbox, excludeBaseUris: options?.excludeBaseUris, }); + observation?.planned(globalThis.Object.keys(inboxes).length); if (globalThis.Object.keys(inboxes).length < 1) { logger.debug("No inboxes found for activity {activityId}.", { activityId: ctx.activityId, activityType: ctx.activityType, identifier: identifier ?? undefined, }); + if (observation != null) await observation.skipped(); return false; } logger.debug("Forwarding activity {activityId} to inboxes:\n{inboxes}", { @@ -6942,10 +6962,13 @@ async function forwardActivityInternal( "set.", ); } + observation?.setPhase("request-preparation"); const promises: Promise[] = []; for (const inbox in inboxes) { - promises.push( - sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, { + const sending = sendActivityToInbox( + inbox, + inboxes[inbox].gatewayInboxes, + { allowPrivateAddress: ctx.federation.allowPrivateAddress, keys, activity: ctx.activity, @@ -6960,9 +6983,15 @@ async function forwardActivityInternal( ctx.federation.firstKnock, { specTtl: ctx.federation.httpMessageSignaturesSpecTtl }, ), - }), + }, ); + if (observation != null) { + // Preserve early rejections while the preparation observer is awaited. + sending.catch(() => {}); + } + promises.push(sending); } + if (observation != null) await observation.prepared("delivery"); await Promise.all(promises); return true; } @@ -6970,14 +6999,17 @@ async function forwardActivityInternal( "Enqueuing activity {activityId} to forward later.", { activityId: ctx.activityId, activity: ctx.activity }, ); + observation?.setPhase("request-preparation"); if (!ctx.federation.manuallyStartQueue) { ctx.federation._startQueueInternal(ctx.data); } + observation?.setPhase("key-resolution"); const keyJwkPairs: SenderKeyJwkPair[] = []; for (const { keyId, privateKey } of keys) { const privateKeyJwk = await exportJwk(privateKey); keyJwkPairs.push({ keyId: keyId.href, privateKey: privateKeyJwk }); } + observation?.setPhase("request-preparation"); const carrier: Record = {}; propagation.inject(context.active(), carrier); const orderingKey = options?.orderingKey; @@ -7010,6 +7042,7 @@ async function forwardActivityInternal( orderingKey: message.orderingKey, }); } + if (observation != null) await observation.prepared("delivery"); const { outboxQueue } = ctx.federation; if (outboxQueue.enqueueMany == null || orderingKey != null) { const promises: PromiseSettledResult[] = await Promise.allSettled( diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 37cad9633..9d4238500 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -11,7 +11,7 @@ type PreparationFailure = Extract< { type: "failed" } >; -/** State owned by one send invocation, never serialized into queues. */ +/** State owned by one outgoing invocation, never serialized into queues. */ export class OutboundBroadcastObservation { readonly #broadcastId = crypto.randomUUID(); #finished = false; @@ -24,6 +24,7 @@ export class OutboundBroadcastObservation { readonly observer: NonNullable< OutboundObservers["onBroadcast"] >, + readonly operation: OutboundBroadcastEvent["operation"], ) {} setPhase(phase: PreparationFailure["failure"]["phase"]): void { @@ -44,25 +45,28 @@ export class OutboundBroadcastObservation { })); } - skipped(): Promise { + skipped( + reason: Extract["reason"] = + "no-destinations", + ): Promise { return this.#finish(Object.freeze({ type: "skipped", - reason: "no-destinations", + reason, })); } - async run( + async run( send: ( observation?: OutboundBroadcastObservation, - ) => Promise, - ): Promise { + ) => Promise, + ): Promise { await this.#notify(Object.freeze({ broadcastId: this.#broadcastId, - operation: "send", + operation: this.operation, stage: "started", })); try { - await send(this); + return await send(this); } catch (error) { const failure: PreparationFailure["failure"] = { kind: this.#phase === "unknown" ? "unknown" : "local", @@ -84,7 +88,7 @@ export class OutboundBroadcastObservation { this.#finished = true; await this.#notify(Object.freeze({ broadcastId: this.#broadcastId, - operation: "send", + operation: this.operation, stage: "preparation-finished", result, })); @@ -107,13 +111,16 @@ export class OutboundBroadcastObservation { } /** Observe preparation without changing the unobserved send's execution path. */ -export function observeOutboundBroadcast( +export function observeOutboundBroadcast( context: Context, observer: OutboundObservers["onBroadcast"], send: ( observation?: OutboundBroadcastObservation, - ) => Promise, -): Promise { + ) => Promise, + operation: OutboundBroadcastEvent["operation"] = "send", +): Promise { if (observer == null) return send(); - return new OutboundBroadcastObservation(context, observer).run(send); + return new OutboundBroadcastObservation(context, observer, operation).run( + send, + ); } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 31f66b45e..1d3458ee1 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -16,8 +16,13 @@ import { } from "node:assert/strict"; import { createFederationBuilder } from "./builder.ts"; import { MemoryKvStore } from "./kv.ts"; -import { ContextImpl, createFederation } from "./middleware.ts"; +import { + ContextImpl, + createFederation, + OutboxContextImpl, +} from "./middleware.ts"; import type { + Federation, Message, MessageQueue, OutboundBroadcastEvent, @@ -951,3 +956,349 @@ test("outbox-context observers report immediate readiness before transport settl }); } }); + +// Keep extension fields and signature-shaped data opaque; forwarding observes +// their existing shape checks, not authentication or a vocabulary round trip. +const forwardingPayload = { + "@context": ["https://www.w3.org/ns/activitystreams", { + extension: "https://example.com/extension", + }], + type: "Create", + id: "https://sender.example/create", + actor: "https://sender.example/actor", + extension: { values: ["original", "payload"] }, + signature: { + type: "RsaSignature2017", + creator: "https://sender.example/key", + signatureValue: "opaque signature", + }, +}; + +function forwardingContext( + federation: Federation, + kind: "inbox" | "outbox", + payload: unknown, + data: TContextData, +) { + const base = federation.createContext(new URL("https://example.com"), data); + ok(base instanceof ContextImpl); + return kind === "inbox" + ? base.toInboxContext( + null, + payload, + forwardingPayload.id, + Create.typeId.href, + ) + : base.toOutboxContext( + "alice", + payload, + forwardingPayload.id, + Create.typeId.href, + ); +} + +test("forwarding observers distinguish skips from preparation failures without extra loading", async (t) => { + for (const kind of ["inbox", "outbox"] as const) { + for (const scenario of ["unsigned", "empty", "failed"] as const) { + await t.step(`${kind}: ${scenario}`, async () => { + let loads = 0; + const failure = Symbol("resolution failed"); + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + documentLoaderFactory: () => () => { + loads++; + throw new Error("extra lookup"); + }, + contextLoaderFactory: () => () => { + loads++; + throw new Error("extra lookup"); + }, + }); + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .mapHandle(() => { + throw failure; + }); + const { signature: _signature, ...unsigned } = forwardingPayload; + const ctx = forwardingContext( + federation, + kind, + scenario === "unsigned" ? unsigned : forwardingPayload, + { label: "app" }, + ); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onBroadcast(context, event) { + contexts.push(context); + events.push(event); + throw new Error("recorder failed"); + }, + }); + const forward = () => + ctx.forwardActivity( + scenario === "failed" ? { username: "alice" } : sender, + scenario === "empty" ? [] : recipient, + { skipIfUnsigned: true }, + ); + if (scenario === "failed") { + await rejects(forward(), (error: unknown) => error === failure); + } else await forward(); + deepStrictEqual( + preparation(events), + scenario === "failed" + ? { + type: "failed", + failure: { + kind: "local", + phase: "sender-resolution", + error: failure, + }, + planning: "not-started", + plannedDestinationCount: 0, + } + : { + type: "skipped", + reason: scenario === "unsigned" ? "unsigned" : "no-destinations", + }, + ); + for (const event of events) strictEqual(event.operation, "forward"); + for (const context of contexts) strictEqual(context, ctx); + strictEqual(ctx.data.label, "app"); + strictEqual(loads, 0); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), false); + } + federation.setOutboundObservers({}); + if (scenario === "failed") { + await rejects(forward(), (error: unknown) => error === failure); + } else await forward(); + strictEqual(events.length, 2); + strictEqual(loads, 0); + }); + } + } +}); + +test("forwarding observers await readiness before enqueue while preserving raw payloads and errors", async (t) => { + for (const batch of [false, true]) { + for (const rejected of [false, true]) { + await t.step( + `${batch ? "batch" : "single"}: ${rejected ? "rejected" : "accepted"}`, + async () => { + const failure = Symbol("enqueue failure"); + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + ...(batch + ? { + enqueueMany(batch: readonly Message[]) { + messages.push(...batch); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + } + : {}), + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + queue: { outbox: queue }, + manuallyStartQueue: true, + activityTransformers: [() => { + throw new Error("forwarding must not transform"); + }], + }); + const ctx = forwardingContext( + federation, + batch ? "outbox" : "inbox", + forwardingPayload, + undefined, + ); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await gate; + } + throw new Error("recorder failed"); + }, + }); + const sending = ctx.forwardActivity(sender, [recipient, recipient], { + skipIfUnsigned: true, + }); + const result = rejected + ? rejects(sending, (error: unknown) => error === failure) + : sending; + await prepared; + strictEqual(messages.length, 0); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), false); + } + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + release(); + await result; + strictEqual(events.length, 2); + strictEqual(messages.length, 1); + ok(messages[0].type === "outbox"); + strictEqual(messages[0].activity, forwardingPayload); + ok(!("broadcastId" in messages[0])); + for (const event of events) strictEqual(event.operation, "forward"); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), !rejected); + } + federation.setOutboundObservers({}); + const unobserved = ctx.forwardActivity(sender, recipient, { + skipIfUnsigned: true, + }); + if (rejected) { + await rejects(unobserved, (error: unknown) => error === failure); + } else await unobserved; + strictEqual(messages.length, 2); + ok(messages[1].type === "outbox"); + strictEqual(messages[1].activity, forwardingPayload); + strictEqual(events.length, 2); + }, + ); + } + } +}); + +test("forwarding observers retain readiness and registration through late immediate results", async (t) => { + for (const status of [202, 503]) { + await t.step(`HTTP ${status}`, async () => { + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let respond!: () => void; + const responseGate = new Promise((resolve) => { + respond = resolve; + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const observerGate = new Promise((resolve) => { + release = resolve; + }); + const [tracerProvider, exporter] = createTestTracerProvider(); + let ended!: () => void; + const concreteSendEnded = new Promise((resolve) => { + ended = resolve; + }); + const exportSpans = exporter.export.bind(exporter); + exporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (spans.some((span) => span.name === "activitypub.send_activity")) { + ended(); + } + }; + let submitted: unknown; + fetchMock.mockGlobal().post( + "https://receiver.example/inbox", + async (call) => { + submitted = JSON.parse(String(call.options.body)); + fetched(); + await responseGate; + return new Response(null, { status }); + }, + ); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + }); + const ctx = forwardingContext( + federation, + status === 202 ? "inbox" : "outbox", + forwardingPayload, + undefined, + ); + const events: OutboundBroadcastEvent[] = []; + let replacements = 0; + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await observerGate; + } + }, + }); + let settled = false; + const sending = ctx.forwardActivity(sender, recipient, { + immediate: true, + skipIfUnsigned: true, + }) + .finally(() => { + settled = true; + }); + const result = status === 202 + ? sending + : rejects(sending, (error: unknown) => { + return error instanceof Error && "statusCode" in error && + error.statusCode === status; + }); + await Promise.all([prepared, fetching]); + deepStrictEqual(submitted, forwardingPayload); + strictEqual(settled, false); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + federation.setOutboundObservers({ + onBroadcast() { + replacements++; + }, + }); + respond(); + await concreteSendEnded; + await new Promise((resolve) => setTimeout(resolve, 0)); + strictEqual(settled, false); + release(); + await result; + strictEqual(events.length, 2); + strictEqual(replacements, 0); + for (const event of events) strictEqual(event.operation, "forward"); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), false); + } + const spans = exporter.getSpans( + status === 202 ? "activitypub.inbox" : "activitypub.outbox", + ); + strictEqual(spans.length, 1); + if (status === 503) { + strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + } + } finally { + respond(); + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 0ee08c84e..d2ed2f4fa 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -16,8 +16,8 @@ export type OutboundPreparationResult = | { /** A deliberate no-send decision. */ readonly type: "skipped"; - /** The completed destination plan contained no obligations. */ - readonly reason: "no-destinations"; + /** An empty completed plan, or an unsigned payload skipped by policy. */ + readonly reason: "no-destinations" | "unsigned"; } | { /** Preparation failed before the producer reached readiness. */ @@ -48,8 +48,7 @@ export type OutboundPreparationResult = }; /** - * A {@link Context.sendActivity} invocation's producer lifecycle, including - * the outbox-context override. + * A send or original-payload forwarding invocation's producer lifecycle. * * Preparation results do not report queue acceptance or delivery completion. * @since 2.5.0 @@ -59,7 +58,7 @@ export type OutboundBroadcastEvent = /** Fresh opaque identity shared by producer events, independent of tracing. */ readonly broadcastId: string; /** The outgoing operation being observed. */ - readonly operation: "send"; + readonly operation: "send" | "forward"; } & ( | { @@ -77,8 +76,8 @@ export type OutboundBroadcastEvent = /** * Optional observers of outgoing activity processing. * - * Currently {@link Context.sendActivity} producer events are observed, including - * the outbox-context override. Callbacks are awaited without a default timeout. + * Send and original-payload forwarding producer events are observed, including + * outbox-context operations. Callbacks are awaited without a default timeout. * Their errors, including errors logging observer failures, do not replace the * send's result. * A callback that never settles delays execution indefinitely. Detached From eb6743ffe08eb2382e29f3e668631783a3d33e99 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 22:56:13 +0900 Subject: [PATCH 06/53] Observe portable replication preparation Emit independent producer events for portable replication while preserving forwarding claims and deadlines. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/federation.ts | 6 +- packages/fedify/src/federation/middleware.ts | 104 +++--- .../src/federation/outbound-observation.ts | 30 +- .../fedify/src/federation/outbound-report.ts | 18 +- .../src/federation/portable-inbox.test.ts | 341 +++++++++++++++++- .../fedify/src/federation/portable-inbox.ts | 100 +++-- 6 files changed, 504 insertions(+), 95 deletions(-) diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 9a03631c3..af0c31d3d 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -91,9 +91,9 @@ export interface Federatable extends TaskRegistry { * later changes to the supplied object do not affect registration. * Available on builders and directly created federations. Building copies * the callbacks, and an active send keeps the callback selected at entry. - * Currently send and original-payload forwarding entry and preparation results - * are observed, including outbox-context operations. Automatic portable - * replication and workers are not observed yet. + * Currently send, original-payload forwarding, and automatic portable + * replication entry and preparation results are observed, including + * outbox-context operations. Workers are not observed yet. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 4d617eb76..22361378d 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -4525,52 +4525,66 @@ export class FederationImpl onNotFound, portableInbox: { recipient, - forward: async (activity, activityId, activityType, signatures) => { - if (this.portableInboxForwarding.maxTargets < 1) return; - if (this.kv.cas == null) { - if (!this.#portableInboxForwardingWarned) { - this.#portableInboxForwardingWarned = true; - logger.warn( - "Activities delivered to FEP-ef61 portable inboxes are not " + - "forwarded to the other gateways, as the key–value store " + - "does not support compare-and-swap (KvStore.cas()), which " + - "is needed to forward each activity at most once.", - ); - } - return; - } - await forwardPortableInboxActivity({ - recipient, - activity, - activityId, - activityType, - excludedOrigins, - baseUrl: context.origin, - kv: this.kv, - kvPrefix: this.kvPrefixes.portableInboxForwarding, - outboxQueue: this.outboxQueue, - startQueue: this.manuallyStartQueue - ? undefined - : () => this._startQueueInternal(contextData), - allowPrivateAddress: this.allowPrivateAddress, - getKeys: () => - this.#getPortableGatewayKeyPairs( - context, - identifier, + forward: (activity, activityId, activityType, signatures) => + observeOutboundBroadcast( + context, + this.outboundObservers.onBroadcast, + async (observation) => { + if (this.portableInboxForwarding.maxTargets < 1) { + if (observation != null) { + await observation.skipped("forwarding-disabled"); + } + return; + } + if (this.kv.cas == null) { + if (!this.#portableInboxForwardingWarned) { + this.#portableInboxForwardingWarned = true; + logger.warn( + "Activities delivered to FEP-ef61 portable inboxes are not " + + "forwarded to the other gateways, as the key–value store " + + "does not support compare-and-swap (KvStore.cas()), which " + + "is needed to forward each activity at most once.", + ); + } + if (observation != null) { + await observation.skipped("forwarding-unavailable"); + } + return; + } + await forwardPortableInboxActivity({ recipient, - ), - specDeterminer: new KvSpecDeterminer( - this.kv, - this.kvPrefixes.httpMessageSignaturesSpec, - this.firstKnock, - { specTtl: this.httpMessageSignaturesSpecTtl }, - ), - signatures, - options: this.portableInboxForwarding, - meterProvider: this.meterProvider, - tracerProvider: this.tracerProvider, - }); - }, + activity, + activityId, + activityType, + excludedOrigins, + baseUrl: context.origin, + kv: this.kv, + kvPrefix: this.kvPrefixes.portableInboxForwarding, + outboxQueue: this.outboxQueue, + startQueue: this.manuallyStartQueue + ? undefined + : () => this._startQueueInternal(contextData), + allowPrivateAddress: this.allowPrivateAddress, + getKeys: () => + this.#getPortableGatewayKeyPairs( + context, + identifier, + recipient, + ), + specDeterminer: new KvSpecDeterminer( + this.kv, + this.kvPrefixes.httpMessageSignaturesSpec, + this.firstKnock, + { specTtl: this.httpMessageSignaturesSpecTtl }, + ), + signatures, + options: this.portableInboxForwarding, + meterProvider: this.meterProvider, + tracerProvider: this.tracerProvider, + }, observation); + }, + "portable-replication", + ), }, }); }, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 9d4238500..03bc76146 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -37,6 +37,10 @@ export class OutboundBroadcastObservation { this.#destinationCount = destinationCount; } + planDestination(): void { + this.#destinationCount++; + } + prepared(preparedFor: "fanout" | "delivery"): Promise { return this.#finish(Object.freeze({ type: "prepared", @@ -68,21 +72,25 @@ export class OutboundBroadcastObservation { try { return await send(this); } catch (error) { - const failure: PreparationFailure["failure"] = { - kind: this.#phase === "unknown" ? "unknown" : "local", - phase: this.#phase, - error, - }; - await this.#finish(Object.freeze({ - type: "failed", - failure: Object.freeze(failure), - planning: this.#planning, - plannedDestinationCount: this.#destinationCount, - })); + await this.failed(error); throw error; } } + failed(error: unknown): Promise { + const failure: PreparationFailure["failure"] = { + kind: this.#phase === "unknown" ? "unknown" : "local", + phase: this.#phase, + error, + }; + return this.#finish(Object.freeze({ + type: "failed", + failure: Object.freeze(failure), + planning: this.#planning, + plannedDestinationCount: this.#destinationCount, + })); + } + async #finish(result: OutboundPreparationResult): Promise { if (this.#finished) return; this.#finished = true; diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index d2ed2f4fa..9b9cd8470 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -16,8 +16,13 @@ export type OutboundPreparationResult = | { /** A deliberate no-send decision. */ readonly type: "skipped"; - /** An empty completed plan, or an unsigned payload skipped by policy. */ - readonly reason: "no-destinations" | "unsigned"; + /** The existing no-send decision made by this producer. */ + readonly reason: + | "no-destinations" + | "unsigned" + | "forwarding-disabled" + | "forwarding-unavailable" + | "no-forwarding-targets"; } | { /** Preparation failed before the producer reached readiness. */ @@ -48,7 +53,7 @@ export type OutboundPreparationResult = }; /** - * A send or original-payload forwarding invocation's producer lifecycle. + * A send, original-payload forwarding, or portable replication producer lifecycle. * * Preparation results do not report queue acceptance or delivery completion. * @since 2.5.0 @@ -58,7 +63,7 @@ export type OutboundBroadcastEvent = /** Fresh opaque identity shared by producer events, independent of tracing. */ readonly broadcastId: string; /** The outgoing operation being observed. */ - readonly operation: "send" | "forward"; + readonly operation: "send" | "forward" | "portable-replication"; } & ( | { @@ -76,8 +81,9 @@ export type OutboundBroadcastEvent = /** * Optional observers of outgoing activity processing. * - * Send and original-payload forwarding producer events are observed, including - * outbox-context operations. Callbacks are awaited without a default timeout. + * Send, original-payload forwarding, and automatic portable replication producer + * events are observed, including outbox-context operations. Callbacks are + * awaited without a default timeout. * Their errors, including errors logging observer failures, do not replace the * send's result. * A callback that never settles delays execution indefinitely. Detached diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index a22ef54b6..7baf53376 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -1,5 +1,9 @@ import type { InboxRequestReport } from "./inbox-report.ts"; -import { mockDocumentLoader, test } from "@fedify/fixture"; +import { + createTestTracerProvider, + mockDocumentLoader, + test, +} from "@fedify/fixture"; import { type Activity, Create, @@ -16,7 +20,13 @@ import { parseIri, type RemoteDocument, } from "@fedify/vocab-runtime"; -import { assert, assertEquals, assertFalse, assertThrows } from "@std/assert"; +import { + assert, + assertEquals, + assertFalse, + assertStrictEquals, + assertThrows, +} from "@std/assert"; import { encodeBase64 } from "byte-encodings/base64"; import fetchMock from "fetch-mock"; import { signRequest, verifyRequestDetailed } from "../sig/http.ts"; @@ -35,6 +45,8 @@ import { type KvKey, type KvStore, MemoryKvStore } from "./kv.ts"; import type { FederationOptions } from "./federation.ts"; import { createFederation } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; +import { observeOutboundBroadcast } from "./outbound-observation.ts"; +import type { OutboundBroadcastEvent } from "./outbound-report.ts"; import { coversDelivery, forwardPortableInboxActivity, @@ -2280,3 +2292,328 @@ test("onRequestFinished covers portable ingress and recipient lookup failures", error: failure, }); }); + +function replicationResult(events: OutboundBroadcastEvent[]) { + assertEquals(events.map((event) => event.stage), [ + "started", + "preparation-finished", + ]); + assertEquals(events.map((event) => event.operation), [ + "portable-replication", + "portable-replication", + ]); + assertEquals(events[0].broadcastId, events[1].broadcastId); + const event = events[1]; + assert(event.stage === "preparation-finished"); + return event.result; +} + +test("portable replication observers distinguish disabled, unavailable, and unclaimed work", async (t) => { + for (const scenario of ["disabled", "unavailable", "no-targets"] as const) { + await t.step(scenario, async () => { + const memory = new MemoryKvStore(); + const kv: KvStore = scenario === "unavailable" + ? { + get: (key) => memory.get(key), + set: (key, value, options) => memory.set(key, value, options), + delete: (key) => memory.delete(key), + list: (prefix) => memory.list(prefix), + } + : memory; + const queue = new RecordingQueue(); + const { federation } = setup({ + kv, + queue, + keyPairs: false, + gateways: scenario === "no-targets" ? [LOCAL] : [LOCAL, GATEWAY2], + options: { + portableInboxForwarding: { + maxTargets: scenario === "disabled" ? 0 : 10, + }, + }, + }); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onBroadcast(ctx, event) { + contexts.push(ctx); + events.push(event); + }, + }); + const response = await federation.fetch( + post(inboxUrl(), await signedFollow()), + { contextData: undefined }, + ); + assertEquals(response.status, 202); + assertEquals(replicationResult(events), { + type: "skipped", + reason: scenario === "disabled" + ? "forwarding-disabled" + : scenario === "unavailable" + ? "forwarding-unavailable" + : "no-forwarding-targets", + }); + assertStrictEquals(contexts[0], contexts[1]); + assertEquals(queue.outbox.length, 0); + }); + } +}); + +test("portable replication observers preserve lazy enqueue, unsigned fallback, and failed claims", async (t) => { + await t.step( + "readiness precedes acceptance without delaying earlier handoffs", + async () => { + const queue = new RecordingQueue(); + let accept!: () => void; + const acceptance = new Promise((resolve) => { + accept = resolve; + }); + queue.enqueue = (message) => { + queue.messages.push(message); + return queue.messages.length === 1 + ? acceptance.then(() => { + throw new Error("acceptance unknown"); + }) + : Promise.resolve(); + }; + const { federation } = setup({ queue, keyPairs: false }); + const events: OutboundBroadcastEvent[] = []; + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const observerGate = new Promise((resolve) => { + release = resolve; + }); + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await observerGate; + } + throw new Error("recorder failed"); + }, + }); + const json = await signedFollow(); + let settled = false; + const pending = federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + }) + .finally(() => { + settled = true; + }); + await prepared; + assertEquals(queue.outbox.length, 1); + assertFalse(settled); + assertEquals(replicationResult(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 2, + }); + accept(); + release(); + assertEquals((await pending).status, 202); + assertEquals(queue.outbox.length, 2); + for (const message of queue.outbox) { + assertEquals(message.activity, json); + assertEquals(message.keys, []); + assertFalse("broadcastId" in message); + } + assertEquals(events.length, 2); + // A swallowed enqueue failure keeps its claim; redelivery is not retried. + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + assertEquals( + (await federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + })).status, + 202, + ); + assertEquals(replicationResult(events.slice(2)), { + type: "skipped", + reason: "no-forwarding-targets", + }); + assert(events[0].broadcastId !== events[2].broadcastId); + assertEquals(queue.outbox.length, 2); + }, + ); + await t.step( + "partial claim failure is incomplete planning and preserves ingress acceptance", + async () => { + const memory = new MemoryKvStore(); + const cas = memory.cas.bind(memory); + const failure = { failed: "second claim" }; + let claims = 0; + memory.cas = (...args) => { + if (args[0][1] === "portableInboxForwarding" && ++claims === 2) { + return Promise.reject(failure); + } + return cas(...args); + }; + const queue = new RecordingQueue(); + const { federation } = setup({ kv: memory, queue, keyPairs: false }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const response = await federation.fetch( + post(inboxUrl(), await signedFollow()), + { contextData: undefined }, + ); + assertEquals(response.status, 202); + assertEquals(replicationResult(events), { + type: "failed", + failure: { + kind: "local", + phase: "destination-planning", + error: failure, + }, + planning: "incomplete", + plannedDestinationCount: 1, + }); + assertEquals(queue.outbox.length, 0); + }, + ); + await t.step( + "a handled key failure prepares unsigned queue messages", + async () => { + const queue = new RecordingQueue(); + const parameters = forwardingParameters({ + outboxQueue: queue, + getKeys: () => Promise.reject(new Error("no keys")), + }); + const ctx = createFederation({ kv: parameters.kv }).createContext( + new URL(LOCAL), + ); + const events: OutboundBroadcastEvent[] = []; + await observeOutboundBroadcast( + ctx, + (_ctx, event) => { + events.push(event); + }, + (observation) => forwardPortableInboxActivity(parameters, observation), + "portable-replication", + ); + assertEquals(replicationResult(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + assertEquals(queue.outbox[0].keys, []); + assertStrictEquals(queue.outbox[0].activity, parameters.activity); + }, + ); +}); + +test("portable replication observers retain preparation after the forwarding deadline", async (t) => { + for (const failed of [false, true]) { + await t.step( + failed ? "late preparation failure" : "late unsigned readiness", + async () => { + let release!: () => void; + const keyGate = new Promise((resolve) => { + release = resolve; + }); + const [tracerProvider, exporter] = createTestTracerProvider(); + let ended!: () => void; + const concreteSendEnded = new Promise((resolve) => { + ended = resolve; + }); + const exportSpans = exporter.export.bind(exporter); + exporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (spans.some((span) => span.name === "activitypub.send_activity")) { + ended(); + } + }; + const failure = { failed: "late local preparation" }; + const parameters = forwardingParameters({ + tracerProvider, + options: resolvePortableInboxForwardingOptions({ + deadline: { milliseconds: 0 }, + }), + getKeys: async () => { + await keyGate; + if (failed) { + Object.defineProperty(parameters.activityId, "href", { + get() { + throw failure; + }, + }); + } + return []; + }, + }); + const ctx = createFederation({ kv: parameters.kv }).createContext( + new URL(LOCAL), + ); + const events: OutboundBroadcastEvent[] = []; + let finished!: () => void; + const preparation = new Promise((resolve) => { + finished = resolve; + }); + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + fetchMock.spyGlobal().post(`begin:${GATEWAY2}/`, () => { + fetched(); + return new Response(null, { status: 503 }); + }); + try { + const forwarded = await observeOutboundBroadcast( + ctx, + (_ctx, event) => { + events.push(event); + if (event.stage === "preparation-finished") finished(); + }, + (observation) => + forwardPortableInboxActivity(parameters, observation), + "portable-replication", + ); + assertEquals(forwarded.length, 1); + assertEquals(events.map((event) => event.stage), ["started"]); + release(); + await preparation; + if (failed) { + assertEquals(replicationResult(events), { + type: "failed", + failure: { + kind: "local", + phase: "request-preparation", + error: failure, + }, + planning: "complete", + plannedDestinationCount: 1, + }); + assertEquals(fetchMock.callHistory.calls().length, 0); + } else { + assertEquals(replicationResult(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + await fetching; + await concreteSendEnded; + // Only now drain the late catch/logging continuation. + await new Promise((resolve) => setTimeout(resolve, 0)); + assertEquals(events.length, 2); + // The next request sees the retained claim and cannot forward again. + assertEquals(await forwardPortableInboxActivity(parameters), []); + } + } finally { + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }, + ); + } +}); diff --git a/packages/fedify/src/federation/portable-inbox.ts b/packages/fedify/src/federation/portable-inbox.ts index 04866a11e..e11455dec 100644 --- a/packages/fedify/src/federation/portable-inbox.ts +++ b/packages/fedify/src/federation/portable-inbox.ts @@ -27,6 +27,7 @@ import type { PortableInboxForwardingOptions } from "./federation.ts"; import type { KvKey, KvStore } from "./kv.ts"; import { recordOutboxEnqueue } from "./metrics.ts"; import type { MessageQueue } from "./mq.ts"; +import type { OutboundBroadcastObservation } from "./outbound-observation.ts"; import type { OutboxMessage, SenderKeyJwkPair } from "./queue.ts"; import { sendActivity, type SenderKeyPair } from "./send.ts"; @@ -291,10 +292,12 @@ export interface ForwardPortableInboxActivityParameters { * other gateways, that gateway has the activity already, so it is claimed * without forwarding the activity to it. See {@link identifySendingGateway}. * @param parameters The parameters. + * @param observation The in-process producer observation, if registered. * @returns The target inbox URLs that the activity was handed off to. */ -export async function forwardPortableInboxActivity( +export async function forwardPortableInboxActivity( parameters: ForwardPortableInboxActivityParameters, + observation?: OutboundBroadcastObservation, ): Promise { const logger = getLogger(["fedify", "federation", "inbox"]); const { @@ -308,10 +311,20 @@ export async function forwardPortableInboxActivity( const options = parameters.options ?? resolvePortableInboxForwardingOptions(); const { maxTargets, ttl } = options; - if (kv.cas == null || maxTargets < 1) return []; + if (kv.cas == null) { + if (observation != null) { + await observation.skipped("forwarding-unavailable"); + } + return []; + } + if (maxTargets < 1) { + if (observation != null) await observation.skipped("forwarding-disabled"); + return []; + } // The deadline covers both identifying the sending gateway and, without // an outbox queue, forwarding: const deadline = Date.now() + options.deadline.total("millisecond"); + observation?.setPhase("destination-planning"); // A compatible activity ID is canonicalized too, so that its equivalent // representations share the same forwarding claims: const canonicalActivityId = getCanonicalPortableId(activityId) ?? @@ -362,9 +375,14 @@ export async function forwardPortableInboxActivity( for (const { gateway, inbox } of targets) { if (await kv.cas(claimKey(gateway), undefined, true, { ttl })) { claimed.push(inbox); + observation?.planDestination(); } } - if (claimed.length < 1) return []; + observation?.planned(claimed.length); + if (claimed.length < 1) { + if (observation != null) await observation.skipped("no-forwarding-targets"); + return []; + } logger.debug( "Forwarding activity {activityId} from the portable inbox {inbox} to " + "other gateways:\n{targets}", @@ -375,9 +393,9 @@ export async function forwardPortableInboxActivity( }, ); if (outboxQueue == null) { - await forwardImmediately(parameters, claimed, deadline); + await forwardImmediately(parameters, claimed, deadline, observation); } else { - await enqueueForwarding(parameters, outboxQueue, claimed); + await enqueueForwarding(parameters, outboxQueue, claimed, observation); } return claimed; } @@ -565,10 +583,11 @@ async function resolveForwardingKey( return key; } -async function forwardImmediately( +async function forwardImmediately( parameters: ForwardPortableInboxActivityParameters, inboxes: readonly URL[], deadlineAt: number, + observation?: OutboundBroadcastObservation, ): Promise { const { activity, @@ -582,27 +601,43 @@ async function forwardImmediately( const logger = getLogger(["fedify", "federation", "inbox"]); const deadline = Math.max(0, deadlineAt - Date.now()); // The deadline also covers resolving the key, which may be slow: - const forwarding = resolveForwardingKey(parameters).then((key) => - Promise.all(inboxes.map((inbox) => - sendActivity({ - activity, - activityId: activityId.href, - activityType, - keys: key == null ? [] : [key], - inbox, - allowPrivateAddress, - specDeterminer, - meterProvider, - tracerProvider, - }).catch((error) => { - logger.error( - "Failed to forward activity {activityId} to {inbox}; it will not " + - "be forwarded to the inbox again:\n{error}", - { activityId: activityId.href, inbox: inbox.href, error }, - ); - }) - )) - ); + observation?.setPhase("key-resolution"); + const forwarding = (async () => { + try { + const key = await resolveForwardingKey(parameters); + observation?.setPhase("request-preparation"); + const sending = inboxes.map((inbox) => + sendActivity({ + activity, + activityId: activityId.href, + activityType, + keys: key == null ? [] : [key], + inbox, + allowPrivateAddress, + specDeterminer, + meterProvider, + tracerProvider, + }).catch((error) => { + logger.error( + "Failed to forward activity {activityId} to {inbox}; it will not " + + "be forwarded to the inbox again:\n{error}", + { activityId: activityId.href, inbox: inbox.href, error }, + ); + }) + ); + if (observation != null) { + // A logging sink may throw while an earlier send settles. Retain its + // rejection until the preparation callback finishes and the sends join. + for (const promise of sending) promise.catch(() => {}); + await observation.prepared("delivery"); + } + await Promise.all(sending); + } catch (error) { + // Preparation can settle after the deadline has returned to the caller. + if (observation != null) await observation.failed(error); + throw error; + } + })(); let timer: ReturnType | undefined; const timedOut = await Promise.race([ forwarding.then(() => false), @@ -620,10 +655,11 @@ async function forwardImmediately( } } -async function enqueueForwarding( +async function enqueueForwarding( parameters: ForwardPortableInboxActivityParameters, outboxQueue: MessageQueue, inboxes: readonly URL[], + observation?: OutboundBroadcastObservation, ): Promise { const { activity, @@ -634,6 +670,7 @@ async function enqueueForwarding( meterProvider, } = parameters; const logger = getLogger(["fedify", "federation", "inbox"]); + observation?.setPhase("key-resolution"); const key = await resolveForwardingKey(parameters); const keys: SenderKeyJwkPair[] = []; if (key != null) { @@ -650,10 +687,12 @@ async function enqueueForwarding( ); } } + observation?.setPhase("request-preparation"); startQueue?.(); const started = new Date().toISOString(); const traceContext: Record = {}; propagation.inject(context.active(), traceContext); + let preparedCount = 0; await Promise.all(inboxes.map(async (inbox) => { const message: OutboxMessage = { type: "outbox", @@ -670,6 +709,11 @@ async function enqueueForwarding( headers: {}, traceContext, }; + if (observation != null && ++preparedCount === inboxes.length) { + // Earlier destinations may already be enqueuing; do not wait for their + // acceptance before reporting the last local message construction. + await observation.prepared("delivery"); + } try { await outboxQueue.enqueue(message); } catch (error) { From 65e6808e412323503ff079af159727badf3de439 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 23:05:35 +0900 Subject: [PATCH 07/53] Cover outbound preparation failure phases Changelog: none Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-report.test.ts | 77 ++++++++++++++++--- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 1d3458ee1..805d786d8 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -307,7 +307,10 @@ test("outbound preparation preserves failure provenance and plan completeness", for ( const phase of [ "sender-resolution", + "key-resolution", + "recipient-resolution", "transformation", + "signing", "destination-planning", "serialization", ] as const @@ -339,6 +342,14 @@ test("outbound preparation preserves failure provenance and plan completeness", value: () => Promise.reject(failure), }); } + if (phase === "signing") { + sentActivity = activity.clone({}); + Object.defineProperty(sentActivity, "getProofs", { + value() { + throw failure; + }, + }); + } if (phase === "destination-planning") { target = recipient.clone({}); Object.defineProperty(target, "inboxId", { @@ -356,18 +367,64 @@ test("outbound preparation preserves failure provenance and plan completeness", throw failure; }); } - await rejects( - federation.createContext(new URL("https://example.com"), undefined) - .sendActivity( - phase === "sender-resolution" ? { username: "alice" } : sender, - target, - sentActivity, - ), - (error: unknown) => error === failure, + let keyDispatches = 0; + if (phase === "key-resolution" || phase === "recipient-resolution") { + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .setKeyPairsDispatcher(() => { + keyDispatches++; + if (phase === "key-resolution") throw failure; + return [{ + privateKey: rsaPrivateKey2, + publicKey: rsaPublicKey2.publicKey!, + }]; + }); + } + if (phase === "recipient-resolution") { + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + () => { + throw failure; + }, + ); + } + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, ); - deepStrictEqual(preparation(events), { + const sending = phase === "recipient-resolution" + ? ctx.sendActivity({ identifier: "alice" }, "followers", sentActivity) + : ctx.sendActivity( + phase === "sender-resolution" + ? { username: "alice" } + : phase === "key-resolution" + ? { identifier: "alice" } + : sender, + target, + sentActivity, + ); + let originalError: unknown; + await rejects(sending, (error: unknown) => { + originalError = error; + if (phase === "key-resolution") { + // Key-dispatcher errors are already handled by getActorKeyPairs(). + // Preserve the public send error from the resulting empty key set. + return error instanceof Error && + error.message === 'No key pair found for actor "alice".'; + } + return error === failure; + }); + if (phase === "key-resolution" || phase === "recipient-resolution") { + strictEqual(keyDispatches, 1); + } + const result = preparation(events); + ok(result.type === "failed"); + strictEqual(result.failure.error, originalError); + deepStrictEqual(result, { type: "failed", - failure: { kind: "local", phase, error: failure }, + failure: { kind: "local", phase, error: originalError }, planning: phase === "serialization" ? "complete" : phase === "destination-planning" From f8776985d9a602c30ab43f5a3bf53ab3ef403bdb Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 23:44:13 +0900 Subject: [PATCH 08/53] Observe producer delivery handoffs Report outbound delivery handoffs and their outcomes to observers, including portable inbox and forwarding paths. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 1 + packages/fedify/src/federation/federation.ts | 3 +- packages/fedify/src/federation/middleware.ts | 301 ++++++++++------- .../src/federation/outbound-observation.ts | 228 +++++++++++-- .../src/federation/outbound-report.test.ts | 304 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 68 ++++ .../src/federation/portable-delivery.test.ts | 31 ++ .../src/federation/portable-inbox.test.ts | 85 ++++- .../fedify/src/federation/portable-inbox.ts | 100 +++--- packages/testing/src/mock.test.ts | 5 +- 10 files changed, 946 insertions(+), 180 deletions(-) diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 6f3f8f7f8..443128b64 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -666,6 +666,7 @@ export class FederationBuilderImpl setOutboundObservers(observers: OutboundObservers): void { this.outboundObservers = globalThis.Object.freeze({ onBroadcast: observers.onBroadcast, + onDelivery: observers.onDelivery, }); } diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index af0c31d3d..32b6064c1 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -93,7 +93,8 @@ export interface Federatable extends TaskRegistry { * the callbacks, and an active send keeps the callback selected at entry. * Currently send, original-payload forwarding, and automatic portable * replication entry and preparation results are observed, including - * outbox-context operations. Workers are not observed yet. + * outbox-context operations and producer-side destination planning and + * handoffs. Workers are not observed yet. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 22361378d..57d7d41ab 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3324,38 +3324,46 @@ export class FederationImpl } const promises: Promise[] = []; for (const inbox in inboxes) { - const sending = sendActivityToInbox( - inbox, - inboxes[inbox].gatewayInboxes, - { - allowPrivateAddress: this.allowPrivateAddress, - keys, - activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, - sharedInbox: inboxes[inbox].sharedInbox, - headers: collectionSync == null ? undefined : new Headers({ - "Collection-Synchronization": - await buildCollectionSynchronizationHeader( - collectionSync, - inboxes[inbox].actorIds, - ), - }), - specDeterminer: new KvSpecDeterminer( - this.kv, - this.kvPrefixes.httpMessageSignaturesSpec, - this.firstKnock, - { specTtl: this.httpMessageSignaturesSpecTtl }, - ), - meterProvider: this.meterProvider, - tracerProvider: this.tracerProvider, - }, - ); + const delivery = observation?.delivery(inbox); + let sending: Promise; + try { + sending = sendActivityToInbox( + inbox, + inboxes[inbox].gatewayInboxes, + { + allowPrivateAddress: this.allowPrivateAddress, + keys, + activity: jsonLd, + activityId: activity.id?.href, + activityType: getTypeId(activity).href, + sharedInbox: inboxes[inbox].sharedInbox, + headers: collectionSync == null ? undefined : new Headers({ + "Collection-Synchronization": + await buildCollectionSynchronizationHeader( + collectionSync, + inboxes[inbox].actorIds, + ), + }), + specDeterminer: new KvSpecDeterminer( + this.kv, + this.kvPrefixes.httpMessageSignaturesSpec, + this.firstKnock, + { specTtl: this.httpMessageSignaturesSpecTtl }, + ), + meterProvider: this.meterProvider, + tracerProvider: this.tracerProvider, + }, + ); + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } if (observation != null) { // The preparation callback may await while an earlier send rejects. // Keep its original rejection for Promise.all without an unhandled one. sending.catch(() => {}); } + if (delivery != null) await delivery.handoff({ type: "direct" }); promises.push(sending); } if (observation != null) await observation.prepared("delivery"); @@ -3378,35 +3386,42 @@ export class FederationImpl propagation.inject(context.active(), carrier); const messages: { message: OutboxMessage; orderingKey?: string }[] = []; for (const inbox in inboxes) { - const { portableInbox, gatewayInboxes } = inboxes[inbox]; - const messageOrderingKey = orderingKey == null - ? undefined - : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; - const message: OutboxMessage = { - type: "outbox", - id: crypto.randomUUID(), - baseUrl: ctx.origin, - keys: keyJwkPairs, - activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, - inbox, - sharedInbox: inboxes[inbox].sharedInbox, - actorIds: [...inboxes[inbox].actorIds], - started: new Date().toISOString(), - attempt: 0, - headers: collectionSync == null ? {} : { - "Collection-Synchronization": - await buildCollectionSynchronizationHeader( - collectionSync, - inboxes[inbox].actorIds, - ), - }, - orderingKey: messageOrderingKey, - ...getPortableMessageFields(portableInbox, gatewayInboxes), - traceContext: carrier, - }; - messages.push({ message, orderingKey: messageOrderingKey }); + try { + const { portableInbox, gatewayInboxes } = inboxes[inbox]; + const messageOrderingKey = orderingKey == null + ? undefined + : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; + const message: OutboxMessage = { + type: "outbox", + id: crypto.randomUUID(), + baseUrl: ctx.origin, + keys: keyJwkPairs, + activity: jsonLd, + activityId: activity.id?.href, + activityType: getTypeId(activity).href, + inbox, + sharedInbox: inboxes[inbox].sharedInbox, + actorIds: [...inboxes[inbox].actorIds], + started: new Date().toISOString(), + attempt: 0, + headers: collectionSync == null ? {} : { + "Collection-Synchronization": + await buildCollectionSynchronizationHeader( + collectionSync, + inboxes[inbox].actorIds, + ), + }, + orderingKey: messageOrderingKey, + ...getPortableMessageFields(portableInbox, gatewayInboxes), + traceContext: carrier, + }; + messages.push({ message, orderingKey: messageOrderingKey }); + } catch (error) { + if (observation != null) { + await observation.delivery(inbox)?.failed(error); + } + throw error; + } } if (observation != null) await observation.prepared("delivery"); const { outboxQueue } = this; @@ -3416,7 +3431,14 @@ export class FederationImpl if (outboxQueue.enqueueMany == null || orderingKey != null) { const promises: PromiseSettledResult[] = await Promise.allSettled( messages.map(async (m) => { - await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }); + await (observation == null + ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) + : observation.enqueue( + [{ inbox: m.message.inbox, queueMessageId: m.message.id }], + "outbox", + () => + outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }), + )); recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message); }), ); @@ -3438,7 +3460,16 @@ export class FederationImpl } } else { try { - await outboxQueue.enqueueMany(messages.map((m) => m.message)); + await (observation == null + ? outboxQueue.enqueueMany(messages.map((m) => m.message)) + : observation.enqueue( + messages.map(({ message }) => ({ + inbox: message.inbox, + queueMessageId: message.id, + })), + "outbox", + () => outboxQueue.enqueueMany!(messages.map((m) => m.message)), + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to send later: {error}", @@ -4528,7 +4559,7 @@ export class FederationImpl forward: (activity, activityId, activityType, signatures) => observeOutboundBroadcast( context, - this.outboundObservers.onBroadcast, + this.outboundObservers, async (observation) => { if (this.portableInboxForwarding.maxTargets < 1) { if (observation != null) { @@ -5692,7 +5723,7 @@ export class ContextImpl implements Context { ): Promise { return observeOutboundBroadcast( this, - this.federation.outboundObservers.onBroadcast, + this.federation.outboundObservers, (observation) => { const tracer = this.tracerProvider.getTracer( metadata.name, @@ -5915,7 +5946,7 @@ export class ContextImpl implements Context { preferSharedInbox: options.preferSharedInbox, excludeBaseUris: options.excludeBaseUris, }); - observation?.planned(globalThis.Object.keys(inboxes).length); + if (observation != null) await observation.planDestinations(inboxes); if (globalThis.Object.keys(inboxes).length < 1) { logger.debug("No inboxes found for activity {activityId}.", { activityId: activity.id?.href, @@ -6000,10 +6031,21 @@ export class ContextImpl implements Context { this.federation._startQueueInternal(this.data); } if (observation != null) await observation.prepared("fanout"); - await this.federation.fanoutQueue.enqueue( - message, - { orderingKey: options.orderingKey }, - ); + await (observation == null + ? this.federation.fanoutQueue.enqueue(message, { + orderingKey: options.orderingKey, + }) + : observation.enqueue( + globalThis.Object.keys(inboxes).map((inbox) => ({ + inbox, + queueMessageId: message.id, + })), + "fanout", + () => + this.federation.fanoutQueue!.enqueue(message, { + orderingKey: options.orderingKey, + }), + )); getFederationMetrics(this.federation.meterProvider).recordQueueTaskEnqueued( { role: "fanout", @@ -6807,7 +6849,7 @@ function forwardActivity( ): Promise { return observeOutboundBroadcast( ctx, - ctx.federation.outboundObservers.onBroadcast, + ctx.federation.outboundObservers, (observation) => { const tracer = ctx.tracerProvider.getTracer( metadata.name, @@ -6949,7 +6991,7 @@ async function forwardActivityInternal( preferSharedInbox: options?.preferSharedInbox, excludeBaseUris: options?.excludeBaseUris, }); - observation?.planned(globalThis.Object.keys(inboxes).length); + if (observation != null) await observation.planDestinations(inboxes); if (globalThis.Object.keys(inboxes).length < 1) { logger.debug("No inboxes found for activity {activityId}.", { activityId: ctx.activityId, @@ -6979,30 +7021,38 @@ async function forwardActivityInternal( observation?.setPhase("request-preparation"); const promises: Promise[] = []; for (const inbox in inboxes) { - const sending = sendActivityToInbox( - inbox, - inboxes[inbox].gatewayInboxes, - { - allowPrivateAddress: ctx.federation.allowPrivateAddress, - keys, - activity: ctx.activity, - activityId: ctx.activityId, - activityType: ctx.activityType, - sharedInbox: inboxes[inbox].sharedInbox, - meterProvider: ctx.meterProvider, - tracerProvider: ctx.tracerProvider, - specDeterminer: new KvSpecDeterminer( - ctx.federation.kv, - ctx.federation.kvPrefixes.httpMessageSignaturesSpec, - ctx.federation.firstKnock, - { specTtl: ctx.federation.httpMessageSignaturesSpecTtl }, - ), - }, - ); + const delivery = observation?.delivery(inbox); + let sending: Promise; + try { + sending = sendActivityToInbox( + inbox, + inboxes[inbox].gatewayInboxes, + { + allowPrivateAddress: ctx.federation.allowPrivateAddress, + keys, + activity: ctx.activity, + activityId: ctx.activityId, + activityType: ctx.activityType, + sharedInbox: inboxes[inbox].sharedInbox, + meterProvider: ctx.meterProvider, + tracerProvider: ctx.tracerProvider, + specDeterminer: new KvSpecDeterminer( + ctx.federation.kv, + ctx.federation.kvPrefixes.httpMessageSignaturesSpec, + ctx.federation.firstKnock, + { specTtl: ctx.federation.httpMessageSignaturesSpecTtl }, + ), + }, + ); + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } if (observation != null) { // Preserve early rejections while the preparation observer is awaited. sending.catch(() => {}); } + if (delivery != null) await delivery.handoff({ type: "direct" }); promises.push(sending); } if (observation != null) await observation.prepared("delivery"); @@ -7030,38 +7080,50 @@ async function forwardActivityInternal( const started = new Date().toISOString(); const messages: { message: OutboxMessage; orderingKey?: string }[] = []; for (const inbox in inboxes) { - const { portableInbox, gatewayInboxes } = inboxes[inbox]; - const message: OutboxMessage = { - type: "outbox", - id: crypto.randomUUID(), - baseUrl: ctx.origin, - keys: keyJwkPairs, - activity: ctx.activity, - activityId: ctx.activityId, - activityType: ctx.activityType, - inbox, - sharedInbox: inboxes[inbox].sharedInbox, - actorIds: [...inboxes[inbox].actorIds], - started, - attempt: 0, - headers: {}, - orderingKey: orderingKey == null - ? undefined - : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`, - ...getPortableMessageFields(portableInbox, gatewayInboxes), - traceContext: carrier, - }; - messages.push({ - message, - orderingKey: message.orderingKey, - }); + try { + const { portableInbox, gatewayInboxes } = inboxes[inbox]; + const message: OutboxMessage = { + type: "outbox", + id: crypto.randomUUID(), + baseUrl: ctx.origin, + keys: keyJwkPairs, + activity: ctx.activity, + activityId: ctx.activityId, + activityType: ctx.activityType, + inbox, + sharedInbox: inboxes[inbox].sharedInbox, + actorIds: [...inboxes[inbox].actorIds], + started, + attempt: 0, + headers: {}, + orderingKey: orderingKey == null + ? undefined + : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`, + ...getPortableMessageFields(portableInbox, gatewayInboxes), + traceContext: carrier, + }; + messages.push({ + message, + orderingKey: message.orderingKey, + }); + } catch (error) { + if (observation != null) await observation.delivery(inbox)?.failed(error); + throw error; + } } if (observation != null) await observation.prepared("delivery"); const { outboxQueue } = ctx.federation; if (outboxQueue.enqueueMany == null || orderingKey != null) { const promises: PromiseSettledResult[] = await Promise.allSettled( messages.map(async (m) => { - await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }); + await (observation == null + ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) + : observation.enqueue( + [{ inbox: m.message.inbox, queueMessageId: m.message.id }], + "outbox", + () => + outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }), + )); recordOutboxEnqueue( ctx.federation.meterProvider, outboxQueue, @@ -7087,7 +7149,16 @@ async function forwardActivityInternal( } } else { try { - await outboxQueue.enqueueMany(messages.map((m) => m.message)); + await (observation == null + ? outboxQueue.enqueueMany(messages.map((m) => m.message)) + : observation.enqueue( + messages.map(({ message }) => ({ + inbox: message.inbox, + queueMessageId: message.id, + })), + "outbox", + () => outboxQueue.enqueueMany!(messages.map((m) => m.message)), + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to forward later:\n{error}", @@ -7227,7 +7298,7 @@ export class OutboxContextImpl extends ContextImpl ): Promise { return observeOutboundBroadcast( this, - this.federation.outboundObservers.onBroadcast, + this.federation.outboundObservers, (observation) => { const tracer = this.tracerProvider.getTracer( metadata.name, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 03bc76146..c1683dca1 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -2,6 +2,7 @@ import { getLogger } from "@logtape/logtape"; import type { Context } from "./context.ts"; import type { OutboundBroadcastEvent, + OutboundDeliveryEvent, OutboundObservers, OutboundPreparationResult, } from "./outbound-report.ts"; @@ -11,6 +12,117 @@ type PreparationFailure = Extract< { type: "failed" } >; +interface PlannedDestination { + readonly actorIds: Iterable; + readonly sharedInbox: boolean; + readonly portableInbox?: string; + readonly mode?: "portable-replication"; +} + +async function notifyObserver( + context: Context, + callback: + | ((context: Context, event: TEvent) => void | Promise) + | undefined, + event: TEvent, + unit: "broadcast" | "delivery", +): Promise { + if (callback == null) return; + try { + await callback(context, event); + } catch (error) { + try { + getLogger(["fedify", "federation", "outbox"]).error( + "An unexpected error occurred in outbound {unit} observer:\n{error}", + { unit, error }, + ); + } catch { + // A logging sink must not replace the send's result either. + } + } +} + +/** One producer's destination, without callbacks or live inputs in queues. */ +class OutboundDeliveryObservation { + #finished = false; + readonly #reference; + readonly #plan; + + constructor( + readonly context: Context, + readonly observer: NonNullable< + OutboundObservers["onDelivery"] + >, + broadcastId: string, + destinationInbox: string, + destination: PlannedDestination, + ) { + this.#reference = Object.freeze({ + broadcast: Object.freeze({ type: "known" as const, id: broadcastId }), + deliveryId: crypto.randomUUID(), + destinationInbox, + ...(destination.portableInbox == null + ? {} + : { portableInbox: destination.portableInbox }), + }); + this.#plan = Object.freeze({ + actorIds: Object.freeze(Array.from(destination.actorIds)), + sharedInbox: destination.sharedInbox, + mode: destination.mode ?? + (destination.portableInbox == null ? "ordinary" : "portable-fallback"), + }); + } + + planned(): Promise { + return notifyObserver( + this.context, + this.observer, + Object.freeze({ + ...this.#reference, + ...this.#plan, + stage: "planned" as const, + }), + "delivery", + ); + } + + handoff( + result: Extract["result"], + ): Promise { + if (this.#finished) return Promise.resolve(); + this.#finished = true; + return notifyObserver( + this.context, + this.observer, + Object.freeze({ + ...this.#reference, + stage: "handoff" as const, + result: Object.freeze(result), + }), + "delivery", + ); + } + + failed(error: unknown): Promise { + if (this.#finished) return Promise.resolve(); + this.#finished = true; + return notifyObserver( + this.context, + this.observer, + Object.freeze({ + ...this.#reference, + stage: "preparation-failed" as const, + failure: Object.freeze({ + kind: "local" as const, + phase: "request-preparation" as const, + error, + }), + }), + "delivery", + ); + } +} + /** State owned by one outgoing invocation, never serialized into queues. */ export class OutboundBroadcastObservation { readonly #broadcastId = crypto.randomUUID(); @@ -19,13 +131,83 @@ export class OutboundBroadcastObservation { #planning: PreparationFailure["planning"] = "not-started"; #destinationCount = 0; + readonly #onBroadcast: OutboundObservers["onBroadcast"]; + readonly #onDelivery: OutboundObservers["onDelivery"]; + readonly #deliveries = new Map< + string, + OutboundDeliveryObservation + >(); + constructor( readonly context: Context, - readonly observer: NonNullable< - OutboundObservers["onBroadcast"] - >, + observers: OutboundObservers, readonly operation: OutboundBroadcastEvent["operation"], - ) {} + ) { + this.#onBroadcast = observers.onBroadcast; + this.#onDelivery = observers.onDelivery; + } + + async planDestinations( + inboxes: Readonly>, + ): Promise { + this.planned(Object.keys(inboxes).length); + if (this.#onDelivery == null) return; + for (const [inbox, destination] of Object.entries(inboxes)) { + this.#deliveries.set( + inbox, + new OutboundDeliveryObservation( + this.context, + this.#onDelivery, + this.#broadcastId, + inbox, + destination, + ), + ); + } + for (const delivery of this.#deliveries.values()) await delivery.planned(); + } + + delivery( + inbox: string, + ): OutboundDeliveryObservation | undefined { + return this.#deliveries.get(inbox); + } + + async enqueue( + entries: readonly { + readonly inbox: string; + readonly queueMessageId: string; + }[], + queue: "fanout" | "outbox", + enqueue: () => Promise, + ): Promise { + let result: TResult; + try { + result = await enqueue(); + } catch (error) { + await Promise.all( + entries.map(({ inbox }) => + this.delivery(inbox)?.handoff({ + type: "enqueue-failed", + acceptance: "unknown", + error, + }) + ), + ); + throw error; + } + // Acceptance is established before unrelated metrics or logging can fail. + await Promise.all( + entries.map(({ inbox, queueMessageId }) => + this.delivery(inbox)?.handoff({ + type: "enqueued", + queue, + queueMessageId, + }) + ), + ); + return result; + } setPhase(phase: PreparationFailure["failure"]["phase"]): void { this.#phase = phase; @@ -37,8 +219,21 @@ export class OutboundBroadcastObservation { this.#destinationCount = destinationCount; } - planDestination(): void { + async planDestination( + inbox: string, + destination: PlannedDestination, + ): Promise { this.#destinationCount++; + if (this.#onDelivery == null) return; + const delivery = new OutboundDeliveryObservation( + this.context, + this.#onDelivery, + this.#broadcastId, + inbox, + destination, + ); + this.#deliveries.set(inbox, delivery); + await delivery.planned(); } prepared(preparedFor: "fanout" | "delivery"): Promise { @@ -102,33 +297,24 @@ export class OutboundBroadcastObservation { })); } - async #notify(event: OutboundBroadcastEvent): Promise { - try { - await this.observer(this.context, event); - } catch (error) { - try { - getLogger(["fedify", "federation", "outbox"]).error( - "An unexpected error occurred in outbound broadcast observer:\n{error}", - { error }, - ); - } catch { - // A logging sink must not replace the send's result either. - } - } + #notify(event: OutboundBroadcastEvent): Promise { + return notifyObserver(this.context, this.#onBroadcast, event, "broadcast"); } } /** Observe preparation without changing the unobserved send's execution path. */ export function observeOutboundBroadcast( context: Context, - observer: OutboundObservers["onBroadcast"], + observers: OutboundObservers, send: ( observation?: OutboundBroadcastObservation, ) => Promise, operation: OutboundBroadcastEvent["operation"] = "send", ): Promise { - if (observer == null) return send(); - return new OutboundBroadcastObservation(context, observer, operation).run( + if (observers.onBroadcast == null && observers.onDelivery == null) { + return send(); + } + return new OutboundBroadcastObservation(context, observers, operation).run( send, ); } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 805d786d8..6d10edff3 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -26,6 +26,7 @@ import type { Message, MessageQueue, OutboundBroadcastEvent, + OutboundDeliveryEvent, OutboundObservers, } from "./mod.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; @@ -525,6 +526,7 @@ test("outbound preparation finishes before immediate sends settle without report releaseObserver = resolve; }); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; const [tracerProvider, spanExporter] = createTestTracerProvider(); let sendEnded!: () => void; const concreteSendEnded = new Promise((resolve) => { @@ -552,6 +554,9 @@ test("outbound preparation finishes before immediate sends settle without report tracerProvider, }); federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, async onBroadcast(_ctx, event) { events.push(event); if (event.stage === "preparation-finished") { @@ -595,6 +600,13 @@ test("outbound preparation finishes before immediate sends settle without report strictEqual(settled, false); releaseObserver(); await result; + deepStrictEqual(deliveries.map((event) => event.stage), [ + "planned", + "handoff", + ]); + const handoff = deliveries[1]; + ok(handoff.stage === "handoff"); + deepStrictEqual(handoff.result, { type: "direct" }); strictEqual(events.length, 2); } finally { releaseResponse(); @@ -614,6 +626,7 @@ test("outbound preparation preserves earlier handoffs when later destination set : "queued messages are not handed off yet", async () => { const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; const messages: Message[] = []; const firstInbox = "https://receiver.example/first-inbox"; const broken = recipient.clone({ @@ -663,6 +676,9 @@ test("outbound preparation preserves earlier handoffs when later destination set }), ); federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, onBroadcast(_ctx, event) { events.push(event); }, @@ -694,6 +710,35 @@ test("outbound preparation preserves earlier handoffs when later destination set // Prove the earlier send still reaches fetch after producer failure. await fetching; } + const planned = deliveries.filter((event) => + event.stage === "planned" + ); + strictEqual(planned.length, 2); + const first = deliveries.filter((event) => + event.destinationInbox === firstInbox + ); + deepStrictEqual( + first.map((event) => event.stage), + immediate ? ["planned", "handoff"] : ["planned"], + ); + if (immediate) { + const handoff = first[1]; + ok(handoff.stage === "handoff"); + deepStrictEqual(handoff.result, { type: "direct" }); + } + const failed = deliveries.find((event) => + event.stage === "preparation-failed" + ); + ok(failed?.stage === "preparation-failed"); + strictEqual( + failed.destinationInbox, + "https://receiver.example/second-inbox", + ); + strictEqual(failed.failure.error, originalError); + strictEqual( + deliveries.filter((event) => event.stage === "handoff").length, + immediate ? 1 : 0, + ); strictEqual(events.length, 2); } finally { fetchMock.hardReset(); @@ -1294,8 +1339,12 @@ test("forwarding observers retain readiness and registration through late immedi undefined, ); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; let replacements = 0; federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, async onBroadcast(_ctx, event) { events.push(event); if (event.stage === "preparation-finished") { @@ -1337,6 +1386,13 @@ test("forwarding observers retain readiness and registration through late immedi strictEqual(settled, false); release(); await result; + deepStrictEqual(deliveries.map((event) => event.stage), [ + "planned", + "handoff", + ]); + const handoff = deliveries[1]; + ok(handoff.stage === "handoff"); + deepStrictEqual(handoff.result, { type: "direct" }); strictEqual(events.length, 2); strictEqual(replacements, 0); for (const event of events) strictEqual(event.operation, "forward"); @@ -1359,3 +1415,251 @@ test("forwarding observers retain readiness and registration through late immedi }); } }); + +test("outbound delivery observers snapshot registration and deduplicated destination facts", async () => { + const events: OutboundDeliveryEvent[] = []; + const contexts: unknown[] = []; + const messages: Message[] = []; + let replacements = 0; + let mutationApplied: boolean | undefined; + const registration: { + onDelivery: NonNullable["onDelivery"]>; + } = { + onDelivery(context, event) { + contexts.push(context); + events.push(event); + if (event.stage === "planned") { + mutationApplied = Reflect.set(event.actorIds, 0, "changed"); + federation.setOutboundObservers({ + onDelivery() { + replacements++; + }, + }); + } + }, + }; + const builder = createFederationBuilder(); + builder.setOutboundObservers(registration); + // The caller's object and later builder changes cannot replace the snapshot. + registration.onDelivery = () => { + replacements++; + }; + const federation = await builder.build({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + builder.setOutboundObservers({}); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const shared = new URL("https://receiver.example/shared"); + const recipients = ["alice", "bob"].map((name) => ({ + id: new URL(`https://receiver.example/${name}`), + inboxId: new URL(`https://receiver.example/${name}/inbox`), + endpoints: { sharedInbox: shared }, + })); + await ctx.sendActivity(sender, [...recipients, recipient], activity, { + preferSharedInbox: true, + }); + strictEqual(mutationApplied, false); + strictEqual(replacements, 0); + strictEqual(events.length, 4); + for (const context of contexts) strictEqual(context, ctx); + const planned = events.filter((event) => event.stage === "planned"); + strictEqual(planned.length, 2); + const sharedPlan = planned.find((event) => + event.destinationInbox === shared.href + ); + ok(sharedPlan?.stage === "planned"); + deepStrictEqual( + sharedPlan.actorIds, + recipients.map((recipient) => recipient.id.href), + ); + strictEqual(sharedPlan.sharedInbox, true); + strictEqual(sharedPlan.mode, "ordinary"); + notStrictEqual(planned[0].deliveryId, planned[1].deliveryId); + strictEqual(planned[0].broadcast.id, planned[1].broadcast.id); + for (const event of events) { + ok(Object.isFrozen(event)); + ok(Object.isFrozen(event.broadcast)); + const plan = planned.find((plan) => plan.deliveryId === event.deliveryId); + ok(plan); + strictEqual(event.destinationInbox, plan.destinationInbox); + if (event.stage === "handoff") { + ok(Object.isFrozen(event.result)); + const result = event.result; + ok(result.type === "enqueued"); + strictEqual(result.queue, "outbox"); + const message = messages.find((message) => + message.id === result.queueMessageId + ); + ok(message?.type === "outbox"); + strictEqual(message.inbox, event.destinationInbox); + ok(!message.actorIds?.includes("changed")); + } + } +}); + +test("outbound delivery observers report queue acceptance separately from producer readiness", async (t) => { + for (const mode of ["single", "batch", "fanout", "forward"] as const) { + for (const rejected of [false, true]) { + await t.step( + `${mode}: ${rejected ? "rejected" : "accepted"}`, + async () => { + const failure = Symbol("enqueue failed"); + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + const fails = rejected && + (mode !== "single" || messages.length === 1); + return fails ? Promise.reject(failure) : Promise.resolve(); + }, + ...(mode === "batch" || mode === "forward" + ? { + enqueueMany(batch: readonly Message[]) { + messages.push(...batch); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + } + : {}), + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: mode === "fanout" ? { fanout: queue } : { outbox: queue }, + }); + const deliveries: OutboundDeliveryEvent[] = []; + const broadcasts: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onDelivery(_ctx, event) { + deliveries.push(event); + throw new Error("recorder failed"); + }, + }); + const targets = [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ]; + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(ctx instanceof ContextImpl); + const sending = mode === "forward" + ? ctx.toInboxContext( + null, + forwardingPayload, + forwardingPayload.id, + Create.typeId.href, + ) + .forwardActivity(sender, targets, { skipIfUnsigned: true }) + : ctx.sendActivity(sender, targets, activity, { + fanout: mode === "fanout" ? "force" : "skip", + }); + if (rejected) { + await rejects(sending, (error: unknown) => error === failure); + } else await sending; + deepStrictEqual(preparation(broadcasts), { + type: "prepared", + preparedFor: mode === "fanout" ? "fanout" : "delivery", + destinationCount: 2, + }); + const planned = deliveries.filter((event) => + event.stage === "planned" + ); + const handoffs = deliveries.filter((event) => + event.stage === "handoff" + ); + strictEqual(planned.length, 2); + strictEqual(handoffs.length, 2); + for (const event of handoffs) { + ok(event.stage === "handoff"); + const fails = rejected && + (mode !== "single" || + event.destinationInbox === recipient.inboxId!.href); + if (fails) { + ok(event.result.type === "enqueue-failed"); + strictEqual(event.result.acceptance, "unknown"); + strictEqual(event.result.error, failure); + } else { + const result = event.result; + ok(result.type === "enqueued"); + strictEqual( + result.queue, + mode === "fanout" ? "fanout" : "outbox", + ); + ok( + messages.some((message) => + message.id === result.queueMessageId + ), + ); + } + strictEqual(event.broadcast.id, broadcasts[0].broadcastId); + strictEqual( + event.deliveryId, + planned.find((plan) => + plan.destinationInbox === event.destinationInbox + )!.deliveryId, + ); + } + }, + ); + } + } +}); + +test("outbound delivery observers retain accepted handoffs after bookkeeping failure", async () => { + const failure = Symbol("metrics failed after acceptance"); + const events: OutboundDeliveryEvent[] = []; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { + outbox: { + enqueue() { + Object.defineProperty(federation, "meterProvider", { + get() { + throw failure; + }, + }); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); + await rejects( + federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, recipient, activity), + (error: unknown) => error === failure, + ); + deepStrictEqual(events.map((event) => event.stage), ["planned", "handoff"]); + const event = events[1]; + ok(event.stage === "handoff"); + ok(event.result.type === "enqueued"); +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 9b9cd8470..cf5022925 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -78,6 +78,65 @@ export type OutboundBroadcastEvent = } ); +/** + * A producer-side destination obligation, not delivery completion. + * + * IDs group this producer's events. Queue execution and identity propagation + * through workers are not observed yet. An unvisited destination stays planned + * when another destination's preparation interrupts the producer. + * @since 2.5.0 + */ +export type OutboundDeliveryEvent = + & { + /** The invocation that established this destination. */ + readonly broadcast: { readonly type: "known"; readonly id: string }; + /** Opaque identity allocated after deduplication or a confirmed claim. */ + readonly deliveryId: string; + /** Planned destination, unchanged by concrete gateway selection. */ + readonly destinationInbox: string; + /** Canonical portable inbox when the selected route represents one. */ + readonly portableInbox?: string; + } + & ( + | { + /** This destination was established by the existing plan. */ + readonly stage: "planned"; + /** Known actors sharing the destination, not an exhaustive audience. */ + readonly actorIds: readonly string[]; + /** Whether the selected destination uses a shared inbox. */ + readonly sharedInbox: boolean; + /** Fallback routes share one obligation; replication targets do not. */ + readonly mode: "ordinary" | "portable-fallback" | "portable-replication"; + } + | { + /** Local execution entry or enqueue settlement, not remote success. */ + readonly stage: "handoff"; + /** Evidence from the actual producer handoff. */ + readonly result: + | { readonly type: "direct" } + | { + readonly type: "enqueued"; + readonly queue: "fanout" | "outbox"; + readonly queueMessageId: string; + } + | { + readonly type: "enqueue-failed"; + readonly acceptance: "unknown"; + readonly error: unknown; + }; + } + | { + /** Destination-local setup failed before entry into concrete sending. */ + readonly stage: "preparation-failed"; + /** The original local setup failure, without inventing an attempt. */ + readonly failure: { + readonly kind: "local"; + readonly phase: "request-preparation"; + readonly error: unknown; + }; + } + ); + /** * Optional observers of outgoing activity processing. * @@ -103,4 +162,13 @@ export interface OutboundObservers { context: Context, event: OutboundBroadcastEvent, ) => void | Promise; + /** + * Observes producer-side planning, handoffs, and destination preparation + * failures. Filtered recipients do not become planned deliveries. This does + * not report concrete attempts, remote success, or worker execution yet. + */ + readonly onDelivery?: ( + context: Context, + event: OutboundDeliveryEvent, + ) => void | Promise; } diff --git a/packages/fedify/src/federation/portable-delivery.test.ts b/packages/fedify/src/federation/portable-delivery.test.ts index 66d5e682e..9c8489ccf 100644 --- a/packages/fedify/src/federation/portable-delivery.test.ts +++ b/packages/fedify/src/federation/portable-delivery.test.ts @@ -7,6 +7,7 @@ import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; import { MemoryKvStore } from "./kv.ts"; import { FederationImpl, InboxContextImpl } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; +import type { OutboundDeliveryEvent } from "./outbound-report.ts"; import { MAX_DELIVERY_GATEWAYS } from "./portable-delivery.ts"; import type { FanoutMessage, Message, OutboxMessage } from "./queue.ts"; import { extractInboxes, SendActivityError } from "./send.ts"; @@ -584,6 +585,12 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { kv: new MemoryKvStore(), allowPrivateAddress: true, }); + const events: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); await createInboxContext(federation).forwardActivity(sender, recipient, { immediate: true, skipIfUnsigned: false, @@ -592,6 +599,15 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { inboxOn("https://gw1.example"), inboxOn("https://gw2.example"), ]); + deepStrictEqual(events.map((event) => event.stage), ["planned", "handoff"]); + strictEqual(events[0].deliveryId, events[1].deliveryId); + for (const event of events) { + strictEqual(event.destinationInbox, inboxOn("https://gw1.example")); + strictEqual(event.portableInbox, PORTABLE_INBOX); + } + const planned = events[0]; + ok(planned.stage === "planned"); + strictEqual(planned.mode, "portable-fallback"); }); await t.step("through the queue", async () => { @@ -604,6 +620,12 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { manuallyStartQueue: true, allowPrivateAddress: true, }); + const events: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); await createInboxContext(federation).forwardActivity(sender, recipient, { orderingKey: "key", skipIfUnsigned: false, @@ -617,6 +639,15 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { inboxOn("https://gw2.example"), ]); strictEqual(queued[0].options?.orderingKey, `key\n${DID}`); + deepStrictEqual(events.map((event) => event.stage), ["planned", "handoff"]); + strictEqual(events[0].deliveryId, events[1].deliveryId); + for (const event of events) { + strictEqual(event.destinationInbox, inboxOn("https://gw1.example")); + strictEqual(event.portableInbox, PORTABLE_INBOX); + } + const planned = events[0]; + ok(planned.stage === "planned"); + strictEqual(planned.mode, "portable-fallback"); }); fetchMock.hardReset(); diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index 7baf53376..9534e7518 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -46,7 +46,10 @@ import type { FederationOptions } from "./federation.ts"; import { createFederation } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; import { observeOutboundBroadcast } from "./outbound-observation.ts"; -import type { OutboundBroadcastEvent } from "./outbound-report.ts"; +import type { + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; import { coversDelivery, forwardPortableInboxActivity, @@ -2333,8 +2336,12 @@ test("portable replication observers distinguish disabled, unavailable, and uncl }, }); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; const contexts: unknown[] = []; federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, onBroadcast(ctx, event) { contexts.push(ctx); events.push(event); @@ -2354,6 +2361,7 @@ test("portable replication observers distinguish disabled, unavailable, and uncl : "no-forwarding-targets", }); assertStrictEquals(contexts[0], contexts[1]); + assertEquals(deliveries, []); assertEquals(queue.outbox.length, 0); }); } @@ -2378,6 +2386,7 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a }; const { federation } = setup({ queue, keyPairs: false }); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; let ready!: () => void; const prepared = new Promise((resolve) => { ready = resolve; @@ -2387,6 +2396,9 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a release = resolve; }); federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, async onBroadcast(_ctx, event) { events.push(event); if (event.stage === "preparation-finished") { @@ -2421,9 +2433,37 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a assertEquals(message.keys, []); assertFalse("broadcastId" in message); } + assertEquals( + deliveries.filter((event) => event.stage === "planned").length, + 2, + ); + const handoffs = deliveries.filter((event) => event.stage === "handoff"); + assertEquals(handoffs.length, 2); + assertEquals( + handoffs.filter((event) => + event.stage === "handoff" && event.result.type === "enqueued" + ).length, + 1, + ); + assertEquals( + handoffs.filter((event) => + event.stage === "handoff" && event.result.type === "enqueue-failed" && + event.result.acceptance === "unknown" + ).length, + 1, + ); + for (const event of deliveries) { + assertEquals(event.portableInbox, `ap+ef61://${did}/users/alice/inbox`); + if (event.stage === "planned") { + assertEquals(event.mode, "portable-replication"); + } + } assertEquals(events.length, 2); // A swallowed enqueue failure keeps its claim; redelivery is not retried. federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, onBroadcast(_ctx, event) { events.push(event); }, @@ -2458,7 +2498,11 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a const queue = new RecordingQueue(); const { federation } = setup({ kv: memory, queue, keyPairs: false }); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, onBroadcast(_ctx, event) { events.push(event); }, @@ -2478,6 +2522,7 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a planning: "incomplete", plannedDestinationCount: 1, }); + assertEquals(deliveries.map((event) => event.stage), ["planned"]); assertEquals(queue.outbox.length, 0); }, ); @@ -2493,10 +2538,16 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a new URL(LOCAL), ); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; await observeOutboundBroadcast( ctx, - (_ctx, event) => { - events.push(event); + { + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast: (_ctx, event) => { + events.push(event); + }, }, (observation) => forwardPortableInboxActivity(parameters, observation), "portable-replication", @@ -2506,6 +2557,14 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a preparedFor: "delivery", destinationCount: 1, }); + assertEquals(deliveries.map((event) => event.stage), [ + "planned", + "handoff", + ]); + const accepted = deliveries[1]; + assert( + accepted.stage === "handoff" && accepted.result.type === "enqueued", + ); assertEquals(queue.outbox[0].keys, []); assertStrictEquals(queue.outbox[0].activity, parameters.activity); }, @@ -2555,6 +2614,7 @@ test("portable replication observers retain preparation after the forwarding dea new URL(LOCAL), ); const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; let finished!: () => void; const preparation = new Promise((resolve) => { finished = resolve; @@ -2570,9 +2630,14 @@ test("portable replication observers retain preparation after the forwarding dea try { const forwarded = await observeOutboundBroadcast( ctx, - (_ctx, event) => { - events.push(event); - if (event.stage === "preparation-finished") finished(); + { + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast: (_ctx, event) => { + events.push(event); + if (event.stage === "preparation-finished") finished(); + }, }, (observation) => forwardPortableInboxActivity(parameters, observation), @@ -2582,7 +2647,15 @@ test("portable replication observers retain preparation after the forwarding dea assertEquals(events.map((event) => event.stage), ["started"]); release(); await preparation; + assertEquals(deliveries.map((event) => event.stage), [ + "planned", + failed ? "preparation-failed" : "handoff", + ]); + assertEquals(deliveries[0].deliveryId, deliveries[1].deliveryId); if (failed) { + const failedDelivery = deliveries[1]; + assert(failedDelivery.stage === "preparation-failed"); + assertStrictEquals(failedDelivery.failure.error, failure); assertEquals(replicationResult(events), { type: "failed", failure: { diff --git a/packages/fedify/src/federation/portable-inbox.ts b/packages/fedify/src/federation/portable-inbox.ts index e11455dec..c8ce77698 100644 --- a/packages/fedify/src/federation/portable-inbox.ts +++ b/packages/fedify/src/federation/portable-inbox.ts @@ -375,7 +375,14 @@ export async function forwardPortableInboxActivity( for (const { gateway, inbox } of targets) { if (await kv.cas(claimKey(gateway), undefined, true, { ttl })) { claimed.push(inbox); - observation?.planDestination(); + if (observation != null) { + await observation.planDestination(inbox.href, { + actorIds: [recipient.actorId.href], + sharedInbox: false, + portableInbox: recipient.canonicalInboxId, + mode: "portable-replication", + }); + } } } observation?.planned(claimed.length); @@ -606,31 +613,39 @@ async function forwardImmediately( try { const key = await resolveForwardingKey(parameters); observation?.setPhase("request-preparation"); - const sending = inboxes.map((inbox) => - sendActivity({ - activity, - activityId: activityId.href, - activityType, - keys: key == null ? [] : [key], - inbox, - allowPrivateAddress, - specDeterminer, - meterProvider, - tracerProvider, - }).catch((error) => { + const sending: Promise[] = []; + for (const inbox of inboxes) { + const delivery = observation?.delivery(inbox.href); + let pending: Promise; + try { + pending = sendActivity({ + activity, + activityId: activityId.href, + activityType, + keys: key == null ? [] : [key], + inbox, + allowPrivateAddress, + specDeterminer, + meterProvider, + tracerProvider, + }); + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } + const settled = pending.catch((error) => { logger.error( "Failed to forward activity {activityId} to {inbox}; it will not " + "be forwarded to the inbox again:\n{error}", { activityId: activityId.href, inbox: inbox.href, error }, ); - }) - ); - if (observation != null) { - // A logging sink may throw while an earlier send settles. Retain its - // rejection until the preparation callback finishes and the sends join. - for (const promise of sending) promise.catch(() => {}); - await observation.prepared("delivery"); + }); + // Callback awaiting must not leave an earlier rejection unhandled. + if (observation != null) settled.catch(() => {}); + sending.push(settled); + if (delivery != null) await delivery.handoff({ type: "direct" }); } + if (observation != null) await observation.prepared("delivery"); await Promise.all(sending); } catch (error) { // Preparation can settle after the deadline has returned to the caller. @@ -694,28 +709,41 @@ async function enqueueForwarding( propagation.inject(context.active(), traceContext); let preparedCount = 0; await Promise.all(inboxes.map(async (inbox) => { - const message: OutboxMessage = { - type: "outbox", - id: crypto.randomUUID(), - baseUrl, - keys, - activity, - activityId: activityId.href, - activityType, - inbox: inbox.href, - sharedInbox: false, - started, - attempt: 0, - headers: {}, - traceContext, - }; + const delivery = observation?.delivery(inbox.href); + let message: OutboxMessage; + try { + message = { + type: "outbox", + id: crypto.randomUUID(), + baseUrl, + keys, + activity, + activityId: activityId.href, + activityType, + inbox: inbox.href, + sharedInbox: false, + started, + attempt: 0, + headers: {}, + traceContext, + }; + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } if (observation != null && ++preparedCount === inboxes.length) { // Earlier destinations may already be enqueuing; do not wait for their // acceptance before reporting the last local message construction. await observation.prepared("delivery"); } try { - await outboxQueue.enqueue(message); + await (observation == null + ? outboxQueue.enqueue(message) + : observation.enqueue( + [{ inbox: inbox.href, queueMessageId: message.id }], + "outbox", + () => outboxQueue.enqueue(message), + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to forward to {inbox}; it " + diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index 5cc118ff8..ca1916bbb 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -209,7 +209,10 @@ test("reset clears sent activities", async () => { test("MockFederation accepts outbound observer registration", () => { const federation = createFederation(); - federation.setOutboundObservers({ onBroadcast: () => {} }); + federation.setOutboundObservers({ + onBroadcast: () => {}, + onDelivery: () => {}, + }); federation.setOutboundObservers({}); }); From 7a5d54dcdeae68c811b3916ac2a0f62f1cdab4b4 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 2 Nov 2026 23:54:33 +0900 Subject: [PATCH 09/53] Document outbound producer observations Describe broadcast and delivery observations, their producer scope, and the distinction between handoff and remote delivery. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 9 ++ changes.d/fedify/outbound-observers.md | 15 +- claude-plugin/skills/fedify/SKILL.md | 44 ++++-- docs/manual/send.md | 134 ++++++++++++++---- .../fedify/src/federation/outbound-report.ts | 5 +- 5 files changed, 156 insertions(+), 51 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 823ec2b1a..f9b92f911 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,6 +48,14 @@ To be released. `idempotency` strategy, as direct processing already did, instead of always deduplicating them per origin. Activities recorded by workers of older versions are still recognized. [[#1290], [#1299]] + - Added `setOutboundObservers()` on federation builders and instances to + observe ordinary `Context.sendActivity()` starts and preparation results + with application context and independent broadcast IDs, without requiring + OpenTelemetry. Register `onBroadcast` to record preparation separately from + queue acceptance and delivery success. Callbacks are awaited and their + errors do not change the send's result. Outbox-context overrides, + forwarding, replication, workers, and HTTP exchanges are not yet observed. + [[#1316] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin @@ -87,6 +95,7 @@ To be released. [#1289]: https://github.com/fedify-dev/fedify/issues/1289 [#1290]: https://github.com/fedify-dev/fedify/issues/1290 [#1299]: https://github.com/fedify-dev/fedify/pull/1299 +[#1316]: https://github.com/fedify-dev/fedify/issues/1316 [#1318]: https://github.com/fedify-dev/fedify/issues/1318 [#1328]: https://github.com/fedify-dev/fedify/pull/1328 [#1353]: https://github.com/fedify-dev/fedify/pull/1353 diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index fa882c5b8..1dae1dc0b 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -2,11 +2,12 @@ links: '#1316': https://github.com/fedify-dev/fedify/issues/1316 --- - - Added `setOutboundObservers()` on federation builders and instances to - observe ordinary `Context.sendActivity()` starts and preparation results - with application context and independent broadcast IDs, without requiring - OpenTelemetry. Register `onBroadcast` to record preparation separately from - queue acceptance and delivery success. Callbacks are awaited and their - errors do not change the send's result. Outbox-context overrides, - forwarding, replication, workers, and HTTP exchanges are not yet observed. + - Added `setOutboundObservers()` on federation builders and instances + for sends, original-payload forwarding, and automatic portable replication. + Register `onBroadcast` to record producer starts and preparation results, + and `onDelivery` for destination planning, direct handoffs, queue acceptance + or uncertainty, and local preparation failures. Callbacks receive + application context and independent grouping IDs without requiring + OpenTelemetry; their errors do not replace the original result. Readiness + and handoff observations do not establish remote delivery success. [[#1316] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index 448af7ffd..d0f63afec 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -244,21 +244,35 @@ deliveries have no retry. > those must be distinct activities. Use a fresh UUID or counter in the > fragment. -Use `federation.setOutboundObservers({ onBroadcast(ctx, event) { ... } })` -(also available on builders) to observe ordinary `Context.sendActivity()` entry -and preparation results. This replaces the whole observer set; `{}` clears it. -Registration and builds copy callbacks, and active sends retain their selected -callback. The callback receives existing context/data without extra loading. -Events share an independent `broadcastId` and work without OpenTelemetry. -Narrow `event.stage` before reading `event.result`, then narrow `result.type` -(`prepared`, `skipped`, or `failed`). Preparation readiness is separate from -queue acceptance and delivery success. Callbacks are awaited without a timeout; -errors are isolated, but a nonsettling callback delays execution indefinitely. -Detached recording must handle its own rejection, shutdown, and persistence. -Outbox-context overrides, forwarding, portable replication, workers, attempts, -HTTP exchanges, and response capture are not observed yet. Mock federations -accept registration without executing observers. Incoming C2S outbox listeners -use the separate `setOutboxListeners()` API. +Use +`federation.setOutboundObservers({ onBroadcast(ctx, event) { ... }, onDelivery(ctx, event) { ... } })` +(also available on builders) for producer observations of +ordinary/outbox-context sends, original-payload forwarding, and automatic +portable replication. Both callbacks are optional. Registration replaces the +whole set; `{}` clears it. Registration and builds copy callbacks, and active +executions retain their selection. Callbacks receive existing context/data +without extra loading and work without OpenTelemetry. + +`onBroadcast` groups start and preparation results with an independent +`broadcastId`; `operation` distinguishes `send`, `forward`, and +`portable-replication`. Narrow `event.stage` before reading `event.result`, then +narrow its `type` (`prepared`, `skipped`, or `failed`). `onDelivery` reports +`planned`, `handoff`, or destination-local `preparation-failed` events with a +`deliveryId` and known `broadcast` reference. Handoffs distinguish `direct`, +`enqueued` (with actual queue/task ID), and `enqueue-failed` (acceptance +unknown). Readiness, queue acceptance, and remote effects are distinct. +Portable fallback keeps one delivery across alternate gateways; replication +claims independent targets. Filtered recipients create no planned delivery. + +Callbacks are awaited without a default timeout; observer/logging errors are +isolated. Nonsettling callbacks can delay execution indefinitely. Portable +forwarding can return at its existing deadline while background work and +observations continue. Detached recording must handle its own rejection, +shutdown, and persistence. Delivery IDs currently group producer-local events; +queue identity propagation/replay, worker execution, attempts, HTTP exchanges, +response capture, cancellation observations, provenance, and trace metadata are +not implemented yet. Mock federations accept registration without executing +observers. Incoming C2S listeners use the separate `setOutboxListeners()` API. See . diff --git a/docs/manual/send.md b/docs/manual/send.md index c6948839f..a81545056 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -59,21 +59,30 @@ async function sendFollow( > the *Context* section. -Observing outgoing preparation ------------------------------- +Observing outgoing activities +----------------------------- *This API is available since Fedify 2.5.0.* Use `Federatable.setOutboundObservers()` on a federation builder or a directly -created federation to observe ordinary `~Context.sendActivity()` invocations: +created federation to observe sends, original-payload forwarding, and automatic +portable-inbox replication. Both callbacks are optional: ~~~~ typescript twoslash -import type { Federation, OutboundBroadcastEvent } from "@fedify/fedify"; +import type { + Federation, + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "@fedify/fedify"; declare const federation: Federation<{ requestId: string }>; declare function recordBroadcast( requestId: string, event: OutboundBroadcastEvent, ): Promise; +declare function recordDelivery( + requestId: string, + event: OutboundDeliveryEvent, +): Promise; // ---cut-before--- federation.setOutboundObservers({ async onBroadcast(ctx, event) { @@ -82,46 +91,117 @@ federation.setOutboundObservers({ console.error(event.result.failure.phase, event.result.failure.error); } }, + async onDelivery(ctx, event) { + await recordDelivery(ctx.data.requestId, event); + if (event.stage === "handoff" && event.result.type === "enqueue-failed") { + console.error("Queue acceptance is unknown", event.result.error); + } + }, }); ~~~~ -Here, `recordBroadcast()` is your application's recording function. The callback -receives the existing context and application data without loading additional -data for observation. Each invocation has a fresh `broadcastId`, -shared by its `started` and `preparation-finished` events. These IDs work -without OpenTelemetry, including when no tracer provider is configured or -sampling is disabled. +Here, `recordBroadcast()` and `recordDelivery()` are your application's +recording functions. Callbacks receive the existing context and application +data without loading additional data for observation. The report fields contain +immutable snapshots of known routing facts; original thrown values remain +available as `unknown` values. + +### Broadcast lifecycle -`started` precedes sender and recipient resolution and signing. The -`preparation-finished` event contains an `OutboundPreparationResult`: +Each invocation has a fresh `broadcastId`, shared by its `started` and +`preparation-finished` events. The `operation` is `send`, `forward`, or +`portable-replication`. Broadcast and delivery identities work without +OpenTelemetry, including when no tracer provider is configured or sampling is +disabled. + +`started` precedes failure-prone preparation. The `preparation-finished` event +contains an `OutboundPreparationResult`: - `prepared` means the current producer reached readiness for fanout or - delivery. `destinationCount` counts deduplicated destinations, not - successful deliveries. Fanout workers still need to prepare deliveries. - - `skipped` with reason `no-destinations` means there was nothing to send. + delivery. `destinationCount` counts deduplicated destinations or confirmed + replication targets, not successful deliveries. Fanout workers still need + to prepare deliveries. + - `skipped` records an existing no-send decision: `no-destinations`, + `unsigned` when forwarding skips an unsigned payload, + `forwarding-disabled`, `forwarding-unavailable` when replication cannot + claim targets, or `no-forwarding-targets` when none remain claimed. - `failed` reports the preparation phase, the original thrown value (which - need not be an `Error`), and the extent of destination planning. + need not be an `Error`), and the extent of destination planning. An + incomplete plan's count covers confirmed obligations so far. + +The `unsigned` reason reflects the existing `skipIfUnsigned` signature/proof +shape check; it does not establish authentication. Preparation readiness does not confirm queue acceptance or successful delivery. Immediate sends can already be running when preparation finishes, and a later -send or enqueue failure does not replace the preparation result. +send or enqueue failure does not replace the preparation result. A crash or +nonsettling operation can leave a start without a later result. + +### Destination observations + +`onDelivery` receives an `OutboundDeliveryEvent` for each established delivery +obligation. Its `broadcast` reference identifies the invocation, and its +`deliveryId` groups events for one deduplicated destination or confirmed +replication target. Different inboxes on the same host remain distinct. + +| Stage | Meaning | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `planned` | The destination was established. `actorIds` lists actors already known to share it; `sharedInbox` describes the selected route. | +| `handoff`, result `direct` | Control entered the direct-send path. This does not confirm an HTTP request or remote success. | +| `handoff`, result `enqueued` | The enqueue resolved. `queue` identifies `fanout` or `outbox`, and `queueMessageId` identifies the accepted task. | +| `handoff`, result `enqueue-failed` | Enqueue rejected. `acceptance` remains `unknown`, because a backend effect may already have happened; `error` retains the original failure. | +| `preparation-failed` | Destination-local setup failed before concrete send entry. This creates no synthetic attempt. | + +`destinationInbox` retains the planned destination. For a portable inbox, +`portableInbox` contains its canonical portable ID. The planned event's `mode` +is `ordinary`, `portable-fallback`, or `portable-replication`. Alternate +fallback gateways share one obligation; automatic replication establishes a +separate obligation for each successfully claimed target. + +Several deliveries can share one accepted fanout task and therefore the same +`queueMessageId`; this task ID is separate from their delivery IDs. Acceptance +is reported before later bookkeeping, so a metrics or logging failure does not +turn an accepted enqueue into an enqueue failure. + +Recipients filtered out before planning create no delivery. Common producer +failures are broadcast failures; a destination-local failure is also reported +for that destination. Earlier handoffs remain valid, and later unvisited +destinations can remain planned without a handoff. Parallel work can interleave +callbacks, so IDs provide grouping rather than a total event order. + +### Registration and callback behavior Each registration call replaces all previous outbound observers; pass `{}` to clear them. Fedify copies the supplied callbacks, and building a federation -copies the builder's registration. An active invocation keeps the callback -selected at entry even if registration changes while it is running. +copies the builder's registration. An active invocation keeps its selected +callbacks even if registration changes while it is running. You can register +only `onBroadcast` or only `onDelivery`. Callbacks are awaited without a default timeout. Their return values and errors cannot veto sending, and callback or error-logging failures do not replace the send's original result. Slow callbacks delay processing; a callback that never -settles delays it indefinitely. If you detach recording work, handle its -rejections and shutdown yourself; Fedify does not guarantee its persistence. - -Currently this API covers ordinary sends only. Sends through an -`OutboxContext` override, forwarding, automatic portable replication, workers, -delivery attempts, HTTP exchanges, and response bodies are not observed yet. -Incoming C2S `setOutboxListeners()` registration is separate. Mock federations -accept outbound observer registration but do not execute these observers. +settles can delay it indefinitely. Existing portable-forwarding deadlines can +return control while background preparation, sending, and observation continue; +a deadline alone does not create a failed preparation result. + +If you detach recording work, handle its rejections and shutdown yourself; +Fedify does not guarantee its persistence. Incoming C2S `setOutboxListeners()` +registration is separate. Mock federations accept outbound observer +registration but do not execute these observers. + +### Current producer scope + +Both callbacks cover ordinary `Context.sendActivity()`, the `OutboxContext` +override, inbox/outbox `forwardActivity()`, and automatic portable replication +at their producer boundaries. Original-payload forwarding and replication keep +the received payload unchanged. Delivery IDs currently group producer-local +events; they are not yet carried through queue continuations or worker replay. + +Worker execution, concrete delivery attempts, HTTP exchanges, response bodies, +explicit cancellation observations, historical provenance, and optional trace +metadata are not reported yet. A direct handoff, accepted task, or prepared +producer must not be treated as proof that the remote application applied an +activity. Specifying a sender diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index cf5022925..c7c43c884 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -145,9 +145,10 @@ export type OutboundDeliveryEvent = * awaited without a default timeout. * Their errors, including errors logging observer failures, do not replace the * send's result. - * A callback that never settles delays execution indefinitely. Detached + * A callback that never settles can delay execution indefinitely. Detached * recording must handle its own rejection and shutdown; no persistence is - * guaranteed. + * guaranteed. Existing portable forwarding deadlines may return while + * background preparation, sending, and observer callbacks continue. * @template TContextData Application context data. * @since 2.5.0 */ From a9e48ec78541788e75dfb3c1cb8b1f3baa522a28 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 00:25:16 +0900 Subject: [PATCH 10/53] Preserve outbound IDs across queue fanout Snapshot outbound identities across queue fanout and retain unknown broadcast recipients when processing legacy messages. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 12 +- .../src/federation/outbound-observation.ts | 198 ++++++++++++++---- .../src/federation/outbound-report.test.ts | 198 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 18 +- .../src/federation/portable-inbox.test.ts | 15 ++ .../fedify/src/federation/portable-inbox.ts | 1 + packages/fedify/src/federation/queue.ts | 14 ++ 7 files changed, 403 insertions(+), 53 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 57d7d41ab..93bf2ac80 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1,6 +1,6 @@ import { observeOutboundBroadcast, - type OutboundBroadcastObservation, + OutboundBroadcastObservation, } from "./outbound-observation.ts"; import { InboxObservation } from "./inbox-observation.ts"; import { type Path, RouterError } from "@fedify/uri-template"; @@ -1389,6 +1389,7 @@ export class FederationImpl data: TContextData, message: FanoutMessage, ): Promise { + const observers = this.outboundObservers; const logger = getLogger(["fedify", "federation", "fanout"]); logger.debug( "Fanning out activity {activityId} to {inboxes} inbox(es)...", @@ -1424,11 +1425,17 @@ export class FederationImpl }), }, ); + const observation = await OutboundBroadcastObservation.resumeFanout( + context, + observers, + message, + ); await this.sendActivity(keys, message.inboxes, activity, { collectionSync: message.collectionSync, orderingKey: message.orderingKey, normalizeExistingProofs: message.normalizeExistingProofs, activityJsonLd: message.activity, + observation, context, }); } @@ -3400,6 +3407,7 @@ export class FederationImpl activityId: activity.id?.href, activityType: getTypeId(activity).href, inbox, + ...observation?.continuation(inbox), sharedInbox: inboxes[inbox].sharedInbox, actorIds: [...inboxes[inbox].actorIds], started: new Date().toISOString(), @@ -6015,6 +6023,7 @@ export class ContextImpl implements Context { [k, { actorIds, sharedInbox, portableInbox, gatewayInboxes }], ) => [k, { actorIds: [...actorIds], + ...observation?.continuation(k), sharedInbox, ...getPortableMessageFields(portableInbox, gatewayInboxes), }]), @@ -7091,6 +7100,7 @@ async function forwardActivityInternal( activityId: ctx.activityId, activityType: ctx.activityType, inbox, + ...observation?.continuation(inbox), sharedInbox: inboxes[inbox].sharedInbox, actorIds: [...inboxes[inbox].actorIds], started, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index c1683dca1..ca138af84 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,5 +1,7 @@ +import { encodeHex } from "byte-encodings/hex"; import { getLogger } from "@logtape/logtape"; import type { Context } from "./context.ts"; +import type { FanoutMessage, SerializedOutboundDelivery } from "./queue.ts"; import type { OutboundBroadcastEvent, OutboundDeliveryEvent, @@ -19,6 +21,62 @@ interface PlannedDestination { readonly mode?: "portable-replication"; } +function snapshotDeliveryIdentity( + identity: SerializedOutboundDelivery, +): SerializedOutboundDelivery { + if ( + identity == null || typeof identity.deliveryId !== "string" || + identity.deliveryId.length < 1 || + typeof identity.destinationInbox !== "string" || + identity.destinationInbox.length < 1 || + identity.broadcast == null || + !(identity.broadcast.type === "unknown" || + identity.broadcast.type === "known" && + typeof identity.broadcast.id === "string" && + identity.broadcast.id.length > 0) || + !(identity.portableInbox == null || + typeof identity.portableInbox === "string") + ) { + throw new TypeError( + "Invalid outbound delivery identity in queue metadata.", + ); + } + return Object.freeze({ + broadcast: Object.freeze( + identity.broadcast.type === "known" + ? { type: "known" as const, id: identity.broadcast.id } + : { type: "unknown" as const }, + ), + deliveryId: identity.deliveryId, + destinationInbox: identity.destinationInbox, + ...(identity.portableInbox == null + ? {} + : { portableInbox: identity.portableInbox }), + }); +} + +async function legacyFanoutIdentity( + taskId: string, + inbox: string, + portableInbox?: string, +): Promise { + // A legacy task establishes destination obligations, not its originating + // invocation. JSON tuple encoding keeps different task/destination pairs + // unambiguous; hashing keeps the derived identity opaque and bounded. + const bytes = new TextEncoder().encode(JSON.stringify([ + "fedify-legacy-fanout-delivery", + taskId, + inbox, + ])); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return { + broadcast: { type: "unknown" }, + deliveryId: `legacy-fanout:${encodeHex(new Uint8Array(digest))}`, + destinationInbox: inbox, + ...(portableInbox == null ? {} : { portableInbox }), + }; +} + async function notifyObserver( context: Context, callback: @@ -45,26 +103,16 @@ async function notifyObserver( /** One producer's destination, without callbacks or live inputs in queues. */ class OutboundDeliveryObservation { #finished = false; - readonly #reference; + readonly #reference: SerializedOutboundDelivery; readonly #plan; constructor( readonly context: Context, - readonly observer: NonNullable< - OutboundObservers["onDelivery"] - >, - broadcastId: string, - destinationInbox: string, + readonly observer: OutboundObservers["onDelivery"], + identity: SerializedOutboundDelivery, destination: PlannedDestination, ) { - this.#reference = Object.freeze({ - broadcast: Object.freeze({ type: "known" as const, id: broadcastId }), - deliveryId: crypto.randomUUID(), - destinationInbox, - ...(destination.portableInbox == null - ? {} - : { portableInbox: destination.portableInbox }), - }); + this.#reference = snapshotDeliveryIdentity(identity); this.#plan = Object.freeze({ actorIds: Object.freeze(Array.from(destination.actorIds)), sharedInbox: destination.sharedInbox, @@ -73,6 +121,10 @@ class OutboundDeliveryObservation { }); } + get identity(): SerializedOutboundDelivery { + return this.#reference; + } + planned(): Promise { return notifyObserver( this.context, @@ -125,7 +177,7 @@ class OutboundDeliveryObservation { /** State owned by one outgoing invocation, never serialized into queues. */ export class OutboundBroadcastObservation { - readonly #broadcastId = crypto.randomUUID(); + readonly #broadcast: SerializedOutboundDelivery["broadcast"]; #finished = false; #phase: PreparationFailure["failure"]["phase"] = "unknown"; #planning: PreparationFailure["planning"] = "not-started"; @@ -142,27 +194,86 @@ export class OutboundBroadcastObservation { readonly context: Context, observers: OutboundObservers, readonly operation: OutboundBroadcastEvent["operation"], + broadcast: SerializedOutboundDelivery["broadcast"] = { + type: "known", + id: crypto.randomUUID(), + }, ) { + this.#broadcast = broadcast; this.#onBroadcast = observers.onBroadcast; this.#onDelivery = observers.onDelivery; } + /** Continue an existing fanout; do not emit a new broadcast invocation. */ + static async resumeFanout( + context: Context, + observers: OutboundObservers, + message: FanoutMessage, + ): Promise | undefined> { + if ( + observers.onDelivery == null && + !Object.values(message.inboxes).some((entry) => entry.outbound != null) + ) { + return undefined; + } + const observation = new OutboundBroadcastObservation( + context, + { onDelivery: observers.onDelivery }, + "send", + { type: "unknown" }, + ); + observation.#finished = true; + observation.planned(Object.keys(message.inboxes).length); + for (const [inbox, destination] of Object.entries(message.inboxes)) { + const identity = destination.outbound ?? + await legacyFanoutIdentity( + message.id, + inbox, + destination.portableInbox, + ); + observation.#deliveries.set( + inbox, + observation.#createDelivery(inbox, destination, identity), + ); + } + return observation; + } + + #createDelivery( + inbox: string, + destination: PlannedDestination, + identity?: SerializedOutboundDelivery, + ): OutboundDeliveryObservation { + return new OutboundDeliveryObservation( + this.context, + this.#onDelivery, + identity ?? { + broadcast: this.#broadcast, + deliveryId: crypto.randomUUID(), + destinationInbox: inbox, + ...(destination.portableInbox == null + ? {} + : { portableInbox: destination.portableInbox }), + }, + destination, + ); + } + + continuation( + inbox: string, + ): { readonly outbound: SerializedOutboundDelivery } | undefined { + const delivery = this.delivery(inbox); + return delivery == null ? undefined : { outbound: delivery.identity }; + } + async planDestinations( inboxes: Readonly>, ): Promise { this.planned(Object.keys(inboxes).length); - if (this.#onDelivery == null) return; for (const [inbox, destination] of Object.entries(inboxes)) { - this.#deliveries.set( - inbox, - new OutboundDeliveryObservation( - this.context, - this.#onDelivery, - this.#broadcastId, - inbox, - destination, - ), - ); + if (!this.#deliveries.has(inbox)) { + this.#deliveries.set(inbox, this.#createDelivery(inbox, destination)); + } } for (const delivery of this.#deliveries.values()) await delivery.planned(); } @@ -224,14 +335,7 @@ export class OutboundBroadcastObservation { destination: PlannedDestination, ): Promise { this.#destinationCount++; - if (this.#onDelivery == null) return; - const delivery = new OutboundDeliveryObservation( - this.context, - this.#onDelivery, - this.#broadcastId, - inbox, - destination, - ); + const delivery = this.#createDelivery(inbox, destination); this.#deliveries.set(inbox, delivery); await delivery.planned(); } @@ -259,11 +363,13 @@ export class OutboundBroadcastObservation { observation?: OutboundBroadcastObservation, ) => Promise, ): Promise { - await this.#notify(Object.freeze({ - broadcastId: this.#broadcastId, - operation: this.operation, - stage: "started", - })); + if (this.#broadcast.type === "known") { + await this.#notify(Object.freeze({ + broadcastId: this.#broadcast.id, + operation: this.operation, + stage: "started", + })); + } try { return await send(this); } catch (error) { @@ -289,12 +395,14 @@ export class OutboundBroadcastObservation { async #finish(result: OutboundPreparationResult): Promise { if (this.#finished) return; this.#finished = true; - await this.#notify(Object.freeze({ - broadcastId: this.#broadcastId, - operation: this.operation, - stage: "preparation-finished", - result, - })); + if (this.#broadcast.type === "known") { + await this.#notify(Object.freeze({ + broadcastId: this.#broadcast.id, + operation: this.operation, + stage: "preparation-finished", + result, + })); + } } #notify(event: OutboundBroadcastEvent): Promise { diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 6d10edff3..92f2a5447 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -498,6 +498,19 @@ test("outbound preparation precedes queue acceptance and is not rewritten by enq await settled; strictEqual(messages.length, 1); strictEqual(messages[0].type, role); + const message = messages[0]; + const identity = message.type === "fanout" + ? message.inboxes[recipient.inboxId!.href].outbound + : message.type === "outbox" + ? message.outbound + : undefined; + ok(identity); + deepStrictEqual(identity.broadcast, { + type: "known", + id: events[0].broadcastId, + }); + strictEqual(identity.destinationInbox, recipient.inboxId!.href); + deepStrictEqual(structuredClone(identity), identity); strictEqual(events.length, 2); ok(!("observation" in messages[0])); }, @@ -1489,6 +1502,8 @@ test("outbound delivery observers snapshot registration and deduplicated destina strictEqual(sharedPlan.sharedInbox, true); strictEqual(sharedPlan.mode, "ordinary"); notStrictEqual(planned[0].deliveryId, planned[1].deliveryId); + ok(planned[0].broadcast.type === "known"); + ok(planned[1].broadcast.type === "known"); strictEqual(planned[0].broadcast.id, planned[1].broadcast.id); for (const event of events) { ok(Object.isFrozen(event)); @@ -1613,7 +1628,27 @@ test("outbound delivery observers report queue acceptance separately from produc ), ); } + ok(event.broadcast.type === "known"); strictEqual(event.broadcast.id, broadcasts[0].broadcastId); + const message = messages.find((message) => + message.type === "fanout" + ? event.destinationInbox in message.inboxes + : message.type === "outbox" && + message.inbox === event.destinationInbox + ); + ok(message); + const identity = message.type === "fanout" + ? message.inboxes[event.destinationInbox].outbound + : message.type === "outbox" + ? message.outbound + : undefined; + deepStrictEqual(identity, { + broadcast: event.broadcast, + deliveryId: event.deliveryId, + destinationInbox: event.destinationInbox, + }); + deepStrictEqual(JSON.parse(JSON.stringify(identity)), identity); + deepStrictEqual(structuredClone(identity), identity); strictEqual( event.deliveryId, planned.find((plan) => @@ -1663,3 +1698,166 @@ test("outbound delivery observers retain accepted handoffs after bookkeeping fai ok(event.stage === "handoff"); ok(event.result.type === "enqueued"); }); + +test("outbound queue identities survive fanout replay and legacy messages", async (t) => { + for (const legacy of [false, true]) { + await t.step( + legacy ? "legacy fanout" : "serialized producer identities", + async () => { + const fanouts: Message[] = []; + const outboxes: Message[] = []; + const recordingQueue = (messages: Message[]): MessageQueue => ({ + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }); + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { + fanout: recordingQueue(fanouts), + outbox: recordingQueue(outboxes), + }, + }); + const broadcasts: OutboundBroadcastEvent[] = []; + // Identity propagation also works when only the broadcast callback exists. + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const targets = [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ]; + await ctx.sendActivity(sender, targets, activity, { fanout: "force" }); + const message = fanouts[0]; + ok(message?.type === "fanout"); + const original = structuredClone(message); + if (legacy) { + for ( + const entry of Object.values(message.inboxes) + ) delete (entry as { outbound?: unknown }).outbound; + } + const events: OutboundDeliveryEvent[] = []; + let newBroadcasts = 0; + federation.setOutboundObservers({ + onBroadcast() { + newBroadcasts++; + }, + onDelivery(_ctx, event) { + events.push(event); + }, + }); + await federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ); + // Preserve metadata even when the consumer's callbacks are disabled. + if (!legacy) federation.setOutboundObservers({}); + await federation.processQueuedTask(undefined, structuredClone(message)); + strictEqual(newBroadcasts, 0); + strictEqual(outboxes.length, 4); + strictEqual(new Set(outboxes.map((message) => message.id)).size, 4); + const identities = new Set(); + for (const inbox of Object.keys(message.inboxes)) { + const children = outboxes.filter((message) => + message.type === "outbox" && message.inbox === inbox + ); + strictEqual(children.length, 2); + ok(children[0].type === "outbox" && children[1].type === "outbox"); + const identity = children[0].outbound; + ok(identity); + identities.add(identity.deliveryId); + deepStrictEqual(children[1].outbound, identity); + strictEqual(identity.destinationInbox, inbox); + deepStrictEqual( + identity.broadcast, + legacy + ? { type: "unknown" } + : { type: "known", id: broadcasts[0].broadcastId }, + ); + if (!legacy) { + deepStrictEqual(identity, original.inboxes[inbox].outbound); + } + for ( + const event of events.filter((event) => + event.destinationInbox === inbox + ) + ) { + strictEqual(event.stage, "handoff"); + strictEqual(event.deliveryId, identity.deliveryId); + deepStrictEqual(event.broadcast, identity.broadcast); + ok(event.stage === "handoff" && event.result.type === "enqueued"); + const queueMessageId = event.result.queueMessageId; + ok(children.some((child) => child.id === queueMessageId)); + } + } + strictEqual(identities.size, 2); + strictEqual(events.length, legacy ? 4 : 2); + if (!legacy) { + const invalid = structuredClone(original); + const first = Object.values(invalid.inboxes)[0]; + ok(first.outbound); + Object.assign(first.outbound, { deliveryId: "" }); + await rejects( + federation.processQueuedTask(undefined, invalid), + TypeError, + ); + strictEqual(outboxes.length, 4); + } + }, + ); + } +}); + +test("outbound observers preserve processing of legacy outbox messages", async () => { + const messages: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await ctx.sendActivity(sender, recipient, activity, { fanout: "skip" }); + const message = messages[0]; + ok(message?.type === "outbox"); + strictEqual(message.outbound, undefined); + federation.setOutboundObservers({ + onDelivery() { + throw new Error("unused worker observer"); + }, + }); + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202); + try { + await federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ); + strictEqual(fetchMock.callHistory.calls().length, 1); + strictEqual(messages.length, 1); + } finally { + fetchMock.hardReset(); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index c7c43c884..c45fc36ce 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -81,15 +81,18 @@ export type OutboundBroadcastEvent = /** * A producer-side destination obligation, not delivery completion. * - * IDs group this producer's events. Queue execution and identity propagation - * through workers are not observed yet. An unvisited destination stays planned - * when another destination's preparation interrupts the producer. + * IDs group producer handoffs and their fanout expansion across queue tasks. + * Full worker execution reports and concrete attempts are not observed yet. + * An unvisited destination stays planned when another destination's preparation + * interrupts the producer. * @since 2.5.0 */ export type OutboundDeliveryEvent = & { - /** The invocation that established this destination. */ - readonly broadcast: { readonly type: "known"; readonly id: string }; + /** The originating invocation, or unknown for legacy fanout work. */ + readonly broadcast: + | { readonly type: "known"; readonly id: string } + | { readonly type: "unknown" }; /** Opaque identity allocated after deduplication or a confirmed claim. */ readonly deliveryId: string; /** Planned destination, unchanged by concrete gateway selection. */ @@ -165,8 +168,9 @@ export interface OutboundObservers { ) => void | Promise; /** * Observes producer-side planning, handoffs, and destination preparation - * failures. Filtered recipients do not become planned deliveries. This does - * not report concrete attempts, remote success, or worker execution yet. + * failures, including fanout continuation handoffs. Filtered recipients do + * not become planned deliveries. Concrete attempts, remote success, and full + * worker execution reports are not observed yet. */ readonly onDelivery?: ( context: Context, diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index 9534e7518..6dfa91f1c 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -2432,6 +2432,21 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a assertEquals(message.activity, json); assertEquals(message.keys, []); assertFalse("broadcastId" in message); + const plan = deliveries.find((event) => + event.stage === "planned" && event.destinationInbox === message.inbox + ); + assert(plan); + assertEquals(message.outbound, { + broadcast: plan.broadcast, + deliveryId: plan.deliveryId, + destinationInbox: plan.destinationInbox, + portableInbox: plan.portableInbox, + }); + assertEquals( + JSON.parse(JSON.stringify(message.outbound)), + message.outbound, + ); + assertEquals(structuredClone(message.outbound), message.outbound); } assertEquals( deliveries.filter((event) => event.stage === "planned").length, diff --git a/packages/fedify/src/federation/portable-inbox.ts b/packages/fedify/src/federation/portable-inbox.ts index c8ce77698..41688acda 100644 --- a/packages/fedify/src/federation/portable-inbox.ts +++ b/packages/fedify/src/federation/portable-inbox.ts @@ -721,6 +721,7 @@ async function enqueueForwarding( activityId: activityId.href, activityType, inbox: inbox.href, + ...observation?.continuation(inbox.href), sharedInbox: false, started, attempt: 0, diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index 02b9af9f3..b4c23c105 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -1,6 +1,16 @@ import type { SerializedInboxAuthentication } from "./inbox-authentication.ts"; import type { SerializedVerifiedActivityView } from "./inbox-view.ts"; +/** Serializable grouping facts, separate from queue task identity. @internal */ +export interface SerializedOutboundDelivery { + readonly broadcast: + | { readonly type: "known"; readonly id: string } + | { readonly type: "unknown" }; + readonly deliveryId: string; + readonly destinationInbox: string; + readonly portableInbox?: string; +} + export interface SenderKeyJwkPair { readonly keyId: string; readonly privateKey: JsonWebKey; @@ -32,6 +42,8 @@ export interface FanoutMessage { { readonly actorIds: readonly string[]; readonly sharedInbox: boolean; + /** Optional for continuations created before outbound observation. */ + readonly outbound?: SerializedOutboundDelivery; /** * The canonical form of the FEP-ef61 portable inbox that the inbox * stands for. @@ -67,6 +79,8 @@ export interface FanoutMessage { export interface OutboxMessage { readonly type: "outbox"; readonly id: ReturnType; + /** Original delivery identity; absent on legacy/unobserved producers. */ + readonly outbound?: SerializedOutboundDelivery; readonly baseUrl: string; readonly keys: readonly SenderKeyJwkPair[]; readonly activity: unknown; From b0ef7c515939c5cb15d64553ac12e688bbc9457c Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 00:38:15 +0900 Subject: [PATCH 11/53] Cover outbound identity across queue retries Verify that retry, hold, defer, and native redelivery paths preserve outbound identity metadata and legacy absence as gateway routes change. https://github.com/fedify-dev/fedify/issues/1316 Changelog: none Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-report.test.ts | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 92f2a5447..d759b86d5 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -29,6 +29,7 @@ import type { OutboundDeliveryEvent, OutboundObservers, } from "./mod.ts"; +import type { OutboxMessage } from "./queue.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; const activity = new Create({ @@ -1861,3 +1862,254 @@ test("outbound observers preserve processing of legacy outbox messages", async ( fetchMock.hardReset(); } }); + +function continuationMessage( + inbox: string, + observed: boolean, + portableInbox?: string, +): OutboxMessage { + return { + type: "outbox", + id: crypto.randomUUID(), + baseUrl: "https://example.com", + keys: [], + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + id: activity.id!.href, + type: "Create", + actor: activity.actorId!.href, + }, + activityId: activity.id!.href, + activityType: Create.typeId.href, + inbox, + sharedInbox: false, + started: new Date().toISOString(), + attempt: 0, + headers: {}, + orderingKey: "original-order", + traceContext: {}, + ...(portableInbox == null ? {} : { portableInbox }), + ...(observed + ? { + outbound: { + broadcast: { type: "known" as const, id: crypto.randomUUID() }, + deliveryId: crypto.randomUUID(), + destinationInbox: inbox, + ...(portableInbox == null ? {} : { portableInbox }), + }, + } + : {}), + }; +} + +test("outbound queue identities survive ordinary retries, holds, and native redelivery", async (t) => { + for (const observed of [false, true]) { + for (const mode of ["retry", "hold", "retry-after", "native"] as const) { + await t.step(`${mode}: ${observed ? "identity" : "legacy"}`, async () => { + const queued: { + message: Message; + options: Parameters[1]; + }[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + manuallyStartQueue: true, + allowPrivateAddress: true, + circuitBreaker: mode === "hold" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + queue: { + outbox: { + nativeRetrial: mode === "native", + enqueue(message, options) { + queued.push({ message, options }); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const message = continuationMessage(recipient.inboxId!.href, observed); + const original = structuredClone(message); + const task: OutboxMessage = JSON.parse(JSON.stringify(message)); + fetchMock.mockGlobal().post( + message.inbox, + mode === "retry-after" + ? { status: 429, headers: { "Retry-After": "3600" } } + : 500, + ); + try { + if (mode === "native") { + await rejects( + federation.processQueuedTask( + undefined, + task, + ), + ); + strictEqual(queued.length, 0); + fetchMock.hardReset(); + fetchMock.mockGlobal().post(message.inbox, 202); + deepStrictEqual(task, original); + const redelivery = { + ...structuredClone(task), + id: crypto.randomUUID(), + }; + await federation.processQueuedTask(undefined, redelivery); + deepStrictEqual(redelivery.outbound, original.outbound); + strictEqual("outbound" in redelivery, observed); + ok(fetchMock.callHistory.calls().length > 0); + strictEqual(queued.length, 0); + } else { + await federation.processQueuedTask( + undefined, + task, + ); + strictEqual(queued.length, 1); + const next = queued[0].message; + ok(next.type === "outbox"); + deepStrictEqual(next.outbound, original.outbound); + strictEqual("outbound" in next, observed); + strictEqual(next.attempt, mode === "hold" ? 0 : 1); + strictEqual(next.circuitHeld, mode === "hold" ? true : undefined); + strictEqual(queued[0].options?.orderingKey, message.orderingKey); + ok(queued[0].options?.delay); + const requests = fetchMock.callHistory.calls().length; + await federation.processQueuedTask(undefined, { + ...structuredClone(next), + id: crypto.randomUUID(), + }); + strictEqual(queued.length, 2); + const replay = queued[1].message; + ok(replay.type === "outbox"); + deepStrictEqual(replay.outbound, original.outbound); + strictEqual("outbound" in replay, observed); + strictEqual(replay.attempt, mode === "hold" ? 0 : 2); + if (mode === "hold") { + strictEqual(fetchMock.callHistory.calls().length, requests); + } + } + deepStrictEqual(task, original); + deepStrictEqual(message, original); + } finally { + fetchMock.hardReset(); + } + }); + } + } +}); + +test("outbound queue identities retain planned destinations through gateway continuations", async (t) => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + "https://gw3.example/inbox", + ]; + const portableInbox = "ap+ef61://z6Mktest/inbox"; + for (const observed of [false, true]) { + for (const mode of ["retry", "hold", "defer", "native"] as const) { + await t.step(`${mode}: ${observed ? "identity" : "legacy"}`, async () => { + const queued: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + manuallyStartQueue: true, + allowPrivateAddress: true, + circuitBreaker: mode === "hold" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + queue: { + outbox: { + nativeRetrial: mode === "native", + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const message = { + ...continuationMessage(gateways[0], observed, portableInbox), + gatewayInboxes: gateways, + }; + const original = structuredClone(message); + const task: OutboxMessage = JSON.parse(JSON.stringify(message)); + fetchMock.mockGlobal(); + for (const [index, inbox] of gateways.entries()) { + fetchMock.post( + inbox, + mode !== "native" && index === 0 + ? 404 + : mode === "defer" + ? { status: 429, headers: { "Retry-After": "3600" } } + : 500, + ); + } + try { + if (mode === "native") { + await rejects( + federation.processQueuedTask( + undefined, + task, + ), + ); + strictEqual(queued.length, 0); + fetchMock.hardReset(); + fetchMock.mockGlobal().post(gateways[0], 202); + deepStrictEqual(task, original); + const redelivery = { + ...structuredClone(task), + id: crypto.randomUUID(), + }; + await federation.processQueuedTask(undefined, redelivery); + deepStrictEqual(redelivery.outbound, original.outbound); + strictEqual("outbound" in redelivery, observed); + ok(fetchMock.callHistory.calls().length > 0); + strictEqual(queued.length, 0); + } else { + await federation.processQueuedTask( + undefined, + task, + ); + strictEqual(queued.length, 1); + const next = queued[0]; + ok(next.type === "outbox"); + strictEqual(next.inbox, gateways[1]); + deepStrictEqual(next.gatewayInboxes, gateways.slice(1)); + deepStrictEqual(next.outbound, original.outbound); + strictEqual("outbound" in next, observed); + strictEqual(next.portableInbox, portableInbox); + if (observed) { + strictEqual(next.outbound?.destinationInbox, gateways[0]); + } + strictEqual(next.attempt, mode === "hold" ? 0 : 1); + strictEqual(next.circuitHeld, mode === "hold" ? true : undefined); + if (mode === "defer") { + deepStrictEqual( + Object.keys(next.gatewayNotBefore ?? {}), + gateways.slice(1), + ); + } + const requests = fetchMock.callHistory.calls().length; + await federation.processQueuedTask(undefined, { + ...structuredClone(next), + id: crypto.randomUUID(), + }); + strictEqual(queued.length, 2); + const replay = queued[1]; + ok(replay.type === "outbox"); + deepStrictEqual(replay.outbound, original.outbound); + strictEqual("outbound" in replay, observed); + strictEqual(replay.inbox, gateways[1]); + strictEqual(replay.attempt, mode === "retry" ? 2 : next.attempt); + if (mode !== "retry") { + strictEqual(fetchMock.callHistory.calls().length, requests); + } + } + deepStrictEqual(task, original); + deepStrictEqual(message, original); + } finally { + fetchMock.hardReset(); + } + }); + } + } +}); From 756eee4d6bd7b2928c9080c81756f4f68e6f94f4 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 01:15:22 +0900 Subject: [PATCH 12/53] Preserve original inbox request provenance Record independent inbox request identities and immutable source facts through immediate contexts and outgoing observations. Keep historical source labels separate from current request labels. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/inbox-observation.ts | 18 +- .../fedify/src/federation/inbox-report.ts | 4 + packages/fedify/src/federation/middleware.ts | 406 +++++++++------ .../src/federation/outbound-observation.ts | 39 +- .../federation/outbound-provenance.test.ts | 484 ++++++++++++++++++ .../src/federation/outbound-provenance.ts | 59 +++ .../fedify/src/federation/outbound-report.ts | 38 ++ .../src/federation/portable-inbox.test.ts | 10 +- 8 files changed, 902 insertions(+), 156 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-provenance.test.ts create mode 100644 packages/fedify/src/federation/outbound-provenance.ts diff --git a/packages/fedify/src/federation/inbox-observation.ts b/packages/fedify/src/federation/inbox-observation.ts index 576c1fe6c..c16888dde 100644 --- a/packages/fedify/src/federation/inbox-observation.ts +++ b/packages/fedify/src/federation/inbox-observation.ts @@ -1,6 +1,8 @@ import { Activity } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; import type { Span } from "@opentelemetry/api"; +import type { OutboundInboxRequest } from "./outbound-report.ts"; +import { withContext } from "@logtape/logtape"; import type { RequestContext } from "./context.ts"; import type { InboxAuthentication, @@ -47,7 +49,10 @@ export class InboxObservation { } }, }; - constructor(readonly inbox: InboxRequestReport["inbox"]) {} + constructor( + readonly inbox: InboxRequestReport["inbox"], + readonly source?: OutboundInboxRequest, + ) {} /** Run an ingress operation and project telemetry; fetch owns completion. */ async run( @@ -123,6 +128,10 @@ export class InboxObservation { ): Promise { if (handler == null) return; const report: InboxRequestReport = { + ...(this.source == null ? {} : { + inboxRequestId: this.source.inboxRequestId, + requestId: this.source.requestId, + }), inbox: this.inbox, payload: this.payload, activity: this.activity, @@ -131,7 +140,12 @@ export class InboxObservation { outcome, }; try { - await handler(context, report); + await withContext( + this.source == null + ? {} + : { inboxRequestId: this.source.inboxRequestId }, + () => handler(context, report), + ); } catch (error) { try { getLogger(["fedify", "federation", "inbox"]).error( diff --git a/packages/fedify/src/federation/inbox-report.ts b/packages/fedify/src/federation/inbox-report.ts index 0d987f090..4637e683a 100644 --- a/packages/fedify/src/federation/inbox-report.ts +++ b/packages/fedify/src/federation/inbox-report.ts @@ -132,6 +132,10 @@ export type InboxRequestOutcome = * @since 2.4.0 */ export interface InboxRequestReport { + /** Server-generated identity; present for routed HTTP ingress. @since 2.5.0 */ + readonly inboxRequestId?: string; + /** Informational logging label of that request, not a unique ID. @since 2.5.0 */ + readonly requestId?: string; /** The delivery route, including a portable gateway route. */ readonly inbox: { readonly kind: "personal" | "shared" | "portable"; diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 93bf2ac80..b43bdb607 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1,3 +1,12 @@ +import { + createInboxRequestSource, + emptyProvenance, + outboundProvenance, +} from "./outbound-provenance.ts"; +import type { + OutboundInboxRequest, + OutboundProvenance, +} from "./outbound-report.ts"; import { observeOutboundBroadcast, OutboundBroadcastObservation, @@ -3104,13 +3113,17 @@ export class FederationImpl #createContext( baseUrl: URL, contextData: TContextData, - opts?: { documentLoader?: DocumentLoader }, + opts?: { + documentLoader?: DocumentLoader; + [outboundProvenance]?: OutboundProvenance; + }, ): ContextImpl; #createContext( request: Request, contextData: TContextData, opts?: { + [outboundProvenance]?: OutboundProvenance; documentLoader?: DocumentLoader; invokedFromActorDispatcher?: { identifier: string }; invokedFromObjectDispatcher?: { @@ -3125,6 +3138,7 @@ export class FederationImpl urlOrRequest: Request | URL, contextData: TContextData, opts: { + [outboundProvenance]?: OutboundProvenance; documentLoader?: DocumentLoader; invokedFromActorDispatcher?: { identifier: string }; invokedFromObjectDispatcher?: { @@ -3145,6 +3159,7 @@ export class FederationImpl } const loaderOptions = this.#getLoaderOptions(url.origin); const ctxOptions: ContextOptions = { + [outboundProvenance]: opts[outboundProvenance], url, federation: this, data: contextData, @@ -3496,9 +3511,16 @@ export class FederationImpl options: FederationFetchOptions, ): Promise { const requestId = getRequestId(request); - return withContext({ requestId }, async () => { + return withContext({ + requestId, + // Each HTTP ingress owns its current identity; async ancestry is not + // evidence that this request came from another inbox request. + inboxRequestId: undefined, + sourceInboxRequestId: undefined, + sourceRequestId: undefined, + }, async () => { const tracer = this._getTracer(); - const metricState: HttpMetricState = {}; + const metricState: HttpMetricState = { requestId }; const metricStart = performance.now(); return await tracer.startActiveSpan( request.method, @@ -3562,7 +3584,14 @@ export class FederationImpl logger.error( "An error occurred while serving request " + "{method} {url}: {error}", - { method: request.method, url: request.url, error }, + { + method: request.method, + url: request.url, + error, + ...(metricState.inboxRequest == null ? {} : { + inboxRequestId: metricState.inboxRequest.inboxRequestId, + }), + }, ); throw error; } @@ -3594,6 +3623,9 @@ export class FederationImpl const url = new URL(request.url); const logTpl = "{method} {path}: {status}"; const values = { + ...(metricState.inboxRequest == null ? {} : { + inboxRequestId: metricState.inboxRequest.inboxRequestId, + }), method: request.method, path: `${url.pathname}${url.search}`, url: request.url, @@ -3680,9 +3712,24 @@ export class FederationImpl metricState.endpoint = getEndpointCategory(route.name); span.updateName(`${request.method} ${route.template}`); const routeName = route.name.replace(/:.*$/, ""); + const source = request.method === "POST" && + (routeName === "inbox" || routeName === "sharedInbox") + ? this.#createInboxRequestSource(request, metricState) + : undefined; + const provenance = globalThis.Object.freeze( + source == null + ? { originatingRequestId: metricState.requestId } + : { inboxRequest: source }, + ); let context: RequestContextImpl; try { - context = this.#createContext(request, contextData); + context = withContext( + source == null ? {} : { inboxRequestId: source.inboxRequestId }, + () => + this.#createContext(request, contextData, { + [outboundProvenance]: provenance, + }), + ); } catch (error) { if ( request.method !== "POST" || @@ -3800,36 +3847,42 @@ export class FederationImpl const observation = new InboxObservation({ kind: routeName === "inbox" ? "personal" : "shared", recipient, - }); + }, source); this.#setInboxCompletion(metricState, observation, () => context); - return await observation.run( - async () => { - if (routeName === "inbox") { - context = this.#createContext(request, contextData, { - documentLoader: await context.getDocumentLoader({ - identifier: recipient!, - }), - }); - } else if (this.sharedInboxKeyDispatcher != null) { - const identity = await this.sharedInboxKeyDispatcher(context); - if (identity != null) { - context = this.#createContext(request, contextData, { - documentLoader: - "identifier" in identity || "username" in identity - ? await context.getDocumentLoader(identity) - : context.getDocumentLoader(identity), + return await withContext( + { inboxRequestId: source!.inboxRequestId }, + () => + observation.run( + async () => { + if (routeName === "inbox") { + context = this.#createContext(request, contextData, { + [outboundProvenance]: context[outboundProvenance], + documentLoader: await context.getDocumentLoader({ + identifier: recipient!, + }), + }); + } else if (this.sharedInboxKeyDispatcher != null) { + const identity = await this.sharedInboxKeyDispatcher(context); + if (identity != null) { + context = this.#createContext(request, contextData, { + [outboundProvenance]: context[outboundProvenance], + documentLoader: + "identifier" in identity || "username" in identity + ? await context.getDocumentLoader(identity) + : context.getDocumentLoader(identity), + }); + } + } + return await this.#handleInbox(request, { + recipient, + context, + contextData, + onNotFound, + observation, }); - } - } - return await this.#handleInbox(request, { - recipient, - context, - contextData, - onNotFound, - observation, - }); - }, - span, + }, + span, + ), ); } switch (routeName) { @@ -3839,6 +3892,7 @@ export class FederationImpl ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier; context = this.#createContext(request, contextData, { + [outboundProvenance]: context[outboundProvenance], invokedFromActorDispatcher: { identifier }, }); return await handleActor(request, { @@ -3855,6 +3909,7 @@ export class FederationImpl const callbacks = this.objectCallbacks[typeId]; const cls = this.objectTypeIds[typeId]; context = this.#createContext(request, contextData, { + [outboundProvenance]: context[outboundProvenance], invokedFromObjectDispatcher: { cls, values: route.values }, }); return await handleObject(request, { @@ -3923,6 +3978,7 @@ export class FederationImpl const identity = await this.sharedInboxKeyDispatcher(context); if (identity != null) { context = this.#createContext(request, contextData, { + [outboundProvenance]: context[outboundProvenance], documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity), @@ -4121,11 +4177,15 @@ export class FederationImpl } return response; } + const provenance = globalThis.Object.freeze({ + originatingRequestId: metricState.requestId, + }); if (collectionRoute != null) { return await this.#fetchPortableCollection(request, portable, route, { onNotFound, onUnauthorized, contextData, + provenance, }); } if (isActor) { @@ -4133,6 +4193,7 @@ export class FederationImpl ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier; const context = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, invokedFromActorDispatcher: { identifier }, portableRequest: portable.portableRequest, }); @@ -4157,6 +4218,7 @@ export class FederationImpl const callbacks = this.objectCallbacks[typeId]; const cls = this.objectTypeIds[typeId]; const context = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, invokedFromObjectDispatcher: { cls, values: route.values }, portableRequest: portable.portableRequest, }); @@ -4180,6 +4242,7 @@ export class FederationImpl onNotFound, onUnauthorized, contextData, + provenance, }: & Required< Pick< @@ -4187,7 +4250,7 @@ export class FederationImpl "onNotFound" | "onUnauthorized" > > - & { contextData: TContextData }, + & { contextData: TContextData; provenance: OutboundProvenance }, ): Promise { const logger = getLogger(["fedify", "federation", "collection"]); const url = new URL(request.url); @@ -4196,8 +4259,11 @@ export class FederationImpl // that its ID comes from the application's data rather than from // the authority of the request, which the portable ID helpers would // otherwise default to: - const ownerContext = this.#createContext(request, contextData); + const ownerContext = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, + }); const context = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, portableRequest: portable.portableRequest, }); const reject = async (reason: string, identifier?: string) => { @@ -4345,6 +4411,19 @@ export class FederationImpl return await onNotFound(request); } + #createInboxRequestSource( + request: Request, + metricState: HttpMetricState, + ): OutboundInboxRequest { + const source = createInboxRequestSource( + request, + metricState.requestId ?? getRequestId(request), + this.origin?.webOrigin, + ); + metricState.inboxRequest = source; + return source; + } + #setInboxCompletion( metricState: HttpMetricState, observation: InboxObservation, @@ -4396,6 +4475,9 @@ export class FederationImpl ): Promise { const unavailableLoader: DocumentLoader = () => Promise.reject(error); const context = new RequestContextImpl({ + [outboundProvenance]: globalThis.Object.freeze({ + inboxRequest: metricState.inboxRequest, + }), url: new URL(request.url), request, federation: this, @@ -4403,7 +4485,7 @@ export class FederationImpl documentLoader: unavailableLoader, contextLoader: unavailableLoader, }); - const observation = new InboxObservation(inbox); + const observation = new InboxObservation(inbox, metricState.inboxRequest); this.#setInboxCompletion(metricState, observation, () => context); return await observation.run( () => Promise.reject(error), @@ -4500,9 +4582,18 @@ export class FederationImpl metricState.endpoint = "inbox"; span.updateName(`${request.method} ${metricState.routeTemplate}`); const identifier = route.values.identifier; + const source = this.#createInboxRequestSource(request, metricState); let context: RequestContextImpl; try { - context = this.#createContext(request, contextData); + context = withContext( + { inboxRequestId: source.inboxRequestId }, + () => + this.#createContext(request, contextData, { + [outboundProvenance]: globalThis.Object.freeze({ + inboxRequest: source, + }), + }), + ); } catch (error) { return await this.#reportInboxPreparationError( request, @@ -4516,118 +4607,123 @@ export class FederationImpl const observation = new InboxObservation({ kind: "portable", recipient: identifier, - }); + }, source); this.#setInboxCompletion(metricState, observation, () => context); - return await observation.run( - async () => { - // The actor is looked up only by its identifier, as for ordinary inbox - // deliveries, so that the identifier alone determines the recipient, - // also for inbox listeners and queued deliveries: - const actor = this.actorCallbacks?.dispatcher == null - ? null - : await this.actorCallbacks.dispatcher(context, identifier); - const localOrigin = context.canonicalOrigin; - const resolution = resolvePortableInboxRecipient( - actor == null || actor instanceof Tombstone ? null : actor, - { - authority: portable.portableRequest.authority, - canonicalInboxId: portable.canonicalId, - localOrigin, - }, - ); - if (resolution.status === "rejected") { - logger.debug( - "Not accepting a delivery to the portable inbox {inbox} on behalf " + - "of the actor {identifier}: {reason}.", - { - inbox: portable.canonicalId, - identifier, - reason: resolution.reason, - }, - ); - return await onNotFound(request); - } - const { recipient } = resolution; - // A portable actor does not necessarily have key pairs for authorized - // fetch, so fall back to the default document loader without them: - if (this.actorCallbacks?.keyPairsDispatcher != null) { - context = this.#createContext(request, contextData, { - documentLoader: await context.getDocumentLoader({ identifier }), - }); - } - const excludedOrigins = [localOrigin, new URL(request.url).origin]; - return await this.#handleInbox(request, { - recipient: identifier, - observation, - context, - contextData, - onNotFound, - portableInbox: { - recipient, - forward: (activity, activityId, activityType, signatures) => - observeOutboundBroadcast( - context, - this.outboundObservers, - async (observation) => { - if (this.portableInboxForwarding.maxTargets < 1) { - if (observation != null) { - await observation.skipped("forwarding-disabled"); - } - return; - } - if (this.kv.cas == null) { - if (!this.#portableInboxForwardingWarned) { - this.#portableInboxForwardingWarned = true; - logger.warn( - "Activities delivered to FEP-ef61 portable inboxes are not " + - "forwarded to the other gateways, as the key–value store " + - "does not support compare-and-swap (KvStore.cas()), which " + - "is needed to forward each activity at most once.", - ); - } - if (observation != null) { - await observation.skipped("forwarding-unavailable"); - } - return; - } - await forwardPortableInboxActivity({ - recipient, - activity, - activityId, - activityType, - excludedOrigins, - baseUrl: context.origin, - kv: this.kv, - kvPrefix: this.kvPrefixes.portableInboxForwarding, - outboxQueue: this.outboxQueue, - startQueue: this.manuallyStartQueue - ? undefined - : () => this._startQueueInternal(contextData), - allowPrivateAddress: this.allowPrivateAddress, - getKeys: () => - this.#getPortableGatewayKeyPairs( - context, - identifier, - recipient, - ), - specDeterminer: new KvSpecDeterminer( - this.kv, - this.kvPrefixes.httpMessageSignaturesSpec, - this.firstKnock, - { specTtl: this.httpMessageSignaturesSpecTtl }, - ), - signatures, - options: this.portableInboxForwarding, - meterProvider: this.meterProvider, - tracerProvider: this.tracerProvider, - }, observation); + return await withContext( + { inboxRequestId: source.inboxRequestId }, + () => + observation.run( + async () => { + // The actor is looked up only by its identifier, as for ordinary inbox + // deliveries, so that the identifier alone determines the recipient, + // also for inbox listeners and queued deliveries: + const actor = this.actorCallbacks?.dispatcher == null + ? null + : await this.actorCallbacks.dispatcher(context, identifier); + const localOrigin = context.canonicalOrigin; + const resolution = resolvePortableInboxRecipient( + actor == null || actor instanceof Tombstone ? null : actor, + { + authority: portable.portableRequest.authority, + canonicalInboxId: portable.canonicalId, + localOrigin, + }, + ); + if (resolution.status === "rejected") { + logger.debug( + "Not accepting a delivery to the portable inbox {inbox} on behalf " + + "of the actor {identifier}: {reason}.", + { + inbox: portable.canonicalId, + identifier, + reason: resolution.reason, }, - "portable-replication", - ), + ); + return await onNotFound(request); + } + const { recipient } = resolution; + // A portable actor does not necessarily have key pairs for authorized + // fetch, so fall back to the default document loader without them: + if (this.actorCallbacks?.keyPairsDispatcher != null) { + context = this.#createContext(request, contextData, { + [outboundProvenance]: context[outboundProvenance], + documentLoader: await context.getDocumentLoader({ identifier }), + }); + } + const excludedOrigins = [localOrigin, new URL(request.url).origin]; + return await this.#handleInbox(request, { + recipient: identifier, + observation, + context, + contextData, + onNotFound, + portableInbox: { + recipient, + forward: (activity, activityId, activityType, signatures) => + observeOutboundBroadcast( + context, + this.outboundObservers, + async (observation) => { + if (this.portableInboxForwarding.maxTargets < 1) { + if (observation != null) { + await observation.skipped("forwarding-disabled"); + } + return; + } + if (this.kv.cas == null) { + if (!this.#portableInboxForwardingWarned) { + this.#portableInboxForwardingWarned = true; + logger.warn( + "Activities delivered to FEP-ef61 portable inboxes are not " + + "forwarded to the other gateways, as the key–value store " + + "does not support compare-and-swap (KvStore.cas()), which " + + "is needed to forward each activity at most once.", + ); + } + if (observation != null) { + await observation.skipped("forwarding-unavailable"); + } + return; + } + await forwardPortableInboxActivity({ + recipient, + activity, + activityId, + activityType, + excludedOrigins, + baseUrl: context.origin, + kv: this.kv, + kvPrefix: this.kvPrefixes.portableInboxForwarding, + outboxQueue: this.outboxQueue, + startQueue: this.manuallyStartQueue + ? undefined + : () => this._startQueueInternal(contextData), + allowPrivateAddress: this.allowPrivateAddress, + getKeys: () => + this.#getPortableGatewayKeyPairs( + context, + identifier, + recipient, + ), + specDeterminer: new KvSpecDeterminer( + this.kv, + this.kvPrefixes.httpMessageSignaturesSpec, + this.firstKnock, + { specTtl: this.httpMessageSignaturesSpecTtl }, + ), + signatures, + options: this.portableInboxForwarding, + meterProvider: this.meterProvider, + tracerProvider: this.tracerProvider, + }, observation); + }, + "portable-replication", + ), + }, + }); }, - }); - }, - span, + span, + ), ); } @@ -4736,7 +4832,11 @@ export class FederationImpl }, ); } - const context = this.#createContext(request, contextData); + const context = this.#createContext(request, contextData, { + [outboundProvenance]: globalThis.Object.freeze({ + originatingRequestId: metricState.requestId, + }), + }); const response = await this.hashlinkMediaDispatcher!(context, media.media); // The onNotFound response is returned as is, since integrations may // recognize it by its identity: @@ -4852,6 +4952,8 @@ type FedifyEndpoint = | "error"; interface HttpMetricState { + requestId?: string; + inboxRequest?: OutboundInboxRequest; inboxCompletion?: { observation: InboxObservation; finish( @@ -4930,6 +5032,7 @@ function getCollectionMetricRoute(routeName: string): } interface ContextOptions { + [outboundProvenance]?: OutboundProvenance; url: URL; federation: FederationImpl; data: TContextData; @@ -4973,6 +5076,7 @@ function createPortableObjectVerifier( } export class ContextImpl implements Context { + readonly [outboundProvenance]: OutboundProvenance; readonly url: URL; readonly federation: FederationImpl; readonly data: TContextData; @@ -4992,9 +5096,11 @@ export class ContextImpl implements Context { documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher, + [outboundProvenance]: provenance = emptyProvenance, }: ContextOptions, ) { this.url = url; + this[outboundProvenance] = provenance; this.federation = federation; this.data = data; this.documentLoader = documentLoader; @@ -5024,6 +5130,7 @@ export class ContextImpl implements Context { clone(data: TContextData): Context { return new ContextImpl({ + [outboundProvenance]: this[outboundProvenance], url: this.url, federation: this.federation, data, @@ -5042,6 +5149,7 @@ export class ContextImpl implements Context { metadata?: InboxContextMetadata, ): InboxContextImpl { return new InboxContextImpl(recipient, activity, activityId, activityType, { + [outboundProvenance]: this[outboundProvenance], url: this.url, federation: this.federation, data: this.data, @@ -5064,6 +5172,7 @@ export class ContextImpl implements Context { activityId, activityType, { + [outboundProvenance]: this[outboundProvenance], url: this.url, federation: this.federation, data: this.data, @@ -6491,6 +6600,7 @@ class RequestContextImpl extends ContextImpl override clone(data: TContextData): RequestContext { return new RequestContextImpl({ + [outboundProvenance]: this[outboundProvenance], url: this.url, federation: this.federation, data, @@ -7226,6 +7336,7 @@ export class InboxContextImpl extends ContextImpl this.activityId, this.activityType, { + [outboundProvenance]: this[outboundProvenance], url: this.url, federation: this.federation, data, @@ -7401,6 +7512,7 @@ export class OutboxContextImpl extends ContextImpl this.activityId, this.activityType, { + [outboundProvenance]: this[outboundProvenance], url: this.url, federation: this.federation, data, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index ca138af84..1331c93c3 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,3 +1,8 @@ +import { + getOutboundProvenance, + snapshotLocalContext, + withOutboundSource, +} from "./outbound-provenance.ts"; import { encodeHex } from "byte-encodings/hex"; import { getLogger } from "@logtape/logtape"; import type { Context } from "./context.ts"; @@ -9,6 +14,9 @@ import type { OutboundPreparationResult, } from "./outbound-report.ts"; +type WithoutSource = T extends unknown ? Omit + : never; + type PreparationFailure = Extract< OutboundPreparationResult, { type: "failed" } @@ -87,7 +95,7 @@ async function notifyObserver( ): Promise { if (callback == null) return; try { - await callback(context, event); + await withOutboundSource(context, () => callback(context, event)); } catch (error) { try { getLogger(["fedify", "federation", "outbox"]).error( @@ -130,6 +138,8 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + provenance: getOutboundProvenance(this.context), + local: snapshotLocalContext(this.context), ...this.#reference, ...this.#plan, stage: "planned" as const, @@ -147,6 +157,8 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + provenance: getOutboundProvenance(this.context), + local: snapshotLocalContext(this.context), ...this.#reference, stage: "handoff" as const, result: Object.freeze(result), @@ -162,6 +174,8 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + provenance: getOutboundProvenance(this.context), + local: snapshotLocalContext(this.context), ...this.#reference, stage: "preparation-failed" as const, failure: Object.freeze({ @@ -405,8 +419,19 @@ export class OutboundBroadcastObservation { } } - #notify(event: OutboundBroadcastEvent): Promise { - return notifyObserver(this.context, this.#onBroadcast, event, "broadcast"); + #notify( + event: WithoutSource, + ): Promise { + return notifyObserver( + this.context, + this.#onBroadcast, + Object.freeze({ + ...event, + provenance: getOutboundProvenance(this.context), + local: snapshotLocalContext(this.context), + }), + "broadcast", + ); } } @@ -420,9 +445,11 @@ export function observeOutboundBroadcast( operation: OutboundBroadcastEvent["operation"] = "send", ): Promise { if (observers.onBroadcast == null && observers.onDelivery == null) { - return send(); + return withOutboundSource(context, () => send()); } - return new OutboundBroadcastObservation(context, observers, operation).run( - send, + return withOutboundSource( + context, + () => + new OutboundBroadcastObservation(context, observers, operation).run(send), ); } diff --git a/packages/fedify/src/federation/outbound-provenance.test.ts b/packages/fedify/src/federation/outbound-provenance.test.ts new file mode 100644 index 000000000..946b4d6e8 --- /dev/null +++ b/packages/fedify/src/federation/outbound-provenance.test.ts @@ -0,0 +1,484 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Note, Person } from "@fedify/vocab"; +import { + computeDigestMultibase, + exportDidKey, + parseIri, +} from "@fedify/vocab-runtime"; +import { + configure, + getLogger, + type LogRecord, + reset, + withContext, +} from "@logtape/logtape"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { + deepStrictEqual, + notStrictEqual, + ok, + rejects, + strictEqual, +} from "node:assert/strict"; +import { signObject } from "../sig/proof.ts"; +import { + ed25519Multikey, + ed25519PrivateKey, + ed25519PublicKey, + rsaPrivateKey2, + rsaPublicKey2, +} from "../testing/keys.ts"; +import type { Context, RequestContext } from "./context.ts"; +import type { InboxRequestReport } from "./inbox-report.ts"; +import { MemoryKvStore } from "./kv.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; +import type { + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; +import type { Message } from "./queue.ts"; + +const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; +const outgoing = new Create({ + id: new URL("https://example.com/outgoing"), + actor: new URL("https://example.com/person2"), +}); +const target = new Person({ + id: new URL("https://remote.example/bob"), + inbox: new URL("https://remote.example/inbox"), +}); +const loaders = { + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + authenticatedDocumentLoaderFactory: () => mockDocumentLoader, +}; +function post(path: string, body: unknown) { + return new Request(`https://inbox.example:8443${path}`, { + method: "POST", + headers: { + "Content-Type": "application/activity+json", + "X-Request-Id": "shared-external-label", + }, + body: JSON.stringify(body), + }); +} +async function payload() { + return await new Create({ + id: new URL("https://example.com/incoming"), + actor: new URL("https://example.com/person2"), + }).toJsonLd({ contextLoader: mockDocumentLoader }); +} + +test("outbound inbox provenance follows clones, conversions, loaders, sends, and forwarding", async (t) => { + for (const path of ["/users/alice/inbox", "/inbox"]) { + await t.step(path, async () => { + const reports: InboxRequestReport[] = []; + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + const messages: Message[] = []; + let retained: Context | undefined; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + origin: "https://canonical.example", + skipSignatureVerification: true, + manuallyStartQueue: true, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + onDelivery(_ctx, event) { + events.push(event); + }, + }); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .setKeyPairsDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return []; + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .setSharedKeyDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return sender; + }) + .on(Create, async (ctx) => { + ok(ctx instanceof ContextImpl); + const clone = ctx.clone(undefined); + ok(clone instanceof ContextImpl); + const outbox = clone.toOutboxContext( + "alice", + {}, + outgoing.id!.href, + Create.typeId.href, + ); + const inbox = outbox.toInboxContext( + null, + {}, + outgoing.id!.href, + Create.typeId.href, + ).clone(undefined); + // All derived contexts share a URL object; historical source facts must not. + ctx.url.host = "changed.example:9443"; + retained = inbox; + await clone.sendActivity(sender, target, outgoing); + await outbox.clone(undefined).sendActivity(sender, target, outgoing); + await inbox.forwardActivity(sender, target, { + skipIfUnsigned: false, + }); + }) + .onRequestFinished(async (ctx, report) => { + reports.push(report); + await ctx.clone(undefined).sendActivity(sender, [], outgoing); + }); + const body = await payload(); + for (let index = 0; index < 2; index++) { + const start = events.length; + strictEqual( + (await federation.fetch(post(path, body), { contextData: undefined })) + .status, + 202, + ); + const report = reports[index]; + ok(report.inboxRequestId); + strictEqual(report.requestId, "shared-external-label"); + const observed = events.slice(start); + ok(observed.length > 0); + for (const event of observed) { + const source = event.provenance.inboxRequest; + ok(source); + strictEqual(source.inboxRequestId, report.inboxRequestId); + strictEqual(source.requestId, report.requestId); + deepStrictEqual(source.context, { + origin: "https://inbox.example:8443", + host: "inbox.example:8443", + hostname: "inbox.example", + canonicalOrigin: "https://canonical.example", + }); + ok(Object.isFrozen(event.provenance)); + ok(Object.isFrozen(source)); + ok(Object.isFrozen(source.context)); + ok(Object.isFrozen(event.local)); + } + } + notStrictEqual(reports[0].inboxRequestId, reports[1].inboxRequestId); + strictEqual(messages.length, 3); // The repeated Activity is an inbox duplicate. + ok(events.some((event) => event.local.host === "changed.example:9443")); + ok(events.some((event) => event.local.host === "inbox.example:8443")); + ok( + events.some((event) => + "operation" in event && event.operation === "forward" + ), + ); + ok(retained); + const count = events.length; + await retained.sendActivity(sender, [], outgoing); + strictEqual( + events[count].provenance.inboxRequest?.inboxRequestId, + reports[0].inboxRequestId, + ); + // Same-looking Request URLs and programmatic inbox conversion establish no HTTP cause. + for ( + const input of [new URL("https://inbox.example:8443"), post(path, body)] + ) { + const independent = input instanceof URL + ? federation.createContext(input, undefined) + : federation.createContext(input, undefined); + ok(independent instanceof ContextImpl); + await independent.toInboxContext( + null, + body, + "https://example.com/manual", + Create.typeId.href, + ).sendActivity(sender, [], outgoing); + deepStrictEqual(events.at(-1)!.provenance, {}); + } + }); + } +}); + +test("outbound inbox provenance exists before loader preparation fails", async () => { + const failure = new Error("loader construction failed"); + const reports: InboxRequestReport[] = []; + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + contextLoaderFactory() { + throw failure; + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .onRequestFinished(async (ctx, report) => { + reports.push(report); + await ctx.sendActivity(sender, [], outgoing); + }); + for (const path of ["/users/alice/inbox", "/inbox"]) { + await rejects( + federation.fetch(post(path, {}), { contextData: undefined }), + (error) => error === failure, + ); + const report = reports.at(-1)!; + ok(report.inboxRequestId); + strictEqual(report.requestId, "shared-external-label"); + strictEqual(report.outcome.type, "exception"); + strictEqual( + events.at(-1)?.provenance.inboxRequest?.inboxRequestId, + report.inboxRequestId, + ); + } + notStrictEqual(reports[0].inboxRequestId, reports[1].inboxRequestId); +}); + +test("outbound provenance keeps source log labels separate from current request labels", async () => { + const records: LogRecord[] = []; + await reset(); + await configure({ + sinks: { buffer: (record) => records.push(record) }, + loggers: [{ category: [], sinks: ["buffer"], lowestLevel: "info" }, { + category: ["logtape", "meta"], + sinks: [], + }], + contextLocalStorage: new AsyncLocalStorage(), + }); + try { + const events: OutboundBroadcastEvent[] = []; + let retained: RequestContext | undefined; + let report: InboxRequestReport | undefined; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + skipSignatureVerification: true, + }); + federation.setActorDispatcher("/users/{identifier}", async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return new Person({}); + }).setKeyPairsDispatcher(() => []); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + getLogger(["test", "provenance"]).info("outbound callback"); + }, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox").on( + Create, + async () => { + await withContext({ + sourceInboxRequestId: "ambient-inbox", + sourceRequestId: "ambient-source", + }, () => + federation.fetch( + new Request("https://inbox.example/users/alice?nested=1", { + headers: { + Accept: "application/activity+json", + "X-Request-Id": "nested-request", + }, + }), + { contextData: undefined }, + )); + }, + ) + .onRequestFinished((ctx, value) => { + retained = ctx; + report = value; + getLogger(["test", "provenance"]).info("inbox callback"); + }); + strictEqual( + (await federation.fetch(post("/users/alice/inbox", await payload()), { + contextData: undefined, + })).status, + 202, + ); + ok(retained && report?.inboxRequestId); + const ingress = records.find((record) => + record.rawMessage === "inbox callback" + ); + ok(ingress); + strictEqual(ingress.properties.inboxRequestId, report.inboxRequestId); + strictEqual(ingress.properties.requestId, report.requestId); + const nested = records.find((record) => + record.properties.url === "https://inbox.example/users/alice?nested=1" && + record.properties.requestId === "nested-request" + ); + ok(nested); + strictEqual(nested.properties.inboxRequestId, undefined); + strictEqual(nested.properties.sourceInboxRequestId, undefined); + strictEqual(nested.properties.sourceRequestId, undefined); + const original = records.find((record) => + record.properties.url === + "https://inbox.example:8443/users/alice/inbox" && + record.properties.status === 202 + ); + ok(original); + strictEqual(original.properties.inboxRequestId, report.inboxRequestId); + await withContext( + { requestId: "unrelated-current" }, + () => retained!.sendActivity(sender, [], outgoing), + ); + const descendant = records.filter((record) => + record.rawMessage === "outbound callback" + ).at(-1); + ok(descendant); + strictEqual(descendant.properties.requestId, "unrelated-current"); + strictEqual(descendant.properties.sourceRequestId, report.requestId); + strictEqual( + descendant.properties.sourceInboxRequestId, + report.inboxRequestId, + ); + const independent = federation.createContext( + new URL("https://inbox.example"), + undefined, + ); + await withContext({ + requestId: "independent-current", + sourceRequestId: "ambient-source", + sourceInboxRequestId: "ambient-inbox", + }, () => independent.sendActivity(sender, [], outgoing)); + deepStrictEqual(events.at(-1)!.provenance, {}); + const independentLog = records.filter((record) => + record.rawMessage === "outbound callback" + ).at(-1); + ok(independentLog); + strictEqual(independentLog.properties.requestId, "independent-current"); + strictEqual(independentLog.properties.sourceRequestId, undefined); + strictEqual(independentLog.properties.sourceInboxRequestId, undefined); + // Ordinary HTTP-derived work retains its logging label without an inbox cause. + const start = events.length; + await federation.fetch( + new Request("https://inbox.example/users/alice", { + headers: { + Accept: "application/activity+json", + "X-Request-Id": "actor-request", + }, + }), + { contextData: undefined }, + ); + ok(events.length > start); + for (const event of events.slice(start)) { + deepStrictEqual(event.provenance, { + originatingRequestId: "actor-request", + }); + } + } finally { + await reset(); + } +}); + +test("outbound provenance does not invent an inbox cause for programmatic routing", async () => { + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox").on( + Create, + async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + }, + ); + const ctx = federation.createContext( + new URL("https://inbox.example"), + undefined, + ); + const signed = await signObject( + new Create({ + id: new URL("https://example.com/programmatic"), + actor: new URL("https://example.com/person2"), + }), + ed25519PrivateKey, + ed25519Multikey.id!, + { contextLoader: mockDocumentLoader }, + ); + strictEqual(await ctx.routeActivity(null, signed), true); + strictEqual(events.length, 2); + for (const event of events) deepStrictEqual(event.provenance, {}); +}); + +test("outbound provenance retains request labels on gateway dispatchers", async (t) => { + const did = await exportDidKey(ed25519PublicKey.publicKey); + const digest = await computeDigestMultibase( + new TextEncoder().encode("gateway media"), + ); + for (const route of ["actor", "object", "collection", "media"] as const) { + await t.step(route, async () => { + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + }); + const send = (ctx: Context) => + ctx.clone(undefined).sendActivity(sender, [], outgoing); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + federation.setActorDispatcher("/users/{identifier}", async (ctx) => { + await send(ctx); + return route === "collection" + ? new Person({ + id: parseIri(`ap+ef61://${did}/users/alice`), + followers: parseIri(`ap+ef61://${did}/users/alice/followers`), + }) + : null; + }); + federation.setObjectDispatcher(Note, "/notes/{id}", async (ctx) => { + await send(ctx); + return null; + }); + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + async (ctx) => { + await send(ctx); + return { items: [] }; + }, + ); + federation.setHashlinkMediaDispatcher(async (ctx) => { + await send(ctx); + return null; + }); + const suffix = route === "actor" + ? `${did}/users/alice` + : route === "object" + ? `${did}/notes/1` + : route === "collection" + ? `${did}/users/alice/followers` + : `hl:${digest}`; + const response = await federation.fetch( + new Request(`https://example.com/.well-known/apgateway/${suffix}`, { + headers: { + Accept: "application/activity+json", + "X-Request-Id": `${route}-request`, + }, + }), + { contextData: undefined }, + ); + strictEqual(response.status, route === "collection" ? 200 : 404); + strictEqual(events.length, route === "collection" ? 4 : 2); + for (const event of events) { + deepStrictEqual(event.provenance, { + originatingRequestId: `${route}-request`, + }); + strictEqual(event.local.host, "example.com"); + } + }); + } +}); diff --git a/packages/fedify/src/federation/outbound-provenance.ts b/packages/fedify/src/federation/outbound-provenance.ts new file mode 100644 index 000000000..711937bbd --- /dev/null +++ b/packages/fedify/src/federation/outbound-provenance.ts @@ -0,0 +1,59 @@ +import { withContext } from "@logtape/logtape"; +import type { Context } from "./context.ts"; +import type { + OutboundInboxRequest, + OutboundLocalContext, + OutboundProvenance, +} from "./outbound-report.ts"; + +// An own symbol property survives context spreads without adding public API. +export const outboundProvenance = Symbol("outboundProvenance"); +export const emptyProvenance: OutboundProvenance = Object.freeze({}); + +export function getOutboundProvenance(context: object): OutboundProvenance { + return (context as { [outboundProvenance]?: OutboundProvenance })[ + outboundProvenance + ] ?? emptyProvenance; +} + +export function snapshotLocalContext( + context: Pick< + Context, + "origin" | "host" | "hostname" | "canonicalOrigin" + >, +): OutboundLocalContext { + return Object.freeze({ + origin: context.origin, + host: context.host, + hostname: context.hostname, + canonicalOrigin: context.canonicalOrigin, + }); +} + +export function createInboxRequestSource( + request: Request, + requestId: string, + canonicalOrigin?: string, +): OutboundInboxRequest { + const url = new URL(request.url); + return Object.freeze({ + inboxRequestId: crypto.randomUUID(), + requestId, + context: snapshotLocalContext({ + origin: url.origin, + host: url.host, + hostname: url.hostname, + canonicalOrigin: canonicalOrigin ?? url.origin, + }), + }); +} + +export function withOutboundSource(context: object, operation: () => T): T { + const source = getOutboundProvenance(context); + // Explicit absence masks unrelated ambient source labels as well. + return withContext({ + sourceInboxRequestId: source.inboxRequest?.inboxRequestId, + sourceRequestId: source.inboxRequest?.requestId ?? + source.originatingRequestId, + }, operation); +} diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index c45fc36ce..c1d142391 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -59,6 +59,7 @@ export type OutboundPreparationResult = * @since 2.5.0 */ export type OutboundBroadcastEvent = + & OutboundSourceMetadata & { /** Fresh opaque identity shared by producer events, independent of tracing. */ readonly broadcastId: string; @@ -88,6 +89,7 @@ export type OutboundBroadcastEvent = * @since 2.5.0 */ export type OutboundDeliveryEvent = + & OutboundSourceMetadata & { /** The originating invocation, or unknown for legacy fanout work. */ readonly broadcast: @@ -177,3 +179,39 @@ export interface OutboundObservers { event: OutboundDeliveryEvent, ) => void | Promise; } +/** URL facts captured independently of a mutable context URL. @since 2.5.0 */ +export interface OutboundLocalContext { + /** Scheme and authority of the context URL. */ + readonly origin: string; + /** URL hostname with its optional port. */ + readonly host: string; + /** URL hostname without a port. */ + readonly hostname: string; + /** Configured canonical web origin, falling back to this context's origin. */ + readonly canonicalOrigin: string; +} + +/** The original routed HTTP inbox request that caused outgoing work. @since 2.5.0 */ +export interface OutboundInboxRequest { + /** Fresh server-generated identity, independent of Activity and trace IDs. */ + readonly inboxRequestId: string; + /** Informational logging label; externally controlled and nonunique. */ + readonly requestId: string; + /** Original request URL facts, preserved separately from current context. */ + readonly context: OutboundLocalContext; +} + +/** Explicit historical cause of outgoing work. @since 2.5.0 */ +export interface OutboundProvenance { + /** Absent for independent contexts, programmatic routing, and unknown work. */ + readonly inboxRequest?: OutboundInboxRequest; + /** Original logging label for request-derived work without an inbox cause. */ + readonly originatingRequestId?: string; +} + +interface OutboundSourceMetadata { + /** Immutable historical cause, distinct from this callback's context. */ + readonly provenance: OutboundProvenance; + /** URL facts of the current callback context. */ + readonly local: OutboundLocalContext; +} diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index 6dfa91f1c..c39de4956 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -2384,7 +2384,7 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a }) : Promise.resolve(); }; - const { federation } = setup({ queue, keyPairs: false }); + const { federation, reports } = setup({ queue, keyPairs: false }); const events: OutboundBroadcastEvent[] = []; const deliveries: OutboundDeliveryEvent[] = []; let ready!: () => void; @@ -2474,6 +2474,14 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a } } assertEquals(events.length, 2); + const source = events[0].provenance.inboxRequest; + assert(source); + assertEquals(source.inboxRequestId, reports[0].inboxRequestId); + assertEquals(source.requestId, reports[0].requestId); + assertEquals(source.context.origin, LOCAL); + for (const event of [...events, ...deliveries]) { + assertEquals(event.provenance.inboxRequest, source); + } // A swallowed enqueue failure keeps its claim; redelivery is not retried. federation.setOutboundObservers({ onDelivery(_ctx, event) { From 1e4af0dd0e3d2d0a94c299c80641108505b74c75 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 01:28:31 +0900 Subject: [PATCH 13/53] Carry request provenance through queues Preserve explicit source snapshots through inbox and outgoing queues, worker reconstruction, and retries. Keep historical source labels separate from current request labels and leave legacy sources unknown. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/inbox.ts | 5 + packages/fedify/src/federation/middleware.ts | 55 +++- .../federation/outbound-provenance.test.ts | 301 ++++++++++++++++++ .../src/federation/outbound-provenance.ts | 63 +++- .../src/federation/outbound-report.test.ts | 20 ++ .../src/federation/portable-inbox.test.ts | 3 + .../fedify/src/federation/portable-inbox.ts | 5 + packages/fedify/src/federation/queue.ts | 7 + 8 files changed, 451 insertions(+), 8 deletions(-) diff --git a/packages/fedify/src/federation/inbox.ts b/packages/fedify/src/federation/inbox.ts index e07be771d..f3a814a86 100644 --- a/packages/fedify/src/federation/inbox.ts +++ b/packages/fedify/src/federation/inbox.ts @@ -1,3 +1,7 @@ +import { + getOutboundProvenance, + serializeProvenance, +} from "./outbound-provenance.ts"; import { getTypeId } from "@fedify/vocab"; import type { Activity } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; @@ -425,6 +429,7 @@ export async function routeActivity( propagation.inject(context.active(), carrier); message = { type: "inbox", + ...serializeProvenance(getOutboundProvenance(ctx)), id: crypto.randomUUID(), baseUrl: ctx.origin, activity: originalJson ?? json, diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index b43bdb607..02752ab99 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1,7 +1,11 @@ import { createInboxRequestSource, emptyProvenance, + getOutboundProvenance, outboundProvenance, + restoreProvenance, + serializeProvenance, + sourceLogContext, } from "./outbound-provenance.ts"; import type { OutboundInboxRequest, @@ -1229,7 +1233,17 @@ export class FederationImpl context.active(), message.traceContext, ); - return withContext({ messageId: message.id }, async () => { + const provenance = message.type === "task" + ? emptyProvenance + : restoreProvenance(message.provenance); + if (message.type !== "task" && message.provenance !== undefined) { + message = { ...message, provenance }; + } + return withContext({ + messageId: message.id, + inboxRequestId: undefined, + ...sourceLogContext(provenance), + }, async () => { if (message.type === "fanout") { const common: QueueTaskCommonAttributes = { role: "fanout", @@ -1244,7 +1258,7 @@ export class FederationImpl if (message.activityId != null) { span.setAttribute("activitypub.activity.id", message.activityId); } - await this.#listenFanoutMessage(contextData, message); + await this.#listenFanoutMessage(contextData, message, provenance); return { outcome: "completed" }; }, classifyAbortableError), ); @@ -1265,7 +1279,12 @@ export class FederationImpl if (message.activityId != null) { span.setAttribute("activitypub.activity.id", message.activityId); } - await this.#listenOutboxMessage(contextData, message, span); + await this.#listenOutboxMessage( + contextData, + message, + span, + provenance, + ); return { outcome: "completed" }; }, classifyAbortableError), ); @@ -1283,6 +1302,7 @@ export class FederationImpl contextData, message, span, + provenance, (activityType) => { common.activityType = activityType; }, @@ -1397,6 +1417,7 @@ export class FederationImpl async #listenFanoutMessage( data: TContextData, message: FanoutMessage, + provenance: OutboundProvenance, ): Promise { const observers = this.outboundObservers; const logger = getLogger(["fedify", "federation", "fanout"]); @@ -1428,6 +1449,7 @@ export class FederationImpl new URL(message.baseUrl), data, { + [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory({ allowPrivateAddress: this.allowPrivateAddress, userAgent: this.userAgent, @@ -1453,6 +1475,7 @@ export class FederationImpl _: TContextData, message: OutboxMessage, span: Span, + provenance: OutboundProvenance, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { @@ -1554,6 +1577,7 @@ export class FederationImpl new URL(message.baseUrl), _, { + [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory(loaderOptions), }, ); @@ -1596,6 +1620,7 @@ export class FederationImpl }, span, { + provenance, keys, logData, getActorIds, @@ -1842,6 +1867,7 @@ export class FederationImpl new URL(message.baseUrl), _, { + [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory(loaderOptions), }, ); @@ -1977,6 +2003,7 @@ export class FederationImpl message: OutboxMessage & { readonly gatewayInboxes: readonly string[] }, span: Span, helpers: { + readonly provenance: OutboundProvenance; readonly keys: readonly SenderKeyPair[]; readonly logData: Record; readonly getActorIds: () => URL[]; @@ -1992,8 +2019,14 @@ export class FederationImpl ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); const circuitLogger = getLogger(["fedify", "federation", "circuit"]); - const { keys, logData, getActorIds, parseActivity, dropHeldOutboxMessage } = - helpers; + const { + provenance, + keys, + logData, + getActorIds, + parseActivity, + dropHeldOutboxMessage, + } = helpers; const circuit = this.outboxQueue == null ? undefined : this.circuitBreaker; const specDeterminer = new KvSpecDeterminer( this.kv, @@ -2251,6 +2284,7 @@ export class FederationImpl new URL(message.baseUrl), contextData, { + [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory( this.#getLoaderOptions(message.baseUrl), ), @@ -2411,11 +2445,14 @@ export class FederationImpl ctxData: TContextData, message: InboxMessage, span: Span, + provenance: OutboundProvenance, onActivityType?: (activityType: string) => void, ): Promise { const logger = getLogger(["fedify", "federation", "inbox"]); const baseUrl = new URL(message.baseUrl); - let context = this.#createContext(baseUrl, ctxData); + let context = this.#createContext(baseUrl, ctxData, { + [outboundProvenance]: provenance, + }); // A portable actor does not necessarily have key pairs for authorized // fetch, so fall back to the default document loader without them: if ( @@ -2423,6 +2460,7 @@ export class FederationImpl this.actorCallbacks?.keyPairsDispatcher != null ) { context = this.#createContext(baseUrl, ctxData, { + [outboundProvenance]: provenance, documentLoader: await context.getDocumentLoader({ identifier: message.identifier, }), @@ -2433,6 +2471,7 @@ export class FederationImpl const identity = await this.sharedInboxKeyDispatcher(context); if (identity != null) { context = this.#createContext(baseUrl, ctxData, { + [outboundProvenance]: provenance, documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity), @@ -3415,6 +3454,7 @@ export class FederationImpl : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; const message: OutboxMessage = { type: "outbox", + ...serializeProvenance(getOutboundProvenance(ctx)), id: crypto.randomUUID(), baseUrl: ctx.origin, keys: keyJwkPairs, @@ -4686,6 +4726,7 @@ export class FederationImpl return; } await forwardPortableInboxActivity({ + provenance: getOutboundProvenance(context), recipient, activity, activityId, @@ -6124,6 +6165,7 @@ export class ContextImpl implements Context { propagation.inject(context.active(), carrier); const message: FanoutMessage = { type: "fanout", + ...serializeProvenance(getOutboundProvenance(this)), id: crypto.randomUUID(), baseUrl: this.origin, keys: keyJwkPairs, @@ -7203,6 +7245,7 @@ async function forwardActivityInternal( const { portableInbox, gatewayInboxes } = inboxes[inbox]; const message: OutboxMessage = { type: "outbox", + ...serializeProvenance(getOutboundProvenance(ctx)), id: crypto.randomUUID(), baseUrl: ctx.origin, keys: keyJwkPairs, diff --git a/packages/fedify/src/federation/outbound-provenance.test.ts b/packages/fedify/src/federation/outbound-provenance.test.ts index 946b4d6e8..ad2f3d5fb 100644 --- a/packages/fedify/src/federation/outbound-provenance.test.ts +++ b/packages/fedify/src/federation/outbound-provenance.test.ts @@ -12,6 +12,7 @@ import { reset, withContext, } from "@logtape/logtape"; +import fetchMock from "fetch-mock"; import { AsyncLocalStorage } from "node:async_hooks"; import { deepStrictEqual, @@ -35,8 +36,10 @@ import { ContextImpl, createFederation } from "./middleware.ts"; import type { OutboundBroadcastEvent, OutboundDeliveryEvent, + OutboundProvenance, } from "./outbound-report.ts"; import type { Message } from "./queue.ts"; +import type { MessageQueue } from "./mq.ts"; const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; const outgoing = new Create({ @@ -482,3 +485,301 @@ test("outbound provenance retains request labels on gateway dispatchers", async }); } }); + +test("queued provenance survives inbox retries, fanout, and outbox worker reconstruction", async (t) => { + const records: LogRecord[] = []; + await reset(); + await configure({ + sinks: { buffer: (record) => records.push(record) }, + loggers: [{ category: [], sinks: ["buffer"], lowestLevel: "info" }, { + category: ["logtape", "meta"], + sinks: [], + }], + contextLocalStorage: new AsyncLocalStorage(), + }); + try { + // Source propagation covers both retry owners. Legacy absence is + // independent of retry ownership, so one legacy execution is sufficient. + for ( + const { legacy, native } of [ + { legacy: false, native: false }, + { legacy: false, native: true }, + { legacy: true, native: false }, + ] + ) { + await t.step( + `${legacy ? "legacy" : "source"}: ${ + native ? "native redelivery" : "scheduled retry" + }`, + async () => { + const messages: Message[] = []; + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + const reports: InboxRequestReport[] = []; + const queue: MessageQueue = { + nativeRetrial: native, + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + origin: "https://canonical.example", + skipSignatureVerification: true, + manuallyStartQueue: true, + queue: { inbox: queue, fanout: queue, outbox: queue }, + }); + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ).setKeyPairsDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return []; + }); + let calls = 0; + const failure = new Error("retry listener"); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .setSharedKeyDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return sender; + }) + .on(Create, async (ctx) => { + getLogger(["test", "queued-source"]).info("queued listener"); + await ctx.sendActivity(sender, [], outgoing); + if (++calls === 1) throw failure; + ok(ctx instanceof ContextImpl); + const clone = ctx.clone(undefined); + await clone.sendActivity(sender, target, outgoing, { + fanout: "force", + }); + const outbox = ctx.toOutboxContext( + "alice", + {}, + outgoing.id!.href, + Create.typeId.href, + ).clone(undefined); + await outbox.sendActivity(sender, target, outgoing, { + fanout: "skip", + }); + await clone.forwardActivity(sender, target, { + skipIfUnsigned: false, + }); + }).onRequestFinished((_ctx, report) => { + reports.push(report); + }); + // Producer metadata must not depend on observer registration. + strictEqual( + (await federation.fetch( + post(native ? "/inbox" : "/users/alice/inbox", await payload()), + { contextData: undefined }, + )).status, + 202, + ); + const produced = messages[0]; + ok(produced.type === "inbox"); + ok(produced.provenance?.inboxRequest); + strictEqual( + produced.provenance.inboxRequest.inboxRequestId, + reports[0].inboxRequestId, + ); + deepStrictEqual( + JSON.parse(JSON.stringify(produced.provenance)), + produced.provenance, + ); + const task = JSON.parse( + JSON.stringify(produced), + ) as typeof produced; + if (legacy) delete (task as { provenance?: unknown }).provenance; + const expected: OutboundProvenance = task.provenance ?? {}; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + onDelivery(_ctx, event) { + events.push(event); + }, + }); + const execute = (message: Message) => + withContext({ + requestId: "worker-current", + inboxRequestId: "unrelated-inbox", + sourceInboxRequestId: "ambient-source", + sourceRequestId: "ambient-label", + }, () => federation.processQueuedTask(undefined, message)); + if (!legacy && !native) { + await rejects(async () => + await execute({ + ...task, + provenance: { + inboxRequest: { + inboxRequestId: "", + requestId: "bad", + context: { + origin: "x", + host: "x", + hostname: "x", + canonicalOrigin: "x", + }, + }, + }, + }), TypeError); + strictEqual(calls, 0); + strictEqual(events.length, 0); + } + messages.length = 0; + const logStart = records.length; + if (native) { + await rejects(execute(task), (error) => error === failure); + } else await execute(task); + const retry = native + ? task + : messages.find((message) => message.type === "inbox"); + ok(retry?.type === "inbox"); + deepStrictEqual(retry.provenance ?? {}, expected); + strictEqual(retry.attempt, native ? 0 : 1); + await execute({ + ...structuredClone(retry), + id: crypto.randomUUID(), + }); + strictEqual(calls, 2); + const fanout = messages.find((message) => message.type === "fanout"); + ok(fanout?.type === "fanout"); + deepStrictEqual(fanout.provenance ?? {}, expected); + await execute(JSON.parse(JSON.stringify(fanout))); + const outgoingTasks = messages.filter((message) => + message.type === "outbox" + ); + strictEqual(outgoingTasks.length, 3); + for (const message of outgoingTasks) { + deepStrictEqual(message.provenance ?? {}, expected); + strictEqual("provenance" in message, !legacy); + } + const consumer = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + origin: "https://consumer.example", + manuallyStartQueue: true, + queue: { outbox: queue }, + }); + consumer.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + let permanent = 0; + consumer.setOutboxPermanentFailureHandler(async (ctx) => { + permanent++; + getLogger(["test", "queued-source"]).info("permanent callback"); + await ctx.clone(undefined).sendActivity(sender, [], outgoing); + }); + fetchMock.mockGlobal().post(target.inboxId!.href, 410); + try { + await withContext({ + requestId: "worker-current", + sourceInboxRequestId: "ambient-source", + sourceRequestId: "ambient-label", + }, () => + consumer.processQueuedTask(undefined, { + ...structuredClone(outgoingTasks[0]), + baseUrl: "https://worker.example", + })); + } finally { + fetchMock.hardReset(); + } + strictEqual(permanent, 1); + ok(events.length > 0); + for (const event of events) { + deepStrictEqual(event.provenance, expected); + ok(Object.isFrozen(event.provenance)); + if (!legacy) { + ok(Object.isFrozen(event.provenance.inboxRequest?.context)); + } + } + strictEqual(events.at(-1)!.local.host, "worker.example"); + strictEqual( + events.at(-1)!.local.canonicalOrigin, + "https://consumer.example", + ); + const workerLogs = records.slice(logStart).filter((record) => + record.rawMessage === "queued listener" || + record.rawMessage === "permanent callback" + ); + strictEqual(workerLogs.length, 3); + for (const record of workerLogs) { + strictEqual(record.properties.requestId, "worker-current"); + strictEqual(record.properties.inboxRequestId, undefined); + strictEqual( + record.properties.sourceInboxRequestId, + expected.inboxRequest?.inboxRequestId, + ); + strictEqual( + record.properties.sourceRequestId, + expected.inboxRequest?.requestId, + ); + } + }, + ); + } + } finally { + await reset(); + } +}); + +test("queued provenance retains a non-inbox HTTP request label without inventing an inbox cause", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { fanout: queue, outbox: queue }, + }); + federation.setActorDispatcher("/users/{identifier}", async (ctx) => { + await ctx.sendActivity(sender, target, outgoing, { fanout: "force" }); + return null; + }); + strictEqual( + (await federation.fetch( + new Request("https://example.com/users/alice", { + headers: { + Accept: "application/activity+json", + "X-Request-Id": "original-http-label", + }, + }), + { contextData: undefined }, + )).status, + 404, + ); + const message = messages[0]; + ok(message?.type === "fanout"); + deepStrictEqual(message.provenance, { + originatingRequestId: "original-http-label", + }); + const events: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); + await withContext( + { requestId: "worker-current", sourceInboxRequestId: "unrelated-inbox" }, + () => + federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ), + ); + strictEqual(events.length, 1); + deepStrictEqual(events[0].provenance, message.provenance); + const child = messages[1]; + ok(child?.type === "outbox"); + deepStrictEqual(child.provenance, message.provenance); +}); diff --git a/packages/fedify/src/federation/outbound-provenance.ts b/packages/fedify/src/federation/outbound-provenance.ts index 711937bbd..8d2160305 100644 --- a/packages/fedify/src/federation/outbound-provenance.ts +++ b/packages/fedify/src/federation/outbound-provenance.ts @@ -51,9 +51,68 @@ export function createInboxRequestSource( export function withOutboundSource(context: object, operation: () => T): T { const source = getOutboundProvenance(context); // Explicit absence masks unrelated ambient source labels as well. - return withContext({ + return withContext(sourceLogContext(source), operation); +} + +/** Restore only explicit source facts; never infer them from queue/trace IDs. */ +export function restoreProvenance(value: unknown): OutboundProvenance { + if (value === undefined) return emptyProvenance; + const record = (v: unknown): v is Record => + v != null && typeof v === "object" && !Array.isArray(v); + const invalid = () => new TypeError("Invalid provenance in queue metadata."); + if (!record(value)) throw invalid(); + if ( + value.originatingRequestId !== undefined && + typeof value.originatingRequestId !== "string" + ) throw invalid(); + let inboxRequest: OutboundInboxRequest | undefined; + if (value.inboxRequest !== undefined) { + const source = value.inboxRequest; + if ( + !record(source) || typeof source.inboxRequestId !== "string" || + source.inboxRequestId.length < 1 || + typeof source.requestId !== "string" || !record(source.context) + ) throw invalid(); + const facts = source.context; + if ( + typeof facts.origin !== "string" || typeof facts.host !== "string" || + typeof facts.hostname !== "string" || + typeof facts.canonicalOrigin !== "string" + ) throw invalid(); + inboxRequest = Object.freeze({ + inboxRequestId: source.inboxRequestId, + requestId: source.requestId, + context: Object.freeze({ + origin: facts.origin, + host: facts.host, + hostname: facts.hostname, + canonicalOrigin: facts.canonicalOrigin, + }), + }); + } + return Object.freeze({ + ...(inboxRequest == null ? {} : { inboxRequest }), + ...(value.originatingRequestId === undefined + ? {} + : { originatingRequestId: value.originatingRequestId }), + }); +} + +/** Omit absent provenance on legacy and independent producers. */ +export function serializeProvenance( + value: OutboundProvenance | undefined, +): { readonly provenance?: OutboundProvenance } { + const provenance = restoreProvenance(value); + return provenance.inboxRequest == null && + provenance.originatingRequestId == null + ? {} + : { provenance }; +} + +export function sourceLogContext(source: OutboundProvenance) { + return { sourceInboxRequestId: source.inboxRequest?.inboxRequestId, sourceRequestId: source.inboxRequest?.requestId ?? source.originatingRequestId, - }, operation); + }; } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index d759b86d5..f1c465ea6 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -1891,6 +1891,18 @@ function continuationMessage( ...(portableInbox == null ? {} : { portableInbox }), ...(observed ? { + provenance: { + inboxRequest: { + inboxRequestId: crypto.randomUUID(), + requestId: "original-request-label", + context: { + origin: "https://source.example:8443", + host: "source.example:8443", + hostname: "source.example", + canonicalOrigin: "https://canonical.example", + }, + }, + }, outbound: { broadcast: { type: "known" as const, id: crypto.randomUUID() }, deliveryId: crypto.randomUUID(), @@ -1967,6 +1979,8 @@ test("outbound queue identities survive ordinary retries, holds, and native rede const next = queued[0].message; ok(next.type === "outbox"); deepStrictEqual(next.outbound, original.outbound); + deepStrictEqual(next.provenance, original.provenance); + strictEqual("provenance" in next, observed); strictEqual("outbound" in next, observed); strictEqual(next.attempt, mode === "hold" ? 0 : 1); strictEqual(next.circuitHeld, mode === "hold" ? true : undefined); @@ -1981,6 +1995,8 @@ test("outbound queue identities survive ordinary retries, holds, and native rede const replay = queued[1].message; ok(replay.type === "outbox"); deepStrictEqual(replay.outbound, original.outbound); + deepStrictEqual(replay.provenance, original.provenance); + strictEqual("provenance" in replay, observed); strictEqual("outbound" in replay, observed); strictEqual(replay.attempt, mode === "hold" ? 0 : 2); if (mode === "hold") { @@ -2075,6 +2091,8 @@ test("outbound queue identities retain planned destinations through gateway cont strictEqual(next.inbox, gateways[1]); deepStrictEqual(next.gatewayInboxes, gateways.slice(1)); deepStrictEqual(next.outbound, original.outbound); + deepStrictEqual(next.provenance, original.provenance); + strictEqual("provenance" in next, observed); strictEqual("outbound" in next, observed); strictEqual(next.portableInbox, portableInbox); if (observed) { @@ -2097,6 +2115,8 @@ test("outbound queue identities retain planned destinations through gateway cont const replay = queued[1]; ok(replay.type === "outbox"); deepStrictEqual(replay.outbound, original.outbound); + deepStrictEqual(replay.provenance, original.provenance); + strictEqual("provenance" in replay, observed); strictEqual("outbound" in replay, observed); strictEqual(replay.inbox, gateways[1]); strictEqual(replay.attempt, mode === "retry" ? 2 : next.attempt); diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index c39de4956..2ed193a8e 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -2479,6 +2479,9 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a assertEquals(source.inboxRequestId, reports[0].inboxRequestId); assertEquals(source.requestId, reports[0].requestId); assertEquals(source.context.origin, LOCAL); + for (const message of queue.outbox) { + assertEquals(message.provenance?.inboxRequest, source); + } for (const event of [...events, ...deliveries]) { assertEquals(event.provenance.inboxRequest, source); } diff --git a/packages/fedify/src/federation/portable-inbox.ts b/packages/fedify/src/federation/portable-inbox.ts index 41688acda..3613564f5 100644 --- a/packages/fedify/src/federation/portable-inbox.ts +++ b/packages/fedify/src/federation/portable-inbox.ts @@ -1,3 +1,5 @@ +import { serializeProvenance } from "./outbound-provenance.ts"; +import type { OutboundProvenance } from "./outbound-report.ts"; import type { Actor, CryptographicKey } from "@fedify/vocab"; import { fromCompatibleEf61Id, @@ -229,6 +231,8 @@ export function coversDelivery(signature: RequestSignature): boolean { * Parameters for {@link forwardPortableInboxActivity}. */ export interface ForwardPortableInboxActivityParameters { + /** Explicit source carried by queued replication work, independent of observers. */ + readonly provenance?: OutboundProvenance; /** The portable inbox that received the activity. */ readonly recipient: PortableInboxRecipient; /** The received activity, exactly as it was received. */ @@ -714,6 +718,7 @@ async function enqueueForwarding( try { message = { type: "outbox", + ...serializeProvenance(parameters.provenance), id: crypto.randomUUID(), baseUrl, keys, diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index b4c23c105..2028c24e4 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -1,3 +1,4 @@ +import type { OutboundProvenance } from "./outbound-report.ts"; import type { SerializedInboxAuthentication } from "./inbox-authentication.ts"; import type { SerializedVerifiedActivityView } from "./inbox-view.ts"; @@ -32,6 +33,8 @@ export type Message = | TaskMessage; export interface FanoutMessage { + /** Historical source facts; absent on legacy/independent messages. */ + readonly provenance?: OutboundProvenance; readonly type: "fanout"; readonly id: ReturnType; readonly baseUrl: string; @@ -77,6 +80,8 @@ export interface FanoutMessage { } export interface OutboxMessage { + /** Historical source facts; absent on legacy/independent messages. */ + readonly provenance?: OutboundProvenance; readonly type: "outbox"; readonly id: ReturnType; /** Original delivery identity; absent on legacy/unobserved producers. */ @@ -148,6 +153,8 @@ export interface TaskMessage { } export interface InboxMessage { + /** Historical source facts; absent on legacy/independent messages. */ + readonly provenance?: OutboundProvenance; readonly type: "inbox"; readonly id: ReturnType; readonly baseUrl: string; From 7da49b126579c1f88aea05c0c2e588f12f968ec1 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 01:37:40 +0900 Subject: [PATCH 14/53] Add application dispatch correlation Accept bounded application dispatch IDs on send and forward options. Preserve exact values through observations and queue continuations without deduplicating sends or changing broadcast identities. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/context.ts | 9 + packages/fedify/src/federation/middleware.ts | 26 +- .../src/federation/outbound-dispatch.test.ts | 366 ++++++++++++++++++ .../src/federation/outbound-observation.ts | 71 +++- .../src/federation/outbound-provenance.ts | 34 +- .../src/federation/outbound-report.test.ts | 1 + .../fedify/src/federation/outbound-report.ts | 2 + 7 files changed, 484 insertions(+), 25 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-dispatch.test.ts diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index ed1ebd19f..50689d9f1 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -1867,6 +1867,15 @@ export type ParseUriResult = * Options for {@link Context.sendActivity} method. */ export interface SendActivityOptions { + /** + * Application-owned correlation for this invocation, never a deduplication + * key. Preserved exactly, including an empty string. At most 1,024 UTF-8 + * bytes; invalid types and oversized values are rejected before preparation. + * Omission means no application dispatch ID for this new invocation. + * @since 2.5.0 + */ + readonly dispatchId?: string; + /** * Whether to prefer the shared inbox for the recipients. */ diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 02752ab99..b710dd8fd 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3250,6 +3250,7 @@ export class FederationImpl orderingKey, context: ctx, observation, + provenance = getOutboundProvenance(ctx), } = options; observation?.setPhase("validation"); if (activity.id == null) { @@ -3454,7 +3455,7 @@ export class FederationImpl : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; const message: OutboxMessage = { type: "outbox", - ...serializeProvenance(getOutboundProvenance(ctx)), + ...serializeProvenance(provenance), id: crypto.randomUUID(), baseUrl: ctx.origin, keys: keyJwkPairs, @@ -3556,6 +3557,7 @@ export class FederationImpl // Each HTTP ingress owns its current identity; async ancestry is not // evidence that this request came from another inbox request. inboxRequestId: undefined, + dispatchId: undefined, sourceInboxRequestId: undefined, sourceRequestId: undefined, }, async () => { @@ -5882,7 +5884,7 @@ export class ContextImpl implements Context { return observeOutboundBroadcast( this, this.federation.outboundObservers, - (observation) => { + (observation, provenance) => { const tracer = this.tracerProvider.getTracer( metadata.name, metadata.version, @@ -5919,6 +5921,7 @@ export class ContextImpl implements Context { options, span, observation, + provenance, ); } catch (e) { span.setStatus({ @@ -5932,6 +5935,8 @@ export class ContextImpl implements Context { }, ); }, + "send", + options.dispatchId, ); } @@ -5946,6 +5951,7 @@ export class ContextImpl implements Context { options: SendActivityOptionsForCollection, span: Span, observation?: OutboundBroadcastObservation, + provenance: OutboundProvenance = getOutboundProvenance(this), ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); observation?.setPhase("sender-resolution"); @@ -6038,6 +6044,7 @@ export class ContextImpl implements Context { const opts: SendActivityInternalOptions = { context: this, observation, + provenance, orderingKey: options.orderingKey, collectionSync, immediate: options.immediate, @@ -6165,7 +6172,7 @@ export class ContextImpl implements Context { propagation.inject(context.active(), carrier); const message: FanoutMessage = { type: "fanout", - ...serializeProvenance(getOutboundProvenance(this)), + ...serializeProvenance(provenance), id: crypto.randomUUID(), baseUrl: this.origin, keys: keyJwkPairs, @@ -7011,7 +7018,7 @@ function forwardActivity( return observeOutboundBroadcast( ctx, ctx.federation.outboundObservers, - (observation) => { + (observation, provenance) => { const tracer = ctx.tracerProvider.getTracer( metadata.name, metadata.version, @@ -7038,6 +7045,7 @@ function forwardActivity( recipients, options, observation, + provenance, ); } catch (e) { span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); @@ -7049,6 +7057,7 @@ function forwardActivity( ); }, "forward", + options?.dispatchId, ); } @@ -7063,6 +7072,7 @@ async function forwardActivityInternal( recipients: Recipient | Recipient[] | "followers", options?: ForwardActivityOptions, observation?: OutboundBroadcastObservation, + provenance: OutboundProvenance = getOutboundProvenance(ctx), ): Promise { const logger = getLogger(["fedify", "federation", loggerCategory]); observation?.setPhase("sender-resolution"); @@ -7245,7 +7255,7 @@ async function forwardActivityInternal( const { portableInbox, gatewayInboxes } = inboxes[inbox]; const message: OutboxMessage = { type: "outbox", - ...serializeProvenance(getOutboundProvenance(ctx)), + ...serializeProvenance(provenance), id: crypto.randomUUID(), baseUrl: ctx.origin, keys: keyJwkPairs, @@ -7463,7 +7473,7 @@ export class OutboxContextImpl extends ContextImpl return observeOutboundBroadcast( this, this.federation.outboundObservers, - (observation) => { + (observation, provenance) => { const tracer = this.tracerProvider.getTracer( metadata.name, metadata.version, @@ -7500,6 +7510,7 @@ export class OutboxContextImpl extends ContextImpl options, span, observation, + provenance, ); if (delivered) this.#deliveryState.delivered = true; } catch (e) { @@ -7514,6 +7525,8 @@ export class OutboxContextImpl extends ContextImpl }, ); }, + "send", + options.dispatchId, ); } @@ -7570,6 +7583,7 @@ export class OutboxContextImpl extends ContextImpl } interface SendActivityInternalOptions { + readonly provenance?: OutboundProvenance; readonly observation?: OutboundBroadcastObservation; readonly immediate?: boolean; readonly collectionSync?: string; diff --git a/packages/fedify/src/federation/outbound-dispatch.test.ts b/packages/fedify/src/federation/outbound-dispatch.test.ts new file mode 100644 index 000000000..cb5db15ae --- /dev/null +++ b/packages/fedify/src/federation/outbound-dispatch.test.ts @@ -0,0 +1,366 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Person } from "@fedify/vocab"; +import { + configure, + getLogger, + type LogRecord, + reset, + withContext, +} from "@logtape/logtape"; +import fetchMock from "fetch-mock"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; +import type { SendActivityOptions } from "./context.ts"; +import type { Federation } from "./federation.ts"; +import { MemoryKvStore } from "./kv.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; +import type { MessageQueue } from "./mq.ts"; +import type { + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; +import type { Message } from "./queue.ts"; +import type { SenderKeyPair } from "./send.ts"; + +const activity = new Create({ + id: new URL("https://example.com/dispatch"), + actor: new URL("https://example.com/person2"), +}); +const target = new Person({ + id: new URL("https://remote.example/bob"), + inbox: new URL("https://remote.example/inbox"), +}); +const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; +const loaders = { + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, +}; +const modes = [ + "send", + "outbox-send", + "inbox-forward", + "outbox-forward", +] as const; +type Mode = typeof modes[number]; +function entry( + federation: Federation, + mode: Mode, + identity: SenderKeyPair | { identifier: string } = sender, +) { + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const outbox = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const inbox = base.toInboxContext( + null, + {}, + activity.id!.href, + Create.typeId.href, + ); + if (mode === "send") { + return { + context: base, + invoke: (options: SendActivityOptions) => + base.sendActivity( + identity, + target, + activity, + Object.assign(options, { fanout: "force" as const }), + ), + }; + } + if (mode === "outbox-send") { + return { + context: outbox, + invoke: (options: SendActivityOptions) => + outbox.sendActivity( + identity, + target, + activity, + Object.assign(options, { fanout: "skip" as const }), + ), + }; + } + const context = mode === "inbox-forward" ? inbox : outbox; + return { + context, + invoke: (options: SendActivityOptions) => + context.forwardActivity( + identity, + target, + Object.assign(options, { skipIfUnsigned: false }), + ), + }; +} + +test("dispatch ID validation precedes send and forward side effects", async (t) => { + for ( + const { mode, observed } of [ + { mode: "send" as const, observed: false }, + ...modes.map((mode) => ({ mode, observed: true })), + ] + ) { + await t.step( + `${mode}: ${observed ? "observed" : "unobserved"}`, + async () => { + let sideEffects = 0; + const queue: MessageQueue = { + enqueue() { + sideEffects++; + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { outbox: queue, fanout: queue }, + }); + federation.setActorDispatcher("/users/{identifier}", () => { + sideEffects++; + return new Person({}); + }).setKeyPairsDispatcher(() => { + sideEffects++; + return []; + }); + const { context, invoke } = entry(federation, mode, { + identifier: "alice", + }); + const provider = context.tracerProvider; + Object.defineProperty(context, "tracerProvider", { + get() { + sideEffects++; + return provider; + }, + }); + const events: OutboundBroadcastEvent[] = []; + let deliveries = 0; + if (observed) { + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + onDelivery() { + deliveries++; + }, + }); + } + for (const value of [null, "x".repeat(1025), "😀".repeat(257)]) { + const start = events.length; + let original: unknown; + await rejects( + invoke({ dispatchId: value as string }), + (error: unknown) => { + original = error; + ok( + value === null + ? error instanceof TypeError + : error instanceof RangeError, + ); + ok((error as Error).message.includes("dispatchId")); + return true; + }, + ); + strictEqual(sideEffects, 0); + strictEqual(deliveries, 0); + if (observed) { + deepStrictEqual(events.slice(start).map((event) => event.stage), [ + "started", + "preparation-finished", + ]); + const finished = events.at(-1)!; + ok( + finished.stage === "preparation-finished" && + finished.result.type === "failed", + ); + strictEqual(finished.result.failure.phase, "validation"); + strictEqual(finished.result.failure.error, original); + strictEqual(finished.provenance.dispatchId, undefined); + } + } + }, + ); + } +}); + +test("dispatch IDs correlate send and forward entries without deduplication or context leakage", async (t) => { + for (const mode of modes) { + await t.step(mode, async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { fanout: queue, outbox: queue }, + }); + const { context, invoke } = entry(federation, mode); + const broadcasts: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const contexts: unknown[] = []; + let current: { dispatchId?: string } = {}; + federation.setOutboundObservers({ + onBroadcast(ctx, event) { + broadcasts.push(event); + contexts.push(ctx); + if (event.stage === "started") { + current.dispatchId = "observer-mutated"; + } + }, + onDelivery(ctx, event) { + deliveries.push(event); + contexts.push(ctx); + }, + }); + const dispatchId = ' app\\dispatch"\n한글 '; + for (const value of [dispatchId, dispatchId, undefined]) { + current = { dispatchId: value }; + await invoke(current); + } + strictEqual(messages.length, 3); + const starts = broadcasts.filter((event) => event.stage === "started"); + strictEqual(starts.length, 3); + strictEqual(new Set(starts.map((event) => event.broadcastId)).size, 3); + for (const ctx of contexts) strictEqual(ctx, context); + for (let index = 0; index < 3; index++) { + const message = messages[index]; + ok(message.type === "outbox" || message.type === "fanout"); + const expected = index < 2 ? dispatchId : undefined; + strictEqual(message.provenance?.dispatchId, expected); + strictEqual(starts[index].provenance.dispatchId, expected); + deepStrictEqual( + JSON.parse(JSON.stringify(message.provenance ?? {})), + message.provenance ?? {}, + ); + const delivery = deliveries[index * 2]; + strictEqual(delivery.provenance.dispatchId, expected); + } + if (mode === "send") { + const fanout = messages[0]; + ok(fanout.type === "fanout"); + await federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(fanout)), + ); + await federation.processQueuedTask(undefined, structuredClone(fanout)); + for (const child of messages.slice(3)) { + ok(child.type === "outbox"); + strictEqual(child.provenance?.dispatchId, dispatchId); + deepStrictEqual( + child.outbound, + fanout.inboxes[target.inboxId!.href].outbound, + ); + } + strictEqual(messages.length, 5); + strictEqual(broadcasts.length, 6); + } + }); + } +}); + +test("dispatch ID byte limits and queue validation preserve exact values", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const producer = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { outbox: queue }, + }); + const { invoke } = entry(producer, "outbox-send"); + const valid = ["", "x".repeat(1024), "😀".repeat(256)]; + for (const dispatchId of valid) await invoke({ dispatchId }); + strictEqual(messages.length, 3); + const records: LogRecord[] = []; + await reset(); + await configure({ + sinks: { buffer: (record) => records.push(record) }, + loggers: [{ + category: ["test", "dispatch"], + sinks: ["buffer"], + lowestLevel: "info", + }, { category: ["logtape", "meta"], sinks: [] }], + contextLocalStorage: new AsyncLocalStorage(), + }); + const consumer = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { outbox: queue }, + }); + const descendants: OutboundBroadcastEvent[] = []; + consumer.setOutboundObservers({ + onBroadcast(_ctx, event) { + descendants.push(event); + }, + }); + consumer.setOutboxPermanentFailureHandler(async (ctx) => { + getLogger(["test", "dispatch"]).info("original dispatch"); + // A new public invocation needs its own application-owned correlation. + await ctx.sendActivity(sender, [], activity); + }); + fetchMock.mockGlobal().post(target.inboxId!.href, 410); + try { + const queued = messages[0]; + ok(queued.type === "outbox"); + for (const dispatchId of [null, "x".repeat(1025), "😀".repeat(257)]) { + await rejects( + async () => + await consumer.processQueuedTask(undefined, { + ...queued, + provenance: { dispatchId: dispatchId as string }, + }), + dispatchId === null ? TypeError : RangeError, + ); + strictEqual(fetchMock.callHistory.calls().length, 0); + } + for (let index = 0; index < messages.length; index++) { + const message = messages[index]; + ok(message.type === "outbox"); + strictEqual(message.provenance?.dispatchId, valid[index]); + await withContext( + { requestId: "current-request", dispatchId: "ambient-dispatch" }, + () => + consumer.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ), + ); + } + const logs = records.filter((record) => + record.rawMessage === "original dispatch" + ); + deepStrictEqual(logs.map((record) => record.properties.dispatchId), valid); + for (const record of logs) { + strictEqual(record.properties.requestId, "current-request"); + } + strictEqual(descendants.length, 6); + for (const event of descendants) { + strictEqual(event.provenance.dispatchId, undefined); + } + } finally { + fetchMock.hardReset(); + await reset(); + } +}); diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 1331c93c3..026b827dc 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,10 +1,11 @@ import { getOutboundProvenance, + invocationProvenance, snapshotLocalContext, - withOutboundSource, + sourceLogContext, } from "./outbound-provenance.ts"; import { encodeHex } from "byte-encodings/hex"; -import { getLogger } from "@logtape/logtape"; +import { getLogger, withContext } from "@logtape/logtape"; import type { Context } from "./context.ts"; import type { FanoutMessage, SerializedOutboundDelivery } from "./queue.ts"; import type { @@ -12,6 +13,7 @@ import type { OutboundDeliveryEvent, OutboundObservers, OutboundPreparationResult, + OutboundProvenance, } from "./outbound-report.ts"; type WithoutSource = T extends unknown ? Omit @@ -85,7 +87,10 @@ async function legacyFanoutIdentity( }; } -async function notifyObserver( +async function notifyObserver< + TContextData, + TEvent extends { readonly provenance: OutboundProvenance }, +>( context: Context, callback: | ((context: Context, event: TEvent) => void | Promise) @@ -95,7 +100,10 @@ async function notifyObserver( ): Promise { if (callback == null) return; try { - await withOutboundSource(context, () => callback(context, event)); + await withContext( + sourceLogContext(event.provenance), + () => callback(context, event), + ); } catch (error) { try { getLogger(["fedify", "federation", "outbox"]).error( @@ -119,6 +127,7 @@ class OutboundDeliveryObservation { readonly observer: OutboundObservers["onDelivery"], identity: SerializedOutboundDelivery, destination: PlannedDestination, + readonly provenance: OutboundProvenance, ) { this.#reference = snapshotDeliveryIdentity(identity); this.#plan = Object.freeze({ @@ -138,7 +147,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ - provenance: getOutboundProvenance(this.context), + provenance: this.provenance, local: snapshotLocalContext(this.context), ...this.#reference, ...this.#plan, @@ -157,7 +166,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ - provenance: getOutboundProvenance(this.context), + provenance: this.provenance, local: snapshotLocalContext(this.context), ...this.#reference, stage: "handoff" as const, @@ -174,7 +183,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ - provenance: getOutboundProvenance(this.context), + provenance: this.provenance, local: snapshotLocalContext(this.context), ...this.#reference, stage: "preparation-failed" as const, @@ -208,6 +217,7 @@ export class OutboundBroadcastObservation { readonly context: Context, observers: OutboundObservers, readonly operation: OutboundBroadcastEvent["operation"], + readonly provenance: OutboundProvenance = getOutboundProvenance(context), broadcast: SerializedOutboundDelivery["broadcast"] = { type: "known", id: crypto.randomUUID(), @@ -234,6 +244,7 @@ export class OutboundBroadcastObservation { context, { onDelivery: observers.onDelivery }, "send", + getOutboundProvenance(context), { type: "unknown" }, ); observation.#finished = true; @@ -270,6 +281,7 @@ export class OutboundBroadcastObservation { : { portableInbox: destination.portableInbox }), }, destination, + this.provenance, ); } @@ -374,7 +386,8 @@ export class OutboundBroadcastObservation { async run( send: ( - observation?: OutboundBroadcastObservation, + observation: OutboundBroadcastObservation | undefined, + provenance: OutboundProvenance, ) => Promise, ): Promise { if (this.#broadcast.type === "known") { @@ -385,7 +398,7 @@ export class OutboundBroadcastObservation { })); } try { - return await send(this); + return await send(this, this.provenance); } catch (error) { await this.failed(error); throw error; @@ -427,7 +440,7 @@ export class OutboundBroadcastObservation { this.#onBroadcast, Object.freeze({ ...event, - provenance: getOutboundProvenance(this.context), + provenance: this.provenance, local: snapshotLocalContext(this.context), }), "broadcast", @@ -440,16 +453,38 @@ export function observeOutboundBroadcast( context: Context, observers: OutboundObservers, send: ( - observation?: OutboundBroadcastObservation, + observation: OutboundBroadcastObservation | undefined, + provenance: OutboundProvenance, ) => Promise, operation: OutboundBroadcastEvent["operation"] = "send", + dispatchId?: string, ): Promise { - if (observers.onBroadcast == null && observers.onDelivery == null) { - return withOutboundSource(context, () => send()); + let provenance: OutboundProvenance; + let invalid: { error: unknown } | undefined; + try { + provenance = invocationProvenance(context, dispatchId); + } catch (error) { + provenance = invocationProvenance(context, undefined); + invalid = { error }; } - return withOutboundSource( - context, - () => - new OutboundBroadcastObservation(context, observers, operation).run(send), - ); + const execute = async ( + observation?: OutboundBroadcastObservation, + ) => { + if (invalid != null) { + observation?.setPhase("validation"); + throw invalid.error; + } + return await send(observation, provenance); + }; + return withContext(sourceLogContext(provenance), () => { + if (observers.onBroadcast == null && observers.onDelivery == null) { + return execute(); + } + return new OutboundBroadcastObservation( + context, + observers, + operation, + provenance, + ).run(execute); + }); } diff --git a/packages/fedify/src/federation/outbound-provenance.ts b/packages/fedify/src/federation/outbound-provenance.ts index 8d2160305..6c1354ac7 100644 --- a/packages/fedify/src/federation/outbound-provenance.ts +++ b/packages/fedify/src/federation/outbound-provenance.ts @@ -54,6 +54,34 @@ export function withOutboundSource(context: object, operation: () => T): T { return withContext(sourceLogContext(source), operation); } +/** The same exact-string budget applies to public input and queued metadata. */ +export function validateDispatchId( + value: unknown, +): asserts value is string | undefined { + if (value === undefined) return; + if (typeof value !== "string") { + throw new TypeError("dispatchId must be a string."); + } + if ( + value.length > 1024 || new TextEncoder().encode(value).byteLength > 1024 + ) { + throw new RangeError("dispatchId must not exceed 1,024 UTF-8 bytes."); + } +} + +/** A new public invocation keeps its HTTP source, with its own dispatch ID. */ +export function invocationProvenance( + context: object, + dispatchId: unknown, +): OutboundProvenance { + validateDispatchId(dispatchId); + const { dispatchId: _previous, ...source } = getOutboundProvenance(context); + return Object.freeze({ + ...source, + ...(dispatchId === undefined ? {} : { dispatchId }), + }); +} + /** Restore only explicit source facts; never infer them from queue/trace IDs. */ export function restoreProvenance(value: unknown): OutboundProvenance { if (value === undefined) return emptyProvenance; @@ -61,6 +89,7 @@ export function restoreProvenance(value: unknown): OutboundProvenance { v != null && typeof v === "object" && !Array.isArray(v); const invalid = () => new TypeError("Invalid provenance in queue metadata."); if (!record(value)) throw invalid(); + validateDispatchId(value.dispatchId); if ( value.originatingRequestId !== undefined && typeof value.originatingRequestId !== "string" @@ -91,6 +120,7 @@ export function restoreProvenance(value: unknown): OutboundProvenance { }); } return Object.freeze({ + ...(value.dispatchId === undefined ? {} : { dispatchId: value.dispatchId }), ...(inboxRequest == null ? {} : { inboxRequest }), ...(value.originatingRequestId === undefined ? {} @@ -103,7 +133,8 @@ export function serializeProvenance( value: OutboundProvenance | undefined, ): { readonly provenance?: OutboundProvenance } { const provenance = restoreProvenance(value); - return provenance.inboxRequest == null && + return provenance.dispatchId === undefined && + provenance.inboxRequest == null && provenance.originatingRequestId == null ? {} : { provenance }; @@ -111,6 +142,7 @@ export function serializeProvenance( export function sourceLogContext(source: OutboundProvenance) { return { + dispatchId: source.dispatchId, sourceInboxRequestId: source.inboxRequest?.inboxRequestId, sourceRequestId: source.inboxRequest?.requestId ?? source.originatingRequestId, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index f1c465ea6..325057b2a 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -1892,6 +1892,7 @@ function continuationMessage( ...(observed ? { provenance: { + dispatchId: "application-dispatch", inboxRequest: { inboxRequestId: crypto.randomUUID(), requestId: "original-request-label", diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index c1d142391..e905e6c4b 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -203,6 +203,8 @@ export interface OutboundInboxRequest { /** Explicit historical cause of outgoing work. @since 2.5.0 */ export interface OutboundProvenance { + /** Application-owned correlation; never a Fedify deduplication key. */ + readonly dispatchId?: string; /** Absent for independent contexts, programmatic routing, and unknown work. */ readonly inboxRequest?: OutboundInboxRequest; /** Original logging label for request-derived work without an inbox cause. */ From d97c7c547f2272f6cd5dc89a1e1a1cdc04b64d14 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 02:09:07 +0900 Subject: [PATCH 15/53] Document outbound identity and correlation Explain queue identity, inbox source, and application dispatch correlation across outgoing work. Keep internal symbol metadata compatible with package declaration generation while preserving context spreads. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 15 +-- changes.d/fedify/outbound-observers.md | 8 +- claude-plugin/skills/fedify/SKILL.md | 29 ++++- docs/manual/context.md | 5 + docs/manual/inbox.md | 10 ++ docs/manual/send.md | 119 +++++++++++++++--- packages/fedify/src/federation/federation.ts | 8 +- .../src/federation/inbox-observation.ts | 2 +- .../fedify/src/federation/inbox-report.ts | 13 +- packages/fedify/src/federation/middleware.ts | 29 ++--- .../src/federation/outbound-provenance.ts | 8 +- .../fedify/src/federation/outbound-report.ts | 11 +- 12 files changed, 199 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f9b92f911..b05d6ca77 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,13 +48,14 @@ To be released. `idempotency` strategy, as direct processing already did, instead of always deduplicating them per origin. Activities recorded by workers of older versions are still recognized. [[#1290], [#1299]] - - Added `setOutboundObservers()` on federation builders and instances to - observe ordinary `Context.sendActivity()` starts and preparation results - with application context and independent broadcast IDs, without requiring - OpenTelemetry. Register `onBroadcast` to record preparation separately from - queue acceptance and delivery success. Callbacks are awaited and their - errors do not change the send's result. Outbox-context overrides, - forwarding, replication, workers, and HTTP exchanges are not yet observed. + - Added `setOutboundObservers()` on federation builders and instances + for sends, original-payload forwarding, and automatic portable replication. + Register `onBroadcast` to record producer starts and preparation results, + and `onDelivery` for destination planning, direct handoffs, queue acceptance + or uncertainty, and local preparation failures. Callbacks receive + application context and independent grouping IDs without requiring + OpenTelemetry; their errors do not replace the original result. Readiness + and handoff observations do not establish remote delivery success. [[#1316] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 1dae1dc0b..041e258f4 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -8,6 +8,10 @@ links: and `onDelivery` for destination planning, direct handoffs, queue acceptance or uncertainty, and local preparation failures. Callbacks receive application context and independent grouping IDs without requiring - OpenTelemetry; their errors do not replace the original result. Readiness - and handoff observations do not establish remote delivery success. + OpenTelemetry; their errors do not replace the original result. Known IDs + and original inbox source survive queue continuations. Send/forward options + accept `dispatchId` for application correlation without deduplication, + limited to 1,024 UTF-8 bytes. Routed inbox reports include `inboxRequestId` + and informational `requestId`. Missing legacy relationships remain unknown. + Readiness and handoff observations do not establish remote delivery success. [[#1316] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index d0f63afec..e973c715b 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -258,7 +258,8 @@ without extra loading and work without OpenTelemetry. `portable-replication`. Narrow `event.stage` before reading `event.result`, then narrow its `type` (`prepared`, `skipped`, or `failed`). `onDelivery` reports `planned`, `handoff`, or destination-local `preparation-failed` events with a -`deliveryId` and known `broadcast` reference. Handoffs distinguish `direct`, +`deliveryId` and a `broadcast` reference (`known` with an `id`, or `unknown` +for legacy fanout work). Handoffs distinguish `direct`, `enqueued` (with actual queue/task ID), and `enqueue-failed` (acceptance unknown). Readiness, queue acceptance, and remote effects are distinct. Portable fallback keeps one delivery across alternate gateways; replication @@ -268,11 +269,27 @@ Callbacks are awaited without a default timeout; observer/logging errors are isolated. Nonsettling callbacks can delay execution indefinitely. Portable forwarding can return at its existing deadline while background work and observations continue. Detached recording must handle its own rejection, -shutdown, and persistence. Delivery IDs currently group producer-local events; -queue identity propagation/replay, worker execution, attempts, HTTP exchanges, -response capture, cancellation observations, provenance, and trace metadata are -not implemented yet. Mock federations accept registration without executing -observers. Incoming C2S listeners use the separate `setOutboxListeners()` API. +shutdown, and persistence. Known IDs survive fanout expansion, retries, holds, +and gateway changes. Fanout expansion continues existing delivery handoffs +without starting another broadcast. Legacy fanout can have an unknown broadcast; +missing source facts remain unavailable, and older consumers can discard new +metadata. + +Events expose `provenance` for preserved source/app correlation and `local` for +current URL facts. A known inbox source includes server-generated +`inboxRequestId`, informational `requestId`, and the original URL snapshot. +It survives context clones/conversions and queues; independent contexts do not +infer it from ambient execution. + +Send/forward options accept `dispatchId`, preserved exactly in event provenance +and queues even without producer observers. Empty strings are valid; the limit +is 1,024 UTF-8 bytes. Invalid types raise `TypeError` and oversized values +`RangeError` before preparation. Reusing an ID sends again with a fresh +broadcast ID. New public calls supply their own app ID; queued continuations +retain it. + +Mock federations accept registration without executing observers. Incoming C2S +listeners use the separate `setOutboxListeners()` API. See . diff --git a/docs/manual/context.md b/docs/manual/context.md index 3f6561d1c..5d4bd8727 100644 --- a/docs/manual/context.md +++ b/docs/manual/context.md @@ -84,6 +84,11 @@ The `Context` object has properties to get the base URL of the current request: For `RequestContext`, there is an additional property named `~RequestContext.url` that contains the full URL of the current request. +Outgoing observations snapshot these values under `event.local`. For work +caused by an inbox request, `event.provenance.inboxRequest.context` retains the +original URL facts through derived contexts and queues. See +[outgoing source and correlation](./send.md#original-source-and-current-context). + [^1]: If no canonical origin is explicitly configured, it is the same as the `Context.origin`. See also the [*Explicitly setting the canonical origin* section](./federation.md#explicitly-setting-the-canonical-origin) in the diff --git a/docs/manual/inbox.md b/docs/manual/inbox.md index 962323d92..81e9aab53 100644 --- a/docs/manual/inbox.md +++ b/docs/manual/inbox.md @@ -215,6 +215,16 @@ covers failures while preparing document loaders or resolving a portable recipient. Inbox collection requests, unmatched routes, programmatic `routeActivity()` calls, and queue workers do not invoke it. +Since Fedify 2.5.0, routed HTTP inbox reports include `report.inboxRequestId` +and `report.requestId`, including preparation failures. The former is a fresh +server-generated ID for each HTTP delivery, even for the same Activity. The +latter is an informational, externally supplied or generated logging label; +it is not unique. These fields are absent when no routed source is known. + +Outgoing work using the handler's context retains this source through derived +contexts and queues. See +[outgoing source and correlation](./send.md#original-source-and-current-context). + `report.payload` is either `unavailable` or `parsed`, whose `value` contains the original JSON. A parsed JSON `null` is distinct from an unavailable body. `report.activity` contains the `Activity` obtained by the existing processing diff --git a/docs/manual/send.md b/docs/manual/send.md index a81545056..2d909954a 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -103,8 +103,9 @@ federation.setOutboundObservers({ Here, `recordBroadcast()` and `recordDelivery()` are your application's recording functions. Callbacks receive the existing context and application data without loading additional data for observation. The report fields contain -immutable snapshots of known routing facts; original thrown values remain -available as `unknown` values. +immutable snapshots of known routing and source facts; original thrown values +remain available as `unknown` values. The context, application data, Activity +objects, and exceptions are not deep-copied by this API. ### Broadcast lifecycle @@ -140,7 +141,8 @@ nonsettling operation can leave a start without a later result. ### Destination observations `onDelivery` receives an `OutboundDeliveryEvent` for each established delivery -obligation. Its `broadcast` reference identifies the invocation, and its +obligation. Its `broadcast` reference is `{ type: "known", id }` when the +invocation is known, or `{ type: "unknown" }` for legacy fanout work. Its `deliveryId` groups events for one deduplicated destination or confirmed replication target. Different inboxes on the same host remain distinct. @@ -169,6 +171,103 @@ for that destination. Earlier handoffs remain valid, and later unvisited destinations can remain planned without a handoff. Parallel work can interleave callbacks, so IDs provide grouping rather than a total event order. +### Identities across queues + +Observed broadcasts and deliveries retain their IDs through fanout expansion, +retries, holds, and gateway changes. Fanout expansion can create new outbox task +IDs without starting another broadcast. Source metadata and `dispatchId` are +serialized even when producer observers are disabled; broadcast/delivery +metadata is allocated when producer observation is enabled. + +| Boundary | Identity behavior | +| ----------------------------------- | --------------------------------------------------------------------------------------------------- | +| Observed send or forward invocation | Fresh `broadcastId`; one `deliveryId` per deduplicated destination or confirmed replication target. | +| Fanout expansion | Existing broadcast/delivery IDs, with new child outbox task IDs. | +| Retry, hold, or deferral | Same delivery identity and provenance. | +| Portable fallback | One delivery across alternate gateways; the planned destination remains unchanged. | +| Portable replication | A separate delivery for each confirmed target claim. | + +~~~~ text +inbox request -> handler -> send/forward -> planned delivery + +-> direct handoff + +-> outbox task -> retry/hold + +-> fanout task -> outbox tasks + +Queue continuations preserve known IDs and the original source. +~~~~ + +IDs provide grouping, not ordering or deduplication. Replayed work can produce +repeated observations. Treat IDs as opaque values. + +Older messages remain processable. Fanout tasks without identity metadata get +stable delivery IDs with `broadcast: { type: "unknown" }`. Missing source and +application correlation remain unavailable. Older consumers can discard these +fields, so mixed-version processing can lose correlation. + +### Original source and current context + +Outgoing events distinguish preserved source facts from current URL facts: + + - `provenance.inboxRequest`, when known, contains the original + `inboxRequestId`, informational `requestId`, and a `context` snapshot. + - `local` describes the context invoking this callback now. + - `provenance.originatingRequestId` retains the logging label for + HTTP-derived work without an inbox cause. + +Both URL snapshots contain `origin`, `host`, `hostname`, and `canonicalOrigin`. +`host` includes an optional port; `hostname` does not. `canonicalOrigin` uses +the configured web origin or falls back to `origin`. + +The original source survives derived sends, forwarding, context clones and +conversions, and queues. Its snapshot stays unchanged if a derived context's +URL changes. Independent contexts do not acquire an inbox cause from ambient +execution. See [observing inbox requests](./inbox.md#observing-inbox-requests) +for the difference between the server-generated `inboxRequestId` and the +informational logging `requestId`. + +Worker logs use `sourceInboxRequestId` and `sourceRequestId` for historical +correlation, preserving an unrelated current `requestId`. + +### Application dispatch correlation + +Send and forward options accept an application-owned `dispatchId`. Fedify +preserves it exactly in `event.provenance.dispatchId` and outgoing queue +continuations. Empty strings are valid; the limit is 1,024 UTF-8 bytes. A +non-string value raises `TypeError`, and an oversized string raises `RangeError` +before preparation. Queue consumers enforce the same limit. + +This example adds application correlation to an inbox-triggered acceptance: + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +declare function saveAcceptance(follow: import("@fedify/vocab").Follow): Promise; +// ---cut-before--- +import { Accept, Follow } from "@fedify/vocab"; + +federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .on(Follow, async (ctx, follow) => { + if (follow.id == null || follow.objectId == null) return; + const actor = ctx.parseUri(follow.objectId); + if (actor?.type !== "actor") return; + const follower = await follow.getActor(ctx); + if (follower == null) return; + const dispatchId = await saveAcceptance(follow); + await ctx.sendActivity( + { identifier: actor.identifier }, + follower, + new Accept({ actor: follow.objectId, object: follow.id }), + { dispatchId }, + ); + }); +~~~~ + +`saveAcceptance()` represents your application's record creation; its ID must +satisfy the byte limit. The new Activity retains the incoming request's source. +Reusing a `dispatchId` sends again with a fresh broadcast ID. A new public call +that omits it does not inherit a previous app ID; queued continuations of the +existing call retain it. + ### Registration and callback behavior Each registration call replaces all previous outbound observers; pass `{}` to @@ -189,20 +288,6 @@ Fedify does not guarantee its persistence. Incoming C2S `setOutboxListeners()` registration is separate. Mock federations accept outbound observer registration but do not execute these observers. -### Current producer scope - -Both callbacks cover ordinary `Context.sendActivity()`, the `OutboxContext` -override, inbox/outbox `forwardActivity()`, and automatic portable replication -at their producer boundaries. Original-payload forwarding and replication keep -the received payload unchanged. Delivery IDs currently group producer-local -events; they are not yet carried through queue continuations or worker replay. - -Worker execution, concrete delivery attempts, HTTP exchanges, response bodies, -explicit cancellation observations, historical provenance, and optional trace -metadata are not reported yet. A direct handoff, accepted task, or prepared -producer must not be treated as proof that the remote application applied an -activity. - Specifying a sender ------------------- diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 32b6064c1..d263482ca 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -91,10 +91,10 @@ export interface Federatable extends TaskRegistry { * later changes to the supplied object do not affect registration. * Available on builders and directly created federations. Building copies * the callbacks, and an active send keeps the callback selected at entry. - * Currently send, original-payload forwarding, and automatic portable - * replication entry and preparation results are observed, including - * outbox-context operations and producer-side destination planning and - * handoffs. Workers are not observed yet. + * Observes send, original-payload forwarding, and automatic portable + * replication preparation, destination planning, and handoffs, including + * outbox-context operations and fanout continuation handoffs. Known IDs, + * original inbox source, and application correlation survive queue work. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/inbox-observation.ts b/packages/fedify/src/federation/inbox-observation.ts index c16888dde..8bd5ae243 100644 --- a/packages/fedify/src/federation/inbox-observation.ts +++ b/packages/fedify/src/federation/inbox-observation.ts @@ -51,7 +51,7 @@ export class InboxObservation { }; constructor( readonly inbox: InboxRequestReport["inbox"], - readonly source?: OutboundInboxRequest, + readonly source: OutboundInboxRequest | undefined = undefined, ) {} /** Run an ingress operation and project telemetry; fetch owns completion. */ diff --git a/packages/fedify/src/federation/inbox-report.ts b/packages/fedify/src/federation/inbox-report.ts index 4637e683a..9fb9b31ea 100644 --- a/packages/fedify/src/federation/inbox-report.ts +++ b/packages/fedify/src/federation/inbox-report.ts @@ -132,9 +132,18 @@ export type InboxRequestOutcome = * @since 2.4.0 */ export interface InboxRequestReport { - /** Server-generated identity; present for routed HTTP ingress. @since 2.5.0 */ + /** + * Fresh server-generated identity for one routed HTTP inbox request, + * including preparation failures and duplicate deliveries. Preserved on + * outgoing causal descendants; absent without a known routed source. + * @since 2.5.0 + */ readonly inboxRequestId?: string; - /** Informational logging label of that request, not a unique ID. @since 2.5.0 */ + /** + * Original informational logging label; externally controlled and nonunique, + * independent of inboxRequestId. Absent without a known routed source. + * @since 2.5.0 + */ readonly requestId?: string; /** The delivery route, including a portable gateway route. */ readonly inbox: { diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index b710dd8fd..336ab676b 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3898,7 +3898,7 @@ export class FederationImpl async () => { if (routeName === "inbox") { context = this.#createContext(request, contextData, { - [outboundProvenance]: context[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(context), documentLoader: await context.getDocumentLoader({ identifier: recipient!, }), @@ -3907,7 +3907,7 @@ export class FederationImpl const identity = await this.sharedInboxKeyDispatcher(context); if (identity != null) { context = this.#createContext(request, contextData, { - [outboundProvenance]: context[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(context), documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) @@ -3934,7 +3934,7 @@ export class FederationImpl ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier; context = this.#createContext(request, contextData, { - [outboundProvenance]: context[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(context), invokedFromActorDispatcher: { identifier }, }); return await handleActor(request, { @@ -3951,7 +3951,7 @@ export class FederationImpl const callbacks = this.objectCallbacks[typeId]; const cls = this.objectTypeIds[typeId]; context = this.#createContext(request, contextData, { - [outboundProvenance]: context[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(context), invokedFromObjectDispatcher: { cls, values: route.values }, }); return await handleObject(request, { @@ -4020,7 +4020,7 @@ export class FederationImpl const identity = await this.sharedInboxKeyDispatcher(context); if (identity != null) { context = this.#createContext(request, contextData, { - [outboundProvenance]: context[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(context), documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity), @@ -4688,7 +4688,7 @@ export class FederationImpl // fetch, so fall back to the default document loader without them: if (this.actorCallbacks?.keyPairsDispatcher != null) { context = this.#createContext(request, contextData, { - [outboundProvenance]: context[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(context), documentLoader: await context.getDocumentLoader({ identifier }), }); } @@ -5119,7 +5119,6 @@ function createPortableObjectVerifier( } export class ContextImpl implements Context { - readonly [outboundProvenance]: OutboundProvenance; readonly url: URL; readonly federation: FederationImpl; readonly data: TContextData; @@ -5143,7 +5142,9 @@ export class ContextImpl implements Context { }: ContextOptions, ) { this.url = url; - this[outboundProvenance] = provenance; + // Keep internal symbol metadata enumerable for context spreads, without + // exposing a computed class field in generated declarations. + globalThis.Object.assign(this, { [outboundProvenance]: provenance }); this.federation = federation; this.data = data; this.documentLoader = documentLoader; @@ -5173,7 +5174,7 @@ export class ContextImpl implements Context { clone(data: TContextData): Context { return new ContextImpl({ - [outboundProvenance]: this[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -5192,7 +5193,7 @@ export class ContextImpl implements Context { metadata?: InboxContextMetadata, ): InboxContextImpl { return new InboxContextImpl(recipient, activity, activityId, activityType, { - [outboundProvenance]: this[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data: this.data, @@ -5215,7 +5216,7 @@ export class ContextImpl implements Context { activityId, activityType, { - [outboundProvenance]: this[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data: this.data, @@ -6649,7 +6650,7 @@ class RequestContextImpl extends ContextImpl override clone(data: TContextData): RequestContext { return new RequestContextImpl({ - [outboundProvenance]: this[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -7389,7 +7390,7 @@ export class InboxContextImpl extends ContextImpl this.activityId, this.activityType, { - [outboundProvenance]: this[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -7568,7 +7569,7 @@ export class OutboxContextImpl extends ContextImpl this.activityId, this.activityType, { - [outboundProvenance]: this[outboundProvenance], + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, diff --git a/packages/fedify/src/federation/outbound-provenance.ts b/packages/fedify/src/federation/outbound-provenance.ts index 6c1354ac7..0007e0f65 100644 --- a/packages/fedify/src/federation/outbound-provenance.ts +++ b/packages/fedify/src/federation/outbound-provenance.ts @@ -7,7 +7,7 @@ import type { } from "./outbound-report.ts"; // An own symbol property survives context spreads without adding public API. -export const outboundProvenance = Symbol("outboundProvenance"); +export const outboundProvenance: unique symbol = Symbol("outboundProvenance"); export const emptyProvenance: OutboundProvenance = Object.freeze({}); export function getOutboundProvenance(context: object): OutboundProvenance { @@ -140,7 +140,11 @@ export function serializeProvenance( : { provenance }; } -export function sourceLogContext(source: OutboundProvenance) { +export function sourceLogContext(source: OutboundProvenance): { + dispatchId: string | undefined; + sourceInboxRequestId: string | undefined; + sourceRequestId: string | undefined; +} { return { dispatchId: source.dispatchId, sourceInboxRequestId: source.inboxRequest?.inboxRequestId, diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index e905e6c4b..55ecb5daa 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -146,8 +146,9 @@ export type OutboundDeliveryEvent = * Optional observers of outgoing activity processing. * * Send, original-payload forwarding, and automatic portable replication producer - * events are observed, including outbox-context operations. Callbacks are - * awaited without a default timeout. + * events are observed, including outbox-context operations and fanout + * continuation handoffs. Known IDs and provenance survive queue continuations. + * Callbacks are awaited without a default timeout. * Their errors, including errors logging observer failures, do not replace the * send's result. * A callback that never settles can delay execution indefinitely. Detached @@ -203,7 +204,11 @@ export interface OutboundInboxRequest { /** Explicit historical cause of outgoing work. @since 2.5.0 */ export interface OutboundProvenance { - /** Application-owned correlation; never a Fedify deduplication key. */ + /** + * Exact application-owned correlation, up to 1,024 UTF-8 bytes; an empty + * string is valid. Never a Fedify deduplication key. Each new public call + * supplies its own value; queued continuations retain it. + */ readonly dispatchId?: string; /** Absent for independent contexts, programmatic routing, and unknown work. */ readonly inboxRequest?: OutboundInboxRequest; From 485ffe6edf0bc223f0805fac0da2af6be6b181b8 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 02:19:52 +0900 Subject: [PATCH 16/53] Timestamp outbound observations Record when each observation occurs so its time remains distinct from the queue identity shared across delivery attempts. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-observation.ts | 7 +- .../src/federation/outbound-report.test.ts | 109 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 2 + 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 026b827dc..01f7a9a80 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -16,7 +16,8 @@ import type { OutboundProvenance, } from "./outbound-report.ts"; -type WithoutSource = T extends unknown ? Omit +type WithoutSource = T extends unknown + ? Omit : never; type PreparationFailure = Extract< @@ -147,6 +148,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), ...this.#reference, @@ -166,6 +168,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), ...this.#reference, @@ -183,6 +186,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), ...this.#reference, @@ -440,6 +444,7 @@ export class OutboundBroadcastObservation { this.#onBroadcast, Object.freeze({ ...event, + observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), }), diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 325057b2a..e603c6c6c 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -29,7 +29,8 @@ import type { OutboundDeliveryEvent, OutboundObservers, } from "./mod.ts"; -import type { OutboxMessage } from "./queue.ts"; +import type { FanoutMessage, OutboxMessage } from "./queue.ts"; +import { OutboundBroadcastObservation } from "./outbound-observation.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; const activity = new Create({ @@ -43,6 +44,112 @@ const options = { contextLoaderFactory: () => mockDocumentLoader, }; +test("outbound observation timestamps snapshot each boundary and refresh on fanout replay", async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + const timestamps: number[] = []; + let now = 1_800_000_000_000; + const originalNow = Temporal.Now.instant; + const observe = async ( + _context: unknown, + event: OutboundBroadcastEvent | OutboundDeliveryEvent, + ) => { + const observedAt = now; + strictEqual(event.observedAt, observedAt); + ok(Object.isFrozen(event)); + ok(Object.isFrozen(event.provenance)); + ok(Object.isFrozen(event.local)); + events.push(event); + timestamps.push(observedAt); + // Time spent awaiting a recorder must not change this event's snapshot. + now += 1_000; + await Promise.resolve(); + strictEqual(event.observedAt, observedAt); + }; + const observers = { onBroadcast: observe, onDelivery: observe }; + const inbox = "https://receiver.example/inbox"; + const otherInbox = "https://receiver.example/other"; + try { + Temporal.Now.instant = () => Temporal.Instant.fromEpochMilliseconds(now); + const observation = new OutboundBroadcastObservation( + ctx, + observers, + "send", + ); + await observation.run(async () => { + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + [otherInbox]: { actorIds: [], sharedInbox: false }, + }); + await observation.delivery(inbox)!.handoff({ type: "direct" }); + await observation.delivery(otherInbox)!.failed(new Error("local setup")); + await observation.prepared("delivery"); + }); + deepStrictEqual(events.map((event) => event.stage), [ + "started", + "planned", + "planned", + "handoff", + "preparation-failed", + "preparation-finished", + ]); + deepStrictEqual(timestamps, [ + 1_800_000_000_000, + 1_800_000_001_000, + 1_800_000_002_000, + 1_800_000_003_000, + 1_800_000_004_000, + 1_800_000_005_000, + ]); + const identity = observation.continuation(inbox)!.outbound; + strictEqual("observedAt" in identity, false); + const message: FanoutMessage = JSON.parse(JSON.stringify({ + type: "fanout", + id: crypto.randomUUID(), + baseUrl: "https://example.com/", + keys: [], + inboxes: { + [inbox]: { actorIds: [], sharedInbox: false, outbound: identity }, + }, + activity: {}, + activityType: Create.typeId.href, + traceContext: {}, + })); + // Replaying the same task preserves grouping, but observes a new instant. + for (let replay = 0; replay < 2; replay++) { + now += 60_000; + const resumed = await OutboundBroadcastObservation.resumeFanout( + ctx, + observers, + message, + ); + ok(resumed); + await resumed.delivery(inbox)!.handoff({ + type: "enqueued", + queue: "outbox", + queueMessageId: crypto.randomUUID(), + }); + const event = events.at(-1)!; + ok("deliveryId" in event); + strictEqual(event.deliveryId, identity.deliveryId); + strictEqual(event.provenance, events[0].provenance); + deepStrictEqual(event.local, events[0].local); + } + deepStrictEqual(timestamps.slice(6), [ + 1_800_000_066_000, + 1_800_000_127_000, + ]); + deepStrictEqual(events.map((event) => event.observedAt), timestamps); + strictEqual("observedAt" in message.inboxes[inbox].outbound!, false); + } finally { + Temporal.Now.instant = originalNow; + } +}); + test("outbound observers report fresh immutable starts without tracing or extra loading", async (t) => { const { AlwaysOffSampler, BasicTracerProvider } = await import( "@opentelemetry/sdk-trace-base" diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 55ecb5daa..953316c3c 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -217,6 +217,8 @@ export interface OutboundProvenance { } interface OutboundSourceMetadata { + /** Unix milliseconds at observation; wall clocks do not ensure global ordering. */ + readonly observedAt: number; /** Immutable historical cause, distinct from this callback's context. */ readonly provenance: OutboundProvenance; /** URL facts of the current callback context. */ From 74a83c6905a6b555431cacdc1e957a09943ffd0e Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 02:23:52 +0900 Subject: [PATCH 17/53] Correlate outbound observations with spans Include an operation's own span IDs when available so observers can correlate events with traces. Omit inherited IDs from no-op spans. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 19 ++- .../src/federation/outbound-observation.ts | 32 +++- .../src/federation/outbound-report.test.ts | 155 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 5 + 4 files changed, 208 insertions(+), 3 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 336ab676b..7c9101b84 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -63,6 +63,7 @@ import { metrics, propagation, type Span, + type SpanContext, SpanKind, SpanStatusCode, trace, @@ -1258,7 +1259,13 @@ export class FederationImpl if (message.activityId != null) { span.setAttribute("activitypub.activity.id", message.activityId); } - await this.#listenFanoutMessage(contextData, message, provenance); + await this.#listenFanoutMessage( + contextData, + message, + provenance, + span, + trace.getSpanContext(extractedContext), + ); return { outcome: "completed" }; }, classifyAbortableError), ); @@ -1418,6 +1425,8 @@ export class FederationImpl data: TContextData, message: FanoutMessage, provenance: OutboundProvenance, + span: Span, + parentSpanContext: SpanContext | undefined, ): Promise { const observers = this.outboundObservers; const logger = getLogger(["fedify", "federation", "fanout"]); @@ -1460,6 +1469,8 @@ export class FederationImpl context, observers, message, + span, + parentSpanContext, ); await this.sendActivity(keys, message.inboxes, activity, { collectionSync: message.collectionSync, @@ -5890,6 +5901,7 @@ export class ContextImpl implements Context { metadata.name, metadata.version, ); + const parentSpanContext = trace.getSpanContext(context.active()); return tracer.startActiveSpan( this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" @@ -5911,6 +5923,7 @@ export class ContextImpl implements Context { }, }, async (span) => { + observation?.setSpan(span, parentSpanContext); try { if (activity.id != null) { span.setAttribute("activitypub.activity.id", activity.id.href); @@ -7024,6 +7037,7 @@ function forwardActivity( metadata.name, metadata.version, ); + const parentSpanContext = trace.getSpanContext(context.active()); return tracer.startActiveSpan( ctx.federation.outboxQueue == null || options?.immediate ? `activitypub.${loggerCategory}` @@ -7035,6 +7049,7 @@ function forwardActivity( attributes: { "activitypub.activity.type": ctx.activityType }, }, async (span) => { + observation?.setSpan(span, parentSpanContext); try { if (ctx.activityId != null) { span.setAttribute("activitypub.activity.id", ctx.activityId); @@ -7479,6 +7494,7 @@ export class OutboxContextImpl extends ContextImpl metadata.name, metadata.version, ); + const parentSpanContext = trace.getSpanContext(context.active()); return tracer.startActiveSpan( this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" @@ -7500,6 +7516,7 @@ export class OutboxContextImpl extends ContextImpl }, }, async (span) => { + observation?.setSpan(span, parentSpanContext); try { if (activity.id != null) { span.setAttribute("activitypub.activity.id", activity.id.href); diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 01f7a9a80..0febc1c5a 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -4,6 +4,11 @@ import { snapshotLocalContext, sourceLogContext, } from "./outbound-provenance.ts"; +import { + isSpanContextValid, + type Span, + type SpanContext, +} from "@opentelemetry/api"; import { encodeHex } from "byte-encodings/hex"; import { getLogger, withContext } from "@logtape/logtape"; import type { Context } from "./context.ts"; @@ -17,7 +22,7 @@ import type { } from "./outbound-report.ts"; type WithoutSource = T extends unknown - ? Omit + ? Omit : never; type PreparationFailure = Extract< @@ -129,6 +134,7 @@ class OutboundDeliveryObservation { identity: SerializedOutboundDelivery, destination: PlannedDestination, readonly provenance: OutboundProvenance, + readonly trace: OutboundBroadcastEvent["trace"], ) { this.#reference = snapshotDeliveryIdentity(identity); this.#plan = Object.freeze({ @@ -148,6 +154,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + ...(this.trace == null ? {} : { trace: this.trace }), observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), @@ -168,6 +175,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + ...(this.trace == null ? {} : { trace: this.trace }), observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), @@ -186,6 +194,7 @@ class OutboundDeliveryObservation { this.context, this.observer, Object.freeze({ + ...(this.trace == null ? {} : { trace: this.trace }), observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), @@ -209,6 +218,7 @@ export class OutboundBroadcastObservation { #phase: PreparationFailure["failure"]["phase"] = "unknown"; #planning: PreparationFailure["planning"] = "not-started"; #destinationCount = 0; + #trace: OutboundBroadcastEvent["trace"]; readonly #onBroadcast: OutboundObservers["onBroadcast"]; readonly #onDelivery: OutboundObservers["onDelivery"]; @@ -237,6 +247,8 @@ export class OutboundBroadcastObservation { context: Context, observers: OutboundObservers, message: FanoutMessage, + span?: Span, + parentSpanContext?: SpanContext, ): Promise | undefined> { if ( observers.onDelivery == null && @@ -251,6 +263,7 @@ export class OutboundBroadcastObservation { getOutboundProvenance(context), { type: "unknown" }, ); + if (span != null) observation.setSpan(span, parentSpanContext); observation.#finished = true; observation.planned(Object.keys(message.inboxes).length); for (const [inbox, destination] of Object.entries(message.inboxes)) { @@ -286,6 +299,7 @@ export class OutboundBroadcastObservation { }, destination, this.provenance, + this.#trace, ); } @@ -350,6 +364,21 @@ export class OutboundBroadcastObservation { return result; } + /** Attach only the span supplied by this execution, never an ambient fallback. */ + setSpan(span: Span, parentSpanContext?: SpanContext): void { + const spanContext = span.spanContext(); + // A no-op tracer can return the parent's context instead of creating a span. + if ( + !isSpanContextValid(spanContext) || spanContext.isRemote || + spanContext.traceId === parentSpanContext?.traceId && + spanContext.spanId === parentSpanContext.spanId + ) return; + this.#trace = Object.freeze({ + traceId: spanContext.traceId, + spanId: spanContext.spanId, + }); + } + setPhase(phase: PreparationFailure["failure"]["phase"]): void { this.#phase = phase; if (phase === "destination-planning") this.#planning = "incomplete"; @@ -444,6 +473,7 @@ export class OutboundBroadcastObservation { this.#onBroadcast, Object.freeze({ ...event, + ...(this.#trace == null ? {} : { trace: this.#trace }), observedAt: Temporal.Now.instant().epochMilliseconds, provenance: this.provenance, local: snapshotLocalContext(this.context), diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index e603c6c6c..b5d066b13 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -5,7 +5,13 @@ import { } from "@fedify/fixture"; import { Create, Person } from "@fedify/vocab"; import { configure, reset } from "@logtape/logtape"; -import { SpanStatusCode } from "@opentelemetry/api"; +import { + isSpanContextValid, + SpanKind, + SpanStatusCode, + trace, + TraceFlags, +} from "@opentelemetry/api"; import fetchMock from "fetch-mock"; import { deepStrictEqual, @@ -196,6 +202,19 @@ test("outbound observers report fresh immutable starts without tracing or extra type: "skipped", reason: "no-destinations", }); + for (const event of events) { + if (event.stage === "started" || tracerProvider == null) { + strictEqual("trace" in event, false); + } else { + ok(event.trace); + ok( + isSpanContextValid({ + ...event.trace, + traceFlags: TraceFlags.NONE, + }), + ); + } + } ok(events[0].broadcastId.length > 0); notStrictEqual(events[0].broadcastId, events[2].broadcastId); strictEqual(loads, 0); @@ -207,6 +226,140 @@ test("outbound observers report fresh immutable starts without tracing or extra } }); +test("outbound trace snapshots identify producer and fanout spans without changing callbacks", async (t) => { + for ( + const mode of [ + "send", + "outbox-send", + "inbox-forward", + "outbox-forward", + ] as const + ) { + await t.step(mode, async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + try { + const queued: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + tracerProvider, + manuallyStartQueue: true, + queue: { fanout: queue, outbox: queue }, + }); + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + let replacements = 0; + const replacement = () => { + replacements++; + }; + const record = async ( + _ctx: unknown, + event: OutboundBroadcastEvent | OutboundDeliveryEvent, + ) => { + events.push(event); + if (event.stage === "started") { + federation.setOutboundObservers({ + onBroadcast: replacement, + onDelivery: replacement, + }); + } + await Promise.resolve(); + }; + const observers = { onBroadcast: record, onDelivery: record }; + federation.setOutboundObservers(observers); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(ctx instanceof ContextImpl); + if (mode.endsWith("forward")) { + await forwardingContext( + federation, + mode === "inbox-forward" ? "inbox" : "outbox", + forwardingPayload, + undefined, + ).forwardActivity(sender, recipient, { skipIfUnsigned: true }); + } else { + const producer = mode === "outbox-send" + ? ctx.toOutboxContext("alice", {}, undefined, Create.typeId.href) + : ctx; + await producer.sendActivity(sender, recipient, activity, { + fanout: mode === "send" ? "force" : "skip", + }); + } + strictEqual(replacements, 0); + strictEqual(events[0].stage, "started"); + strictEqual("trace" in events[0], false); + ok(events.length > 2); + const producerEvents = events.length; + if (mode === "send") { + federation.setOutboundObservers(observers); + const task = queued.find((message) => message.type === "fanout"); + ok(task); + await federation.processQueuedTask(undefined, structuredClone(task)); + ok(events.length > producerEvents); + } + for (const [index, event] of events.entries()) { + if (index === 0) continue; + ok(event.trace); + ok(Object.isFrozen(event.trace)); + const span = exporter.spans.find((span) => + span.spanContext().spanId === event.trace!.spanId + ); + ok(span); + deepStrictEqual(event.trace, { + traceId: span.spanContext().traceId, + spanId: span.spanContext().spanId, + }); + strictEqual( + span.kind, + index < producerEvents ? SpanKind.PRODUCER : SpanKind.CONSUMER, + ); + } + } finally { + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound trace snapshots omit invalid and inherited span identities", async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const parent = { + traceId: "11111111111111111111111111111111", + spanId: "2222222222222222", + traceFlags: TraceFlags.NONE, + }; + for ( + const spanContext of [ + { ...parent, traceId: "00000000000000000000000000000000" }, + parent, + { ...parent, isRemote: true }, + ] + ) { + const events: OutboundBroadcastEvent[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onBroadcast(_ctx, event) { + events.push(event); + }, + }, "send"); + observation.setSpan(trace.wrapSpanContext(spanContext), parent); + await observation.skipped(); + strictEqual(events.length, 1); + strictEqual("trace" in events[0], false); + } +}); + test("outbound observers await entry before preparation and retain the selected callback", async () => { const federation = createFederation(options); let preparations = 0; diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 953316c3c..a75b64689 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -219,6 +219,11 @@ export interface OutboundProvenance { interface OutboundSourceMetadata { /** Unix milliseconds at observation; wall clocks do not ensure global ordering. */ readonly observedAt: number; + /** Valid IDs of this operation's own span, when one is available. */ + readonly trace?: { + readonly traceId: string; + readonly spanId: string; + }; /** Immutable historical cause, distinct from this callback's context. */ readonly provenance: OutboundProvenance; /** URL facts of the current callback context. */ From b631da530cd022fca1ef9cc085ef21bad1c30a8e Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 02:33:23 +0900 Subject: [PATCH 18/53] Observe concrete outbound send attempts Report terminal results for concrete send attempts so observers can distinguish delivery outcomes from producer readiness. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 1 + packages/fedify/src/federation/middleware.ts | 3 + .../src/federation/outbound-observation.ts | 148 ++++++++- .../src/federation/outbound-report.test.ts | 314 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 74 +++++ packages/fedify/src/federation/send.ts | 92 ++++- 6 files changed, 606 insertions(+), 26 deletions(-) diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 443128b64..2f3766a6f 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -667,6 +667,7 @@ export class FederationBuilderImpl this.outboundObservers = globalThis.Object.freeze({ onBroadcast: observers.onBroadcast, onDelivery: observers.onDelivery, + onAttempt: observers.onAttempt, }); } diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 7c9101b84..3b259fbfb 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3404,6 +3404,9 @@ export class FederationImpl inbox, inboxes[inbox].gatewayInboxes, { + observation: inboxes[inbox].portableInbox == null + ? delivery + : undefined, allowPrivateAddress: this.allowPrivateAddress, keys, activity: jsonLd, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 0febc1c5a..032506913 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -14,6 +14,7 @@ import { getLogger, withContext } from "@logtape/logtape"; import type { Context } from "./context.ts"; import type { FanoutMessage, SerializedOutboundDelivery } from "./queue.ts"; import type { + OutboundAttemptReport, OutboundBroadcastEvent, OutboundDeliveryEvent, OutboundObservers, @@ -102,7 +103,7 @@ async function notifyObserver< | ((context: Context, event: TEvent) => void | Promise) | undefined, event: TEvent, - unit: "broadcast" | "delivery", + unit: "broadcast" | "delivery" | "attempt", ): Promise { if (callback == null) return; try { @@ -122,6 +123,111 @@ async function notifyObserver< } } +function snapshotTrace( + span: Span, + parentSpanContext?: SpanContext, +): OutboundBroadcastEvent["trace"] { + const spanContext = span.spanContext(); + // A no-op tracer can return the parent's context instead of creating a span. + if ( + !isSpanContextValid(spanContext) || spanContext.isRemote || + spanContext.traceId === parentSpanContext?.traceId && + spanContext.spanId === parentSpanContext.spanId + ) return undefined; + return Object.freeze({ + traceId: spanContext.traceId, + spanId: spanContext.spanId, + }); +} + +type AttemptFailure = Extract< + OutboundAttemptReport["outcome"], + { type: "failed" } +>["failure"]; +type AttemptFacts = Pick< + OutboundAttemptReport, + "attemptId" | "inboxUrl" | "observedAt" | "trace" | "outcome" +>; + +/** State owned by one concrete send, never persisted in queue metadata. */ +export class OutboundAttemptObservation { + readonly #attemptId = crypto.randomUUID(); + readonly #failures: AttemptFailure[] = []; + #response: { readonly status: number } | undefined; + #trace: OutboundAttemptReport["trace"]; + #finished = false; + + constructor( + readonly inboxUrl: string, + readonly report: (facts: AttemptFacts) => Promise, + ) {} + + setSpan(span: Span, parentSpanContext?: SpanContext): void { + this.#trace = snapshotTrace(span, parentSpanContext); + } + + response(status: number): void { + this.#response = Object.freeze({ status }); + } + + failure(failure: AttemptFailure): void { + this.#failures.push(Object.freeze(failure)); + } + + unknown(error: unknown): void { + if ( + !this.#failures.some((failure) => + failure.error === error || failure.phase !== "cleanup" + ) + ) { + this.failure({ kind: "unknown", phase: "unknown", error }); + } + } + + succeeded(): Promise { + return this.#finish({ type: "succeeded", response: this.#response! }); + } + + failed(error: unknown): Promise { + let failure: AttemptFailure; + let diagnostics: AttemptFailure[]; + if ( + this.#response != null && + (this.#response.status < 200 || this.#response.status >= 300) + ) { + failure = this.#failures.find((failure) => failure.kind === "http") ?? + Object.freeze({ kind: "http", phase: "response" }); + diagnostics = this.#failures.filter((failure) => failure.kind !== "http"); + } else { + failure = this.#failures.find((failure) => failure.phase !== "cleanup") ?? + this.#failures[0] ?? + Object.freeze({ kind: "unknown", phase: "unknown", error }); + diagnostics = this.#failures.filter((item) => item !== failure); + } + return this.#finish({ + type: "failed", + failure, + error, + ...(this.#response == null ? {} : { response: this.#response }), + ...(diagnostics.length === 0 + ? {} + : { diagnostics: Object.freeze(diagnostics) }), + }); + } + + #finish(outcome: OutboundAttemptReport["outcome"]): Promise { + if (this.#finished) return Promise.resolve(); + this.#finished = true; + return this.report({ + attemptId: this.#attemptId, + inboxUrl: this.inboxUrl, + observedAt: Temporal.Now.instant().epochMilliseconds, + ...(this.#trace == null ? {} : { trace: this.#trace }), + outcome: Object.freeze(outcome), + }); + } +} + /** One producer's destination, without callbacks or live inputs in queues. */ class OutboundDeliveryObservation { #finished = false; @@ -135,6 +241,7 @@ class OutboundDeliveryObservation { destination: PlannedDestination, readonly provenance: OutboundProvenance, readonly trace: OutboundBroadcastEvent["trace"], + readonly onAttempt: OutboundObservers["onAttempt"], ) { this.#reference = snapshotDeliveryIdentity(identity); this.#plan = Object.freeze({ @@ -145,6 +252,25 @@ class OutboundDeliveryObservation { }); } + startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { + if (this.onAttempt == null) return undefined; + return new OutboundAttemptObservation( + inboxUrl, + (facts) => + notifyObserver( + this.context, + this.onAttempt, + Object.freeze({ + ...this.#reference, + ...facts, + provenance: this.provenance, + local: snapshotLocalContext(this.context), + }), + "attempt", + ), + ); + } + get identity(): SerializedOutboundDelivery { return this.#reference; } @@ -222,6 +348,7 @@ export class OutboundBroadcastObservation { readonly #onBroadcast: OutboundObservers["onBroadcast"]; readonly #onDelivery: OutboundObservers["onDelivery"]; + readonly #onAttempt: OutboundObservers["onAttempt"]; readonly #deliveries = new Map< string, OutboundDeliveryObservation @@ -240,6 +367,7 @@ export class OutboundBroadcastObservation { this.#broadcast = broadcast; this.#onBroadcast = observers.onBroadcast; this.#onDelivery = observers.onDelivery; + this.#onAttempt = observers.onAttempt; } /** Continue an existing fanout; do not emit a new broadcast invocation. */ @@ -300,6 +428,7 @@ export class OutboundBroadcastObservation { destination, this.provenance, this.#trace, + this.#onAttempt, ); } @@ -366,17 +495,7 @@ export class OutboundBroadcastObservation { /** Attach only the span supplied by this execution, never an ambient fallback. */ setSpan(span: Span, parentSpanContext?: SpanContext): void { - const spanContext = span.spanContext(); - // A no-op tracer can return the parent's context instead of creating a span. - if ( - !isSpanContextValid(spanContext) || spanContext.isRemote || - spanContext.traceId === parentSpanContext?.traceId && - spanContext.spanId === parentSpanContext.spanId - ) return; - this.#trace = Object.freeze({ - traceId: spanContext.traceId, - spanId: spanContext.spanId, - }); + this.#trace = snapshotTrace(span, parentSpanContext); } setPhase(phase: PreparationFailure["failure"]["phase"]): void { @@ -512,7 +631,10 @@ export function observeOutboundBroadcast( return await send(observation, provenance); }; return withContext(sourceLogContext(provenance), () => { - if (observers.onBroadcast == null && observers.onDelivery == null) { + if ( + observers.onBroadcast == null && observers.onDelivery == null && + observers.onAttempt == null + ) { return execute(); } return new OutboundBroadcastObservation( diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index b5d066b13..6b9ab2957 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -4,6 +4,7 @@ import { test, } from "@fedify/fixture"; import { Create, Person } from "@fedify/vocab"; +import { FetchError, UrlError } from "@fedify/vocab-runtime"; import { configure, reset } from "@logtape/logtape"; import { isSpanContextValid, @@ -31,11 +32,16 @@ import type { Federation, Message, MessageQueue, + OutboundAttemptReport, OutboundBroadcastEvent, OutboundDeliveryEvent, OutboundObservers, } from "./mod.ts"; import type { FanoutMessage, OutboxMessage } from "./queue.ts"; +import { + sendActivity as sendConcreteActivity, + SendActivityError, +} from "./send.ts"; import { OutboundBroadcastObservation } from "./outbound-observation.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; @@ -156,6 +162,314 @@ test("outbound observation timestamps snapshot each boundary and refresh on fano } }); +test("outbound attempts report ordinary and outbox sends with attempt-only builder registration", async (t) => { + for (const kind of ["ordinary", "outbox"] as const) { + for (const status of [202, 503, "policy"] as const) { + await t.step(`${kind}: ${status}`, async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + try { + const builder = createFederationBuilder(); + const reports: OutboundAttemptReport[] = []; + const contexts: unknown[] = []; + let recorded = false; + let replacementCalls = 0; + builder.setOutboundObservers({ + async onAttempt(context, report) { + contexts.push(context); + reports.push(report); + await Promise.resolve(); + recorded = true; + throw new Error("observer rejection must be isolated"); + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + tracerProvider, + allowPrivateAddress: status !== "policy", + }); + builder.setOutboundObservers({ + onAttempt() { + replacementCalls++; + }, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = kind === "ordinary" + ? base + : base.toOutboxContext("alice", {}, undefined, Create.typeId.href); + const target = status === "policy" + ? recipient.clone({ + inbox: new URL("http://127.0.0.1/inbox"), + }) + : recipient; + if (status !== "policy") { + fetchMock.mockGlobal().post(target.inboxId!.href, status); + } + const sending = ctx.sendActivity(sender, target, activity, { + immediate: true, + dispatchId: "application-dispatch", + }); + let outward: unknown; + if (status === 202) await sending; + else { + await rejects(sending, (error: unknown) => { + outward = error; + return status === "policy" + ? error instanceof UrlError + : error instanceof SendActivityError && + error.statusCode === status; + }); + } + strictEqual(recorded, true); + strictEqual(replacementCalls, 0); + strictEqual(reports.length, 1); + strictEqual(contexts[0], ctx); + const report = reports[0]; + ok(report.attemptId.length > 0); + ok(report.deliveryId.length > 0); + ok(report.broadcast.type === "known"); + strictEqual(report.destinationInbox, target.inboxId!.href); + strictEqual(report.inboxUrl, target.inboxId!.href); + strictEqual(report.provenance.dispatchId, "application-dispatch"); + ok(Object.isFrozen(report)); + ok(Object.isFrozen(report.outcome)); + ok(Object.isFrozen(report.local)); + ok(Object.isFrozen(report.provenance)); + const span = exporter.getSpan("activitypub.send_activity"); + ok(span); + deepStrictEqual(report.trace, { + traceId: span.spanContext().traceId, + spanId: span.spanContext().spanId, + }); + if (status === 202) { + deepStrictEqual(report.outcome, { + type: "succeeded", + response: { status }, + }); + } else { + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, outward); + strictEqual(report.outcome.failure.error, outward); + strictEqual( + report.outcome.failure.kind, + status === "policy" ? "policy" : "http", + ); + if (status === "policy") { + strictEqual(report.outcome.response, undefined); + } else deepStrictEqual(report.outcome.response, { status }); + } + } finally { + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } + } +}); + +test("outbound attempts await the selected observer without inventing pre-entry attempts", async () => { + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + const reports: OutboundAttemptReport[] = []; + let replacements = 0; + let settled = false; + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + if (event.stage === "started") { + federation.setOutboundObservers({ + onAttempt() { + replacements++; + }, + }); + } + }, + async onAttempt(_ctx, report) { + reports.push(report); + entered(); + await gate; + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sending = ctx.sendActivity(sender, recipient, activity, { + immediate: true, + }) + .finally(() => { + settled = true; + }); + await observing; + strictEqual(settled, false); + strictEqual(replacements, 0); + release(); + await sending; + strictEqual(reports.length, 1); + await ctx.sendActivity(sender, [], activity); + await rejects( + ctx.sendActivity(sender, recipient, new Create({})), + TypeError, + ); + strictEqual(replacements, 0); + strictEqual(reports.length, 1); + } finally { + release(); + fetchMock.hardReset(); + } +}); + +test("outbound attempts preserve boundary failures and allocate fresh IDs per concrete entry", async (t) => { + for ( + const phase of [ + "success", + "serialization", + "dns", + "request", + "cleanup", + "http-cleanup", + "signing", + ] as const + ) { + await t.step(phase, async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const reports: OutboundAttemptReport[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + reports.push(report); + }, + onDelivery(_ctx, event) { + deliveries.push(event); + }, + }, "send"); + const inbox = "https://receiver.example/inbox"; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = phase === "dns" + ? new UrlError(inbox, { reason: "dns" }) + : Symbol(phase); + const delivery = observation.delivery(inbox)!; + let submissions = 0; + const send = () => + sendConcreteActivity({ + observation: delivery, + activity: phase === "serialization" + ? { + toJSON() { + throw failure; + }, + } + : {}, + keys: phase === "signing" ? [sender] : [], + inbox: new URL(inbox), + specDeterminer: { + determineSpec() { + throw failure; + }, + rememberSpec() { + return Promise.resolve(); + }, + }, + }, { + openHop(url) { + if (phase === "dns") return Promise.reject(failure); + return Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return phase === "request" + ? Promise.reject(failure) + : Promise.resolve( + new Response(null, { + status: phase === "http-cleanup" ? 503 : 202, + }), + ); + }, + close() { + return phase === "cleanup" || phase === "http-cleanup" + ? Promise.reject(failure) + : Promise.resolve(); + }, + }); + }, + }); + let outward: unknown; + if (phase === "success") { + await send(); + await send(); + strictEqual(reports.length, 2); + notStrictEqual(reports[0].attemptId, reports[1].attemptId); + strictEqual(reports[0].deliveryId, reports[1].deliveryId); + strictEqual(submissions, 2); + return; + } + await rejects(send(), (error: unknown) => { + outward = error; + return phase === "serialization" + ? error === failure + : error instanceof FetchError; + }); + strictEqual(reports.length, 1); + const report = reports[0]; + strictEqual(report.deliveryId, deliveries[0].deliveryId); + strictEqual(report.outcome.type, "failed"); + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, outward); + if (phase === "http-cleanup") { + deepStrictEqual(report.outcome.failure, { + kind: "http", + phase: "response", + }); + deepStrictEqual(report.outcome.diagnostics, [{ + kind: "local", + phase: "cleanup", + error: failure, + }]); + deepStrictEqual(report.outcome.response, { status: 503 }); + } else { + deepStrictEqual(report.outcome.failure, { + kind: phase === "dns" || phase === "request" + ? "transport" + : phase === "signing" + ? "unknown" + : "local", + phase: phase === "dns" + ? "validation" + : phase === "signing" + ? "unknown" + : phase, + error: failure, + }); + } + strictEqual( + submissions, + ["request", "cleanup", "http-cleanup"].includes(phase) ? 1 : 0, + ); + strictEqual("trace" in report, false); + }); + } +}); + test("outbound observers report fresh immutable starts without tracing or extra loading", async (t) => { const { AlwaysOffSampler, BasicTracerProvider } = await import( "@opentelemetry/sdk-trace-base" diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index a75b64689..abfd676d7 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -142,6 +142,71 @@ export type OutboundDeliveryEvent = } ); +type OutboundAttemptFailure = + | { + readonly kind: "http"; + readonly phase: "response"; + readonly error?: unknown; + } + | { + readonly kind: "transport"; + readonly phase: "validation" | "request"; + readonly error: unknown; + } + | { + readonly kind: "policy"; + readonly phase: "validation"; + readonly error: unknown; + } + | { + readonly kind: "local"; + readonly phase: "serialization" | "request-preparation" | "cleanup"; + readonly error: unknown; + } + | { + readonly kind: "unknown"; + readonly phase: "unknown"; + readonly error: unknown; + }; + +/** + * The terminal result of one ordinary immediate concrete send. + * + * A send that fails before submitting HTTP still constitutes an attempt. + * Forwarding, portable gateways, and queued worker sends are not observed yet. + * @since 2.5.0 + */ +export type OutboundAttemptReport = OutboundSourceMetadata & { + /** Fresh opaque identity allocated at concrete sender entry. */ + readonly attemptId: string; + /** The originating invocation, independent of tracing. */ + readonly broadcast: OutboundDeliveryEvent["broadcast"]; + /** Destination obligation shared with producer delivery events. */ + readonly deliveryId: string; + /** Original planned inbox, independent of the concrete route. */ + readonly destinationInbox: string; + /** Inbox selected for this invocation of the concrete sender. */ + readonly inboxUrl: string; + /** Settled execution result, separate from producer preparation. */ + readonly outcome: + | { + readonly type: "succeeded"; + /** Final response selected by the sender. */ + readonly response: { readonly status: number }; + } + | { + readonly type: "failed"; + /** Failure evidence captured before broad sender error wrapping. */ + readonly failure: OutboundAttemptFailure; + /** Exact value escaping the sender, possibly different from failure.error. */ + readonly error: unknown; + /** Final response when the sender reached final-response consumption. */ + readonly response?: { readonly status: number }; + /** Additional failures, without replacing the primary evidence. */ + readonly diagnostics?: readonly OutboundAttemptFailure[]; + }; +}; + /** * Optional observers of outgoing activity processing. * @@ -159,6 +224,15 @@ export type OutboundDeliveryEvent = * @since 2.5.0 */ export interface OutboundObservers { + /** + * Observes settlement of ordinary immediate sends, including pre-fetch + * failures. Callback errors cannot replace the sender's result. + */ + readonly onAttempt?: ( + context: Context, + report: OutboundAttemptReport, + ) => void | Promise; + /** * Observes invocation entry and the current producer's preparation result. * The existing context is passed without loading data for observation. diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index d8356c498..07c1841fe 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -8,6 +8,7 @@ import { import { getLogger, withContext } from "@logtape/logtape"; import { type Attributes, + context, type MeterProvider, type Span, SpanKind, @@ -22,6 +23,7 @@ import { type HttpMessageSignaturesSpecDeterminer, } from "../sig/http.ts"; import { isPortableUri } from "../sig/portable-key-id.ts"; +import type { OutboundAttemptObservation } from "./outbound-observation.ts"; import { getDurationMs, getFederationMetrics } from "./metrics.ts"; import { mergeGatewayInboxes, @@ -205,6 +207,11 @@ export interface SenderKeyPair { * Parameters for {@link sendActivity}. */ export interface SendActivityParameters { + /** In-process destination observation; never serialized. @internal */ + readonly observation?: { + startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined; + }; + /** * The activity to send. */ @@ -296,6 +303,8 @@ export function sendActivity( options: SendActivityParameters, deps: SendActivityDependencies = {}, ): Promise { + const attempt = options.observation?.startAttempt(options.inbox.href); + const parentSpanContext = trace.getSpanContext(context.active()); const tracerProvider = options.tracerProvider ?? trace.getTracerProvider(); const tracer = tracerProvider.getTracer(metadata.name, metadata.version); return tracer.startActiveSpan( @@ -307,6 +316,7 @@ export function sendActivity( }, }, async (span) => { + attempt?.setSpan(span, parentSpanContext); if (options.activityId != null) { span.setAttribute("activitypub.activity.id", options.activityId); } @@ -318,10 +328,17 @@ export function sendActivity( await withContext( { traceId: spanContext.traceId, spanId: spanContext.spanId }, () => - sendActivityInternal({ ...options, tracerProvider }, span, deps), + sendActivityInternal( + { ...options, tracerProvider }, + span, + deps, + attempt, + ), ); + await attempt?.succeeded(); } catch (e) { span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); + await attempt?.failed(e); throw e; } finally { span.end(); @@ -463,6 +480,7 @@ async function sendActivityInternal( }: SendActivityParameters, span: Span, deps: SendActivityDependencies, + attempt?: OutboundAttemptObservation, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); const federationMetrics = getFederationMetrics(meterProvider); @@ -476,17 +494,40 @@ async function sendActivityInternal( session = await openUnvalidatedHop(url); } catch (error) { if (error instanceof UrlError && error.reason === "dns") { + attempt?.failure({ kind: "transport", phase: "validation", error }); logger.error("DNS resolution failed for URL: {url}", { url, error }); const failure = new FetchError(url, error.message); failure.cause = error; throw failure; } if (error instanceof UrlError) { + if (error.reason === "disallowed") { + attempt?.failure({ kind: "policy", phase: "validation", error }); + } logger.error("Disallowed private URL: {url}", { url, error }); } throw error; } - return session; + if (attempt == null) return session; + return { + url: session.url, + async fetch(input, init) { + try { + return await session.fetch(input, init); + } catch (error) { + attempt.failure({ kind: "transport", phase: "request", error }); + throw error; + } + }, + async close() { + try { + await session.close(); + } catch (error) { + attempt.failure({ kind: "local", phase: "cleanup", error }); + throw error; + } + }, + }; } let initialSession: PublicFetchSession; @@ -514,14 +555,32 @@ async function sendActivityInternal( return openHop(url); }; try { - headers = new Headers(headers); - headers.set("Content-Type", "application/activity+json"); - const request = new Request(inbox, { - method: "POST", - headers, - body: JSON.stringify(activity), - redirect: "manual", - }); + let request: Request; + let body: string | undefined; + try { + headers = new Headers(headers); + headers.set("Content-Type", "application/activity+json"); + } catch (error) { + attempt?.failure({ kind: "local", phase: "request-preparation", error }); + throw error; + } + try { + body = JSON.stringify(activity); + } catch (error) { + attempt?.failure({ kind: "local", phase: "serialization", error }); + throw error; + } + try { + request = new Request(inbox, { + method: "POST", + headers, + body, + redirect: "manual", + }); + } catch (error) { + attempt?.failure({ kind: "local", phase: "request-preparation", error }); + throw error; + } let rsaKey: SenderKeyPair | null = null; for (const key of keys) { if (key.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") { @@ -549,22 +608,27 @@ async function sendActivityInternal( }, ); } + const consume = (response: Response) => { + attempt?.response(response.status); + return consumeSendActivityResponse(response, inbox); + }; let outcome: SendActivityOutcome; try { outcome = rsaKey == null ? await fetchWithValidatedRedirects( request, openDeliveryHop, - (response) => consumeSendActivityResponse(response, inbox), + consume, ) : await doubleKnockScoped( request, rsaKey, { tracerProvider, specDeterminer }, openDeliveryHop, - (response) => consumeSendActivityResponse(response, inbox), + consume, ); } catch (error) { + attempt?.unknown(error); // A refused redirect remains a policy error rather than a transport // failure, just as a refused initial inbox does. const failure = error instanceof FetchError || error instanceof UrlError @@ -590,7 +654,7 @@ async function sendActivityInternal( "{statusText}):\n{error}", { activityId, inbox: inbox.href, status, statusText, error }, ); - throw new SendActivityError( + const failure = new SendActivityError( inbox, status, `Failed to send activity ${activityId} to ${inbox.href} ` + @@ -598,6 +662,8 @@ async function sendActivityInternal( error, responseHeaders, ); + attempt?.failure({ kind: "http", phase: "response", error: failure }); + throw failure; } deliverySuccess = true; From 522d0240ae0f7bc9f36dfdd3612eb5e896f3f017 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 02:49:04 +0900 Subject: [PATCH 19/53] Observe forwarding and gateway attempts Report concrete forwarding and gateway attempts separately to preserve delivery grouping and capture late replication completion. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 5 +- .../src/federation/outbound-report.test.ts | 21 +++ .../fedify/src/federation/outbound-report.ts | 8 +- .../src/federation/portable-delivery.test.ts | 122 +++++++++++++++++- .../src/federation/portable-inbox.test.ts | 116 +++++++++++++++++ .../fedify/src/federation/portable-inbox.ts | 1 + 6 files changed, 264 insertions(+), 9 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 3b259fbfb..ab460a18b 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3404,9 +3404,7 @@ export class FederationImpl inbox, inboxes[inbox].gatewayInboxes, { - observation: inboxes[inbox].portableInbox == null - ? delivery - : undefined, + observation: delivery, allowPrivateAddress: this.allowPrivateAddress, keys, activity: jsonLd, @@ -7218,6 +7216,7 @@ async function forwardActivityInternal( inbox, inboxes[inbox].gatewayInboxes, { + observation: delivery, allowPrivateAddress: ctx.federation.allowPrivateAddress, keys, activity: ctx.activity, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 6b9ab2957..3831ed982 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -1928,8 +1928,12 @@ test("forwarding observers retain readiness and registration through late immedi ); const events: OutboundBroadcastEvent[] = []; const deliveries: OutboundDeliveryEvent[] = []; + const attempts: OutboundAttemptReport[] = []; let replacements = 0; federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, onDelivery(_ctx, event) { deliveries.push(event); }, @@ -1964,6 +1968,9 @@ test("forwarding observers retain readiness and registration through late immedi destinationCount: 1, }); federation.setOutboundObservers({ + onAttempt() { + replacements++; + }, onBroadcast() { replacements++; }, @@ -1981,6 +1988,20 @@ test("forwarding observers retain readiness and registration through late immedi const handoff = deliveries[1]; ok(handoff.stage === "handoff"); deepStrictEqual(handoff.result, { type: "direct" }); + strictEqual(attempts.length, 1); + const attempt = attempts[0]; + strictEqual(attempt.deliveryId, handoff.deliveryId); + deepStrictEqual(attempt.broadcast, handoff.broadcast); + strictEqual(attempt.inboxUrl, recipient.inboxId!.href); + strictEqual( + attempt.outcome.type, + status === 202 ? "succeeded" : "failed", + ); + deepStrictEqual(attempt.outcome.response, { status }); + if (attempt.outcome.type === "failed") { + strictEqual(attempt.outcome.failure.kind, "http"); + strictEqual(attempt.outcome.failure.error, attempt.outcome.error); + } strictEqual(events.length, 2); strictEqual(replacements, 0); for (const event of events) strictEqual(event.operation, "forward"); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index abfd676d7..6e7593583 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -170,10 +170,10 @@ type OutboundAttemptFailure = }; /** - * The terminal result of one ordinary immediate concrete send. + * The terminal result of one immediate concrete send. * * A send that fails before submitting HTTP still constitutes an attempt. - * Forwarding, portable gateways, and queued worker sends are not observed yet. + * Queued worker sends are not observed yet. * @since 2.5.0 */ export type OutboundAttemptReport = OutboundSourceMetadata & { @@ -185,6 +185,8 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { readonly deliveryId: string; /** Original planned inbox, independent of the concrete route. */ readonly destinationInbox: string; + /** Canonical portable inbox when the planned destination represents one. */ + readonly portableInbox?: string; /** Inbox selected for this invocation of the concrete sender. */ readonly inboxUrl: string; /** Settled execution result, separate from producer preparation. */ @@ -225,7 +227,7 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { */ export interface OutboundObservers { /** - * Observes settlement of ordinary immediate sends, including pre-fetch + * Observes settlement of immediate sends and forwarding, including pre-fetch * failures. Callback errors cannot replace the sender's result. */ readonly onAttempt?: ( diff --git a/packages/fedify/src/federation/portable-delivery.test.ts b/packages/fedify/src/federation/portable-delivery.test.ts index 9c8489ccf..f41b41e60 100644 --- a/packages/fedify/src/federation/portable-delivery.test.ts +++ b/packages/fedify/src/federation/portable-delivery.test.ts @@ -2,12 +2,21 @@ import { mockDocumentLoader, test } from "@fedify/fixture"; import { Create, Endpoints, Note, Person, type Recipient } from "@fedify/vocab"; import { parseIri } from "@fedify/vocab-runtime"; import fetchMock from "fetch-mock"; -import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { + deepStrictEqual, + notStrictEqual, + ok, + rejects, + strictEqual, +} from "node:assert/strict"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; import { MemoryKvStore } from "./kv.ts"; -import { FederationImpl, InboxContextImpl } from "./middleware.ts"; +import { ContextImpl, FederationImpl, InboxContextImpl } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; -import type { OutboundDeliveryEvent } from "./outbound-report.ts"; +import type { + OutboundAttemptReport, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; import { MAX_DELIVERY_GATEWAYS } from "./portable-delivery.ts"; import type { FanoutMessage, Message, OutboxMessage } from "./queue.ts"; import { extractInboxes, SendActivityError } from "./send.ts"; @@ -541,6 +550,113 @@ test("Context.sendActivity() to a portable inbox", async (t) => { fetchMock.hardReset(); }); +test("portable fallback attempts retain one delivery across concrete gateway sends", async (t) => { + for ( + const operation of ["send", "inbox-forward", "outbox-forward"] as const + ) { + for (const accepted of [true, false]) { + await t.step( + `${operation}: ${ + accepted ? "accepted fallback" : "exhausted gateways" + }`, + async () => { + const raw = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: "https://example.com/activities/forwarded", + actor: "https://example.com/person2", + extra: "preserve received payload", + }; + const reports: OutboundAttemptReport[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const submitted: unknown[] = []; + fetchMock.mockGlobal() + .post("begin:https://gw1.example/", (call) => { + submitted.push(JSON.parse(String(call.options.body))); + return 503; + }) + .post("begin:https://gw2.example/", (call) => { + submitted.push(JSON.parse(String(call.options.body))); + return accepted ? 202 : 502; + }); + try { + const federation = new FederationImpl({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onAttempt(_ctx, report) { + reports.push(report); + throw new Error( + "attempt observer cannot stop gateway fallback", + ); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + ); + ok(ctx instanceof ContextImpl); + const target = portableActor({ + gateways: ["https://gw1.example", "https://gw2.example"], + }); + const pending = operation === "send" + ? ctx.sendActivity(sender, target, createActivity(), { + immediate: true, + }) + : (operation === "inbox-forward" + ? ctx.toInboxContext(null, raw, raw.id, Create.typeId.href) + : ctx.toOutboxContext("alice", raw, raw.id, Create.typeId.href)) + .forwardActivity(sender, target, { + immediate: true, + skipIfUnsigned: false, + }); + if (accepted) await pending; + else {await rejects(pending, (error: unknown) => + error instanceof SendActivityError && + error.statusCode === 502);} + strictEqual(reports.length, 2); + deepStrictEqual( + reports.map((report) => report.inboxUrl), + [ + inboxOn("https://gw1.example"), + inboxOn("https://gw2.example"), + ], + ); + notStrictEqual(reports[0].attemptId, reports[1].attemptId); + strictEqual(reports[0].deliveryId, reports[1].deliveryId); + deepStrictEqual(reports[0].broadcast, reports[1].broadcast); + for (const report of reports) { + strictEqual( + report.destinationInbox, + inboxOn("https://gw1.example"), + ); + strictEqual(report.portableInbox, PORTABLE_INBOX); + strictEqual(report.deliveryId, deliveries[0].deliveryId); + } + strictEqual(reports[0].outcome.type, "failed"); + strictEqual( + reports[1].outcome.type, + accepted ? "succeeded" : "failed", + ); + deepStrictEqual(reports.map((report) => report.outcome.response), [ + { status: 503 }, + { status: accepted ? 202 : 502 }, + ]); + if (operation !== "send") { + for (const payload of submitted) deepStrictEqual(payload, raw); + } + } finally { + fetchMock.hardReset(); + } + }, + ); + } + } +}); + test("InboxContext.forwardActivity() to a portable inbox", async (t) => { fetchMock.spyGlobal(); const activity = { diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index 2ed193a8e..5919ccfae 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -47,6 +47,7 @@ import { createFederation } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; import { observeOutboundBroadcast } from "./outbound-observation.ts"; import type { + OutboundAttemptReport, OutboundBroadcastEvent, OutboundDeliveryEvent, } from "./outbound-report.ts"; @@ -2597,6 +2598,121 @@ test("portable replication observers preserve lazy enqueue, unsigned fallback, a ); }); +test("portable replication attempts settle independently after the forwarding deadline", async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let failed!: () => void; + const failureReported = new Promise((resolve) => { + failed = resolve; + }); + let succeeded!: () => void; + const successReported = new Promise((resolve) => { + succeeded = resolve; + }); + let ended!: () => void; + const concreteSendsEnded = new Promise((resolve) => { + ended = resolve; + }); + const exportSpans = exporter.export.bind(exporter); + exporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (exporter.getSpans("activitypub.send_activity").length === 2) ended(); + }; + const attempts: OutboundAttemptReport[] = []; + const submitted: unknown[] = []; + fetchMock.spyGlobal() + .post(`begin:${GATEWAY2}/`, (call) => { + submitted.push(JSON.parse(String(call.options.body))); + return 503; + }) + .post(`begin:${GATEWAY3}/`, async (call) => { + submitted.push(JSON.parse(String(call.options.body))); + fetched(); + await gate; + return 202; + }); + try { + const { federation, reports } = setup({ + keyPairs: false, + options: { + allowPrivateAddress: true, + tracerProvider, + portableInboxForwarding: { deadline: { milliseconds: 0 } }, + }, + }); + let replacements = 0; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + if (report.outcome.type === "failed") failed(); + else succeeded(); + throw new Error("replication observer failure must be isolated"); + }, + }); + const json = await signedFollow(); + const response = await federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + }); + assertEquals(response.status, 202); + await Promise.all([fetching, failureReported]); + // The pending gateway has no synthetic timeout attempt outcome. + assertEquals(attempts.length, 1); + assertEquals(attempts[0].outcome.type, "failed"); + assertEquals(attempts[0].outcome.response, { status: 503 }); + federation.setOutboundObservers({ + onAttempt() { + replacements++; + }, + }); + release(); + await Promise.all([successReported, concreteSendsEnded]); + assertEquals(attempts.length, 2); + assertEquals(attempts[1].outcome, { + type: "succeeded", + response: { status: 202 }, + }); + assertEquals(replacements, 0); + assert(attempts[0].attemptId !== attempts[1].attemptId); + assert(attempts[0].deliveryId !== attempts[1].deliveryId); + assertEquals(attempts[0].broadcast, attempts[1].broadcast); + for (const report of attempts) { + assertEquals(report.destinationInbox, report.inboxUrl); + assertEquals(report.portableInbox, `ap+ef61://${did}/users/alice/inbox`); + assertEquals( + report.provenance.inboxRequest?.inboxRequestId, + reports[0].inboxRequestId, + ); + assert(report.provenance.inboxRequest); + const span = exporter.spans.find((span) => + span.spanContext().spanId === report.trace?.spanId + ); + assert(span); + assertEquals(span.name, "activitypub.send_activity"); + } + assertEquals(submitted, [json, json]); + // Swallowed send failures retain their claims, rather than causing retries. + assertEquals( + (await federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + })).status, + 202, + ); + assertEquals(submitted.length, 2); + assertEquals(replacements, 0); + } finally { + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } +}); + test("portable replication observers retain preparation after the forwarding deadline", async (t) => { for (const failed of [false, true]) { await t.step( diff --git a/packages/fedify/src/federation/portable-inbox.ts b/packages/fedify/src/federation/portable-inbox.ts index 3613564f5..5c1c729da 100644 --- a/packages/fedify/src/federation/portable-inbox.ts +++ b/packages/fedify/src/federation/portable-inbox.ts @@ -623,6 +623,7 @@ async function forwardImmediately( let pending: Promise; try { pending = sendActivity({ + observation: delivery, activity, activityId: activityId.href, activityType, From 6ee883e8622ddf627f2300bf888520f61a8a19d5 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 03:04:39 +0900 Subject: [PATCH 20/53] Observe queued outbound send attempts Preserve delivery grouping while recording fresh attempts for retries and omit unknown destinations from legacy queued messages. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 41 ++- .../src/federation/outbound-observation.ts | 110 +++++-- .../src/federation/outbound-report.test.ts | 305 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 15 +- 4 files changed, 441 insertions(+), 30 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index ab460a18b..265697a94 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -13,6 +13,7 @@ import type { } from "./outbound-report.ts"; import { observeOutboundBroadcast, + OutboundAttemptObserver, OutboundBroadcastObservation, } from "./outbound-observation.ts"; import { InboxObservation } from "./inbox-observation.ts"; @@ -1483,11 +1484,12 @@ export class FederationImpl } async #listenOutboxMessage( - _: TContextData, + contextData: TContextData, message: OutboxMessage, span: Span, provenance: OutboundProvenance, ): Promise { + const onAttempt = this.outboundObservers.onAttempt; const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { keyIds: message.keys.map((pair) => pair.keyId), @@ -1513,6 +1515,30 @@ export class FederationImpl keys.push(pair); } const loaderOptions = this.#getLoaderOptions(message.baseUrl); + let observationContext: ContextImpl | undefined; + let observationDocumentLoader: DocumentLoader | undefined; + let observationContextLoader: DocumentLoader | undefined; + const observation = OutboundAttemptObserver.resumeOutbox( + () => + observationContext ??= new ContextImpl({ + [outboundProvenance]: provenance, + federation: this, + url: new URL("/", message.baseUrl), + data: contextData, + // Observing settlement must not create loaders or perform lookups. + documentLoader: (...args) => + (observationDocumentLoader ??= this.documentLoaderFactory( + loaderOptions, + ))(...args), + contextLoader: (...args) => + (observationContextLoader ??= this.contextLoaderFactory( + loaderOptions, + ))(...args), + }), + onAttempt, + message, + provenance, + ); let parsedActorIds: URL[] | undefined; const getActorIds = () => { parsedActorIds ??= (message.actorIds ?? []).flatMap((id) => { @@ -1586,7 +1612,7 @@ export class FederationImpl if (this.outboxPermanentFailureHandler != null) { const ctx = this.#createContext( new URL(message.baseUrl), - _, + contextData, { [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory(loaderOptions), @@ -1624,13 +1650,14 @@ export class FederationImpl const { gatewayInboxes } = message; if (gatewayInboxes != null && gatewayInboxes.length > 1) { await this.#listenGatewayOutboxMessage( - _, + contextData, { ...message, gatewayInboxes, }, span, { + observation, provenance, keys, logData, @@ -1685,6 +1712,7 @@ export class FederationImpl } } await sendActivity({ + observation, allowPrivateAddress: this.allowPrivateAddress, keys, activity: message.activity, @@ -1724,7 +1752,7 @@ export class FederationImpl } try { return getRemoteHost(new URL(message.inbox)); - } catch (_) { + } catch { logger.warn( "Invalid inbox URL in queued outbox message: {inbox}", logData, @@ -1876,7 +1904,7 @@ export class FederationImpl if (this.outboxPermanentFailureHandler != null) { const ctx = this.#createContext( new URL(message.baseUrl), - _, + contextData, { [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory(loaderOptions), @@ -2014,6 +2042,7 @@ export class FederationImpl message: OutboxMessage & { readonly gatewayInboxes: readonly string[] }, span: Span, helpers: { + readonly observation: SendActivityParameters["observation"]; readonly provenance: OutboundProvenance; readonly keys: readonly SenderKeyPair[]; readonly logData: Record; @@ -2031,6 +2060,7 @@ export class FederationImpl const logger = getLogger(["fedify", "federation", "outbox"]); const circuitLogger = getLogger(["fedify", "federation", "circuit"]); const { + observation, provenance, keys, logData, @@ -2138,6 +2168,7 @@ export class FederationImpl } try { await sendActivity({ + observation, allowPrivateAddress: this.allowPrivateAddress, keys, activity: message.activity, diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 032506913..352487aa6 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -12,7 +12,11 @@ import { import { encodeHex } from "byte-encodings/hex"; import { getLogger, withContext } from "@logtape/logtape"; import type { Context } from "./context.ts"; -import type { FanoutMessage, SerializedOutboundDelivery } from "./queue.ts"; +import type { + FanoutMessage, + OutboxMessage, + SerializedOutboundDelivery, +} from "./queue.ts"; import type { OutboundAttemptReport, OutboundBroadcastEvent, @@ -228,11 +232,81 @@ export class OutboundAttemptObservation { } } +type AttemptIdentity = Pick< + OutboundAttemptReport, + "broadcast" | "deliveryId" | "destinationInbox" | "portableInbox" +>; + +/** Selected callback and restored grouping, shared by concrete gateway attempts. */ +export class OutboundAttemptObserver { + readonly #queue: OutboundAttemptReport["queue"]; + + constructor( + readonly context: () => Context, + readonly callback: OutboundObservers["onAttempt"], + readonly identity: AttemptIdentity, + readonly provenance: OutboundProvenance, + queue?: OutboundAttemptReport["queue"], + ) { + this.#queue = queue == null ? undefined : Object.freeze({ ...queue }); + } + + static resumeOutbox( + context: () => Context, + callback: OutboundObservers["onAttempt"], + message: OutboxMessage, + provenance: OutboundProvenance, + ): OutboundAttemptObserver | undefined { + if (callback == null) return undefined; + const identity = message.outbound == null + ? Object.freeze({ + broadcast: Object.freeze({ type: "unknown" as const }), + deliveryId: message.id, + ...(message.portableInbox == null + ? {} + : { portableInbox: message.portableInbox }), + }) + : snapshotDeliveryIdentity(message.outbound); + return new OutboundAttemptObserver( + context, + callback, + identity, + provenance, + { + queueMessageId: message.id, + queueAttempt: message.attempt, + }, + ); + } + + startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { + if (this.callback == null) return undefined; + const context = this.context(); + return new OutboundAttemptObservation( + inboxUrl, + (facts) => + notifyObserver( + context, + this.callback, + Object.freeze({ + ...this.identity, + ...facts, + ...(this.#queue == null ? {} : { queue: this.#queue }), + provenance: this.provenance, + local: snapshotLocalContext(context), + }), + "attempt", + ), + ); + } +} + /** One producer's destination, without callbacks or live inputs in queues. */ class OutboundDeliveryObservation { #finished = false; readonly #reference: SerializedOutboundDelivery; readonly #plan; + readonly #attempts: OutboundAttemptObserver; constructor( readonly context: Context, @@ -241,9 +315,17 @@ class OutboundDeliveryObservation { destination: PlannedDestination, readonly provenance: OutboundProvenance, readonly trace: OutboundBroadcastEvent["trace"], - readonly onAttempt: OutboundObservers["onAttempt"], + onAttempt: OutboundObservers["onAttempt"], + queue?: OutboundAttemptReport["queue"], ) { this.#reference = snapshotDeliveryIdentity(identity); + this.#attempts = new OutboundAttemptObserver( + () => this.context, + onAttempt, + this.#reference, + this.provenance, + queue, + ); this.#plan = Object.freeze({ actorIds: Object.freeze(Array.from(destination.actorIds)), sharedInbox: destination.sharedInbox, @@ -253,22 +335,7 @@ class OutboundDeliveryObservation { } startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { - if (this.onAttempt == null) return undefined; - return new OutboundAttemptObservation( - inboxUrl, - (facts) => - notifyObserver( - this.context, - this.onAttempt, - Object.freeze({ - ...this.#reference, - ...facts, - provenance: this.provenance, - local: snapshotLocalContext(this.context), - }), - "attempt", - ), - ); + return this.#attempts.startAttempt(inboxUrl); } get identity(): SerializedOutboundDelivery { @@ -363,6 +430,7 @@ export class OutboundBroadcastObservation { type: "known", id: crypto.randomUUID(), }, + readonly queue: OutboundAttemptReport["queue"] = undefined, ) { this.#broadcast = broadcast; this.#onBroadcast = observers.onBroadcast; @@ -379,17 +447,18 @@ export class OutboundBroadcastObservation { parentSpanContext?: SpanContext, ): Promise | undefined> { if ( - observers.onDelivery == null && + observers.onDelivery == null && observers.onAttempt == null && !Object.values(message.inboxes).some((entry) => entry.outbound != null) ) { return undefined; } const observation = new OutboundBroadcastObservation( context, - { onDelivery: observers.onDelivery }, + { onDelivery: observers.onDelivery, onAttempt: observers.onAttempt }, "send", getOutboundProvenance(context), { type: "unknown" }, + { queueMessageId: message.id }, ); if (span != null) observation.setSpan(span, parentSpanContext); observation.#finished = true; @@ -429,6 +498,7 @@ export class OutboundBroadcastObservation { this.provenance, this.#trace, this.#onAttempt, + this.queue, ); } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 3831ed982..052c1af14 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -2440,7 +2440,11 @@ test("outbound observers preserve processing of legacy outbox messages", async ( const message = messages[0]; ok(message?.type === "outbox"); strictEqual(message.outbound, undefined); + const attempts: OutboundAttemptReport[] = []; federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, onDelivery() { throw new Error("unused worker observer"); }, @@ -2453,6 +2457,11 @@ test("outbound observers preserve processing of legacy outbox messages", async ( ); strictEqual(fetchMock.callHistory.calls().length, 1); strictEqual(messages.length, 1); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].deliveryId, message.id); + deepStrictEqual(attempts[0].broadcast, { type: "unknown" }); + strictEqual("destinationInbox" in attempts[0], false); + deepStrictEqual(attempts[0].provenance, {}); } finally { fetchMock.hardReset(); } @@ -2510,6 +2519,201 @@ function continuationMessage( }; } +test("outbound worker attempts restore current context without creating loaders", async () => { + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + let loads = 0; + const unexpectedLoader = () => { + loads++; + throw new Error("observation must not create loaders"); + }; + Object.defineProperties(federation, { + documentLoaderFactory: { value: unexpectedLoader }, + contextLoaderFactory: { value: unexpectedLoader }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + const data = { label: "current worker" }; + const reports: OutboundAttemptReport[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onAttempt(ctx, report) { + reports.push(report); + contexts.push(ctx.data, ctx.clone(data).data); + throw new Error("worker observer failure must be isolated"); + }, + }); + fetchMock.mockGlobal().post(message.inbox, 202); + try { + await federation.processQueuedTask(data, structuredClone(message)); + strictEqual(loads, 0); + deepStrictEqual(contexts, [data, data]); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].provenance, message.provenance); + strictEqual(reports[0].local.origin, new URL(message.baseUrl).origin); + deepStrictEqual(reports[0].queue, { + queueMessageId: message.id, + queueAttempt: 0, + }); + strictEqual(reports[0].outcome.type, "succeeded"); + } finally { + fetchMock.hardReset(); + } +}); + +test("outbound fanout workers report fresh attempts with restored grouping and current data", async (t) => { + for (const legacy of [false, true]) { + await t.step(legacy ? "legacy fanout" : "populated identity", async () => { + const messages: Message[] = []; + const [tracerProvider, exporter] = createTestTracerProvider(); + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + manuallyStartQueue: true, + queue: { + fanout: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + if (!legacy) federation.setOutboundObservers({ onBroadcast() {} }); + const ctx = federation.createContext(new URL("https://example.com"), { + label: "producer", + }); + await ctx.sendActivity(sender, recipient, activity, { + fanout: "force", + dispatchId: "original-dispatch", + }); + const message = messages[0]; + ok(message?.type === "fanout"); + const identity = message.inboxes[recipient.inboxId!.href].outbound; + strictEqual(identity == null, legacy); + const reports: OutboundAttemptReport[] = []; + const contexts: unknown[] = []; + const data = { label: "worker" }; + federation.setOutboundObservers({ + onAttempt(context, report) { + reports.push(report); + contexts.push(context.data); + }, + }); + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202); + try { + await federation.processQueuedTask(data, structuredClone(message)); + await federation.processQueuedTask(data, structuredClone(message)); + strictEqual(reports.length, 2); + deepStrictEqual(contexts, [data, data]); + notStrictEqual(reports[0].attemptId, reports[1].attemptId); + strictEqual(reports[0].deliveryId, reports[1].deliveryId); + if (identity != null) { + strictEqual(reports[0].deliveryId, identity.deliveryId); + } + for (const report of reports) { + strictEqual(report.destinationInbox, recipient.inboxId!.href); + deepStrictEqual( + report.broadcast, + identity?.broadcast ?? { type: "unknown" }, + ); + strictEqual(report.provenance.dispatchId, "original-dispatch"); + deepStrictEqual(report.queue, { queueMessageId: message.id }); + const span = exporter.spans.find((span) => + span.spanContext().spanId === report.trace?.spanId + ); + ok(span); + strictEqual(span.name, "activitypub.send_activity"); + } + } finally { + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound workers omit attempts for key import, fanout parsing, and held-task drops", async (t) => { + for ( + const phase of [ + "key-import", + "fanout-parsing", + "drop", + "gateway-drop", + ] as const + ) { + await t.step(phase, async () => { + const messages: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: { heldActivityTtl: { seconds: 1 } }, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const original = continuationMessage(recipient.inboxId!.href, true); + const message: Message = phase === "fanout-parsing" + ? { + type: "fanout", + id: original.id, + baseUrl: original.baseUrl, + keys: [], + inboxes: { [original.inbox]: { actorIds: [], sharedInbox: false } }, + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + }, + activityType: Create.typeId.href, + traceContext: {}, + } + : phase === "key-import" + ? { + ...original, + keys: [{ keyId: sender.keyId.href, privateKey: { kty: "invalid" } }], + } + : { + ...original, + circuitHeld: true, + circuitHeldSince: "2000-01-01T00:00:00Z", + ...(phase === "gateway-drop" + ? { + gatewayInboxes: [original.inbox, "https://gateway.example/inbox"], + } + : {}), + }; + fetchMock.mockGlobal(); + try { + const processing = federation.processQueuedTask(undefined, message); + if (phase === "key-import" || phase === "fanout-parsing") { + await rejects(processing); + } else await processing; + strictEqual(attempts.length, 0); + strictEqual(fetchMock.callHistory.calls().length, 0); + strictEqual(messages.length, 0); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + test("outbound queue identities survive ordinary retries, holds, and native redelivery", async (t) => { for (const observed of [false, true]) { for (const mode of ["retry", "hold", "retry-after", "native"] as const) { @@ -2537,6 +2741,12 @@ test("outbound queue identities survive ordinary retries, holds, and native rede }, }); const message = continuationMessage(recipient.inboxId!.href, observed); + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); const original = structuredClone(message); const task: OutboxMessage = JSON.parse(JSON.stringify(message)); fetchMock.mockGlobal().post( @@ -2599,6 +2809,45 @@ test("outbound queue identities survive ordinary retries, holds, and native rede strictEqual(fetchMock.callHistory.calls().length, requests); } } + strictEqual(attempts.length, mode === "hold" ? 1 : 2); + strictEqual( + new Set(attempts.map((report) => report.attemptId)).size, + attempts.length, + ); + for (const report of attempts) { + strictEqual( + report.destinationInbox, + original.outbound?.destinationInbox, + ); + deepStrictEqual( + report.broadcast, + original.outbound?.broadcast ?? { type: "unknown" }, + ); + strictEqual( + report.provenance.dispatchId, + original.provenance?.dispatchId, + ); + deepStrictEqual( + report.provenance.inboxRequest, + original.provenance?.inboxRequest, + ); + strictEqual(report.local.origin, new URL(message.baseUrl).origin); + ok(report.queue); + strictEqual( + report.deliveryId, + observed + ? original.outbound!.deliveryId + : report.queue.queueMessageId, + ); + ok(Object.isFrozen(report.queue)); + } + strictEqual(attempts[0].queue?.queueAttempt, 0); + if (mode !== "hold") { + strictEqual( + attempts[1].queue?.queueAttempt, + mode === "native" ? 0 : 1, + ); + } deepStrictEqual(task, original); deepStrictEqual(message, original); } finally { @@ -2642,6 +2891,12 @@ test("outbound queue identities retain planned destinations through gateway cont ...continuationMessage(gateways[0], observed, portableInbox), gatewayInboxes: gateways, }; + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); const original = structuredClone(message); const task: OutboxMessage = JSON.parse(JSON.stringify(message)); fetchMock.mockGlobal(); @@ -2720,6 +2975,56 @@ test("outbound queue identities retain planned destinations through gateway cont strictEqual(fetchMock.callHistory.calls().length, requests); } } + strictEqual( + attempts.length, + mode === "native" ? 4 : mode === "retry" ? 5 : 3, + ); + strictEqual( + new Set(attempts.map((report) => report.attemptId)).size, + attempts.length, + ); + for (const report of attempts) { + strictEqual( + report.destinationInbox, + original.outbound?.destinationInbox, + ); + deepStrictEqual( + report.broadcast, + original.outbound?.broadcast ?? { type: "unknown" }, + ); + strictEqual( + report.provenance.dispatchId, + original.provenance?.dispatchId, + ); + deepStrictEqual( + report.provenance.inboxRequest, + original.provenance?.inboxRequest, + ); + strictEqual(report.local.origin, new URL(message.baseUrl).origin); + ok(report.queue); + strictEqual( + report.deliveryId, + observed + ? original.outbound!.deliveryId + : report.queue.queueMessageId, + ); + ok(Object.isFrozen(report.queue)); + } + strictEqual(attempts[0].queue?.queueAttempt, 0); + deepStrictEqual( + attempts.slice(0, 3).map((report) => report.inboxUrl), + gateways, + ); + for (const report of attempts) { + strictEqual(report.portableInbox, portableInbox); + } + if (mode === "retry") { + deepStrictEqual( + attempts.slice(3).map((report) => report.inboxUrl), + gateways.slice(1), + ); + strictEqual(attempts[3].queue?.queueAttempt, 1); + } deepStrictEqual(task, original); deepStrictEqual(message, original); } finally { diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 6e7593583..9987ea895 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -170,10 +170,9 @@ type OutboundAttemptFailure = }; /** - * The terminal result of one immediate concrete send. + * The terminal result of one concrete send. * * A send that fails before submitting HTTP still constitutes an attempt. - * Queued worker sends are not observed yet. * @since 2.5.0 */ export type OutboundAttemptReport = OutboundSourceMetadata & { @@ -183,12 +182,18 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { readonly broadcast: OutboundDeliveryEvent["broadcast"]; /** Destination obligation shared with producer delivery events. */ readonly deliveryId: string; - /** Original planned inbox, independent of the concrete route. */ - readonly destinationInbox: string; + /** Original planned inbox; unavailable on legacy outbox tasks. */ + readonly destinationInbox?: string; /** Canonical portable inbox when the planned destination represents one. */ readonly portableInbox?: string; /** Inbox selected for this invocation of the concrete sender. */ readonly inboxUrl: string; + /** Current queue execution; absent on immediate sends. */ + readonly queue?: { + readonly queueMessageId: string; + /** Present when the queue message carries a Fedify retry counter. */ + readonly queueAttempt?: number; + }; /** Settled execution result, separate from producer preparation. */ readonly outcome: | { @@ -227,7 +232,7 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { */ export interface OutboundObservers { /** - * Observes settlement of immediate sends and forwarding, including pre-fetch + * Observes settlement of concrete sends and forwarding, including pre-fetch * failures. Callback errors cannot replace the sender's result. */ readonly onAttempt?: ( From 208795da67b8075cc5ac0db915723bf814533817 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 03:20:48 +0900 Subject: [PATCH 21/53] Observe unsigned outbound HTTP exchanges Preserve actual redirect and transport facts when observing unsigned outbound HTTP exchanges. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 1 + packages/fedify/src/federation/middleware.ts | 3 +- .../src/federation/outbound-observation.ts | 94 +++++++- .../src/federation/outbound-report.test.ts | 215 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 48 ++++ packages/fedify/src/federation/send.ts | 7 +- 6 files changed, 361 insertions(+), 7 deletions(-) diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 2f3766a6f..40c68b7e0 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -668,6 +668,7 @@ export class FederationBuilderImpl onBroadcast: observers.onBroadcast, onDelivery: observers.onDelivery, onAttempt: observers.onAttempt, + onHttpExchange: observers.onHttpExchange, }); } diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 265697a94..1567bcfe7 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1489,7 +1489,7 @@ export class FederationImpl span: Span, provenance: OutboundProvenance, ): Promise { - const onAttempt = this.outboundObservers.onAttempt; + const { onAttempt, onHttpExchange } = this.outboundObservers; const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { keyIds: message.keys.map((pair) => pair.keyId), @@ -1538,6 +1538,7 @@ export class FederationImpl onAttempt, message, provenance, + onHttpExchange, ); let parsedActorIds: URL[] | undefined; const getActorIds = () => { diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 352487aa6..2bedce29b 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -21,6 +21,7 @@ import type { OutboundAttemptReport, OutboundBroadcastEvent, OutboundDeliveryEvent, + OutboundHttpExchangeReport, OutboundObservers, OutboundPreparationResult, OutboundProvenance, @@ -107,7 +108,7 @@ async function notifyObserver< | ((context: Context, event: TEvent) => void | Promise) | undefined, event: TEvent, - unit: "broadcast" | "delivery" | "attempt", + unit: "broadcast" | "delivery" | "attempt" | "http-exchange", ): Promise { if (callback == null) return; try { @@ -153,6 +154,16 @@ type AttemptFacts = Pick< "attemptId" | "inboxUrl" | "observedAt" | "trace" | "outcome" >; +type ExchangeFacts = Pick< + OutboundHttpExchangeReport, + | "attemptId" + | "inboxUrl" + | "exchangeIndex" + | "observedAt" + | "request" + | "outcome" +>; + /** State owned by one concrete send, never persisted in queue metadata. */ export class OutboundAttemptObservation { readonly #attemptId = crypto.randomUUID(); @@ -160,16 +171,62 @@ export class OutboundAttemptObservation { #response: { readonly status: number } | undefined; #trace: OutboundAttemptReport["trace"]; #finished = false; + #exchangeCount = 0; constructor( readonly inboxUrl: string, readonly report: (facts: AttemptFacts) => Promise, + readonly reportExchange: + | ((facts: ExchangeFacts) => Promise) + | undefined = undefined, ) {} setSpan(span: Span, parentSpanContext?: SpanContext): void { this.#trace = snapshotTrace(span, parentSpanContext); } + async fetch( + request: Request, + submit: () => Promise, + ): Promise { + const exchangeIndex = this.#exchangeCount++; + const snapshot = Object.freeze({ + url: request.url, + method: request.method, + }); + let response: Response; + try { + response = await submit(); + } catch (error) { + await this.#exchange(exchangeIndex, snapshot, { type: "failed", error }); + throw error; + } + await this.#exchange(exchangeIndex, snapshot, { + type: "response", + response: Object.freeze({ + status: response.status, + statusText: response.statusText, + ...(response.url === "" ? {} : { url: response.url }), + }), + }); + return response; + } + + #exchange( + exchangeIndex: number, + request: OutboundHttpExchangeReport["request"], + outcome: OutboundHttpExchangeReport["outcome"], + ): Promise { + return this.reportExchange?.({ + attemptId: this.#attemptId, + inboxUrl: this.inboxUrl, + exchangeIndex, + observedAt: Temporal.Now.instant().epochMilliseconds, + request, + outcome: Object.freeze(outcome), + }) ?? Promise.resolve(); + } + response(status: number): void { this.#response = Object.freeze({ status }); } @@ -247,6 +304,8 @@ export class OutboundAttemptObserver { readonly identity: AttemptIdentity, readonly provenance: OutboundProvenance, queue?: OutboundAttemptReport["queue"], + readonly onHttpExchange: OutboundObservers["onHttpExchange"] = + undefined, ) { this.#queue = queue == null ? undefined : Object.freeze({ ...queue }); } @@ -256,8 +315,9 @@ export class OutboundAttemptObserver { callback: OutboundObservers["onAttempt"], message: OutboxMessage, provenance: OutboundProvenance, + onHttpExchange?: OutboundObservers["onHttpExchange"], ): OutboundAttemptObserver | undefined { - if (callback == null) return undefined; + if (callback == null && onHttpExchange == null) return undefined; const identity = message.outbound == null ? Object.freeze({ broadcast: Object.freeze({ type: "unknown" as const }), @@ -276,11 +336,12 @@ export class OutboundAttemptObserver { queueMessageId: message.id, queueAttempt: message.attempt, }, + onHttpExchange, ); } startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { - if (this.callback == null) return undefined; + if (this.callback == null && this.onHttpExchange == null) return undefined; const context = this.context(); return new OutboundAttemptObservation( inboxUrl, @@ -297,6 +358,19 @@ export class OutboundAttemptObserver { }), "attempt", ), + (facts) => + notifyObserver( + context, + this.onHttpExchange, + Object.freeze({ + ...this.identity, + ...facts, + ...(this.#queue == null ? {} : { queue: this.#queue }), + provenance: this.provenance, + local: snapshotLocalContext(context), + }), + "http-exchange", + ), ); } } @@ -316,6 +390,7 @@ class OutboundDeliveryObservation { readonly provenance: OutboundProvenance, readonly trace: OutboundBroadcastEvent["trace"], onAttempt: OutboundObservers["onAttempt"], + onHttpExchange: OutboundObservers["onHttpExchange"], queue?: OutboundAttemptReport["queue"], ) { this.#reference = snapshotDeliveryIdentity(identity); @@ -325,6 +400,7 @@ class OutboundDeliveryObservation { this.#reference, this.provenance, queue, + onHttpExchange, ); this.#plan = Object.freeze({ actorIds: Object.freeze(Array.from(destination.actorIds)), @@ -416,6 +492,7 @@ export class OutboundBroadcastObservation { readonly #onBroadcast: OutboundObservers["onBroadcast"]; readonly #onDelivery: OutboundObservers["onDelivery"]; readonly #onAttempt: OutboundObservers["onAttempt"]; + readonly #onHttpExchange: OutboundObservers["onHttpExchange"]; readonly #deliveries = new Map< string, OutboundDeliveryObservation @@ -436,6 +513,7 @@ export class OutboundBroadcastObservation { this.#onBroadcast = observers.onBroadcast; this.#onDelivery = observers.onDelivery; this.#onAttempt = observers.onAttempt; + this.#onHttpExchange = observers.onHttpExchange; } /** Continue an existing fanout; do not emit a new broadcast invocation. */ @@ -448,13 +526,18 @@ export class OutboundBroadcastObservation { ): Promise | undefined> { if ( observers.onDelivery == null && observers.onAttempt == null && + observers.onHttpExchange == null && !Object.values(message.inboxes).some((entry) => entry.outbound != null) ) { return undefined; } const observation = new OutboundBroadcastObservation( context, - { onDelivery: observers.onDelivery, onAttempt: observers.onAttempt }, + { + onDelivery: observers.onDelivery, + onAttempt: observers.onAttempt, + onHttpExchange: observers.onHttpExchange, + }, "send", getOutboundProvenance(context), { type: "unknown" }, @@ -498,6 +581,7 @@ export class OutboundBroadcastObservation { this.provenance, this.#trace, this.#onAttempt, + this.#onHttpExchange, this.queue, ); } @@ -703,7 +787,7 @@ export function observeOutboundBroadcast( return withContext(sourceLogContext(provenance), () => { if ( observers.onBroadcast == null && observers.onDelivery == null && - observers.onAttempt == null + observers.onAttempt == null && observers.onHttpExchange == null ) { return execute(); } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 052c1af14..e15849790 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -35,6 +35,7 @@ import type { OutboundAttemptReport, OutboundBroadcastEvent, OutboundDeliveryEvent, + OutboundHttpExchangeReport, OutboundObservers, } from "./mod.ts"; import type { FanoutMessage, OutboxMessage } from "./queue.ts"; @@ -162,6 +163,220 @@ test("outbound observation timestamps snapshot each boundary and refresh on fano } }); +test("outbound exchange-only observers await snapshots across unsigned worker redirects", async () => { + const builder = createFederationBuilder<{ label: string }>(); + const exchanges: OutboundHttpExchangeReport[] = []; + const contexts: unknown[] = []; + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let replacements = 0; + builder.setOutboundObservers({ + async onHttpExchange(ctx, report) { + exchanges.push(report); + contexts.push(ctx.data); + entered(); + await gate; + throw new Error("exchange callback rejection must be isolated"); + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + builder.setOutboundObservers({ + onHttpExchange() { + replacements++; + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + const data = { label: "worker" }; + const next = new URL("/next", message.inbox).href; + const bodies: unknown[] = []; + fetchMock.mockGlobal() + .post(message.inbox, (call) => { + bodies.push(JSON.parse(String(call.options.body))); + return { status: 303, headers: { Location: next } }; + }) + .get(next, 202); + let settled = false; + const processing = federation.processQueuedTask(data, message) + .finally(() => { + settled = true; + }); + try { + await observing; + strictEqual(settled, false); + strictEqual(fetchMock.callHistory.calls().length, 1); + strictEqual(exchanges.length, 1); + strictEqual( + Reflect.set(exchanges[0].request, "url", "https://changed.example"), + false, + ); + federation.setOutboundObservers({ + onHttpExchange() { + replacements++; + }, + }); + release(); + await processing; + strictEqual(replacements, 0); + strictEqual(exchanges.length, 2); + deepStrictEqual(contexts, [data, data]); + deepStrictEqual(bodies, [message.activity]); + deepStrictEqual(exchanges.map((report) => report.exchangeIndex), [0, 1]); + deepStrictEqual(exchanges.map((report) => report.request), [ + { url: message.inbox, method: "POST" }, + { url: next, method: "GET" }, + ]); + strictEqual(exchanges[0].attemptId, exchanges[1].attemptId); + for (const [index, report] of exchanges.entries()) { + strictEqual(report.deliveryId, message.outbound!.deliveryId); + deepStrictEqual(report.broadcast, message.outbound!.broadcast); + deepStrictEqual(report.provenance, message.provenance); + deepStrictEqual(report.queue, { + queueMessageId: message.id, + queueAttempt: message.attempt, + }); + strictEqual(report.outcome.type, "response"); + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, index === 0 ? 303 : 202); + ok(Object.isFrozen(report)); + ok(Object.isFrozen(report.request)); + ok(Object.isFrozen(report.outcome)); + ok(Object.isFrozen(report.outcome.response)); + strictEqual("trace" in report, false); + } + strictEqual(fetchMock.callHistory.calls().length, 2); + } finally { + release(); + await processing; + fetchMock.hardReset(); + } +}); + +test("outbound unsigned exchanges report only actual submissions and retain received responses", async (t) => { + for ( + const phase of [ + "http", + "request", + "policy", + "serialization", + "redirect-policy", + "redirect-cleanup", + "final-cleanup", + ] as const + ) { + await t.step(phase, async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + throw new Error("recorder must not change the exchange result"); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = phase.includes("policy") + ? new UrlError("destination denied") + : undefined; + let submissions = 0; + let closed = 0; + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: phase === "serialization" + ? { + toJSON() { + throw failure; + }, + } + : {}, + keys: [], + inbox: new URL(inbox), + }, { + openHop(url) { + if ( + phase === "policy" || phase === "redirect-policy" && url !== inbox + ) { + return Promise.reject(failure); + } + return Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + if (phase === "request") return Promise.reject(failure); + const redirect = phase.startsWith("redirect-"); + return Promise.resolve( + new Response(null, { + status: redirect ? 307 : phase === "http" ? 503 : 202, + statusText: "Observed response", + headers: redirect ? { Location: "/next" } : {}, + }), + ); + }, + close() { + closed++; + return phase.endsWith("cleanup") + ? Promise.reject(failure) + : Promise.resolve(); + }, + }); + }, + }), + (error: unknown) => { + outward = error; + return true; + }, + ); + strictEqual(attempts.length, 1); + ok(attempts[0].outcome.type === "failed"); + strictEqual(attempts[0].outcome.error, outward); + const count = phase === "policy" || phase === "serialization" ? 0 : 1; + strictEqual(submissions, count); + strictEqual(exchanges.length, count); + if (count === 0) return; + const report = exchanges[0]; + strictEqual(report.attemptId, attempts[0].attemptId); + strictEqual(report.exchangeIndex, 0); + deepStrictEqual(report.request, { url: inbox, method: "POST" }); + if (phase === "request") { + strictEqual(report.outcome.type, "failed"); + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, failure); + } else { + ok(report.outcome.type === "response"); + deepStrictEqual(report.outcome.response, { + status: phase.startsWith("redirect-") + ? 307 + : phase === "http" + ? 503 + : 202, + statusText: "Observed response", + }); + } + ok(closed > 0); + }); + } +}); + test("outbound attempts report ordinary and outbox sends with attempt-only builder registration", async (t) => { for (const kind of ["ordinary", "outbox"] as const) { for (const status of [202, 503, "policy"] as const) { diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 9987ea895..1324bd731 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -214,6 +214,45 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { }; }; +/** + * One actual unsigned fetch submission within a concrete send attempt. + * + * A redirect response remains an observed response even if a later hop fails. + * This report does not establish the attempt's final response. Signed exchanges + * and response body/header capture are not observed yet. + * @since 2.5.0 + */ +export type OutboundHttpExchangeReport = + & OutboundSourceMetadata + & Pick< + OutboundAttemptReport, + | "attemptId" + | "broadcast" + | "deliveryId" + | "destinationInbox" + | "portableInbox" + | "inboxUrl" + | "queue" + > + & { + /** Zero-based submission order within this attempt. */ + readonly exchangeIndex: number; + /** Immutable facts of the request actually submitted. */ + readonly request: { readonly url: string; readonly method: string }; + /** A received response, including non-2xx, or the original fetch rejection. */ + readonly outcome: + | { + readonly type: "response"; + readonly response: { + readonly status: number; + readonly statusText: string; + /** Response URL when supplied by fetch. */ + readonly url?: string; + }; + } + | { readonly type: "failed"; readonly error: unknown }; + }; + /** * Optional observers of outgoing activity processing. * @@ -231,6 +270,15 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { * @since 2.5.0 */ export interface OutboundObservers { + /** + * Observes an unsigned fetch after response headers or rejection, before + * subsequent redirect handling. Callback errors do not replace transport. + */ + readonly onHttpExchange?: ( + context: Context, + report: OutboundHttpExchangeReport, + ) => void | Promise; + /** * Observes settlement of concrete sends and forwarding, including pre-fetch * failures. Callback errors cannot replace the sender's result. diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 07c1841fe..7f7537f8e 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -619,6 +619,7 @@ async function sendActivityInternal( request, openDeliveryHop, consume, + attempt, ) : await doubleKnockScoped( request, @@ -763,6 +764,7 @@ export async function fetchWithValidatedRedirects( request: Request, openHop: DoubleKnockOpenHop, consume: (response: Response) => Promise, + attempt?: OutboundAttemptObservation, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); let body: string | undefined = await request.clone().text(); @@ -771,7 +773,10 @@ export async function fetchWithValidatedRedirects( let response: Response; try { // Bun also needs the redirect option on fetch() itself. - response = await session.fetch(request, { redirect: "manual" }); + response = await (attempt == null + ? session.fetch(request, { redirect: "manual" }) + : attempt.fetch(request, () => + session.fetch(request, { redirect: "manual" }))); } catch (error) { await closeAfterError(session, logger); throw error; From 97b5837689bc6904461e782434943450f1b40edd Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 03:28:09 +0900 Subject: [PATCH 22/53] Observe signed outbound HTTP exchanges Report actual signed requests and responses across challenges, fallbacks, and redirects while preserving original failure evidence. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-report.test.ts | 283 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 16 +- packages/fedify/src/federation/send.ts | 5 + packages/fedify/src/sig/http.test.ts | 52 ++++ packages/fedify/src/sig/http.ts | 129 ++++++-- 5 files changed, 444 insertions(+), 41 deletions(-) diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index e15849790..26f43589a 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -377,6 +377,273 @@ test("outbound unsigned exchanges report only actual submissions and retain rece } }); +test("outbound signed exchanges follow actual challenge, fallback, and redirect submissions", async (t) => { + const challenge = + 'sig1=("@method" "@target-uri");created;nonce="observer-nonce"'; + for ( + const mode of [ + "success", + "fallback", + "challenge", + "challenge-fallback", + "challenge-redirect", + "unfulfillable", + ] as const + ) { + await t.step(mode, async () => { + const inbox = recipient.inboxId!.href; + const next = new URL("/next", inbox).href; + const responses: ResponseInit[] = mode === "success" + ? [{ status: 202 }] + : mode === "fallback" + ? [{ status: 401 }, { status: 202 }] + : mode === "unfulfillable" + ? [ + { + status: 401, + headers: { + "Accept-Signature": 'sig1=("@method" "x-absent");created', + }, + }, + { status: 202 }, + ] + : [ + { status: 401, headers: { "Accept-Signature": challenge } }, + ...(mode === "challenge" + ? [{ status: 202 }] + : mode === "challenge-fallback" + ? [{ status: 401 }, { status: 202 }] + : [{ status: 307, headers: { Location: next } }, { status: 202 }]), + ]; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + throw new Error("observer must not alter signature negotiation"); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const requests: Request[] = []; + const origins: string[] = []; + const payload = { id: "https://example.com/activity", extra: "original" }; + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: payload, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { + determineSpec(origin) { + origins.push(origin); + return "rfc9421"; + }, + rememberSpec() {}, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch(input) { + ok(input instanceof Request); + const response = responses[requests.length]; + requests.push(input); + ok( + response, + "observation must not create extra HTTP submissions", + ); + return Promise.resolve(new Response(null, response)); + }, + close: () => Promise.resolve(), + }), + }); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(exchanges.length, responses.length); + strictEqual(requests.length, responses.length); + strictEqual(origins.length, mode === "challenge-redirect" ? 2 : 1); + for (const [index, report] of exchanges.entries()) { + strictEqual(report.exchangeIndex, index); + strictEqual(report.attemptId, attempts[0].attemptId); + deepStrictEqual(report.request, { + url: requests[index].url, + method: requests[index].method, + }); + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, responses[index].status); + deepStrictEqual( + JSON.parse(await requests[index].clone().text()), + payload, + ); + ok(requests[index].headers.has("Signature")); + } + ok(requests[0].headers.has("Signature-Input")); + if (mode.startsWith("challenge")) { + ok( + requests[1].headers.get("Signature-Input")?.includes( + 'nonce="observer-nonce"', + ), + ); + } + if ( + mode === "fallback" || mode === "challenge-fallback" || + mode === "unfulfillable" + ) { + strictEqual(requests.at(-1)!.headers.has("Signature-Input"), false); + } + if (mode === "challenge-redirect") strictEqual(requests[2].url, next); + }); + } +}); + +test("outbound signed exchanges preserve earlier responses when subsequent operations fail", async (t) => { + for ( + const phase of [ + "spec-lookup", + "signing", + "later-spec-lookup", + "later-signing", + "post-response", + "request", + "handled-signing", + ] as const + ) { + await t.step(phase, async () => { + const inbox = recipient.inboxId!.href; + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = Symbol(phase); + const keyId = new URL(sender.keyId); + const href = keyId.href; + let submissions = 0; + let lookups = 0; + Object.defineProperty(keyId, "href", { + get() { + if ( + phase === "signing" || phase === "later-signing" && submissions > 0 + ) throw failure; + return href; + }, + }); + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [{ ...sender, keyId }], + inbox: new URL(inbox), + specDeterminer: { + determineSpec() { + lookups++; + if ( + phase === "spec-lookup" || + phase === "later-spec-lookup" && lookups > 1 + ) throw failure; + return "rfc9421"; + }, + rememberSpec() { + if (phase === "post-response") return Promise.reject(failure); + }, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + if ( + phase === "request" || + phase === "handled-signing" && submissions > 1 + ) return Promise.reject(failure); + return Promise.resolve( + new Response(null, { + status: phase === "later-spec-lookup" + ? 307 + : phase === "post-response" + ? 202 + : 401, + headers: phase === "later-spec-lookup" + ? { Location: "/next" } + : phase === "handled-signing" + ? { + "Accept-Signature": + 'sig1=("@method" "x-absent");created', + } + : {}, + }), + ); + }, + close: () => Promise.resolve(), + }), + }), + (error: unknown) => { + outward = error; + return error instanceof FetchError; + }, + ); + strictEqual(exchanges.length, submissions); + strictEqual(attempts.length, 1); + const outcome = attempts[0].outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + strictEqual(outcome.failure.error, failure); + const transport = phase === "request" || phase === "handled-signing"; + strictEqual(outcome.failure.kind, transport ? "transport" : "local"); + strictEqual( + outcome.failure.phase, + transport ? "request" : phase.replace("later-", ""), + ); + const count = phase === "spec-lookup" || phase === "signing" + ? 0 + : phase === "handled-signing" + ? 2 + : 1; + strictEqual(submissions, count); + for (const [index, report] of exchanges.entries()) { + strictEqual(report.attemptId, attempts[0].attemptId); + strictEqual(report.exchangeIndex, index); + if (transport && index === count - 1) { + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, failure); + } else { + ok(report.outcome.type === "response"); + strictEqual( + report.outcome.response.status, + phase === "later-spec-lookup" + ? 307 + : phase === "post-response" + ? 202 + : 401, + ); + } + } + }); + } +}); + test("outbound attempts report ordinary and outbox sends with attempt-only builder registration", async (t) => { for (const kind of ["ordinary", "outbox"] as const) { for (const status of [202, 503, "policy"] as const) { @@ -557,7 +824,7 @@ test("outbound attempts preserve boundary failures and allocate fresh IDs per co "request", "cleanup", "http-cleanup", - "signing", + "spec-lookup", ] as const ) { await t.step(phase, async () => { @@ -595,7 +862,7 @@ test("outbound attempts preserve boundary failures and allocate fresh IDs per co }, } : {}, - keys: phase === "signing" ? [sender] : [], + keys: phase === "spec-lookup" ? [sender] : [], inbox: new URL(inbox), specDeterminer: { determineSpec() { @@ -663,16 +930,8 @@ test("outbound attempts preserve boundary failures and allocate fresh IDs per co deepStrictEqual(report.outcome.response, { status: 503 }); } else { deepStrictEqual(report.outcome.failure, { - kind: phase === "dns" || phase === "request" - ? "transport" - : phase === "signing" - ? "unknown" - : "local", - phase: phase === "dns" - ? "validation" - : phase === "signing" - ? "unknown" - : phase, + kind: phase === "dns" || phase === "request" ? "transport" : "local", + phase: phase === "dns" ? "validation" : phase, error: failure, }); } diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 1324bd731..0301dc84c 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -160,7 +160,13 @@ type OutboundAttemptFailure = } | { readonly kind: "local"; - readonly phase: "serialization" | "request-preparation" | "cleanup"; + readonly phase: + | "serialization" + | "request-preparation" + | "cleanup" + | "signing" + | "spec-lookup" + | "post-response"; readonly error: unknown; } | { @@ -215,11 +221,11 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { }; /** - * One actual unsigned fetch submission within a concrete send attempt. + * One actual fetch submission within a concrete send attempt. * * A redirect response remains an observed response even if a later hop fails. - * This report does not establish the attempt's final response. Signed exchanges - * and response body/header capture are not observed yet. + * This report does not establish the attempt's final response. Response + * body/header capture is not observed yet. * @since 2.5.0 */ export type OutboundHttpExchangeReport = @@ -271,7 +277,7 @@ export type OutboundHttpExchangeReport = */ export interface OutboundObservers { /** - * Observes an unsigned fetch after response headers or rejection, before + * Observes a fetch after response headers or rejection, before * subsequent redirect handling. Callback errors do not replace transport. */ readonly onHttpExchange?: ( diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 7f7537f8e..85ac91e2a 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -627,6 +627,11 @@ async function sendActivityInternal( { tracerProvider, specDeterminer }, openDeliveryHop, consume, + attempt == null ? undefined : { + fetch: (request, submit) => attempt.fetch(request, submit), + error: (phase, error) => + attempt.failure({ kind: "local", phase, error }), + }, ); } catch (error) { attempt?.unknown(error); diff --git a/packages/fedify/src/sig/http.test.ts b/packages/fedify/src/sig/http.test.ts index 1ef7cfd82..1431b8e08 100644 --- a/packages/fedify/src/sig/http.test.ts +++ b/packages/fedify/src/sig/http.test.ts @@ -3280,6 +3280,58 @@ const lifecycleIdentity = { privateKey: rsaPrivateKey2, }; +test("doubleKnockScoped() observes each actual GET and HEAD transport retry", async (t) => { + for (const method of ["GET", "HEAD"]) { + await t.step(method, async () => { + const url = "https://example.com/transport-observation"; + const observed: Request[] = []; + let submissions = 0; + let closes = 0; + const result = await doubleKnockScoped( + new Request(url, { method }), + { + keyId: rsaPublicKey2.id!, + privateKey: rsaPrivateKey2, + }, + {}, + () => + Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return submissions === 1 + ? Promise.reject(new TypeError("temporary failure")) + : Promise.resolve(new Response(null, { status: 202 })); + }, + close() { + closes++; + return Promise.resolve(); + }, + }), + (response) => Promise.resolve(response.status), + { + fetch(request, submit) { + observed.push(request); + return submit(); + }, + error() { + throw new Error("transport retry must not be a local failure"); + }, + }, + ); + assertEquals(result, 202); + assertEquals(submissions, 2); + assertEquals(observed.length, 2); + assert( + observed[0] === observed[1], + "transport retry reuses the signed request", + ); + assertEquals(observed[0].method, method); + assertEquals(closes, 1); + }); + } +}); + test("doubleKnockScoped() keeps challenge retries and fallback in the validated hop", async () => { const url = "https://example.com/lifecycle/challenge-fallback"; const hops = recordHops((_, index): RecordedResponseInit => diff --git a/packages/fedify/src/sig/http.ts b/packages/fedify/src/sig/http.ts index ff825fbff..4f54a29c3 100644 --- a/packages/fedify/src/sig/http.ts +++ b/packages/fedify/src/sig/http.ts @@ -2110,24 +2110,67 @@ async function createRedirectRequest( }); } +/** In-process hooks for scoped outbound delivery, outside the public options. */ +export interface DoubleKnockObservation { + fetch(request: Request, submit: () => Promise): Promise; + error( + phase: "signing" | "spec-lookup" | "post-response", + error: unknown, + ): void; +} + +function observeDoubleKnockOperation( + observation: DoubleKnockObservation | undefined, + phase: "signing" | "spec-lookup" | "post-response", + operation: () => T | Promise, +): T | Promise { + if (observation == null) return operation(); + try { + return Promise.resolve(operation()).catch((error) => { + observation.error(phase, error); + throw error; + }); + } catch (error) { + observation.error(phase, error); + throw error; + } +} + +function submitDoubleKnockRequest( + session: PublicFetchSession, + request: Request, + signal: AbortSignal | undefined, + observation: DoubleKnockObservation | undefined, +): Promise { + const submit = () => + session.fetch(request, { + // Since Bun has a bug that ignores the `Request.redirect` option, + // to work around it we specify `redirect: "manual"` here too: + // https://github.com/oven-sh/bun/issues/10754 + redirect: "manual", + signal, + }); + return observation == null ? submit() : observation.fetch(request, submit); +} + async function fetchDoubleKnockRequest( session: PublicFetchSession, request: Request, signedRequest: Request, - signal?: AbortSignal, + signal: AbortSignal | undefined, + observation?: DoubleKnockObservation, ): Promise { const maxAttempts = request.method === "GET" || request.method === "HEAD" ? 2 : 1; for (let attempt = 1;; attempt++) { try { - return await session.fetch(signedRequest, { - // Since Bun has a bug that ignores the `Request.redirect` option, - // to work around it we specify `redirect: "manual"` here too: - // https://github.com/oven-sh/bun/issues/10754 - redirect: "manual", + return await submitDoubleKnockRequest( + session, + signedRequest, signal, - }); + observation, + ); } catch (error) { const abortedSignal = getAbortedSignal( signal, @@ -2269,12 +2312,14 @@ export async function doubleKnockScoped( options: DoubleKnockOptions, openHop: DoubleKnockOpenHop, consume: (response: Response) => Promise, + observation?: DoubleKnockObservation, ): Promise { const { response, session } = await doubleKnockCore( request, identity, options, openHop, + observation, ); try { return await consume(response); @@ -2332,6 +2377,7 @@ async function doubleKnockCore( identity: { keyId: URL; privateKey: CryptoKey }, options: DoubleKnockOptions, openHop: DoubleKnockOpenHop, + observation?: DoubleKnockObservation, ): Promise { const { specDeterminer, log, tracerProvider, signal } = options; signal?.throwIfAborted(); @@ -2352,13 +2398,22 @@ async function doubleKnockCore( const origin = new URL(request.url).origin; const firstTrySpec: HttpMessageSignaturesSpec = specDeterminer == null ? "rfc9421" - : await specDeterminer.determineSpec(origin); + : await observeDoubleKnockOperation( + observation, + "spec-lookup", + () => specDeterminer.determineSpec(origin), + ); signal?.throwIfAborted(); - let signedRequest = await signRequest( - request, - identity.privateKey, - identity.keyId, - { spec: firstTrySpec, tracerProvider, body }, + let signedRequest = await observeDoubleKnockOperation( + observation, + "signing", + () => + signRequest( + request, + identity.privateKey, + identity.keyId, + { spec: firstTrySpec, tracerProvider, body }, + ), ); signal?.throwIfAborted(); log?.(signedRequest); @@ -2367,6 +2422,7 @@ async function doubleKnockCore( request, signedRequest, signal, + observation, ); const firstRedirected = isRedirectResponse(response); if ( @@ -2436,14 +2492,22 @@ async function doubleKnockCore( cancelResponseBody(response, request.url); signedRequest = challengeRequest; log?.(signedRequest); - response = await session.fetch(signedRequest, { - redirect: "manual", + response = await submitDoubleKnockRequest( + session, + signedRequest, signal, - }); + observation, + ); } if (fulfilled && !isRedirectResponse(response)) { if (response.status < 300) { - await specDeterminer?.rememberSpec(origin, "rfc9421"); + await observeDoubleKnockOperation( + observation, + "post-response", + () => + specDeterminer?.rememberSpec(origin, "rfc9421") ?? + Promise.resolve(), + ); return { response, session }; } if (response.status !== 400 && response.status !== 401) { @@ -2471,11 +2535,16 @@ async function doubleKnockCore( // The retry goes to the same URL, so it uses the same session: cancelResponseBody(response, request.url); signal?.throwIfAborted(); - signedRequest = await signRequest( - request, - identity.privateKey, - identity.keyId, - { spec, tracerProvider, body }, + signedRequest = await observeDoubleKnockOperation( + observation, + "signing", + () => + signRequest( + request, + identity.privateKey, + identity.keyId, + { spec, tracerProvider, body }, + ), ); signal?.throwIfAborted(); log?.(signedRequest); @@ -2484,16 +2553,28 @@ async function doubleKnockCore( request, signedRequest, signal, + observation, ); if ( !isRedirectResponse(response) && response.status !== 400 && response.status !== 401 ) { - await specDeterminer?.rememberSpec(origin, spec); + await observeDoubleKnockOperation( + observation, + "post-response", + () => + specDeterminer?.rememberSpec(origin, spec) ?? Promise.resolve(), + ); } } } else if (!firstRedirected) { - await specDeterminer?.rememberSpec(origin, firstTrySpec); + await observeDoubleKnockOperation( + observation, + "post-response", + () => + specDeterminer?.rememberSpec(origin, firstTrySpec) ?? + Promise.resolve(), + ); } if (!isRedirectResponse(response)) return { response, session }; From 99f945b50176a0f1d4e9c51fad0b91498517af23 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 03:37:26 +0900 Subject: [PATCH 23/53] Preserve outbound response finality Each fetch response is an observed fact, but a challenge or redirect can lead to another request. Select the final response in HTTP control flow so a later signing failure cannot promote an intermediate response, and cache or cleanup failures cannot erase selected HTTP evidence. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-observation.ts | 56 ++- .../src/federation/outbound-report.test.ts | 324 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 37 +- .../src/federation/portable-delivery.test.ts | 17 +- .../src/federation/portable-inbox.test.ts | 12 +- packages/fedify/src/federation/send.ts | 5 +- packages/fedify/src/sig/http.test.ts | 1 + packages/fedify/src/sig/http.ts | 13 +- 8 files changed, 424 insertions(+), 41 deletions(-) diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 2bedce29b..b6afcb9d1 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -151,7 +151,12 @@ type AttemptFailure = Extract< >["failure"]; type AttemptFacts = Pick< OutboundAttemptReport, - "attemptId" | "inboxUrl" | "observedAt" | "trace" | "outcome" + | "attemptId" + | "inboxUrl" + | "httpExchangeCount" + | "observedAt" + | "trace" + | "outcome" >; type ExchangeFacts = Pick< @@ -164,11 +169,17 @@ type ExchangeFacts = Pick< | "outcome" >; +type AttemptResponse = OutboundAttemptReport["outcome"]["response"]; +type FinalResponse = Extract; +type ObservedResponse = Pick; + /** State owned by one concrete send, never persisted in queue metadata. */ export class OutboundAttemptObservation { readonly #attemptId = crypto.randomUUID(); readonly #failures: AttemptFailure[] = []; - #response: { readonly status: number } | undefined; + #response: AttemptResponse = Object.freeze({ type: "not-observed" }); + #finalResponse: FinalResponse | undefined; + readonly #responses = new WeakMap(); #trace: OutboundAttemptReport["trace"]; #finished = false; #exchangeCount = 0; @@ -201,13 +212,20 @@ export class OutboundAttemptObservation { await this.#exchange(exchangeIndex, snapshot, { type: "failed", error }); throw error; } + const metadata = Object.freeze({ + status: response.status, + statusText: response.statusText, + ...(response.url === "" ? {} : { url: response.url }), + }); + const observed = Object.freeze({ exchangeIndex, metadata }); + this.#responses.set(response, observed); + this.#response = Object.freeze({ + type: "undetermined", + lastObserved: observed, + }); await this.#exchange(exchangeIndex, snapshot, { type: "response", - response: Object.freeze({ - status: response.status, - statusText: response.statusText, - ...(response.url === "" ? {} : { url: response.url }), - }), + response: metadata, }); return response; } @@ -227,8 +245,18 @@ export class OutboundAttemptObservation { }) ?? Promise.resolve(); } - response(status: number): void { - this.#response = Object.freeze({ status }); + response(response: Response, disposition: "final" | "intermediate"): void { + const observed = this.#responses.get(response); + if (observed == null) return; + if (disposition === "final") { + this.#finalResponse = Object.freeze({ type: "final", ...observed }); + this.#response = this.#finalResponse; + } else { + this.#response = Object.freeze({ + type: "intermediate-only", + lastObserved: observed, + }); + } } failure(failure: AttemptFailure): void { @@ -246,15 +274,16 @@ export class OutboundAttemptObservation { } succeeded(): Promise { - return this.#finish({ type: "succeeded", response: this.#response! }); + return this.#finish({ type: "succeeded", response: this.#finalResponse! }); } failed(error: unknown): Promise { let failure: AttemptFailure; let diagnostics: AttemptFailure[]; if ( - this.#response != null && - (this.#response.status < 200 || this.#response.status >= 300) + this.#response.type === "final" && + (this.#response.metadata.status < 200 || + this.#response.metadata.status >= 300) ) { failure = this.#failures.find((failure) => failure.kind === "http") ?? Object.freeze({ kind: "http", phase: "response" }); @@ -269,7 +298,7 @@ export class OutboundAttemptObservation { type: "failed", failure, error, - ...(this.#response == null ? {} : { response: this.#response }), + response: this.#response, ...(diagnostics.length === 0 ? {} : { diagnostics: Object.freeze(diagnostics) }), @@ -282,6 +311,7 @@ export class OutboundAttemptObservation { return this.report({ attemptId: this.#attemptId, inboxUrl: this.inboxUrl, + httpExchangeCount: this.#exchangeCount, observedAt: Temporal.Now.instant().epochMilliseconds, ...(this.#trace == null ? {} : { trace: this.#trace }), outcome: Object.freeze(outcome), diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 26f43589a..1ace9c3db 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -352,6 +352,23 @@ test("outbound unsigned exchanges report only actual submissions and retain rece const count = phase === "policy" || phase === "serialization" ? 0 : 1; strictEqual(submissions, count); strictEqual(exchanges.length, count); + strictEqual(attempts[0].httpExchangeCount, count); + const evidence = attempts[0].outcome.response; + if (count === 0 || phase === "request") { + deepStrictEqual(evidence, { type: "not-observed" }); + } else if (phase.startsWith("redirect-")) { + deepStrictEqual(evidence, { + type: "intermediate-only", + lastObserved: { + exchangeIndex: 0, + metadata: { status: 307, statusText: "Observed response" }, + }, + }); + } else { + ok(evidence.type === "final"); + strictEqual(evidence.exchangeIndex, 0); + strictEqual(evidence.metadata.status, phase === "http" ? 503 : 202); + } if (count === 0) return; const report = exchanges[0]; strictEqual(report.attemptId, attempts[0].attemptId); @@ -469,6 +486,13 @@ test("outbound signed exchanges follow actual challenge, fallback, and redirect strictEqual(attempts[0].outcome.type, "succeeded"); strictEqual(exchanges.length, responses.length); strictEqual(requests.length, responses.length); + strictEqual(attempts[0].httpExchangeCount, requests.length); + const final = attempts[0].outcome.response; + ok(final.type === "final"); + strictEqual(final.exchangeIndex, requests.length - 1); + ok(Object.isFrozen(final)); + ok(Object.isFrozen(final.metadata)); + strictEqual(final.metadata.status, 202); strictEqual(origins.length, mode === "challenge-redirect" ? 2 : 1); for (const [index, report] of exchanges.entries()) { strictEqual(report.exchangeIndex, index); @@ -622,6 +646,19 @@ test("outbound signed exchanges preserve earlier responses when subsequent opera ? 2 : 1; strictEqual(submissions, count); + strictEqual(attempts[0].httpExchangeCount, count); + if (phase === "post-response") { + ok(outcome.response.type === "final"); + strictEqual(outcome.response.exchangeIndex, 0); + strictEqual(outcome.response.metadata.status, 202); + } else if (phase.startsWith("later-") || phase === "handled-signing") { + ok(outcome.response.type === "intermediate-only"); + strictEqual(outcome.response.lastObserved.exchangeIndex, 0); + strictEqual( + outcome.response.lastObserved.metadata.status, + phase === "later-spec-lookup" ? 307 : 401, + ); + } else deepStrictEqual(outcome.response, { type: "not-observed" }); for (const [index, report] of exchanges.entries()) { strictEqual(report.attemptId, attempts[0].attemptId); strictEqual(report.exchangeIndex, index); @@ -644,6 +681,261 @@ test("outbound signed exchanges preserve earlier responses when subsequent opera } }); +test("outbound attempt finality preserves HTTP, cache, and cleanup failures together", async (t) => { + for (const status of [202, 503]) { + for (const phase of ["cache", "cleanup", "both"] as const) { + await t.step(`${status}: ${phase}`, async () => { + const inbox = recipient.inboxId!.href; + const reports: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + reports.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const cacheError = Symbol("cache failure"); + const cleanupError = Symbol("cleanup failure"); + let submissions = 0; + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() { + if (phase !== "cleanup") throw cacheError; + }, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + return Promise.resolve( + new Response(null, { + status: ++submissions === 1 ? 401 : status, + statusText: "Observed response", + }), + ); + }, + close() { + return phase === "cache" + ? Promise.resolve() + : Promise.reject(cleanupError); + }, + }), + }), + (error: unknown) => { + outward = error; + return error instanceof FetchError; + }, + ); + strictEqual(reports.length, 1); + strictEqual(submissions, 2); + const report = reports[0]; + strictEqual(report.httpExchangeCount, 2); + const outcome = report.outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + strictEqual( + (outward as FetchError).cause, + phase === "cleanup" ? cleanupError : cacheError, + ); + deepStrictEqual(outcome.response, { + type: "final", + exchangeIndex: 1, + metadata: { status, statusText: "Observed response" }, + }); + const diagnostics = []; + if (phase !== "cleanup") { + diagnostics.push({ + kind: "local", + phase: "post-response", + error: cacheError, + }); + } + if (phase !== "cache") { + diagnostics.push({ + kind: "local", + phase: "cleanup", + error: cleanupError, + }); + } + if (status === 503) { + deepStrictEqual(outcome.failure, { kind: "http", phase: "response" }); + deepStrictEqual(outcome.diagnostics, diagnostics); + } else { + deepStrictEqual(outcome.failure, diagnostics[0]); + deepStrictEqual( + outcome.diagnostics, + diagnostics.length > 1 ? diagnostics.slice(1) : undefined, + ); + } + ok(Object.isFrozen(outcome.response)); + ok(Object.isFrozen(outcome.response.metadata)); + if (outcome.diagnostics != null) { + ok(Object.isFrozen(outcome.diagnostics)); + } + strictEqual(exchanges.length, 2); + ok(exchanges[1].outcome.type === "response"); + deepStrictEqual( + exchanges[1].outcome.response, + outcome.response.metadata, + ); + }); + } + } +}); + +test("outbound attempt finality follows branch decisions rather than status or last index", async (t) => { + for ( + const mode of [ + "challenge-cache", + "final-401", + "unsigned-300", + "signed-300", + "undetermined", + ] as const + ) { + await t.step(mode, async () => { + const inbox = recipient.inboxId!.href; + const reports: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + reports.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = Symbol(mode); + const denied = new UrlError("redirect denied"); + const keyId = new URL(sender.keyId); + const href = keyId.href; + let submissions = 0; + let cacheWrites = 0; + Object.defineProperty(keyId, "href", { + get() { + if (mode === "undetermined" && submissions > 0) throw failure; + return href; + }, + }); + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: mode === "unsigned-300" ? [] : [{ ...sender, keyId }], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() { + cacheWrites++; + if (mode === "challenge-cache") throw failure; + }, + }, + }, { + openHop(url) { + if (url !== inbox) return Promise.reject(denied); + return Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return Promise.resolve( + new Response(null, { + status: mode.endsWith("300") + ? 300 + : mode === "challenge-cache" && submissions > 1 + ? 202 + : 401, + statusText: "Observed response", + headers: mode.endsWith("300") + ? { Location: "/next" } + : mode === "challenge-cache" || mode === "undetermined" + ? { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created', + } + : {}, + }), + ); + }, + close: () => Promise.resolve(), + }); + }, + }), + (error: unknown) => { + outward = error; + return true; + }, + ); + strictEqual(reports.length, 1); + const report = reports[0]; + strictEqual(report.httpExchangeCount, submissions); + strictEqual( + submissions, + mode === "challenge-cache" || mode === "final-401" ? 2 : 1, + ); + strictEqual(exchanges.length, submissions); + const outcome = report.outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + const evidence = outcome.response; + if (mode === "signed-300" || mode === "undetermined") { + ok( + evidence.type === + (mode === "signed-300" ? "intermediate-only" : "undetermined"), + ); + strictEqual(evidence.lastObserved.exchangeIndex, 0); + strictEqual( + evidence.lastObserved.metadata.status, + mode === "signed-300" ? 300 : 401, + ); + strictEqual( + outcome.failure.kind, + mode === "signed-300" ? "policy" : "unknown", + ); + } else { + ok(evidence.type === "final"); + strictEqual(evidence.exchangeIndex, submissions - 1); + strictEqual( + evidence.metadata.status, + mode === "challenge-cache" ? 202 : mode === "final-401" ? 401 : 300, + ); + strictEqual( + outcome.failure.kind, + mode === "challenge-cache" ? "local" : "http", + ); + } + if (mode === "final-401") strictEqual(cacheWrites, 0); + if (mode === "challenge-cache" || mode === "undetermined") { + strictEqual(outcome.failure.error, failure); + } + }); + } +}); + test("outbound attempts report ordinary and outbox sends with attempt-only builder registration", async (t) => { for (const kind of ["ordinary", "outbox"] as const) { for (const status of [202, 503, "policy"] as const) { @@ -726,11 +1018,15 @@ test("outbound attempts report ordinary and outbox sends with attempt-only build traceId: span.spanContext().traceId, spanId: span.spanContext().spanId, }); + strictEqual( + report.httpExchangeCount, + status === "policy" ? 0 : status === 202 ? 1 : 2, + ); if (status === 202) { - deepStrictEqual(report.outcome, { - type: "succeeded", - response: { status }, - }); + strictEqual(report.outcome.type, "succeeded"); + ok(report.outcome.response.type === "final"); + strictEqual(report.outcome.response.metadata.status, status); + strictEqual(report.outcome.response.exchangeIndex, 0); } else { ok(report.outcome.type === "failed"); strictEqual(report.outcome.error, outward); @@ -740,8 +1036,14 @@ test("outbound attempts report ordinary and outbox sends with attempt-only build status === "policy" ? "policy" : "http", ); if (status === "policy") { - strictEqual(report.outcome.response, undefined); - } else deepStrictEqual(report.outcome.response, { status }); + deepStrictEqual(report.outcome.response, { + type: "not-observed", + }); + } else { + ok(report.outcome.response.type === "final"); + strictEqual(report.outcome.response.metadata.status, status); + strictEqual(report.outcome.response.exchangeIndex, 1); + } } } finally { fetchMock.hardReset(); @@ -927,7 +1229,11 @@ test("outbound attempts preserve boundary failures and allocate fresh IDs per co phase: "cleanup", error: failure, }]); - deepStrictEqual(report.outcome.response, { status: 503 }); + deepStrictEqual(report.outcome.response, { + type: "final", + exchangeIndex: 0, + metadata: { status: 503, statusText: "" }, + }); } else { deepStrictEqual(report.outcome.failure, { kind: phase === "dns" || phase === "request" ? "transport" : "local", @@ -2471,7 +2777,9 @@ test("forwarding observers retain readiness and registration through late immedi attempt.outcome.type, status === 202 ? "succeeded" : "failed", ); - deepStrictEqual(attempt.outcome.response, { status }); + ok(attempt.outcome.response.type === "final"); + strictEqual(attempt.outcome.response.metadata.status, status); + strictEqual(attempt.httpExchangeCount, status === 202 ? 1 : 2); if (attempt.outcome.type === "failed") { strictEqual(attempt.outcome.failure.kind, "http"); strictEqual(attempt.outcome.failure.error, attempt.outcome.error); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 0301dc84c..78c2559fb 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -175,6 +175,28 @@ type OutboundAttemptFailure = readonly error: unknown; }; +interface OutboundHttpResponseMetadata { + readonly status: number; + readonly statusText: string; + /** Response URL when supplied by fetch. */ + readonly url?: string; +} + +type OutboundAttemptResponse = + | { + readonly type: "final"; + readonly exchangeIndex: number; + readonly metadata: OutboundHttpResponseMetadata; + } + | { readonly type: "not-observed" } + | { + readonly type: "intermediate-only" | "undetermined"; + readonly lastObserved: { + readonly exchangeIndex: number; + readonly metadata: OutboundHttpResponseMetadata; + }; + }; + /** * The terminal result of one concrete send. * @@ -194,6 +216,8 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { readonly portableInbox?: string; /** Inbox selected for this invocation of the concrete sender. */ readonly inboxUrl: string; + /** Actual fetch submissions, including challenges and redirects; may be zero. */ + readonly httpExchangeCount: number; /** Current queue execution; absent on immediate sends. */ readonly queue?: { readonly queueMessageId: string; @@ -205,7 +229,7 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { | { readonly type: "succeeded"; /** Final response selected by the sender. */ - readonly response: { readonly status: number }; + readonly response: Extract; } | { readonly type: "failed"; @@ -213,8 +237,8 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { readonly failure: OutboundAttemptFailure; /** Exact value escaping the sender, possibly different from failure.error. */ readonly error: unknown; - /** Final response when the sender reached final-response consumption. */ - readonly response?: { readonly status: number }; + /** Explicit final or last-observed response evidence, including absence. */ + readonly response: OutboundAttemptResponse; /** Additional failures, without replacing the primary evidence. */ readonly diagnostics?: readonly OutboundAttemptFailure[]; }; @@ -249,12 +273,7 @@ export type OutboundHttpExchangeReport = readonly outcome: | { readonly type: "response"; - readonly response: { - readonly status: number; - readonly statusText: string; - /** Response URL when supplied by fetch. */ - readonly url?: string; - }; + readonly response: OutboundHttpResponseMetadata; } | { readonly type: "failed"; readonly error: unknown }; }; diff --git a/packages/fedify/src/federation/portable-delivery.test.ts b/packages/fedify/src/federation/portable-delivery.test.ts index f41b41e60..662ab15de 100644 --- a/packages/fedify/src/federation/portable-delivery.test.ts +++ b/packages/fedify/src/federation/portable-delivery.test.ts @@ -641,10 +641,19 @@ test("portable fallback attempts retain one delivery across concrete gateway sen reports[1].outcome.type, accepted ? "succeeded" : "failed", ); - deepStrictEqual(reports.map((report) => report.outcome.response), [ - { status: 503 }, - { status: accepted ? 202 : 502 }, - ]); + for (const [index, report] of reports.entries()) { + const response = report.outcome.response; + ok(response.type === "final"); + strictEqual( + response.metadata.status, + index === 0 ? 503 : accepted ? 202 : 502, + ); + strictEqual( + report.httpExchangeCount, + index === 1 && accepted ? 1 : 2, + ); + strictEqual(response.exchangeIndex, report.httpExchangeCount - 1); + } if (operation !== "send") { for (const payload of submitted) deepStrictEqual(payload, raw); } diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index 5919ccfae..d4dde27dc 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -2665,7 +2665,9 @@ test("portable replication attempts settle independently after the forwarding de // The pending gateway has no synthetic timeout attempt outcome. assertEquals(attempts.length, 1); assertEquals(attempts[0].outcome.type, "failed"); - assertEquals(attempts[0].outcome.response, { status: 503 }); + assert(attempts[0].outcome.response.type === "final"); + assertEquals(attempts[0].outcome.response.metadata.status, 503); + assertEquals(attempts[0].httpExchangeCount, 1); federation.setOutboundObservers({ onAttempt() { replacements++; @@ -2674,10 +2676,10 @@ test("portable replication attempts settle independently after the forwarding de release(); await Promise.all([successReported, concreteSendsEnded]); assertEquals(attempts.length, 2); - assertEquals(attempts[1].outcome, { - type: "succeeded", - response: { status: 202 }, - }); + assertEquals(attempts[1].outcome.type, "succeeded"); + assert(attempts[1].outcome.response.type === "final"); + assertEquals(attempts[1].outcome.response.metadata.status, 202); + assertEquals(attempts[1].httpExchangeCount, 1); assertEquals(replacements, 0); assert(attempts[0].attemptId !== attempts[1].attemptId); assert(attempts[0].deliveryId !== attempts[1].deliveryId); diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 85ac91e2a..c06c8fa33 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -609,7 +609,6 @@ async function sendActivityInternal( ); } const consume = (response: Response) => { - attempt?.response(response.status); return consumeSendActivityResponse(response, inbox); }; let outcome: SendActivityOutcome; @@ -629,6 +628,8 @@ async function sendActivityInternal( consume, attempt == null ? undefined : { fetch: (request, submit) => attempt.fetch(request, submit), + response: (response, disposition) => + attempt.response(response, disposition), error: (phase, error) => attempt.failure({ kind: "local", phase, error }), }, @@ -790,6 +791,7 @@ export async function fetchWithValidatedRedirects( if ( ![301, 302, 303, 307, 308].includes(response.status) || location == null ) { + attempt?.response(response, "final"); try { return await consume(response); } catch (error) { @@ -804,6 +806,7 @@ export async function fetchWithValidatedRedirects( } } // The current hop is closed before the next one is validated and opened: + attempt?.response(response, "intermediate"); cancelResponseBody(response, request.url); await session.close(); if (redirects >= 20) throw new TypeError("Too many redirects"); diff --git a/packages/fedify/src/sig/http.test.ts b/packages/fedify/src/sig/http.test.ts index 1431b8e08..d84fa379e 100644 --- a/packages/fedify/src/sig/http.test.ts +++ b/packages/fedify/src/sig/http.test.ts @@ -3314,6 +3314,7 @@ test("doubleKnockScoped() observes each actual GET and HEAD transport retry", as observed.push(request); return submit(); }, + response() {}, error() { throw new Error("transport retry must not be a local failure"); }, diff --git a/packages/fedify/src/sig/http.ts b/packages/fedify/src/sig/http.ts index 4f54a29c3..ecc9da6fe 100644 --- a/packages/fedify/src/sig/http.ts +++ b/packages/fedify/src/sig/http.ts @@ -2113,6 +2113,7 @@ async function createRedirectRequest( /** In-process hooks for scoped outbound delivery, outside the public options. */ export interface DoubleKnockObservation { fetch(request: Request, submit: () => Promise): Promise; + response(response: Response, disposition: "final" | "intermediate"): void; error( phase: "signing" | "spec-lookup" | "post-response", error: unknown, @@ -2487,6 +2488,7 @@ async function doubleKnockCore( } } if (fulfilled && challengeRequest != null) { + observation?.response(response, "intermediate"); signal?.throwIfAborted(); // The challenge retry stays on the current validated hop. cancelResponseBody(response, request.url); @@ -2501,6 +2503,7 @@ async function doubleKnockCore( } if (fulfilled && !isRedirectResponse(response)) { if (response.status < 300) { + observation?.response(response, "final"); await observeDoubleKnockOperation( observation, "post-response", @@ -2511,6 +2514,7 @@ async function doubleKnockCore( return { response, session }; } if (response.status !== 400 && response.status !== 401) { + observation?.response(response, "final"); return { response, session }; } } @@ -2518,6 +2522,7 @@ async function doubleKnockCore( } if (!isRedirectResponse(response)) { + observation?.response(response, "intermediate"); // verification failed; retry with the other spec of HTTP Signatures // (double-knocking; see https://swicg.github.io/activitypub-http-signature/#how-to-upgrade-supported-versions) const spec = firstTrySpec === "draft-cavage-http-signatures-12" @@ -2559,6 +2564,7 @@ async function doubleKnockCore( !isRedirectResponse(response) && response.status !== 400 && response.status !== 401 ) { + observation?.response(response, "final"); await observeDoubleKnockOperation( observation, "post-response", @@ -2568,6 +2574,7 @@ async function doubleKnockCore( } } } else if (!firstRedirected) { + observation?.response(response, "final"); await observeDoubleKnockOperation( observation, "post-response", @@ -2576,10 +2583,14 @@ async function doubleKnockCore( Promise.resolve(), ); } - if (!isRedirectResponse(response)) return { response, session }; + if (!isRedirectResponse(response)) { + observation?.response(response, "final"); + return { response, session }; + } // Follow redirects manually to get the final URL. The current hop is // closed before the next one is validated and opened: + observation?.response(response, "intermediate"); cancelResponseBody(response, request.url); await session.close(); if (redirected >= maximumRedirection) { From 687b93a08283039a3308baca9405fc608c9feab2 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 03:47:32 +0900 Subject: [PATCH 24/53] Document outbound attempt and HTTP reports Explain producer readiness, concrete delivery attempts, actual HTTP fetches, and selected final responses so observers can interpret reports. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 8 +- changes.d/fedify/outbound-observers.md | 13 +- claude-plugin/skills/fedify/SKILL.md | 41 +++++- docs/manual/send.md | 128 ++++++++++++++++-- packages/fedify/src/federation/federation.ts | 6 +- .../fedify/src/federation/outbound-report.ts | 52 +++++-- packages/fedify/src/federation/send.ts | 1 + packages/fedify/src/sig/http.ts | 1 + 8 files changed, 211 insertions(+), 39 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b05d6ca77..973aa57a2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -54,8 +54,12 @@ To be released. and `onDelivery` for destination planning, direct handoffs, queue acceptance or uncertainty, and local preparation failures. Callbacks receive application context and independent grouping IDs without requiring - OpenTelemetry; their errors do not replace the original result. Readiness - and handoff observations do not establish remote delivery success. + OpenTelemetry; their errors do not replace the original result. Known IDs + and original inbox source survive queue continuations. Send/forward options + accept `dispatchId` for application correlation without deduplication, + limited to 1,024 UTF-8 bytes. Routed inbox reports include `inboxRequestId` + and informational `requestId`. Missing legacy relationships remain unknown. + Readiness and handoff observations do not establish remote delivery success. [[#1316] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 041e258f4..79adb67d8 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -6,12 +6,17 @@ links: for sends, original-payload forwarding, and automatic portable replication. Register `onBroadcast` to record producer starts and preparation results, and `onDelivery` for destination planning, direct handoffs, queue acceptance - or uncertainty, and local preparation failures. Callbacks receive - application context and independent grouping IDs without requiring + or uncertainty, and local preparation failures. Use `onAttempt` for concrete + send results and exact HTTP exchange counts, including queued retries and + gateway fallback, and `onHttpExchange` for each actual signed or unsigned + fetch. Reports distinguish received responses from the selected final + response and retain HTTP evidence alongside later local failures. They + include observation timestamps and optional operation trace IDs. Callbacks + receive application context and independent grouping IDs without requiring OpenTelemetry; their errors do not replace the original result. Known IDs and original inbox source survive queue continuations. Send/forward options accept `dispatchId` for application correlation without deduplication, limited to 1,024 UTF-8 bytes. Routed inbox reports include `inboxRequestId` and informational `requestId`. Missing legacy relationships remain unknown. - Readiness and handoff observations do not establish remote delivery success. - [[#1316] by Jiwon Kwon] + Readiness and handoff observations do not establish remote delivery + success. [[#1316] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index e973c715b..d5932d108 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -248,10 +248,12 @@ Use `federation.setOutboundObservers({ onBroadcast(ctx, event) { ... }, onDelivery(ctx, event) { ... } })` (also available on builders) for producer observations of ordinary/outbox-context sends, original-payload forwarding, and automatic -portable replication. Both callbacks are optional. Registration replaces the -whole set; `{}` clears it. Registration and builds copy callbacks, and active -executions retain their selection. Callbacks receive existing context/data -without extra loading and work without OpenTelemetry. +portable replication. Add `onAttempt(ctx, report)` and +`onHttpExchange(ctx, report)` for concrete send and actual fetch results. All +callbacks are optional. Registration replaces the whole set; `{}` clears it. +Registration and builds copy callbacks, and active executions retain their +selection. Callbacks receive existing context/data without extra loading and +work without OpenTelemetry. `onBroadcast` groups start and preparation results with an independent `broadcastId`; `operation` distinguishes `send`, `forward`, and @@ -265,6 +267,25 @@ unknown). Readiness, queue acceptance, and remote effects are distinct. Portable fallback keeps one delivery across alternate gateways; replication claims independent targets. Filtered recipients create no planned delivery. +`onAttempt` emits one terminal `OutboundAttemptReport` per concrete send, +including workers and each gateway/retry, with a fresh `attemptId` under its +delivery. `httpExchangeCount` counts actual fetches, including challenges and +redirects; pre-fetch failures have zero. `onHttpExchange` emits immutable +`OutboundHttpExchangeReport` snapshots of actual request URL/method and response +status/text/available URL, or the original fetch error. `exchangeIndex` starts +at zero per attempt. No live transport objects, bodies, or headers are exposed. + +For attempt reports, narrow `report.outcome.response.type`: `final` has an +`exchangeIndex` and +`metadata`; `intermediate-only` and `undetermined` have `lastObserved`; +`not-observed` has no response. The last received challenge or redirect is not +necessarily final. A selected non-OK final response stays primary `http` failure +when cache/cleanup also fails; local errors appear in `diagnostics`. A final 2xx +plus a local error is still a failed attempt. Preserve `failure.error` +separately from the outward `outcome.error`; HTTP failure evidence may lack an +exception. Holds, drops, and failures before sender entry produce no attempt, +and attempt reports do not establish worker scheduling results. + Callbacks are awaited without a default timeout; observer/logging errors are isolated. Nonsettling callbacks can delay execution indefinitely. Portable forwarding can return at its existing deadline while background work and @@ -272,14 +293,20 @@ observations continue. Detached recording must handle its own rejection, shutdown, and persistence. Known IDs survive fanout expansion, retries, holds, and gateway changes. Fanout expansion continues existing delivery handoffs without starting another broadcast. Legacy fanout can have an unknown broadcast; -missing source facts remain unavailable, and older consumers can discard new -metadata. +legacy outbox reports use the current task UUID as `deliveryId` and omit an +unavailable original `destinationInbox`. Missing source facts remain +unavailable, and older consumers can discard new metadata. Attempt/exchange +`queue` identifies the current message and available Fedify retry counter; +workers receive current context data without observation-only loader calls. Events expose `provenance` for preserved source/app correlation and `local` for current URL facts. A known inbox source includes server-generated `inboxRequestId`, informational `requestId`, and the original URL snapshot. It survives context clones/conversions and queues; independent contexts do not -infer it from ambient execution. +infer it from ambient execution. `observedAt` is current Unix milliseconds, +not a global ordering guarantee. Optional `trace` carries only valid IDs of the +operation's own span, never an ambient parent fallback; grouping works without +tracing. Entry events and exchanges omit trace when no own span exists. Send/forward options accept `dispatchId`, preserved exactly in event provenance and queues even without producer observers. Empty strings are valid; the limit diff --git a/docs/manual/send.md b/docs/manual/send.md index 2d909954a..2ade23dc8 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -66,7 +66,8 @@ Observing outgoing activities Use `Federatable.setOutboundObservers()` on a federation builder or a directly created federation to observe sends, original-payload forwarding, and automatic -portable-inbox replication. Both callbacks are optional: +portable-inbox replication. Each callback is optional. This example records +producer preparation and delivery handoffs: ~~~~ typescript twoslash import type { @@ -101,11 +102,13 @@ federation.setOutboundObservers({ ~~~~ Here, `recordBroadcast()` and `recordDelivery()` are your application's -recording functions. Callbacks receive the existing context and application -data without loading additional data for observation. The report fields contain -immutable snapshots of known routing and source facts; original thrown values -remain available as `unknown` values. The context, application data, Activity -objects, and exceptions are not deep-copied by this API. +recording functions. Callbacks receive context and application data without +loading additional data for observation. Queued work uses the current worker's +context and data; the original HTTP cause is recorded separately in +`provenance`. The report fields contain immutable snapshots of known routing +and source facts; original thrown values remain available as `unknown` values. +The context, application data, Activity objects, and exceptions are not +deep-copied by this API. ### Broadcast lifecycle @@ -171,6 +174,103 @@ for that destination. Earlier handoffs remain valid, and later unvisited destinations can remain planned without a handoff. Parallel work can interleave callbacks, so IDs provide grouping rather than a total event order. +### Attempts and HTTP exchanges + +`onAttempt` receives an `OutboundAttemptReport` once a concrete send settles, +including immediate sends, forwarding, portable replication, and queue workers. +Each entry into the sender gets a fresh `attemptId` under its `deliveryId`. +Alternate gateways and retries create new attempts; signature challenges and +redirects can create several fetches within one attempt. `inboxUrl` is the +inbox selected for that attempt, while `request.url` on an exchange identifies +the actual URL fetched at that step. + +`onHttpExchange` receives an `OutboundHttpExchangeReport` for each actual fetch +submission. `exchangeIndex` starts at zero within its `attemptId`. Its `request` +copies the submitted URL and method, including those of signed requests. A +`response` outcome records status, status text, and the response URL when fetch +supplies it; a `failed` outcome retains the original fetch rejection. A non-2xx +response is still a received response. Neither live transport objects nor +response bodies or headers are exposed. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + onHttpExchange(_ctx, report) { + console.log(report.attemptId, report.exchangeIndex, report.request.url); + if (report.outcome.type === "response") { + console.log("HTTP response", report.outcome.response.status); + } + }, + onAttempt(_ctx, report) { + console.log(report.attemptId, report.httpExchangeCount); + const response = report.outcome.response; + if (response.type === "final") { + console.log("Final response", response.exchangeIndex, response.metadata.status); + } + if (report.outcome.type === "failed") { + console.error(report.outcome.failure.kind, report.outcome.failure.error); + } + }, +}); +~~~~ + +An attempt can fail before fetch, for example during URL validation or signing. +It still has an attempt ID, with `httpExchangeCount: 0` and no exchange report. +Failures before sender entry, such as queue key import or fanout parsing, and +worker holds, drops, or deferrals do not create attempts. Attempt reports do not +establish a worker's scheduling result. + +### Response versus final response + +A response means one fetch received HTTP headers. A final response is the +response that the HTTP flow selected as the attempt's result. For example: + +~~~~ text +401 challenge -> retry -> 202 accepted +exchange 0 exchange 1 (final) +~~~~ + +If signing the retry fails, the 401 stays intermediate; being the last response +does not make it final. An attempt's `outcome.response` makes that distinction +explicit: + +| Type | Evidence | +| ------------------- | -------------------------------------------------------------------------------------------------- | +| `final` | The flow selected this response. `exchangeIndex` and immutable `metadata` identify it. | +| `not-observed` | No response headers were received, including pre-fetch failures and rejected fetches. | +| `intermediate-only` | The flow chose to continue past the received response. `lastObserved` retains its index/metadata. | +| `undetermined` | Execution failed before finality was established. `lastObserved` retains the known response facts. | + +`httpExchangeCount` counts actual fetch submissions, including challenge, +redirect, and transport retries. It does not count signing or preparation work. +A `succeeded` attempt has a successful final HTTP response after the send +execution settles; it does not establish subsequent remote application work. + +A selected final response survives later cache or cleanup failures. With a +final non-OK response, `failure.kind` remains `http` and `diagnostics` retains +additional local failures. With a final 2xx response and a local failure, the +attempt is `failed` with the 2xx evidence retained. `failure.error` preserves +original operation evidence when available; an HTTP failure can lack an error +if execution failed before constructing its usual HTTP exception. The separate +`outcome.error` is the exact value that escaped the sender, including any +existing wrapping. Observer errors never become attempt diagnostics. + +### Observation time and tracing + +Each event or report includes `observedAt`, Unix milliseconds captured at its +observation boundary before awaiting the callback. Replay gets a new observation +time while retaining known grouping and historical source facts. Wall clocks do +not establish a global order across workers. + +Optional `trace` contains valid `traceId` and `spanId` from the observed +operation's own span, including unsampled spans. Fedify omits it when no valid +own span exists. Producer entry precedes span creation, and exchanges have no +dedicated span; their reports omit `trace`. Ambient parent/server spans are not +substituted. Grouping IDs and observations work independently of tracing or +exporter storage. + ### Identities across queues Observed broadcasts and deliveries retain their IDs through fanout expansion, @@ -200,7 +300,10 @@ IDs provide grouping, not ordering or deduplication. Replayed work can produce repeated observations. Treat IDs as opaque values. Older messages remain processable. Fanout tasks without identity metadata get -stable delivery IDs with `broadcast: { type: "unknown" }`. Missing source and +stable delivery IDs with `broadcast: { type: "unknown" }`. Legacy outbox attempt +reports use the current task UUID as `deliveryId` and unknown broadcast +membership; `destinationInbox` is absent when the original plan is unavailable. +An explicit canonical `portableInbox` can still be known. Missing source and application correlation remain unavailable. Older consumers can discard these fields, so mixed-version processing can lose correlation. @@ -272,16 +375,19 @@ existing call retain it. Each registration call replaces all previous outbound observers; pass `{}` to clear them. Fedify copies the supplied callbacks, and building a federation -copies the builder's registration. An active invocation keeps its selected -callbacks even if registration changes while it is running. You can register -only `onBroadcast` or only `onDelivery`. +copies the builder's registration. An active invocation or worker execution +keeps its selected callbacks even if registration changes while it is running. +You can register any callback independently. On attempt/exchange reports, +`queue` contains the current `queueMessageId` and an available Fedify +`queueAttempt` counter. Native redelivery can repeat that counter while +creating fresh attempts. Callbacks are awaited without a default timeout. Their return values and errors cannot veto sending, and callback or error-logging failures do not replace the send's original result. Slow callbacks delay processing; a callback that never settles can delay it indefinitely. Existing portable-forwarding deadlines can return control while background preparation, sending, and observation continue; -a deadline alone does not create a failed preparation result. +a deadline alone does not create a failed preparation result or attempt outcome. If you detach recording work, handle its rejections and shutdown yourself; Fedify does not guarantee its persistence. Incoming C2S `setOutboxListeners()` diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index d263482ca..a03eac93f 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -93,8 +93,10 @@ export interface Federatable extends TaskRegistry { * the callbacks, and an active send keeps the callback selected at entry. * Observes send, original-payload forwarding, and automatic portable * replication preparation, destination planning, and handoffs, including - * outbox-context operations and fanout continuation handoffs. Known IDs, - * original inbox source, and application correlation survive queue work. + * outbox-context operations and fanout continuation handoffs. Separate + * callbacks report concrete send attempts and actual HTTP exchanges, including + * queued retries and gateway fallback. Known IDs, original inbox source, and + * application correlation survive queue work; workers use current context data. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 78c2559fb..af5b5489d 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -83,7 +83,8 @@ export type OutboundBroadcastEvent = * A producer-side destination obligation, not delivery completion. * * IDs group producer handoffs and their fanout expansion across queue tasks. - * Full worker execution reports and concrete attempts are not observed yet. + * Concrete sends use separate {@link OutboundAttemptReport} values; these + * events do not establish delivery completion or worker scheduling outcomes. * An unvisited destination stays planned when another destination's preparation * interrupts the producer. * @since 2.5.0 @@ -176,7 +177,9 @@ type OutboundAttemptFailure = }; interface OutboundHttpResponseMetadata { + /** Actual HTTP status returned by fetch. */ readonly status: number; + /** Copied HTTP status text. */ readonly statusText: string; /** Response URL when supplied by fetch. */ readonly url?: string; @@ -184,13 +187,21 @@ interface OutboundHttpResponseMetadata { type OutboundAttemptResponse = | { + /** The HTTP flow selected this response before follow-up work. */ readonly type: "final"; + /** Actual submission index of the selected response. */ readonly exchangeIndex: number; + /** Frozen response facts, retained across later local failures. */ readonly metadata: OutboundHttpResponseMetadata; } - | { readonly type: "not-observed" } | { + /** No response headers were received, including zero-fetch attempts. */ + readonly type: "not-observed"; + } + | { + /** Continued past the response, or failed before deciding finality. */ readonly type: "intermediate-only" | "undetermined"; + /** Latest received response, without a finality claim. */ readonly lastObserved: { readonly exchangeIndex: number; readonly metadata: OutboundHttpResponseMetadata; @@ -200,7 +211,10 @@ type OutboundAttemptResponse = /** * The terminal result of one concrete send. * - * A send that fails before submitting HTTP still constitutes an attempt. + * A send that fails before submitting HTTP still constitutes an attempt, with + * zero exchanges. Each gateway send and retry gets a fresh attempt identity. + * Final response evidence follows the HTTP flow's selection, not the last + * observed status; later cache/cleanup failures can still fail the execution. * @since 2.5.0 */ export type OutboundAttemptReport = OutboundSourceMetadata & { @@ -233,7 +247,11 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { } | { readonly type: "failed"; - /** Failure evidence captured before broad sender error wrapping. */ + /** + * Bounded failure kind and operation phase, captured before broad wrapping. + * Unknown provenance stays unknown; exception names do not establish it. + * Final non-OK HTTP evidence remains primary alongside local diagnostics. + */ readonly failure: OutboundAttemptFailure; /** Exact value escaping the sender, possibly different from failure.error. */ readonly error: unknown; @@ -248,8 +266,9 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { * One actual fetch submission within a concrete send attempt. * * A redirect response remains an observed response even if a later hop fails. - * This report does not establish the attempt's final response. Response - * body/header capture is not observed yet. + * This report does not establish the attempt's final response. Metadata is + * copied before callback awaiting; no live Request/Response/Headers or body + * capture is exposed. Exchange callbacks may run before the attempt settles. * @since 2.5.0 */ export type OutboundHttpExchangeReport = @@ -283,7 +302,10 @@ export type OutboundHttpExchangeReport = * * Send, original-payload forwarding, and automatic portable replication producer * events are observed, including outbox-context operations and fanout - * continuation handoffs. Known IDs and provenance survive queue continuations. + * continuation handoffs. Concrete attempts and actual fetch exchanges include + * queue workers, signature challenges, redirects, and alternate gateways. + * Known IDs and provenance survive queue continuations; callback context data + * belongs to the current execution. Each callback is independently optional. * Callbacks are awaited without a default timeout. * Their errors, including errors logging observer failures, do not replace the * send's result. @@ -296,8 +318,9 @@ export type OutboundHttpExchangeReport = */ export interface OutboundObservers { /** - * Observes a fetch after response headers or rejection, before - * subsequent redirect handling. Callback errors do not replace transport. + * Observes each actual fetch after response headers or rejection, before + * subsequent challenge/redirect handling. Non-2xx responses are still response + * outcomes. Pre-fetch failures emit no exchange. Errors cannot veto transport. */ readonly onHttpExchange?: ( context: Context, @@ -305,8 +328,10 @@ export interface OutboundObservers { ) => void | Promise; /** - * Observes settlement of concrete sends and forwarding, including pre-fetch - * failures. Callback errors cannot replace the sender's result. + * Observes settlement of concrete sends and forwarding, including workers + * and pre-fetch failures. Holds, drops, and failures before sender entry emit + * no attempt. Final HTTP evidence, local diagnostics, and the outward error + * remain distinct. Callback errors cannot replace the sender's result. */ readonly onAttempt?: ( context: Context, @@ -326,8 +351,9 @@ export interface OutboundObservers { /** * Observes producer-side planning, handoffs, and destination preparation * failures, including fanout continuation handoffs. Filtered recipients do - * not become planned deliveries. Concrete attempts, remote success, and full - * worker execution reports are not observed yet. + * not become planned deliveries. Concrete attempts and exchanges have their + * own callbacks. These events do not report remote success or worker + * scheduling outcomes. */ readonly onDelivery?: ( context: Context, diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index c06c8fa33..fd2c94829 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -764,6 +764,7 @@ async function consumeSendActivityResponse( * @param request The request to send. * @param openHop The function to open a session for each hop. * @param consume The function to consume the final response. + * @param attempt The in-process attempt observation, if registered. * @returns The value returned by `consume`. */ export async function fetchWithValidatedRedirects( diff --git a/packages/fedify/src/sig/http.ts b/packages/fedify/src/sig/http.ts index ecc9da6fe..94d1ec412 100644 --- a/packages/fedify/src/sig/http.ts +++ b/packages/fedify/src/sig/http.ts @@ -2305,6 +2305,7 @@ export async function doubleKnock( * @param openHop The function to open a session for each hop. The session of * a hop is closed before the next hop is opened. * @param consume The function to consume the final response. + * @param observation The in-process transport/finality hooks, if registered. * @returns The value returned by `consume`. */ export async function doubleKnockScoped( From cd5294828c201204a6ebec5e6e339f3d06c6b03e Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:12:06 +0900 Subject: [PATCH 25/53] Observe outbound worker settlement Report worker completion and concrete attempt counts so observers can distinguish settlement from delivery success and scheduling. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 1 + packages/fedify/src/federation/middleware.ts | 241 ++++++++--- .../src/federation/outbound-observation.ts | 156 ++++++- .../src/federation/outbound-report.test.ts | 406 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 49 +++ packages/fedify/src/federation/send.ts | 3 + 6 files changed, 768 insertions(+), 88 deletions(-) diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 40c68b7e0..986a7ea9a 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -669,6 +669,7 @@ export class FederationBuilderImpl onDelivery: observers.onDelivery, onAttempt: observers.onAttempt, onHttpExchange: observers.onHttpExchange, + onWorker: observers.onWorker, }); } diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 1567bcfe7..9ab64c836 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -9,12 +9,14 @@ import { } from "./outbound-provenance.ts"; import type { OutboundInboxRequest, + OutboundObservers, OutboundProvenance, } from "./outbound-report.ts"; import { observeOutboundBroadcast, OutboundAttemptObserver, OutboundBroadcastObservation, + OutboundWorkerObservation, } from "./outbound-observation.ts"; import { InboxObservation } from "./inbox-observation.ts"; import { type Path, RouterError } from "@fedify/uri-template"; @@ -1247,55 +1249,85 @@ export class FederationImpl ...sourceLogContext(provenance), }, async () => { if (message.type === "fanout") { + const observers = this.outboundObservers; + const worker = this.#createOutboundWorkerObservation( + contextData, + message, + provenance, + observers.onWorker, + ); const common: QueueTaskCommonAttributes = { role: "fanout", queue: this.fanoutQueue, activityType: message.activityType, }; - await this.#runWorkerSpan( - "activitypub.fanout", - { "activitypub.activity.type": message.activityType }, - extractedContext, - this.#instrumentWorkerBody(common, async (span) => { - if (message.activityId != null) { - span.setAttribute("activitypub.activity.id", message.activityId); - } - await this.#listenFanoutMessage( - contextData, - message, - provenance, - span, - trace.getSpanContext(extractedContext), - ); - return { outcome: "completed" }; - }, classifyAbortableError), - ); + const execute = () => + this.#runWorkerSpan( + "activitypub.fanout", + { "activitypub.activity.type": message.activityType }, + extractedContext, + this.#instrumentWorkerBody(common, async (span) => { + if (message.activityId != null) { + span.setAttribute( + "activitypub.activity.id", + message.activityId, + ); + } + await this.#listenFanoutMessage( + contextData, + message, + provenance, + span, + trace.getSpanContext(extractedContext), + observers, + worker, + ); + return { outcome: "completed" }; + }, classifyAbortableError), + worker, + ); + await (worker == null ? execute() : worker.run(execute)); } else if (message.type === "outbox") { + const observers = this.outboundObservers; + const worker = this.#createOutboundWorkerObservation( + contextData, + message, + provenance, + observers.onWorker, + ); const common: QueueTaskCommonAttributes = { role: "outbox", queue: this.outboxQueue, activityType: message.activityType, }; - await this.#runWorkerSpan( - "activitypub.outbox", - { - "activitypub.activity.type": message.activityType, - "activitypub.activity.retries": message.attempt, - }, - extractedContext, - this.#instrumentWorkerBody(common, async (span) => { - if (message.activityId != null) { - span.setAttribute("activitypub.activity.id", message.activityId); - } - await this.#listenOutboxMessage( - contextData, - message, - span, - provenance, - ); - return { outcome: "completed" }; - }, classifyAbortableError), - ); + const execute = () => + this.#runWorkerSpan( + "activitypub.outbox", + { + "activitypub.activity.type": message.activityType, + "activitypub.activity.retries": message.attempt, + }, + extractedContext, + this.#instrumentWorkerBody(common, async (span) => { + if (message.activityId != null) { + span.setAttribute( + "activitypub.activity.id", + message.activityId, + ); + } + await this.#listenOutboxMessage( + contextData, + message, + span, + provenance, + observers, + worker, + ); + return { outcome: "completed" }; + }, classifyAbortableError), + worker, + ); + await (worker == null ? execute() : worker.run(execute)); } else if (message.type === "inbox") { const common: QueueTaskCommonAttributes = { role: "inbox", @@ -1342,6 +1374,39 @@ export class FederationImpl }); } + #createOutboundWorkerObservation( + contextData: TContextData, + message: FanoutMessage | OutboxMessage, + provenance: OutboundProvenance, + callback: OutboundObservers["onWorker"], + ): OutboundWorkerObservation | undefined { + if (callback == null) return undefined; + let documentLoader: DocumentLoader | undefined; + let contextLoader: DocumentLoader | undefined; + return new OutboundWorkerObservation( + () => { + const loaderOptions = this.#getLoaderOptions(message.baseUrl); + return new ContextImpl({ + [outboundProvenance]: provenance, + federation: this, + url: new URL("/", message.baseUrl), + data: contextData, + documentLoader: (...args) => + (documentLoader ??= this.documentLoaderFactory(loaderOptions))( + ...args, + ), + contextLoader: (...args) => + (contextLoader ??= this.contextLoaderFactory(loaderOptions))( + ...args, + ), + }); + }, + callback, + message, + provenance, + ); + } + /** * Opens a CONSUMER span for a queue worker and scopes the extracted trace * context around its body. The body—typically built by @@ -1358,12 +1423,14 @@ export class FederationImpl spanAttributes: Attributes, extractedContext: ReturnType, instrumentedBody: (span: Span) => () => Promise, + worker?: OutboundWorkerObservation, ): Promise { await this._getTracer().startActiveSpan( spanName, { kind: SpanKind.CONSUMER, attributes: spanAttributes }, extractedContext, async (span: Span) => { + worker?.setSpan(span, trace.getSpanContext(extractedContext)); const spanCtx = span.spanContext(); return await withContext( { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, @@ -1428,8 +1495,9 @@ export class FederationImpl provenance: OutboundProvenance, span: Span, parentSpanContext: SpanContext | undefined, + observers: OutboundObservers, + worker?: OutboundWorkerObservation, ): Promise { - const observers = this.outboundObservers; const logger = getLogger(["fedify", "federation", "fanout"]); logger.debug( "Fanning out activity {activityId} to {inboxes} inbox(es)...", @@ -1438,34 +1506,47 @@ export class FederationImpl inboxes: globalThis.Object.keys(message.inboxes).length, }, ); - const keys: SenderKeyPair[] = await Promise.all( - message.keys.map(async ({ keyId, privateKey }) => ({ - keyId: new URL(keyId), - privateKey: await importJwk(privateKey, "private"), - })), - ); - const activity = await Activity.fromJsonLd(message.activity, { - contextLoader: this.contextLoaderFactory({ - allowPrivateAddress: this.allowPrivateAddress, - userAgent: this.userAgent, - }), - documentLoader: this.documentLoaderFactory({ - allowPrivateAddress: this.allowPrivateAddress, - userAgent: this.userAgent, - }), - tracerProvider: this.tracerProvider, - }); - const context = this.#createContext( - new URL(message.baseUrl), - data, - { - [outboundProvenance]: provenance, + const importKeys = () => + Promise.all( + message.keys.map(async ({ keyId, privateKey }) => ({ + keyId: new URL(keyId), + privateKey: await importJwk(privateKey, "private"), + })), + ); + const keys: SenderKeyPair[] = worker == null + ? await importKeys() + : await worker.prepare("key-import", importKeys); + const parseActivity = () => + Activity.fromJsonLd(message.activity, { + contextLoader: this.contextLoaderFactory({ + allowPrivateAddress: this.allowPrivateAddress, + userAgent: this.userAgent, + }), documentLoader: this.documentLoaderFactory({ allowPrivateAddress: this.allowPrivateAddress, userAgent: this.userAgent, }), - }, - ); + tracerProvider: this.tracerProvider, + }); + const activity = worker == null + ? await parseActivity() + : await worker.prepare("activity-parsing", parseActivity); + const createContext = () => + this.#createContext( + new URL(message.baseUrl), + data, + { + [outboundProvenance]: provenance, + documentLoader: this.documentLoaderFactory({ + allowPrivateAddress: this.allowPrivateAddress, + userAgent: this.userAgent, + }), + }, + ); + const context = worker == null + ? createContext() + : await worker.prepare("context-preparation", createContext); + worker?.useContext(context); const observation = await OutboundBroadcastObservation.resumeFanout( context, observers, @@ -1479,6 +1560,7 @@ export class FederationImpl normalizeExistingProofs: message.normalizeExistingProofs, activityJsonLd: message.activity, observation, + worker, context, }); } @@ -1488,8 +1570,10 @@ export class FederationImpl message: OutboxMessage, span: Span, provenance: OutboundProvenance, + observers: OutboundObservers, + worker?: OutboundWorkerObservation, ): Promise { - const { onAttempt, onHttpExchange } = this.outboundObservers; + const { onAttempt, onHttpExchange } = observers; const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { keyIds: message.keys.map((pair) => pair.keyId), @@ -1502,10 +1586,13 @@ export class FederationImpl const keys: SenderKeyPair[] = []; let rsaKeyPair: SenderKeyPair | null = null; for (const { keyId, privateKey } of message.keys) { - const pair: SenderKeyPair = { + const importPair = async (): Promise => ({ keyId: new URL(keyId), privateKey: await importJwk(privateKey, "private"), - }; + }); + const pair = worker == null + ? await importPair() + : await worker.prepare("key-import", importPair); if ( rsaKeyPair == null && pair.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" @@ -1520,7 +1607,7 @@ export class FederationImpl let observationContextLoader: DocumentLoader | undefined; const observation = OutboundAttemptObserver.resumeOutbox( () => - observationContext ??= new ContextImpl({ + worker?.getContext() ?? (observationContext ??= new ContextImpl({ [outboundProvenance]: provenance, federation: this, url: new URL("/", message.baseUrl), @@ -1534,7 +1621,7 @@ export class FederationImpl (observationContextLoader ??= this.contextLoaderFactory( loaderOptions, ))(...args), - }), + })), onAttempt, message, provenance, @@ -1659,6 +1746,7 @@ export class FederationImpl span, { observation, + worker, provenance, keys, logData, @@ -1713,6 +1801,7 @@ export class FederationImpl } } await sendActivity({ + attemptStarted: worker?.attemptStarted, observation, allowPrivateAddress: this.allowPrivateAddress, keys, @@ -1721,7 +1810,10 @@ export class FederationImpl activityType: message.activityType, inbox, sharedInbox: message.sharedInbox, - headers: new Headers(message.headers), + headers: worker == null + ? new Headers(message.headers) + : await worker.prepare("request-preparation", () => + new Headers(message.headers)), specDeterminer: new KvSpecDeterminer( this.kv, this.kvPrefixes.httpMessageSignaturesSpec, @@ -2044,6 +2136,7 @@ export class FederationImpl span: Span, helpers: { readonly observation: SendActivityParameters["observation"]; + readonly worker: OutboundWorkerObservation | undefined; readonly provenance: OutboundProvenance; readonly keys: readonly SenderKeyPair[]; readonly logData: Record; @@ -2062,6 +2155,7 @@ export class FederationImpl const circuitLogger = getLogger(["fedify", "federation", "circuit"]); const { observation, + worker, provenance, keys, logData, @@ -2169,6 +2263,7 @@ export class FederationImpl } try { await sendActivity({ + attemptStarted: worker?.attemptStarted, observation, allowPrivateAddress: this.allowPrivateAddress, keys, @@ -2177,7 +2272,10 @@ export class FederationImpl activityType: message.activityType, inbox, sharedInbox: message.sharedInbox, - headers: new Headers(message.headers), + headers: worker == null + ? new Headers(message.headers) + : await worker.prepare("request-preparation", () => + new Headers(message.headers)), specDeterminer, meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, @@ -3293,6 +3391,7 @@ export class FederationImpl orderingKey, context: ctx, observation, + worker, provenance = getOutboundProvenance(ctx), } = options; observation?.setPhase("validation"); @@ -3436,6 +3535,7 @@ export class FederationImpl inbox, inboxes[inbox].gatewayInboxes, { + attemptStarted: worker?.attemptStarted, observation: delivery, allowPrivateAddress: this.allowPrivateAddress, keys, @@ -7635,6 +7735,7 @@ export class OutboxContextImpl extends ContextImpl } interface SendActivityInternalOptions { + readonly worker?: OutboundWorkerObservation; readonly provenance?: OutboundProvenance; readonly observation?: OutboundBroadcastObservation; readonly immediate?: boolean; diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index b6afcb9d1..3d8016d5e 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -25,6 +25,7 @@ import type { OutboundObservers, OutboundPreparationResult, OutboundProvenance, + OutboundWorkerReport, } from "./outbound-report.ts"; type WithoutSource = T extends unknown @@ -108,7 +109,7 @@ async function notifyObserver< | ((context: Context, event: TEvent) => void | Promise) | undefined, event: TEvent, - unit: "broadcast" | "delivery" | "attempt" | "http-exchange", + unit: "broadcast" | "delivery" | "attempt" | "http-exchange" | "worker", ): Promise { if (callback == null) return; try { @@ -117,14 +118,18 @@ async function notifyObserver< () => callback(context, event), ); } catch (error) { - try { - getLogger(["fedify", "federation", "outbox"]).error( - "An unexpected error occurred in outbound {unit} observer:\n{error}", - { unit, error }, - ); - } catch { - // A logging sink must not replace the send's result either. - } + logObserverFailure(unit, error); + } +} + +function logObserverFailure(unit: string, error: unknown): void { + try { + getLogger(["fedify", "federation", "outbox"]).error( + "An unexpected error occurred in outbound {unit} observer:\n{error}", + { unit, error }, + ); + } catch { + // A logging sink must not replace the execution's result either. } } @@ -324,6 +329,129 @@ type AttemptIdentity = Pick< "broadcast" | "deliveryId" | "destinationInbox" | "portableInbox" >; +function snapshotOutboxIdentity(message: OutboxMessage): AttemptIdentity { + return message.outbound == null + ? Object.freeze({ + broadcast: Object.freeze({ type: "unknown" as const }), + deliveryId: message.id, + ...(message.portableInbox == null + ? {} + : { portableInbox: message.portableInbox }), + }) + : snapshotDeliveryIdentity(message.outbound); +} + +type WorkerPreparationFailure = Extract< + OutboundWorkerReport["result"], + { type: "preparation-failed" } +>["failure"]; + +/** Per-execution state, independent of concrete attempt/exchange callbacks. */ +export class OutboundWorkerObservation { + #attemptsStarted = 0; + #trace: OutboundWorkerReport["trace"]; + #preparationFailure: WorkerPreparationFailure | undefined; + readonly #queue: OutboundWorkerReport["queue"]; + readonly #delivery: AttemptIdentity | undefined; + readonly #role: OutboundWorkerReport["role"]; + #currentContext: Context | undefined; + + constructor( + readonly context: () => Context, + readonly callback: OutboundObservers["onWorker"], + message: FanoutMessage | OutboxMessage, + readonly provenance: OutboundProvenance, + ) { + this.#role = message.type; + this.#queue = Object.freeze({ + queueMessageId: message.id, + ...(message.type === "outbox" ? { queueAttempt: message.attempt } : {}), + }); + if (message.type === "outbox") { + try { + this.#delivery = snapshotOutboxIdentity(message); + } catch { + // Invalid optional grouping cannot change the worker's own execution. + this.#delivery = undefined; + } + } + } + + readonly attemptStarted: () => void = () => { + this.#attemptsStarted++; + }; + + getContext(): Context { + return this.#currentContext ??= this.context(); + } + + useContext(context: Context): void { + this.#currentContext = context; + } + + setSpan(span: Span, parentSpanContext?: SpanContext): void { + this.#trace = snapshotTrace(span, parentSpanContext); + } + + async prepare( + phase: WorkerPreparationFailure["phase"], + operation: () => TResult | Promise, + ): Promise { + try { + const result = await operation(); + this.#preparationFailure = undefined; + return result; + } catch (error) { + this.#preparationFailure = Object.freeze({ kind: "local", phase, error }); + throw error; + } + } + + async run(operation: () => Promise): Promise { + try { + await operation(); + } catch (error) { + await this.#notify( + this.#preparationFailure == null || + !Object.is(this.#preparationFailure.error, error) + ? { type: "failed", error } + : { + type: "preparation-failed", + failure: this.#preparationFailure, + error, + }, + ); + throw error; + } + await this.#notify({ type: "completed" }); + } + + async #notify(result: OutboundWorkerReport["result"]): Promise { + const observedAt = Temporal.Now.instant().epochMilliseconds; + try { + const context = this.getContext(); + await notifyObserver( + context, + this.callback, + Object.freeze({ + role: this.#role, + queue: this.#queue, + ...(this.#delivery == null ? {} : { delivery: this.#delivery }), + attemptsStarted: this.#attemptsStarted, + observedAt, + ...(this.#trace == null ? {} : { trace: this.#trace }), + provenance: this.provenance, + local: snapshotLocalContext(context), + result: Object.freeze(result), + }), + "worker", + ); + } catch (error) { + logObserverFailure("worker", error); + } + } +} + /** Selected callback and restored grouping, shared by concrete gateway attempts. */ export class OutboundAttemptObserver { readonly #queue: OutboundAttemptReport["queue"]; @@ -348,15 +476,7 @@ export class OutboundAttemptObserver { onHttpExchange?: OutboundObservers["onHttpExchange"], ): OutboundAttemptObserver | undefined { if (callback == null && onHttpExchange == null) return undefined; - const identity = message.outbound == null - ? Object.freeze({ - broadcast: Object.freeze({ type: "unknown" as const }), - deliveryId: message.id, - ...(message.portableInbox == null - ? {} - : { portableInbox: message.portableInbox }), - }) - : snapshotDeliveryIdentity(message.outbound); + const identity = snapshotOutboxIdentity(message); return new OutboundAttemptObserver( context, callback, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 1ace9c3db..6221cf968 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -1,4 +1,5 @@ import { + createTestMeterProvider, createTestTracerProvider, mockDocumentLoader, test, @@ -26,6 +27,7 @@ import { MemoryKvStore } from "./kv.ts"; import { ContextImpl, createFederation, + FederationImpl, OutboxContextImpl, } from "./middleware.ts"; import type { @@ -37,6 +39,7 @@ import type { OutboundDeliveryEvent, OutboundHttpExchangeReport, OutboundObservers, + OutboundWorkerReport, } from "./mod.ts"; import type { FanoutMessage, OutboxMessage } from "./queue.ts"; import { @@ -3301,6 +3304,409 @@ function continuationMessage( }; } +test("outbound worker-only observers count real sends and report local settlement", async (t) => { + for ( + const mode of [ + "success", + "handled", + "native", + "gateway", + "recovered-preparation", + ] as const + ) { + await t.step(mode, async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + const builder = createFederationBuilder<{ label: string }>(); + const reports: OutboundWorkerReport[] = []; + const contexts: unknown[] = []; + const queued: Message[] = []; + let replacements = 0; + builder.setOutboundObservers({ + async onWorker(ctx, report) { + reports.push(report); + contexts.push(ctx.data); + await Promise.resolve(); + throw new Error("worker observer error must not replace settlement"); + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + tracerProvider, + queue: { + outbox: { + nativeRetrial: mode === "native" || + mode === "recovered-preparation", + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + builder.setOutboundObservers({ + onWorker() { + replacements++; + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + if (mode === "gateway" || mode === "recovered-preparation") { + Object.assign(message, { + gatewayInboxes: [message.inbox, "https://gateway.example/inbox"], + }); + } + if (mode === "recovered-preparation") { + let reads = 0; + Object.assign(message, { + headers: new Proxy({}, { + get(target, property, receiver) { + if (property === Symbol.iterator && ++reads === 1) { + throw new Error("first gateway header preparation"); + } + return Reflect.get(target, property, receiver); + }, + }), + }); + } + const data = { label: "worker data" }; + fetchMock.mockGlobal().post( + message.inbox, + mode === "success" ? 202 : 503, + ); + if (mode === "gateway" || mode === "recovered-preparation") { + fetchMock.post( + "https://gateway.example/inbox", + mode === "gateway" ? 202 : 503, + ); + } + let outward: unknown; + try { + const processing = federation.processQueuedTask(data, message); + if (mode === "native" || mode === "recovered-preparation") { + await rejects(processing, (error: unknown) => { + outward = error; + return error instanceof SendActivityError; + }); + } else await processing; + strictEqual(replacements, 0); + strictEqual(reports.length, 1); + deepStrictEqual(contexts, [data]); + const report = reports[0]; + strictEqual(report.role, "outbox"); + strictEqual(report.attemptsStarted, mode === "gateway" ? 2 : 1); + deepStrictEqual(report.queue, { + queueMessageId: message.id, + queueAttempt: 0, + }); + deepStrictEqual(report.delivery, message.outbound); + deepStrictEqual(report.provenance, message.provenance); + strictEqual(report.local.origin, new URL(message.baseUrl).origin); + if (mode === "native" || mode === "recovered-preparation") { + deepStrictEqual(report.result, { type: "failed", error: outward }); + } else deepStrictEqual(report.result, { type: "completed" }); + strictEqual(queued.length, mode === "handled" ? 1 : 0); + const workerSpan = exporter.getSpan("activitypub.outbox"); + ok(workerSpan); + deepStrictEqual(report.trace, { + traceId: workerSpan.spanContext().traceId, + spanId: workerSpan.spanContext().spanId, + }); + ok(Object.isFrozen(report)); + ok(Object.isFrozen(report.queue)); + ok(Object.isFrozen(report.result)); + } finally { + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound worker observers report explicit key and fanout parse failures with zero sends", async (t) => { + for (const mode of ["key-import", "activity-parsing", "non-error"] as const) { + const phase = mode === "non-error" ? "activity-parsing" : mode; + await t.step(mode, async () => { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + const reports: OutboundWorkerReport[] = []; + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const original = continuationMessage(recipient.inboxId!.href, true); + let loaderCalls = 0; + Object.defineProperties(federation, { + documentLoaderFactory: { + value: () => { + loaderCalls++; + return mockDocumentLoader; + }, + }, + contextLoaderFactory: { + value: () => { + loaderCalls++; + if (mode === "non-error") throw NaN; + return mockDocumentLoader; + }, + }, + }); + const message: Message = phase === "key-import" + ? { + ...original, + keys: [{ keyId: sender.keyId.href, privateKey: { kty: "invalid" } }], + } + : { + type: "fanout", + id: original.id, + baseUrl: original.baseUrl, + keys: [], + inboxes: {}, + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + }, + activityType: Create.typeId.href, + traceContext: {}, + provenance: original.provenance, + }; + let outward: unknown; + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => { + outward = error; + return true; + }, + ); + strictEqual(reports.length, 1); + const report = reports[0]; + strictEqual(report.attemptsStarted, 0); + deepStrictEqual(report.result, { + type: "preparation-failed", + failure: { kind: "local", phase, error: outward }, + error: outward, + }); + deepStrictEqual(report.provenance, original.provenance); + strictEqual( + loaderCalls, + mode === "non-error" ? 1 : phase === "key-import" ? 0 : 2, + ); + if (phase === "activity-parsing") { + strictEqual(report.role, "fanout"); + strictEqual(report.delivery, undefined); + } + }); + } +}); + +test("outbound worker reports distinguish fanout child tasks from concrete sends", async (t) => { + for (const queued of [false, true]) { + await t.step(queued ? "child outbox tasks" : "direct sends", async () => { + const fanouts: Message[] = []; + const children: Message[] = []; + const queue = (messages: Message[]): MessageQueue => ({ + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }); + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { + fanout: queue(fanouts), + ...(queued ? { outbox: queue(children) } : {}), + }, + }); + const reports: OutboundWorkerReport[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onWorker(ctx, report) { + reports.push(report); + contexts.push(ctx.data); + }, + }); + const ctx = federation.createContext(new URL("https://example.com"), { + label: "producer", + }); + const targets = [ + recipient, + recipient.clone({ inbox: new URL("https://receiver.example/second") }), + ]; + await ctx.sendActivity(sender, targets, activity, { + fanout: "force", + dispatchId: "original invocation", + }); + strictEqual(reports.length, 0); + const task = fanouts[0]; + ok(task?.type === "fanout"); + const data = { label: "fanout worker" }; + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202).post( + "https://receiver.example/second", + 202, + ); + try { + await federation.processQueuedTask(data, task); + strictEqual(reports.length, 1); + strictEqual(reports[0].role, "fanout"); + strictEqual(reports[0].delivery, undefined); + strictEqual(reports[0].attemptsStarted, queued ? 0 : 2); + strictEqual(children.length, queued ? 2 : 0); + deepStrictEqual(reports[0].queue, { queueMessageId: task.id }); + deepStrictEqual(reports[0].result, { type: "completed" }); + strictEqual(reports[0].provenance.dispatchId, "original invocation"); + deepStrictEqual(contexts, [data]); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("outbound worker callbacks retain selection, await settlement, and isolate logging failures", async () => { + let logs = 0; + await configure({ + reset: true, + sinks: { + broken() { + logs++; + throw new Error("logging failure"); + }, + }, + loggers: [{ + category: ["fedify", "federation", "outbox"], + sinks: ["broken"], + lowestLevel: "error", + }, { category: ["logtape", "meta"], sinks: [] }], + }); + let respond!: () => void; + const responseGate = new Promise((resolve) => { + respond = resolve; + }); + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let replacements = 0; + let settled = false; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + async onWorker(_ctx, report) { + reports.push(report); + entered(); + await gate; + throw new Error("observer failure"); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, false); + fetchMock.mockGlobal().post(message.inbox, async () => { + fetched(); + await responseGate; + return 202; + }); + const processing = federation.processQueuedTask(undefined, message).finally( + () => { + settled = true; + }, + ); + try { + await fetching; + federation.setOutboundObservers({ + onWorker() { + replacements++; + }, + }); + respond(); + await observing; + strictEqual(settled, false); + release(); + await processing; + strictEqual(reports.length, 1); + strictEqual(replacements, 0); + strictEqual(logs, 1); + deepStrictEqual(reports[0].delivery, { + broadcast: { type: "unknown" }, + deliveryId: message.id, + }); + } finally { + respond(); + release(); + await processing; + fetchMock.hardReset(); + await reset(); + } +}); + +test("outbound worker failure after a completed sender does not rewrite its attempt", async () => { + const [meterProvider] = createTestMeterProvider(); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + meterProvider, + }); + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskOutcome; + const failure = Symbol("worker bookkeeping after sender settlement"); + Object.defineProperty(metrics, "recordQueueTaskOutcome", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + const attempts: OutboundAttemptReport[] = []; + const workers: OutboundWorkerReport[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onAttempt(ctx, report) { + attempts.push(report); + contexts.push(ctx); + }, + onWorker(ctx, report) { + workers.push(report); + contexts.push(ctx); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + fetchMock.mockGlobal().post(message.inbox, 202); + try { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(workers.length, 1); + strictEqual(workers[0].attemptsStarted, 1); + deepStrictEqual(workers[0].result, { type: "failed", error: failure }); + strictEqual(contexts[0], contexts[1]); + } finally { + Object.defineProperty(metrics, "recordQueueTaskOutcome", { + value: original, + }); + fetchMock.hardReset(); + } +}); + test("outbound worker attempts restore current context without creating loaders", async () => { const federation = createFederation<{ label: string }>({ kv: new MemoryKvStore(), diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index af5b5489d..3e3edb584 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -297,6 +297,45 @@ export type OutboundHttpExchangeReport = | { readonly type: "failed"; readonly error: unknown }; }; +/** + * One fanout or outbox worker execution after its local processing settles. + * + * Completion does not establish HTTP success or scheduling acceptance. Actions + * are reported separately when their observation is implemented. + * @since 2.5.0 + */ +export type OutboundWorkerReport = OutboundSourceMetadata & { + /** Current outgoing queue role; inbox/custom-task workers are separate. */ + readonly role: "fanout" | "outbox"; + /** Actual task and available Fedify retry counter for this execution. */ + readonly queue: NonNullable; + /** Concrete sender entries, independent of enabled attempt/exchange callbacks. */ + readonly attemptsStarted: number; + /** Known outbox obligation; absent for fanout or unavailable metadata. */ + readonly delivery?: Pick< + OutboundAttemptReport, + "broadcast" | "deliveryId" | "destinationInbox" | "portableInbox" + >; + /** Actual local settlement, separate from individual attempts. */ + readonly result: + | { readonly type: "completed" } + | { readonly type: "failed"; readonly error: unknown } + | { + readonly type: "preparation-failed"; + readonly failure: { + readonly kind: "local"; + readonly phase: + | "key-import" + | "activity-parsing" + | "context-preparation" + | "request-preparation"; + readonly error: unknown; + }; + /** Exact error escaping the worker preparation operation. */ + readonly error: unknown; + }; +}; + /** * Optional observers of outgoing activity processing. * @@ -317,6 +356,16 @@ export type OutboundHttpExchangeReport = * @since 2.5.0 */ export interface OutboundObservers { + /** + * Observes terminal outgoing worker processing with current context/data. + * Errors are isolated; results do not replace attempt outcomes or establish + * future backend execution. Scheduling actions are not observed here yet. + */ + readonly onWorker?: ( + context: Context, + report: OutboundWorkerReport, + ) => void | Promise; + /** * Observes each actual fetch after response headers or rejection, before * subsequent challenge/redirect handling. Non-2xx responses are still response diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index fd2c94829..38913d659 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -207,6 +207,8 @@ export interface SenderKeyPair { * Parameters for {@link sendActivity}. */ export interface SendActivityParameters { + /** Count actual sender entry for the current outgoing worker. @internal */ + readonly attemptStarted?: () => void; /** In-process destination observation; never serialized. @internal */ readonly observation?: { startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined; @@ -303,6 +305,7 @@ export function sendActivity( options: SendActivityParameters, deps: SendActivityDependencies = {}, ): Promise { + options.attemptStarted?.(); const attempt = options.observation?.startAttempt(options.inbox.href); const parentSpanContext = trace.getSpanContext(context.active()); const tracerProvider = options.tracerProvider ?? trace.getTracerProvider(); From 4e5d7798573917e08e0a72379e70e095527d5cc6 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:22:34 +0900 Subject: [PATCH 26/53] Report ordinary outbox worker actions Report actual enqueues, uncertain failures, and local abandonment so observers can distinguish them from worker completion and backend replay. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 124 +++++--- .../src/federation/outbound-observation.ts | 79 +++++ .../src/federation/outbound-report.test.ts | 294 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 41 ++- 4 files changed, 488 insertions(+), 50 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 9ab64c836..1d7caf09a 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1574,6 +1574,7 @@ export class FederationImpl worker?: OutboundWorkerObservation, ): Promise { const { onAttempt, onHttpExchange } = observers; + if ((message.gatewayInboxes?.length ?? 0) <= 1) worker?.enableActions(); const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { keyIds: message.keys.map((pair) => pair.keyId), @@ -1669,10 +1670,18 @@ export class FederationImpl circuitHeld: true, circuitHeldSince: heldSince.toString(), } satisfies OutboxMessage; - await outboxQueue.enqueue(heldMessage, { + const enqueueOptions = { delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, - }); + }; + await (worker == null + ? outboxQueue.enqueue(heldMessage, enqueueOptions) + : worker.enqueue( + "defer", + heldMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(heldMessage, enqueueOptions), + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -1722,6 +1731,7 @@ export class FederationImpl actorIds: getActorIds(), }); } catch (handlerError) { + worker?.dropFailure([inbox.href], handlerError); logger.error( "An unexpected error occurred in " + "outboxPermanentFailureHandler:\n{error}", @@ -1782,14 +1792,18 @@ export class FederationImpl return; } if (decision.type === "drop") { - const activity = await parseActivity(); - await dropHeldOutboxMessage( - circuit, - remoteHost, - inbox, - decision.heldSince, - activity, - ); + worker?.abandon("held-ttl-expired", [inbox.href]); + const drop = async () => { + const activity = await parseActivity(); + await dropHeldOutboxMessage( + circuit, + remoteHost, + inbox, + decision.heldSince, + activity, + ); + }; + await (worker == null ? drop() : worker.drop([inbox.href], drop)); return; } if (decision.stateChange != null) { @@ -1968,13 +1982,18 @@ export class FederationImpl } if (circuitDrop != null) { - await dropHeldOutboxMessage( - circuitDrop.circuit, - circuitDrop.remoteHost, - circuitDrop.inbox, - circuitDrop.heldSince, - activity, - ); + worker?.abandon("held-ttl-expired", [circuitDrop.inbox.href]); + const drop = () => + dropHeldOutboxMessage( + circuitDrop.circuit, + circuitDrop.remoteHost, + circuitDrop.inbox, + circuitDrop.heldSince, + activity, + ); + await (worker == null + ? drop() + : worker.drop([circuitDrop.inbox.href], drop)); return; } @@ -1982,6 +2001,7 @@ export class FederationImpl if ( isPermanentFailure ) { + worker?.abandon("permanent-failure", [message.inbox]); this.metrics.recordPermanentFailure( error.inbox, error.statusCode, @@ -1995,30 +2015,34 @@ export class FederationImpl }, ); if (this.outboxPermanentFailureHandler != null) { - const ctx = this.#createContext( - new URL(message.baseUrl), - contextData, - { - [outboundProvenance]: provenance, - documentLoader: this.documentLoaderFactory(loaderOptions), - }, - ); - try { - await this.outboxPermanentFailureHandler(ctx, { - reason: "http", - inbox: new URL(message.inbox), - activity, - error, - statusCode: error.statusCode, - actorIds: getActorIds(), - }); - } catch (handlerError) { - logger.error( - "An unexpected error occurred in " + - "outboxPermanentFailureHandler:\n{error}", - { ...logData, error: handlerError }, + const drop = async () => { + const ctx = this.#createContext( + new URL(message.baseUrl), + contextData, + { + [outboundProvenance]: provenance, + documentLoader: this.documentLoaderFactory(loaderOptions), + }, ); - } + try { + await this.outboxPermanentFailureHandler!(ctx, { + reason: "http", + inbox: new URL(message.inbox), + activity, + error, + statusCode: error.statusCode, + actorIds: getActorIds(), + }); + } catch (handlerError) { + worker?.dropFailure([message.inbox], handlerError); + logger.error( + "An unexpected error occurred in " + + "outboxPermanentFailureHandler:\n{error}", + { ...logData, error: handlerError }, + ); + } + }; + await (worker == null ? drop() : worker.drop([message.inbox], drop)); } recordOutboxActivity( this.meterProvider, @@ -2076,13 +2100,18 @@ export class FederationImpl } satisfies OutboxMessage; const { outboxQueue } = this; if (outboxQueue != null) { - await outboxQueue.enqueue( - retryMessage, - { - delay: clampNegativeDelay(delay), - orderingKey: message.orderingKey, - }, - ); + const enqueueOptions = { + delay: clampNegativeDelay(delay), + orderingKey: message.orderingKey, + }; + await (worker == null + ? outboxQueue.enqueue(retryMessage, enqueueOptions) + : worker.enqueue( + "retry", + retryMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(retryMessage, enqueueOptions), + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -2098,6 +2127,7 @@ export class FederationImpl ); } } else { + worker?.abandon("retry-policy-declined", [message.inbox]); logger.error( "Failed to send activity {activityId} to {inbox} after {attempt} " + "attempts; giving up:\n{error}", diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 3d8016d5e..61f744358 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -346,9 +346,13 @@ type WorkerPreparationFailure = Extract< { type: "preparation-failed" } >["failure"]; +type WorkerAction = NonNullable[number]; +type WorkerAbandonment = Extract; + /** Per-execution state, independent of concrete attempt/exchange callbacks. */ export class OutboundWorkerObservation { #attemptsStarted = 0; + #actions: WorkerAction[] | undefined; #trace: OutboundWorkerReport["trace"]; #preparationFailure: WorkerPreparationFailure | undefined; readonly #queue: OutboundWorkerReport["queue"]; @@ -381,6 +385,78 @@ export class OutboundWorkerObservation { this.#attemptsStarted++; }; + enableActions(): void { + this.#actions ??= []; + } + + async enqueue( + action: "retry" | "defer", + message: OutboxMessage, + delay: Temporal.Duration, + enqueue: () => Promise, + ): Promise { + if (this.#actions == null) return await enqueue(); + const facts = { + action, + queue: Object.freeze({ + queueMessageId: message.id, + queueAttempt: message.attempt, + }), + inboxUrls: Object.freeze([message.inbox]), + delay: delay.toString(), + }; + try { + await enqueue(); + } catch (error) { + this.#actions.push( + Object.freeze({ + type: "action-failed", + ...facts, + acceptance: "unknown" as const, + error, + }), + ); + throw error; + } + this.#actions.push(Object.freeze({ type: "scheduled", ...facts })); + } + + abandon( + reason: WorkerAbandonment["reason"], + inboxUrls: readonly string[], + ): void { + this.#actions?.push( + Object.freeze({ + type: "abandoned", + reason, + inboxUrls: Object.freeze([...inboxUrls]), + }), + ); + } + + dropFailure(inboxUrls: readonly string[], error: unknown): void { + this.#actions?.push( + Object.freeze({ + type: "action-failed", + action: "drop", + inboxUrls: Object.freeze([...inboxUrls]), + error, + }), + ); + } + + async drop( + inboxUrls: readonly string[], + operation: () => Promise, + ): Promise { + try { + await operation(); + } catch (error) { + this.dropFailure(inboxUrls, error); + throw error; + } + } + getContext(): Context { return this.#currentContext ??= this.context(); } @@ -438,6 +514,9 @@ export class OutboundWorkerObservation { queue: this.#queue, ...(this.#delivery == null ? {} : { delivery: this.#delivery }), attemptsStarted: this.#attemptsStarted, + ...(this.#actions == null + ? {} + : { actions: Object.freeze([...this.#actions]) }), observedAt, ...(this.#trace == null ? {} : { trace: this.#trace }), provenance: this.provenance, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 6221cf968..6246d418c 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -3304,6 +3304,295 @@ function continuationMessage( }; } +test("ordinary outbox worker actions record actual retry and hold enqueues", async (t) => { + for (const mode of ["retry", "retry-after", "hold"] as const) { + await t.step(mode, async () => { + const queued: { + message: Message; + options: Parameters[1]; + }[] = []; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => Temporal.Duration.from({ milliseconds: 250 }), + circuitBreaker: mode === "hold" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + queue: { + outbox: { + enqueue(message, options) { + queued.push({ message, options }); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + Object.assign(message, { + outbound: { + ...message.outbound!, + destinationInbox: "https://original.example/inbox", + }, + }); + fetchMock.mockGlobal().post( + message.inbox, + mode === "retry-after" + ? { + status: 429, + headers: { "Retry-After": "1" }, + } + : 503, + ); + try { + await federation.processQueuedTask(undefined, message); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 1); + deepStrictEqual(reports[0].result, { type: "completed" }); + strictEqual(queued.length, 1); + const task = queued[0].message; + ok(task.type === "outbox"); + const action = reports[0].actions?.[0]; + ok(action?.type === "scheduled"); + strictEqual(action.action, mode === "hold" ? "defer" : "retry"); + deepStrictEqual(action.queue, { + queueMessageId: task.id, + queueAttempt: mode === "hold" ? 0 : 1, + }); + strictEqual(reports[0].queue.queueAttempt, 0); + strictEqual(action.delay, queued[0].options?.delay?.toString()); + deepStrictEqual(action.inboxUrls, [message.inbox]); + strictEqual( + reports[0].delivery?.destinationInbox, + "https://original.example/inbox", + ); + strictEqual(reports[0].actions?.length, 1); + ok(Object.isFrozen(reports[0].actions)); + ok(Object.isFrozen(action)); + ok(Object.isFrozen(action.queue)); + ok(Object.isFrozen(action.inboxUrls)); + if (mode === "retry-after") { + strictEqual( + Temporal.Duration.from(action.delay).total("milliseconds"), + 1_000, + ); + } + if (mode === "hold") { + const requests = fetchMock.callHistory.calls().length; + await federation.processQueuedTask(undefined, structuredClone(task)); + strictEqual(reports.length, 2); + strictEqual(reports[1].attemptsStarted, 0); + strictEqual(fetchMock.callHistory.calls().length, requests); + const held = reports[1].actions?.[0]; + ok(held?.type === "scheduled"); + strictEqual(held.action, "defer"); + strictEqual(held.queue.queueAttempt, 0); + } + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("ordinary outbox worker actions preserve enqueue uncertainty and accepted bookkeeping failures", async (t) => { + for (const mode of ["rejected", "accepted"] as const) { + await t.step(mode, async () => { + const [meterProvider] = createTestMeterProvider(); + const failure = Symbol(mode); + const queued: Message[] = []; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + meterProvider, + circuitBreaker: false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return mode === "rejected" + ? Promise.reject(failure) + : Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskEnqueued; + if (mode === "accepted") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + } + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + fetchMock.mockGlobal().post(message.inbox, 503); + try { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + strictEqual(queued.length, 1); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 1); + deepStrictEqual(reports[0].result, { type: "failed", error: failure }); + const action = reports[0].actions?.[0]; + ok(action); + strictEqual(reports[0].actions?.length, 1); + strictEqual( + action.type, + mode === "rejected" ? "action-failed" : "scheduled", + ); + ok( + action.type === "scheduled" || + action.type === "action-failed" && action.action === "retry", + ); + strictEqual(action.action, "retry"); + deepStrictEqual(action.queue, { + queueMessageId: queued[0].id, + queueAttempt: 1, + }); + if (action.type === "action-failed") { + ok("acceptance" in action); + strictEqual(action.acceptance, "unknown"); + strictEqual(action.error, failure); + } else strictEqual("acceptance" in action, false); + } finally { + if (mode === "accepted") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + value: original, + }); + } + fetchMock.hardReset(); + } + }); + } +}); + +test("ordinary outbox worker actions retain local abandonment and failed drop steps", async (t) => { + for ( + const mode of [ + "permanent", + "declined", + "ttl", + "ttl-parse", + "ttl-handler", + "permanent-handler", + ] as const + ) { + await t.step(mode, async () => { + const reports: OutboundWorkerReport[] = []; + const queued: Message[] = []; + const ttl = mode.startsWith("ttl"); + const failure = Symbol(mode); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => null, + circuitBreaker: ttl ? { heldActivityTtl: { seconds: 1 } } : false, + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + if (mode.endsWith("handler")) { + federation.setOutboxPermanentFailureHandler(() => { + throw failure; + }); + } + if (mode === "ttl-parse") { + Object.defineProperty(federation, "documentLoaderFactory", { + value: () => { + throw failure; + }, + }); + } + const original = continuationMessage(recipient.inboxId!.href, true); + const message = ttl + ? { + ...original, + circuitHeld: true as const, + circuitHeldSince: "2000-01-01T00:00:00Z", + } + : original; + fetchMock.mockGlobal().post( + message.inbox, + mode.startsWith("permanent") ? 404 : 503, + ); + try { + const processing = federation.processQueuedTask(undefined, message); + if (mode === "ttl-parse") { + await rejects(processing, (error: unknown) => error === failure); + } else await processing; + strictEqual(reports.length, 1); + strictEqual(queued.length, 0); + const report = reports[0]; + strictEqual(report.attemptsStarted, ttl ? 0 : 1); + deepStrictEqual( + report.result, + mode === "ttl-parse" + ? { type: "failed", error: failure } + : { type: "completed" }, + ); + const actions = report.actions; + ok(actions); + const abandoned = actions[0]; + ok(abandoned.type === "abandoned"); + strictEqual( + abandoned.reason, + ttl + ? "held-ttl-expired" + : mode === "declined" + ? "retry-policy-declined" + : "permanent-failure", + ); + deepStrictEqual(abandoned.inboxUrls, [message.inbox]); + if (mode.endsWith("handler") || mode === "ttl-parse") { + strictEqual(actions.length, 2); + const failed = actions[1]; + ok(failed.type === "action-failed" && failed.action === "drop"); + strictEqual(failed.error, failure); + strictEqual("acceptance" in failed, false); + } else strictEqual(actions.length, 1); + if (ttl) strictEqual(fetchMock.callHistory.calls().length, 0); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + test("outbound worker-only observers count real sends and report local settlement", async (t) => { for ( const mode of [ @@ -3395,6 +3684,11 @@ test("outbound worker-only observers count real sends and report local settlemen deepStrictEqual(contexts, [data]); const report = reports[0]; strictEqual(report.role, "outbox"); + if (mode === "gateway" || mode === "recovered-preparation") { + strictEqual(report.actions, undefined); + } else if (mode === "handled") { + ok(report.actions?.[0].type === "scheduled"); + } else deepStrictEqual(report.actions, []); strictEqual(report.attemptsStarted, mode === "gateway" ? 2 : 1); deepStrictEqual(report.queue, { queueMessageId: message.id, diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 3e3edb584..a4947e1de 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -297,11 +297,43 @@ export type OutboundHttpExchangeReport = | { readonly type: "failed"; readonly error: unknown }; }; +type OutboundWorkerSchedule = { + readonly action: "retry" | "defer"; + /** Actual continuation task and Fedify retry counter. */ + readonly queue: NonNullable; + /** Actual affected routes, copied independently of the planned delivery. */ + readonly inboxUrls: readonly string[]; + /** ISO 8601 duration passed to enqueue after delay normalization. */ + readonly delay: string; +}; + +type OutboundWorkerAction = + | (OutboundWorkerSchedule & { readonly type: "scheduled" }) + | (OutboundWorkerSchedule & { + readonly type: "action-failed"; + readonly acceptance: "unknown"; + readonly error: unknown; + }) + | { + readonly type: "abandoned"; + readonly reason: + | "held-ttl-expired" + | "permanent-failure" + | "retry-policy-declined"; + readonly inboxUrls: readonly string[]; + } + | { + readonly type: "action-failed"; + readonly action: "drop"; + readonly inboxUrls: readonly string[]; + readonly error: unknown; + }; + /** * One fanout or outbox worker execution after its local processing settles. * - * Completion does not establish HTTP success or scheduling acceptance. Actions - * are reported separately when their observation is implemented. + * Completion does not establish HTTP success. Scheduling/drop action evidence + * is separate from the execution result and backend acknowledgment. * @since 2.5.0 */ export type OutboundWorkerReport = OutboundSourceMetadata & { @@ -311,6 +343,8 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { readonly queue: NonNullable; /** Concrete sender entries, independent of enabled attempt/exchange callbacks. */ readonly attemptsStarted: number; + /** Ordinary outbox action facts; omitted for other worker paths. */ + readonly actions?: readonly OutboundWorkerAction[]; /** Known outbox obligation; absent for fanout or unavailable metadata. */ readonly delivery?: Pick< OutboundAttemptReport, @@ -359,7 +393,8 @@ export interface OutboundObservers { /** * Observes terminal outgoing worker processing with current context/data. * Errors are isolated; results do not replace attempt outcomes or establish - * future backend execution. Scheduling actions are not observed here yet. + * future backend execution. Ordinary outbox action facts are included; + * gateway and fanout actions are not observed yet. */ readonly onWorker?: ( context: Context, From 40c35a5a3a5de5736952790c807e287283f678b6 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:31:46 +0900 Subject: [PATCH 27/53] Report partial gateway worker actions Distinguish per-route abandonment and actual continuation from whole-delivery outcomes in outbound reports. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 103 +++-- .../src/federation/outbound-observation.ts | 9 +- .../src/federation/outbound-report.test.ts | 358 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 12 +- 4 files changed, 439 insertions(+), 43 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 1d7caf09a..4aca2bcf2 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1574,7 +1574,7 @@ export class FederationImpl worker?: OutboundWorkerObservation, ): Promise { const { onAttempt, onHttpExchange } = observers; - if ((message.gatewayInboxes?.length ?? 0) <= 1) worker?.enableActions(); + worker?.enableActions(); const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { keyIds: message.keys.map((pair) => pair.keyId), @@ -2213,6 +2213,7 @@ export class FederationImpl readonly retryAfter?: Temporal.Duration; } const survivors: Survivor[] = []; + const permanentRoutes = new Set(); let lastError: unknown; let lastPermanent: { error: SendActivityError; inbox: URL } | undefined; let activity: Activity | undefined; @@ -2223,13 +2224,19 @@ export class FederationImpl inbox: URL, heldSince: Temporal.Instant, ) => { - await dropHeldOutboxMessage( - circuit, - remoteHost, - inbox, - heldSince, - await getActivity(), + const routes = message.gatewayInboxes.filter((route) => + !permanentRoutes.has(route) ); + worker?.abandon("held-ttl-expired", routes); + const execute = async () => + dropHeldOutboxMessage( + circuit, + remoteHost, + inbox, + heldSince, + await getActivity(), + ); + await (worker == null ? execute() : worker.drop(routes, execute)); }; // Checked up front, as gateways deferred by Retry-After are skipped // without asking the circuit breaker, which would otherwise tell: @@ -2378,6 +2385,8 @@ export class FederationImpl ); } if (isPermanentFailure) { + permanentRoutes.add(inboxHref); + worker?.abandon("permanent-failure", [inboxHref]); logger.warn( "Permanent delivery failure for activity {activityId} to " + "{inbox} ({status}); not trying the gateway again.", @@ -2451,32 +2460,38 @@ export class FederationImpl { ...logData, portableInbox: message.portableInbox }, ); if (this.outboxPermanentFailureHandler != null) { - const ctx = this.#createContext( - new URL(message.baseUrl), - contextData, - { - [outboundProvenance]: provenance, - documentLoader: this.documentLoaderFactory( - this.#getLoaderOptions(message.baseUrl), - ), - }, - ); - try { - await this.outboxPermanentFailureHandler(ctx, { - reason: "http", - inbox, - activity: await getActivity(), - error, - statusCode: error.statusCode, - actorIds: getActorIds(), - }); - } catch (handlerError) { - logger.error( - "An unexpected error occurred in " + - "outboxPermanentFailureHandler:\n{error}", - { ...logData, error: handlerError }, + const drop = async () => { + const ctx = this.#createContext( + new URL(message.baseUrl), + contextData, + { + [outboundProvenance]: provenance, + documentLoader: this.documentLoaderFactory( + this.#getLoaderOptions(message.baseUrl), + ), + }, ); - } + try { + await this.outboxPermanentFailureHandler!(ctx, { + reason: "http", + inbox, + activity: await getActivity(), + error, + statusCode: error.statusCode, + actorIds: getActorIds(), + }); + } catch (handlerError) { + worker?.dropFailure(message.gatewayInboxes, handlerError); + logger.error( + "An unexpected error occurred in " + + "outboxPermanentFailureHandler:\n{error}", + { ...logData, error: handlerError }, + ); + } + }; + await (worker == null + ? drop() + : worker.drop(message.gatewayInboxes, drop)); } recordOutboxActivity( this.meterProvider, @@ -2537,10 +2552,18 @@ export class FederationImpl "until one of its gateways can be tried again.", { ...logData, portableInbox: message.portableInbox }, ); - await outboxQueue.enqueue(heldMessage, { + const enqueueOptions = { delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, - }); + }; + await (worker == null + ? outboxQueue.enqueue(heldMessage, enqueueOptions) + : worker.enqueue( + "defer", + heldMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(heldMessage, enqueueOptions), + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -2567,6 +2590,10 @@ export class FederationImpl throw lastError; } if (policyDelay == null || outboxQueue == null) { + worker?.abandon( + outboxQueue == null ? "queue-unavailable" : "retry-policy-declined", + survivors.map((route) => route.inbox), + ); logger.error( "Failed to send activity {activityId} to every gateway of " + "{portableInbox} after {attempt} attempts; giving up:\n{error}", @@ -2593,10 +2620,14 @@ export class FederationImpl "{portableInbox} (attempt #{attempt}); retry...:\n{error}", { ...logData, portableInbox: message.portableInbox, error: lastError }, ); - await outboxQueue.enqueue(retryMessage, { + const enqueueOptions = { delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, - }); + }; + await (worker == null + ? outboxQueue.enqueue(retryMessage, enqueueOptions) + : worker.enqueue("retry", retryMessage, enqueueOptions.delay, () => + outboxQueue.enqueue(retryMessage, enqueueOptions))); this.metrics.recordQueueTaskEnqueued( { role: "outbox", diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 61f744358..4ac32bd2b 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -402,8 +402,15 @@ export class OutboundWorkerObservation { queueMessageId: message.id, queueAttempt: message.attempt, }), - inboxUrls: Object.freeze([message.inbox]), + inboxUrls: Object.freeze( + message.gatewayInboxes != null && message.gatewayInboxes.length > 1 + ? [...message.gatewayInboxes] + : [message.inbox], + ), delay: delay.toString(), + ...(message.gatewayNotBefore == null ? {} : { + notBefore: Object.freeze({ ...message.gatewayNotBefore }), + }), }; try { await enqueue(); diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 6246d418c..75e22d3e8 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -3304,6 +3304,309 @@ function continuationMessage( }; } +test("gateway worker actions retain route abandonment without claiming an unqueued deferral", async () => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + "https://gw3.example/inbox", + ]; + const queued: Message[] = []; + const workers: OutboundWorkerReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: false, + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + workers.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + }; + fetchMock.mockGlobal().post(gateways[0], 404) + .post(gateways[1], { status: 429, headers: { "Retry-After": "3600" } }) + .post(gateways[2], 202); + try { + await federation.processQueuedTask(undefined, message); + strictEqual(workers.length, 1); + strictEqual(workers[0].attemptsStarted, 3); + deepStrictEqual(workers[0].result, { type: "completed" }); + deepStrictEqual(workers[0].actions, [{ + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [gateways[0]], + }]); + strictEqual(queued.length, 0); + deepStrictEqual(attempts.map((report) => report.outcome.type), [ + "failed", + "failed", + "succeeded", + ]); + for (const report of attempts) { + strictEqual(report.deliveryId, message.outbound!.deliveryId); + } + } finally { + fetchMock.hardReset(); + } +}); + +test("gateway worker actions preserve abandonment and available drop errors", async (t) => { + const gateways = ["https://gw1.example/inbox", "https://gw2.example/inbox"]; + for ( + const mode of [ + "permanent", + "permanent-handler", + "declined", + "queue-unavailable", + "ttl", + "ttl-parse", + ] as const + ) { + await t.step(mode, async () => { + const ttl = mode.startsWith("ttl"); + const failure = Symbol(mode); + const reports: OutboundWorkerReport[] = []; + const queued: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: ttl ? { heldActivityTtl: { seconds: 1 } } : false, + outboxRetryPolicy: () => + mode === "declined" ? null : Temporal.Duration.from({ seconds: 1 }), + ...(mode === "queue-unavailable" ? {} : { + queue: { + outbox: { + enqueue(message: Message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }), + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + if (mode === "permanent-handler") { + federation.setOutboxPermanentFailureHandler(() => { + throw failure; + }); + } + if (mode === "ttl-parse") { + Object.defineProperty(federation, "documentLoaderFactory", { + value: () => { + throw failure; + }, + }); + } + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + ...(ttl + ? { + circuitHeld: true as const, + circuitHeldSince: "2000-01-01T00:00:00Z", + } + : {}), + }; + fetchMock.mockGlobal(); + for (const inbox of gateways) { + fetchMock.post(inbox, mode.startsWith("permanent") ? 404 : 503); + } + try { + const processing = federation.processQueuedTask(undefined, message); + if (mode === "ttl-parse") { + await rejects(processing, (error: unknown) => error === failure); + } else await processing; + strictEqual(reports.length, 1); + strictEqual(queued.length, 0); + const report = reports[0]; + strictEqual(report.attemptsStarted, ttl ? 0 : 2); + deepStrictEqual( + report.result, + mode === "ttl-parse" + ? { type: "failed", error: failure } + : { type: "completed" }, + ); + const actions = report.actions; + ok(actions); + const abandonments = actions.filter((action) => + action.type === "abandoned" + ); + if (mode.startsWith("permanent")) { + deepStrictEqual( + abandonments, + gateways.map((inbox) => ({ + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [inbox], + })), + ); + } else {deepStrictEqual(abandonments, [{ + type: "abandoned", + reason: ttl + ? "held-ttl-expired" + : mode === "declined" + ? "retry-policy-declined" + : "queue-unavailable", + inboxUrls: gateways, + }]);} + const failed = actions.filter((action) => + action.type === "action-failed" + ); + if (mode === "permanent-handler" || mode === "ttl-parse") { + strictEqual(failed.length, 1); + strictEqual(failed[0].action, "drop"); + strictEqual(failed[0].error, failure); + deepStrictEqual(failed[0].inboxUrls, gateways); + strictEqual("acceptance" in failed[0], false); + } else strictEqual(failed.length, 0); + if (ttl) strictEqual(fetchMock.callHistory.calls().length, 0); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("gateway continuation failures preserve partial actions and settled attempts", async (t) => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + "https://gw3.example/inbox", + ]; + for ( + const mode of ["retry-reject", "defer-reject", "retry-bookkeeping"] as const + ) { + await t.step(mode, async () => { + const [meterProvider] = createTestMeterProvider(); + const failure = Symbol(mode); + const queued: Message[] = []; + const workers: OutboundWorkerReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + meterProvider, + circuitBreaker: mode === "defer-reject" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return mode === "retry-bookkeeping" + ? Promise.resolve() + : Promise.reject(failure); + }, + listen: () => Promise.resolve(), + }, + }, + }); + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskEnqueued; + if (mode === "retry-bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + } + federation.setOutboundObservers({ + onWorker(_ctx, report) { + workers.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + }; + fetchMock.mockGlobal().post(gateways[0], 404).post(gateways[1], 503).post( + gateways[2], + 503, + ); + try { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + strictEqual(queued.length, 1); + strictEqual(workers.length, 1); + strictEqual(workers[0].attemptsStarted, 3); + deepStrictEqual(workers[0].result, { type: "failed", error: failure }); + const actions = workers[0].actions; + ok(actions); + strictEqual(actions.length, 2); + deepStrictEqual(actions[0], { + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [gateways[0]], + }); + const action = actions[1]; + ok( + action.type === "scheduled" || + action.type === "action-failed" && action.action !== "drop", + ); + strictEqual(action.action, mode === "defer-reject" ? "defer" : "retry"); + deepStrictEqual(action.inboxUrls, gateways.slice(1)); + const task = queued[0]; + ok(task.type === "outbox"); + deepStrictEqual(action.queue, { + queueMessageId: task.id, + queueAttempt: task.attempt, + }); + if (mode === "retry-bookkeeping") strictEqual(action.type, "scheduled"); + else { + ok(action.type === "action-failed"); + strictEqual(action.acceptance, "unknown"); + strictEqual(action.error, failure); + } + strictEqual(attempts.length, 3); + for (const report of attempts) { + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.failure.kind, "http"); + } + } finally { + if (mode === "retry-bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + value: original, + }); + } + fetchMock.hardReset(); + } + }); + } +}); + test("ordinary outbox worker actions record actual retry and hold enqueues", async (t) => { for (const mode of ["retry", "retry-after", "hold"] as const) { await t.step(mode, async () => { @@ -3685,7 +3988,7 @@ test("outbound worker-only observers count real sends and report local settlemen const report = reports[0]; strictEqual(report.role, "outbox"); if (mode === "gateway" || mode === "recovered-preparation") { - strictEqual(report.actions, undefined); + deepStrictEqual(report.actions, []); } else if (mode === "handled") { ok(report.actions?.[0].type === "scheduled"); } else deepStrictEqual(report.actions, []); @@ -4351,6 +4654,8 @@ test("outbound queue identities retain planned destinations through gateway cont for (const mode of ["retry", "hold", "defer", "native"] as const) { await t.step(`${mode}: ${observed ? "identity" : "legacy"}`, async () => { const queued: Message[] = []; + const enqueueOptions: Parameters[1][] = []; + const workers: OutboundWorkerReport[] = []; const federation = createFederation({ kv: new MemoryKvStore(), manuallyStartQueue: true, @@ -4361,8 +4666,9 @@ test("outbound queue identities retain planned destinations through gateway cont queue: { outbox: { nativeRetrial: mode === "native", - enqueue(message) { + enqueue(message, options) { queued.push(message); + enqueueOptions.push(options); return Promise.resolve(); }, listen: () => Promise.resolve(), @@ -4375,6 +4681,9 @@ test("outbound queue identities retain planned destinations through gateway cont }; const attempts: OutboundAttemptReport[] = []; federation.setOutboundObservers({ + onWorker(_ctx, report) { + workers.push(report); + }, onAttempt(_ctx, report) { attempts.push(report); }, @@ -4507,6 +4816,51 @@ test("outbound queue identities retain planned destinations through gateway cont ); strictEqual(attempts[3].queue?.queueAttempt, 1); } + strictEqual(workers.length, 2); + for (const [index, report] of workers.entries()) { + if (mode === "native") { + deepStrictEqual(report.actions, []); + strictEqual( + report.result.type, + index === 0 ? "failed" : "completed", + ); + strictEqual(report.attemptsStarted, index === 0 ? 3 : 1); + continue; + } + const actions = report.actions; + ok(actions); + if (index === 0) { + deepStrictEqual(actions[0], { + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [gateways[0]], + }); + } + strictEqual(actions.length, index === 0 ? 2 : 1); + const action = actions.at(-1)!; + ok(action.type === "scheduled"); + const task = queued[index]; + ok(task.type === "outbox"); + strictEqual( + action.action, + mode === "hold" || index === 1 && mode === "defer" + ? "defer" + : "retry", + ); + deepStrictEqual(action.inboxUrls, gateways.slice(1)); + deepStrictEqual(action.queue, { + queueMessageId: task.id, + queueAttempt: task.attempt, + }); + strictEqual(action.delay, enqueueOptions[index]?.delay?.toString()); + deepStrictEqual(action.notBefore, task.gatewayNotBefore); + if (action.notBefore != null) ok(Object.isFrozen(action.notBefore)); + strictEqual( + report.attemptsStarted, + index === 0 ? 3 : mode === "retry" ? 2 : 0, + ); + deepStrictEqual(report.result, { type: "completed" }); + } deepStrictEqual(task, original); deepStrictEqual(message, original); } finally { diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index a4947e1de..8bbecb7e0 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -305,6 +305,8 @@ type OutboundWorkerSchedule = { readonly inboxUrls: readonly string[]; /** ISO 8601 duration passed to enqueue after delay normalization. */ readonly delay: string; + /** Per-route ISO instants retained by the scheduled continuation, if any. */ + readonly notBefore?: Readonly>; }; type OutboundWorkerAction = @@ -319,7 +321,9 @@ type OutboundWorkerAction = readonly reason: | "held-ttl-expired" | "permanent-failure" - | "retry-policy-declined"; + | "retry-policy-declined" + | "queue-unavailable"; + /** Routes locally abandoned; other routes of the delivery may continue. */ readonly inboxUrls: readonly string[]; } | { @@ -343,7 +347,7 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { readonly queue: NonNullable; /** Concrete sender entries, independent of enabled attempt/exchange callbacks. */ readonly attemptsStarted: number; - /** Ordinary outbox action facts; omitted for other worker paths. */ + /** Outbox action facts, including gateway subsets; omitted for fanout. */ readonly actions?: readonly OutboundWorkerAction[]; /** Known outbox obligation; absent for fanout or unavailable metadata. */ readonly delivery?: Pick< @@ -393,8 +397,8 @@ export interface OutboundObservers { /** * Observes terminal outgoing worker processing with current context/data. * Errors are isolated; results do not replace attempt outcomes or establish - * future backend execution. Ordinary outbox action facts are included; - * gateway and fanout actions are not observed yet. + * future backend execution. Outbox/gateway action facts are included; + * fanout actions are not observed yet. */ readonly onWorker?: ( context: Context, From 27b906fd1d825f7abd03a9bd0dc902953213ceda Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:38:29 +0900 Subject: [PATCH 28/53] Report actual fanout and direct actions Preserve confirmed actions and stable identities across replay when queue acceptance is partial or unknown. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 146 ++++--- .../src/federation/outbound-observation.ts | 59 ++- .../src/federation/outbound-report.test.ts | 368 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 32 +- packages/fedify/src/federation/send.ts | 4 +- 5 files changed, 541 insertions(+), 68 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 4aca2bcf2..1c4b28626 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1498,6 +1498,7 @@ export class FederationImpl observers: OutboundObservers, worker?: OutboundWorkerObservation, ): Promise { + worker?.enableActions(); const logger = getLogger(["fedify", "federation", "fanout"]); logger.debug( "Fanning out activity {activityId} to {inboxes} inbox(es)...", @@ -3592,11 +3593,27 @@ export class FederationImpl const delivery = observation?.delivery(inbox); let sending: Promise; try { + const createHeaders = collectionSync == null + ? undefined + : async () => + new Headers({ + "Collection-Synchronization": + await buildCollectionSynchronizationHeader( + collectionSync, + inboxes[inbox].actorIds, + ), + }); sending = sendActivityToInbox( inbox, inboxes[inbox].gatewayInboxes, { - attemptStarted: worker?.attemptStarted, + attemptStarted: worker == null + ? undefined + : (target) => + worker.directAttemptStarted( + target, + observation?.continuation(inbox)?.outbound, + ), observation: delivery, allowPrivateAddress: this.allowPrivateAddress, keys, @@ -3604,13 +3621,11 @@ export class FederationImpl activityId: activity.id?.href, activityType: getTypeId(activity).href, sharedInbox: inboxes[inbox].sharedInbox, - headers: collectionSync == null ? undefined : new Headers({ - "Collection-Synchronization": - await buildCollectionSynchronizationHeader( - collectionSync, - inboxes[inbox].actorIds, - ), - }), + headers: createHeaders == null + ? undefined + : await (worker == null + ? createHeaders() + : worker.prepare("request-preparation", createHeaders)), specDeterminer: new KvSpecDeterminer( this.kv, this.kvPrefixes.httpMessageSignaturesSpec, @@ -3654,37 +3669,44 @@ export class FederationImpl const messages: { message: OutboxMessage; orderingKey?: string }[] = []; for (const inbox in inboxes) { try { - const { portableInbox, gatewayInboxes } = inboxes[inbox]; - const messageOrderingKey = orderingKey == null - ? undefined - : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; - const message: OutboxMessage = { - type: "outbox", - ...serializeProvenance(provenance), - id: crypto.randomUUID(), - baseUrl: ctx.origin, - keys: keyJwkPairs, - activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, - inbox, - ...observation?.continuation(inbox), - sharedInbox: inboxes[inbox].sharedInbox, - actorIds: [...inboxes[inbox].actorIds], - started: new Date().toISOString(), - attempt: 0, - headers: collectionSync == null ? {} : { - "Collection-Synchronization": - await buildCollectionSynchronizationHeader( - collectionSync, - inboxes[inbox].actorIds, - ), - }, - orderingKey: messageOrderingKey, - ...getPortableMessageFields(portableInbox, gatewayInboxes), - traceContext: carrier, + const prepareMessage = async () => { + const { portableInbox, gatewayInboxes } = inboxes[inbox]; + const messageOrderingKey = orderingKey == null + ? undefined + : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; + const message: OutboxMessage = { + type: "outbox", + ...serializeProvenance(provenance), + id: crypto.randomUUID(), + baseUrl: ctx.origin, + keys: keyJwkPairs, + activity: jsonLd, + activityId: activity.id?.href, + activityType: getTypeId(activity).href, + inbox, + ...observation?.continuation(inbox), + sharedInbox: inboxes[inbox].sharedInbox, + actorIds: [...inboxes[inbox].actorIds], + started: new Date().toISOString(), + attempt: 0, + headers: collectionSync == null ? {} : { + "Collection-Synchronization": + await buildCollectionSynchronizationHeader( + collectionSync, + inboxes[inbox].actorIds, + ), + }, + orderingKey: messageOrderingKey, + ...getPortableMessageFields(portableInbox, gatewayInboxes), + traceContext: carrier, + }; + return { message, orderingKey: messageOrderingKey }; }; - messages.push({ message, orderingKey: messageOrderingKey }); + messages.push( + await (worker == null + ? prepareMessage() + : worker.prepare("request-preparation", prepareMessage)), + ); } catch (error) { if (observation != null) { await observation.delivery(inbox)?.failed(error); @@ -3700,14 +3722,21 @@ export class FederationImpl if (outboxQueue.enqueueMany == null || orderingKey != null) { const promises: PromiseSettledResult[] = await Promise.allSettled( messages.map(async (m) => { - await (observation == null - ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) - : observation.enqueue( - [{ inbox: m.message.inbox, queueMessageId: m.message.id }], - "outbox", - () => - outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }), - )); + const enqueue = () => + worker == null + ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) + : worker.enqueueChildren( + [m.message], + () => + outboxQueue.enqueue(m.message, { + orderingKey: m.orderingKey, + }), + ); + await (observation == null ? enqueue() : observation.enqueue( + [{ inbox: m.message.inbox, queueMessageId: m.message.id }], + "outbox", + enqueue, + )); recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message); }), ); @@ -3729,16 +3758,21 @@ export class FederationImpl } } else { try { - await (observation == null - ? outboxQueue.enqueueMany(messages.map((m) => m.message)) - : observation.enqueue( - messages.map(({ message }) => ({ - inbox: message.inbox, - queueMessageId: message.id, - })), - "outbox", - () => outboxQueue.enqueueMany!(messages.map((m) => m.message)), - )); + const enqueue = () => + worker == null + ? outboxQueue.enqueueMany!(messages.map((m) => m.message)) + : worker.enqueueChildren( + messages.map((m) => m.message), + () => outboxQueue.enqueueMany!(messages.map((m) => m.message)), + ); + await (observation == null ? enqueue() : observation.enqueue( + messages.map(({ message }) => ({ + inbox: message.inbox, + queueMessageId: message.id, + })), + "outbox", + enqueue, + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to send later: {error}", diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 4ac32bd2b..fe43fc9ff 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -389,6 +389,59 @@ export class OutboundWorkerObservation { this.#actions ??= []; } + directAttemptStarted( + inboxUrl: string, + identity?: SerializedOutboundDelivery, + ): void { + this.attemptStarted(); + this.#actions?.push(Object.freeze({ + type: "handoff", + action: "direct", + inboxUrls: Object.freeze([inboxUrl]), + ...(identity == null + ? {} + : { delivery: snapshotDeliveryIdentity(identity) }), + })); + } + + async enqueueChildren( + messages: readonly OutboxMessage[], + enqueue: () => Promise, + ): Promise { + if (this.#actions == null) return await enqueue(); + const facts = messages.map((message) => ({ + action: "enqueue" as const, + queue: Object.freeze({ + queueMessageId: message.id, + queueAttempt: message.attempt, + }), + inboxUrls: Object.freeze( + message.gatewayInboxes != null && message.gatewayInboxes.length > 1 + ? [...message.gatewayInboxes] + : [message.inbox], + ), + ...(message.outbound == null + ? {} + : { delivery: snapshotDeliveryIdentity(message.outbound) }), + })); + try { + await enqueue(); + } catch (error) { + for (const fact of facts) { + this.#actions.push(Object.freeze({ + type: "action-failed", + ...fact, + acceptance: "unknown" as const, + error, + })); + } + throw error; + } + for (const fact of facts) { + this.#actions.push(Object.freeze({ type: "scheduled", ...fact })); + } + } + async enqueue( action: "retry" | "defer", message: OutboxMessage, @@ -521,9 +574,7 @@ export class OutboundWorkerObservation { queue: this.#queue, ...(this.#delivery == null ? {} : { delivery: this.#delivery }), attemptsStarted: this.#attemptsStarted, - ...(this.#actions == null - ? {} - : { actions: Object.freeze([...this.#actions]) }), + actions: Object.freeze([...(this.#actions ?? [])]), observedAt, ...(this.#trace == null ? {} : { trace: this.#trace }), provenance: this.provenance, @@ -762,7 +813,7 @@ export class OutboundBroadcastObservation { ): Promise | undefined> { if ( observers.onDelivery == null && observers.onAttempt == null && - observers.onHttpExchange == null && + observers.onHttpExchange == null && observers.onWorker == null && !Object.values(message.inboxes).some((entry) => entry.outbound != null) ) { return undefined; diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 75e22d3e8..ad8816466 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -3670,6 +3670,7 @@ test("ordinary outbox worker actions record actual retry and hold enqueues", asy queueAttempt: mode === "hold" ? 0 : 1, }); strictEqual(reports[0].queue.queueAttempt, 0); + ok(action.action === "retry" || action.action === "defer"); strictEqual(action.delay, queued[0].options?.delay?.toString()); deepStrictEqual(action.inboxUrls, [message.inbox]); strictEqual( @@ -4155,6 +4156,26 @@ test("outbound worker reports distinguish fanout child tasks from concrete sends strictEqual(reports[0].delivery, undefined); strictEqual(reports[0].attemptsStarted, queued ? 0 : 2); strictEqual(children.length, queued ? 2 : 0); + strictEqual(reports[0].actions.length, 2); + for (const action of reports[0].actions) { + strictEqual(action.type, queued ? "scheduled" : "handoff"); + ok(action.type === "scheduled" || action.type === "handoff"); + strictEqual(action.action, queued ? "enqueue" : "direct"); + const inbox = action.inboxUrls[0]; + strictEqual(action.delivery?.destinationInbox, inbox); + deepStrictEqual(action.delivery?.broadcast, { type: "unknown" }); + ok(action.delivery?.deliveryId); + if (action.type === "scheduled") { + const child = children.find((child) => + child.id === action.queue.queueMessageId + ); + ok(child?.type === "outbox"); + deepStrictEqual(action.delivery, child.outbound); + } + ok(Object.isFrozen(action)); + ok(Object.isFrozen(action.inboxUrls)); + ok(Object.isFrozen(action.delivery)); + } deepStrictEqual(reports[0].queue, { queueMessageId: task.id }); deepStrictEqual(reports[0].result, { type: "completed" }); strictEqual(reports[0].provenance.dispatchId, "original invocation"); @@ -4852,6 +4873,7 @@ test("outbound queue identities retain planned destinations through gateway cont queueMessageId: task.id, queueAttempt: task.attempt, }); + ok(action.action === "retry" || action.action === "defer"); strictEqual(action.delay, enqueueOptions[index]?.delay?.toString()); deepStrictEqual(action.notBefore, task.gatewayNotBefore); if (action.notBefore != null) ok(Object.isFrozen(action.notBefore)); @@ -4870,3 +4892,349 @@ test("outbound queue identities retain planned destinations through gateway cont } } }); + +test("fanout worker actions retain child enqueue settlement and replay identities", async (t) => { + for ( + const mode of [ + "individual", + "partial", + "batch", + "batch-rejected", + "ordered", + "bookkeeping", + ] as const + ) { + await t.step(mode, async () => { + const fanouts: Message[] = []; + const submitted: OutboxMessage[] = []; + const accepted: OutboxMessage[] = []; + const reports: OutboundWorkerReport[] = []; + const broadcasts: OutboundBroadcastEvent[] = []; + const failure = Symbol(mode); + const [meterProvider] = createTestMeterProvider(); + let batches = 0; + const outbox: MessageQueue = { + enqueue(message) { + ok(message.type === "outbox"); + submitted.push(message); + if ( + mode === "partial" && reports.length === 0 && + message.inbox.endsWith("/second") + ) { + return Promise.reject(failure); + } + accepted.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + if (["batch", "batch-rejected", "ordered"].includes(mode)) { + outbox.enqueueMany = (messages) => { + batches++; + for (const message of messages) { + ok(message.type === "outbox"); + submitted.push(message); + } + // A rejected batch may already have accepted part of its input. + accepted.push( + ...submitted.slice(0, mode === "batch-rejected" ? 1 : 2), + ); + return mode === "batch-rejected" + ? Promise.reject(failure) + : Promise.resolve(); + }; + } + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + meterProvider, + queue: { + fanout: { + enqueue(message) { + fanouts.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + outbox, + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onWorker(_ctx, report) { + reports.push(report); + }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + sender, + [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ], + activity, + { + fanout: "force", + ...(mode === "ordered" ? { orderingKey: "conversation" } : {}), + }, + ); + const task = fanouts[0]; + ok(task?.type === "fanout"); + const producerEvents = [...broadcasts]; + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskEnqueued; + if (mode === "bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + } + try { + for ( + let replay = 0; + replay < (mode === "individual" || mode === "partial" ? 2 : 1); + replay++ + ) { + submitted.length = 0; + accepted.length = 0; + const processing = federation.processQueuedTask( + undefined, + structuredClone(task), + ); + const rejected = mode === "partial" && replay === 0 || + mode === "batch-rejected" || mode === "bookkeeping"; + if (rejected) { + await rejects( + processing, + (error: unknown) => + mode === "bookkeeping" + ? error instanceof AggregateError && + error.errors.every((item) => item === failure) + : error === failure, + ); + } else await processing; + const report = reports[replay]; + strictEqual(report.result.type, rejected ? "failed" : "completed"); + strictEqual(report.attemptsStarted, 0); + strictEqual(report.delivery, undefined); + strictEqual(report.actions.length, 2); + strictEqual(submitted.length, 2); + strictEqual( + accepted.length, + mode === "partial" && replay === 0 || mode === "batch-rejected" + ? 1 + : 2, + ); + for (const child of submitted) { + const action = report.actions.find((action) => + "queue" in action && action.queue.queueMessageId === child.id + ); + ok( + action?.type === "scheduled" || action?.type === "action-failed", + ); + ok(action.action === "enqueue"); + deepStrictEqual(action.queue, { + queueMessageId: child.id, + queueAttempt: 0, + }); + deepStrictEqual(action.inboxUrls, [child.inbox]); + deepStrictEqual( + action.delivery, + task.inboxes[child.inbox].outbound, + ); + deepStrictEqual(action.delivery, child.outbound); + strictEqual("delay" in action, false); + strictEqual( + action.type, + mode === "batch-rejected" || + mode === "partial" && replay === 0 && + child.inbox.endsWith("/second") + ? "action-failed" + : "scheduled", + ); + if (action.type === "action-failed") { + strictEqual(action.acceptance, "unknown"); + strictEqual(action.error, failure); + } + if (replay > 0) { + const earlier = reports[0].actions.find((earlier) => + "delivery" in earlier && + earlier.delivery?.destinationInbox === child.inbox + ); + ok( + (earlier?.type === "scheduled" || + earlier?.type === "action-failed") && + earlier.action === "enqueue", + ); + notStrictEqual( + action.queue.queueMessageId, + earlier.queue.queueMessageId, + ); + deepStrictEqual(action.delivery, earlier.delivery); + } + } + deepStrictEqual(broadcasts, producerEvents); + } + strictEqual( + batches, + mode === "batch" || mode === "batch-rejected" ? 1 : 0, + ); + } finally { + if (mode === "bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: original, + }); + } + } + }); + } +}); + +test("fanout worker actions preserve earlier direct entries across later setup failure", async (t) => { + for (const queued of [false, true]) { + await t.step( + queued + ? "prepare all children before enqueue" + : "earlier send still settles", + async () => { + const fanouts: Message[] = []; + const children: Message[] = []; + const reports: OutboundWorkerReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const broadcasts: OutboundBroadcastEvent[] = []; + const queue = (messages: Message[]): MessageQueue => ({ + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { + fanout: queue(fanouts), + ...(queued ? { outbox: queue(children) } : {}), + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onWorker(_ctx, report) { + reports.push(report); + }, + }); + await federation.createContext( + new URL("https://example.com"), + undefined, + ) + .sendActivity( + sender, + [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ], + activity, + { fanout: "force" }, + ); + const originalTask = fanouts[0]; + ok(originalTask?.type === "fanout"); + const secondInbox = "https://receiver.example/second"; + const task: FanoutMessage = { + ...originalTask, + collectionSync: "https://example.com/followers", + inboxes: { + ...originalTask.inboxes, + [secondInbox]: { + ...originalTask.inboxes[secondInbox], + actorIds: ["invalid actor URL"], + }, + }, + }; + const producerEvents = [...broadcasts]; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let observed!: () => void; + const settled = new Promise((resolve) => { + observed = resolve; + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onWorker(_ctx, report) { + reports.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + observed(); + }, + }); + fetchMock.mockGlobal().post(recipient.inboxId!.href, async () => { + await gate; + return 202; + }); + try { + let failure: unknown; + await rejects( + federation.processQueuedTask(undefined, task), + (error: unknown) => { + failure = error; + return error instanceof TypeError; + }, + ); + strictEqual(reports.length, 1); + const report = reports[0]; + deepStrictEqual(report.result, { + type: "preparation-failed", + error: failure, + failure: { + kind: "local", + phase: "request-preparation", + error: failure, + }, + }); + strictEqual(report.attemptsStarted, queued ? 0 : 1); + strictEqual(children.length, 0); + strictEqual(report.actions.length, queued ? 0 : 1); + if (!queued) { + const action = report.actions[0]; + ok(action.type === "handoff"); + strictEqual(action.action, "direct"); + deepStrictEqual(action.inboxUrls, [recipient.inboxId!.href]); + deepStrictEqual( + action.delivery, + task.inboxes[recipient.inboxId!.href].outbound, + ); + strictEqual(attempts.length, 0); + release(); + await settled; + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(reports.length, 1); + } + deepStrictEqual(broadcasts, producerEvents); + } finally { + release(); + fetchMock.hardReset(); + } + }, + ); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 8bbecb7e0..3c5c49e5d 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -309,13 +309,33 @@ type OutboundWorkerSchedule = { readonly notBefore?: Readonly>; }; +type OutboundWorkerChild = { + readonly action: "enqueue"; + /** Actual child task identity and initial Fedify retry counter. */ + readonly queue: NonNullable; + /** Routes submitted with this child task. */ + readonly inboxUrls: readonly string[]; + /** Stable child obligation, independent of its task identity. */ + readonly delivery?: OutboundWorkerReport["delivery"]; +}; + type OutboundWorkerAction = - | (OutboundWorkerSchedule & { readonly type: "scheduled" }) - | (OutboundWorkerSchedule & { + | ((OutboundWorkerSchedule | OutboundWorkerChild) & { + readonly type: "scheduled"; + }) + | ((OutboundWorkerSchedule | OutboundWorkerChild) & { readonly type: "action-failed"; readonly acceptance: "unknown"; readonly error: unknown; }) + | { + readonly type: "handoff"; + readonly action: "direct"; + /** Actual concrete sender target; entry does not establish HTTP success. */ + readonly inboxUrls: readonly string[]; + /** Planned obligation retained across gateway fallback targets. */ + readonly delivery?: OutboundWorkerReport["delivery"]; + } | { readonly type: "abandoned"; readonly reason: @@ -347,8 +367,8 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { readonly queue: NonNullable; /** Concrete sender entries, independent of enabled attempt/exchange callbacks. */ readonly attemptsStarted: number; - /** Outbox action facts, including gateway subsets; omitted for fanout. */ - readonly actions?: readonly OutboundWorkerAction[]; + /** Observed scheduling, direct-entry, and local abandonment facts. */ + readonly actions: readonly OutboundWorkerAction[]; /** Known outbox obligation; absent for fanout or unavailable metadata. */ readonly delivery?: Pick< OutboundAttemptReport, @@ -397,8 +417,8 @@ export interface OutboundObservers { /** * Observes terminal outgoing worker processing with current context/data. * Errors are isolated; results do not replace attempt outcomes or establish - * future backend execution. Outbox/gateway action facts are included; - * fanout actions are not observed yet. + * future backend execution. Actions retain actual accepted/uncertain child + * handoffs and retry/defer/drop facts, separately from attempt outcomes. */ readonly onWorker?: ( context: Context, diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 38913d659..4ad9e44b0 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -208,7 +208,7 @@ export interface SenderKeyPair { */ export interface SendActivityParameters { /** Count actual sender entry for the current outgoing worker. @internal */ - readonly attemptStarted?: () => void; + readonly attemptStarted?: (inboxUrl: string) => void; /** In-process destination observation; never serialized. @internal */ readonly observation?: { startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined; @@ -305,7 +305,7 @@ export function sendActivity( options: SendActivityParameters, deps: SendActivityDependencies = {}, ): Promise { - options.attemptStarted?.(); + options.attemptStarted?.(options.inbox.href); const attempt = options.observation?.startAttempt(options.inbox.href); const parentSpanContext = trace.getSpanContext(context.active()); const tracerProvider = options.tracerProvider ?? trace.getTracerProvider(); From 6705edfc5a829ac20e8c1d74c68e9cdbbf81f696 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:45:10 +0900 Subject: [PATCH 29/53] Document outgoing worker reports Explain local settlement, delivery attempts, and queue actions so observers can interpret worker reports correctly. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 13 ++- changes.d/fedify/outbound-observers.md | 7 +- claude-plugin/skills/fedify/SKILL.md | 23 +++++ docs/manual/mq.md | 23 +++++ docs/manual/send.md | 92 ++++++++++++++++++- packages/fedify/src/federation/federation.ts | 5 +- .../fedify/src/federation/outbound-report.ts | 25 ++++- 7 files changed, 173 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 973aa57a2..38367449e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -52,15 +52,20 @@ To be released. for sends, original-payload forwarding, and automatic portable replication. Register `onBroadcast` to record producer starts and preparation results, and `onDelivery` for destination planning, direct handoffs, queue acceptance - or uncertainty, and local preparation failures. Callbacks receive - application context and independent grouping IDs without requiring + or uncertainty, and local preparation failures. Use `onAttempt` for concrete + send results and exact HTTP exchange counts, including queued retries and + gateway fallback, and `onHttpExchange` for each actual signed or unsigned + fetch. Reports distinguish received responses from the selected final + response and retain HTTP evidence alongside later local failures. They + include observation timestamps and optional operation trace IDs. Callbacks + receive application context and independent grouping IDs without requiring OpenTelemetry; their errors do not replace the original result. Known IDs and original inbox source survive queue continuations. Send/forward options accept `dispatchId` for application correlation without deduplication, limited to 1,024 UTF-8 bytes. Routed inbox reports include `inboxRequestId` and informational `requestId`. Missing legacy relationships remain unknown. - Readiness and handoff observations do not establish remote delivery success. - [[#1316] by Jiwon Kwon\] + Readiness and handoff observations do not establish remote delivery + success. [[#1316] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 79adb67d8..4bde643dd 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -10,7 +10,12 @@ links: send results and exact HTTP exchange counts, including queued retries and gateway fallback, and `onHttpExchange` for each actual signed or unsigned fetch. Reports distinguish received responses from the selected final - response and retain HTTP evidence alongside later local failures. They + response and retain HTTP evidence alongside later local failures. Use + `onWorker` for fanout/outbox execution results, concrete send counts, and + actual child enqueue, retry, deferral, or abandonment actions. Worker + reports retain accepted children through partial failures and distinguish + unknown queue acceptance from confirmed scheduling; local completion does + not establish backend acknowledgment or remote success. Reports include observation timestamps and optional operation trace IDs. Callbacks receive application context and independent grouping IDs without requiring OpenTelemetry; their errors do not replace the original result. Known IDs diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index d5932d108..c3d6ce973 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -286,6 +286,29 @@ separately from the outward `outcome.error`; HTTP failure evidence may lack an exception. Holds, drops, and failures before sender entry produce no attempt, and attempt reports do not establish worker scheduling results. +`onWorker(ctx, report)` reports each settled fanout/outbox execution, including +its instrumentation, with current worker context/data. It works independently +of attempt/exchange callbacks. `role` and `queue` identify the current worker; +`attemptsStarted` counts concrete sender entries, never child enqueues. Narrow +`result.type`: `completed` is local normal return (including handled failures +and abandonment), `failed` retains the escaping error, and `preparation-failed` +identifies the actual preparation phase. Outbox `delivery` identifies its +obligation; fanout has no single delivery. + +Read `actions` separately from the worker result. `scheduled` records a resolved +`enqueue`, `retry`, or `defer`; its `queue` identifies the actual task/counter. +Retry/defer includes ISO `delay` and optional per-route `notBefore` constraints. +`action-failed` retains enqueue rejection with `acceptance: "unknown"`, or a +failed `drop` step. `handoff`/`direct` records concrete fanout sender entry; +`abandoned` records a local reason. Each action lists affected `inboxUrls`, so +gateway outcomes can differ by route. Child enqueues and direct handoffs carry +known delivery references. Accepted children survive sibling/bookkeeping +failure; a rejected batch leaves all submitted children uncertain. Replay keeps +delivery IDs with fresh child task IDs, without another broadcast. Worker +settlement does not establish backend acknowledgment, a scheduled native retry, +or remote success; later direct attempts may settle after a setup failure has +already ended the worker. Do not infer scheduling from an attempt result. + Callbacks are awaited without a default timeout; observer/logging errors are isolated. Nonsettling callbacks can delay execution indefinitely. Portable forwarding can return at its existing deadline while background work and diff --git a/docs/manual/mq.md b/docs/manual/mq.md index 2cb99c78f..c46788af5 100644 --- a/docs/manual/mq.md +++ b/docs/manual/mq.md @@ -1297,6 +1297,29 @@ Optimized performance : Backend-specific optimizations for retry logic. +Observing outgoing workers +-------------------------- + +*This API is available since Fedify 2.5.0.* + +Register `onWorker` through `Federatable.setOutboundObservers()` to observe +local fanout and outbox execution, including calls to +`Federation.processQueuedTask()`. Reports include the current task, concrete +send count, local result, and actual child enqueue, retry, deferral, or +abandonment actions. They receive the current worker context and data, with +known delivery identities and source metadata restored from the queued message. +Inbox and custom-task processing use separate mechanisms. + +A completed worker report means the local path returned normally; it can include +a handled send failure or abandonment. An accepted enqueue can survive a later +worker failure, and a rejected enqueue or batch does not prove the backend +accepted nothing. Reports do not establish backend acknowledgment or future +native retry execution. Use attempt observers for concrete send results and +queue depth for messages still waiting. See +[worker execution and actions](./send.md#worker-execution-and-actions) for +report variants and a typed example. + + Queue depth reporting --------------------- diff --git a/docs/manual/send.md b/docs/manual/send.md index 2ade23dc8..598b6e76b 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -222,6 +222,90 @@ Failures before sender entry, such as queue key import or fanout parsing, and worker holds, drops, or deferrals do not create attempts. Attempt reports do not establish a worker's scheduling result. +### Worker execution and actions + +`onWorker` receives one `OutboundWorkerReport` after a fanout or outbox worker's +local execution settles, including its existing metrics and cleanup. It receives +the current worker context and application data. It also works when attempt and +exchange observers are disabled; inbox and custom-task workers are separate. + +`role` identifies `fanout` or `outbox`. `queue` identifies the current task and, +for outbox tasks, its Fedify retry counter. `attemptsStarted` counts actual +concrete sender entries: expanding fanout into child tasks counts zero, while +direct sends and alternate gateways each count when entered. An outbox report's +`delivery` identifies its obligation when available. Fanout has no single +`delivery`; its child actions carry their own delivery references. + +Narrow `result.type` to read the execution result: + +| Type | Meaning | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `completed` | The local worker path returned normally, including handled send failures and abandonment. | +| `failed` | Processing escaped with the original `error`, even if an earlier attempt succeeded. | +| `preparation-failed` | Key import, activity parsing, context creation, or request preparation failed; `failure.phase` identifies the operation. | + +`actions` records actual local decisions and enqueue settlements independently +of the result. An empty array means no such action was observed in that +execution. Narrow an action's `type`, then its `action` when present: + +| Type | Evidence | +| --------------- | --------------------------------------------------------------------------------------------------------------- | +| `scheduled` | An `enqueue`, `retry`, or `defer` call resolved. `queue` identifies the submitted task and counter. | +| `action-failed` | An enqueue rejected, with `acceptance: "unknown"` and the original `error`, or an available `drop` step failed. | +| `handoff` | A fanout worker entered a concrete sender (`action: "direct"`); this does not establish HTTP success. | +| `abandoned` | The local path stopped work for the listed routes, with a `reason` such as `permanent-failure`. | + +Each action's `inboxUrls` contains its affected routes. Gateway abandonment can +remove one route while another succeeds or is rescheduled. Retry/defer actions +include the actual ISO 8601 `delay` passed to the queue and any per-route +`notBefore` constraints retained in the continuation. Child `enqueue` actions +and direct handoffs include their known `delivery` references. Constructing a +child or deciding to retry does not itself establish an enqueue. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + onWorker(_ctx, report) { + console.log(report.role, report.queue.queueMessageId, report.attemptsStarted); + if (report.result.type === "preparation-failed") { + console.error(report.result.failure.phase, report.result.error); + } + for (const action of report.actions) { + if (action.type === "scheduled") { + console.log(action.action, action.queue.queueMessageId, action.inboxUrls); + if (action.action === "enqueue") { + console.log(action.delivery?.deliveryId); + } else { + console.log(action.delay, action.notBefore); + } + } else if (action.type === "action-failed") { + console.error(action.action, action.error); + if (action.action !== "drop") { + console.error("Queue acceptance is unknown", action.queue.queueMessageId); + } + } + } + }, +}); +~~~~ + +Accepted child tasks remain in the report when another child fails or later +bookkeeping throws. A rejected batch leaves acceptance unknown for every +submitted child, even if the backend accepted some of them. Fanout replay keeps +known delivery IDs while creating fresh child task IDs, without another +broadcast or a change to the producer's preparation result. A later destination +setup failure can leave earlier direct sends running; the worker report captures +entries observed by its settlement, while those attempt callbacks may arrive +later. + +Worker settlement does not establish backend acknowledgment, a future native +retry, or remote application effects. A native rethrow records failure without +claiming a scheduled continuation; native redelivery can reuse the task and +Fedify retry counter. See [message queues](./mq.md#observing-outgoing-workers) +for how this relates to queue processing. + ### Response versus final response A response means one fetch received HTTP headers. A final response is the @@ -377,10 +461,10 @@ Each registration call replaces all previous outbound observers; pass `{}` to clear them. Fedify copies the supplied callbacks, and building a federation copies the builder's registration. An active invocation or worker execution keeps its selected callbacks even if registration changes while it is running. -You can register any callback independently. On attempt/exchange reports, -`queue` contains the current `queueMessageId` and an available Fedify -`queueAttempt` counter. Native redelivery can repeat that counter while -creating fresh attempts. +You can register any callback independently, including `onWorker` alone. On +attempt/exchange reports, `queue` contains the current `queueMessageId` and an +available Fedify `queueAttempt` counter. Native redelivery can repeat that +counter while creating fresh attempts. Callbacks are awaited without a default timeout. Their return values and errors cannot veto sending, and callback or error-logging failures do not replace the diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index a03eac93f..80951a21a 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -95,7 +95,10 @@ export interface Federatable extends TaskRegistry { * replication preparation, destination planning, and handoffs, including * outbox-context operations and fanout continuation handoffs. Separate * callbacks report concrete send attempts and actual HTTP exchanges, including - * queued retries and gateway fallback. Known IDs, original inbox source, and + * queued retries and gateway fallback. Worker reports retain local execution + * results and actual enqueue, retry, deferral, and abandonment actions, + * separately from attempts and backend acknowledgment. + * Known IDs, original inbox source, and * application correlation survive queue work; workers use current context data. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 3c5c49e5d..730804e93 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -356,8 +356,11 @@ type OutboundWorkerAction = /** * One fanout or outbox worker execution after its local processing settles. * - * Completion does not establish HTTP success. Scheduling/drop action evidence - * is separate from the execution result and backend acknowledgment. + * Emitted after the local worker path and its instrumentation settle. A normal + * return can include handled send failures or abandonment; completion does not + * establish HTTP success or backend acknowledgment. Actions retain local route + * decisions and actual enqueue settlement, separately from attempt outcomes. + * A native rethrow does not establish a scheduled continuation or future replay. * @since 2.5.0 */ export type OutboundWorkerReport = OutboundSourceMetadata & { @@ -367,14 +370,24 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { readonly queue: NonNullable; /** Concrete sender entries, independent of enabled attempt/exchange callbacks. */ readonly attemptsStarted: number; - /** Observed scheduling, direct-entry, and local abandonment facts. */ + /** + * Observed enqueue settlement, concrete direct-entry, and local abandonment. + * Accepted actions survive later worker failure; rejected batches leave each + * submitted child uncertain. Empty when no action was observed by settlement. + */ readonly actions: readonly OutboundWorkerAction[]; - /** Known outbox obligation; absent for fanout or unavailable metadata. */ + /** + * Outbox obligation, with unavailable legacy relationships omitted. + * Fanout has no single delivery; its child actions carry their own references. + */ readonly delivery?: Pick< OutboundAttemptReport, "broadcast" | "deliveryId" | "destinationInbox" | "portableInbox" >; - /** Actual local settlement, separate from individual attempts. */ + /** + * Actual local settlement, separate from individual attempts and queue actions. + * Preparation failures identify the operation that produced the escaping error. + */ readonly result: | { readonly type: "completed" } | { readonly type: "failed"; readonly error: unknown } @@ -401,6 +414,8 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * events are observed, including outbox-context operations and fanout * continuation handoffs. Concrete attempts and actual fetch exchanges include * queue workers, signature challenges, redirects, and alternate gateways. + * Worker reports describe local fanout/outbox settlement and actual queue + * actions; they do not establish backend acknowledgment or future execution. * Known IDs and provenance survive queue continuations; callback context data * belongs to the current execution. Each callback is independently optional. * Callbacks are awaited without a default timeout. From 01eead515edbe477546956b864d4db6e7ce682cf Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:52:12 +0900 Subject: [PATCH 30/53] Capture outgoing exchange headers Allow opt-in header snapshots with mandatory redaction, independent budgets, and atomic configuration validation. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 3 + packages/fedify/src/federation/middleware.ts | 1 + .../fedify/src/federation/outbound-headers.ts | 86 ++++ .../src/federation/outbound-observation.ts | 37 +- .../src/federation/outbound-report.test.ts | 379 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 38 +- 6 files changed, 540 insertions(+), 4 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-headers.ts diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 986a7ea9a..113ae4402 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -1,3 +1,4 @@ +import { snapshotHeaderPolicy } from "./outbound-headers.ts"; import type { OutboundObservers } from "./outbound-report.ts"; import type { InboxRequestFinishedHandler } from "./inbox-report.ts"; import { @@ -665,6 +666,8 @@ export class FederationBuilderImpl setOutboundObservers(observers: OutboundObservers): void { this.outboundObservers = globalThis.Object.freeze({ + requestHeaders: snapshotHeaderPolicy(observers.requestHeaders), + responseHeaders: snapshotHeaderPolicy(observers.responseHeaders), onBroadcast: observers.onBroadcast, onDelivery: observers.onDelivery, onAttempt: observers.onAttempt, diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 1c4b28626..62cf65953 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1628,6 +1628,7 @@ export class FederationImpl message, provenance, onHttpExchange, + observers, ); let parsedActorIds: URL[] | undefined; const getActorIds = () => { diff --git a/packages/fedify/src/federation/outbound-headers.ts b/packages/fedify/src/federation/outbound-headers.ts new file mode 100644 index 000000000..afbd6d172 --- /dev/null +++ b/packages/fedify/src/federation/outbound-headers.ts @@ -0,0 +1,86 @@ +import type { + OutboundHeaderPolicy, + OutboundHeaderSnapshot, +} from "./outbound-report.ts"; + +const protectedHeaders = new Set([ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "signature", + "signature-input", + "authentication-info", + "proxy-authentication-info", + "x-api-key", + "x-auth-token", +]); + +function snapshotNames(names: readonly string[]): readonly string[] { + if (!Array.isArray(names)) { + throw new TypeError("Outbound header names must be an array."); + } + const normalized = new Set(); + for (const name of names) { + if ( + typeof name !== "string" || + !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name) || name.includes("*") + ) { + throw new TypeError("Outbound header names must be literal HTTP tokens."); + } + normalized.add(name.toLowerCase()); + } + return Object.freeze([...normalized].sort()); +} + +/** Validate/copy policy before registration replaces the selected observers. */ +export function snapshotHeaderPolicy( + policy: OutboundHeaderPolicy | undefined, +): OutboundHeaderPolicy | undefined { + if (policy === undefined) return undefined; + if (policy == null || typeof policy !== "object") { + throw new TypeError("Outbound header policy must be an object."); + } + const { allow, redact, maxBytes } = policy; + if (typeof maxBytes !== "number") { + throw new TypeError("Outbound header maxBytes must be a number."); + } + if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) { + throw new RangeError( + "Outbound header maxBytes must be a positive safe integer.", + ); + } + return Object.freeze({ + allow: snapshotNames(allow), + ...(redact === undefined ? {} : { redact: snapshotNames(redact) }), + maxBytes, + }); +} + +/** Copy only selected/redacted entries; budgets never truncate a header value. */ +export function snapshotHeaders( + headers: Headers, + policy: OutboundHeaderPolicy, +): OutboundHeaderSnapshot { + const redact = new Set([...protectedHeaders, ...(policy.redact ?? [])]); + const entries: (readonly [string, string])[] = []; + const encoder = new TextEncoder(); + let bytes = 0; + let limitReached = false; + for (const name of policy.allow) { + if (!headers.has(name)) continue; + // Preserve the runtime's entry order without retaining unselected values. + for (const [key, rawValue] of headers) { + if (key !== name) continue; + const value = redact.has(name) ? "[redacted]" : rawValue; + const size = encoder.encode(`${name}: ${value}\r\n`).byteLength; + if (size > policy.maxBytes - bytes) { + limitReached = true; + continue; + } + entries.push(Object.freeze([name, value] as const)); + bytes += size; + } + } + return Object.freeze({ entries: Object.freeze(entries), limitReached }); +} diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index fe43fc9ff..6bbd03086 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,3 +1,4 @@ +import { snapshotHeaders } from "./outbound-headers.ts"; import { getOutboundProvenance, invocationProvenance, @@ -164,6 +165,11 @@ type AttemptFacts = Pick< | "outcome" >; +type HeaderPolicies = Pick< + OutboundObservers, + "requestHeaders" | "responseHeaders" +>; + type ExchangeFacts = Pick< OutboundHttpExchangeReport, | "attemptId" @@ -195,6 +201,7 @@ export class OutboundAttemptObservation { readonly reportExchange: | ((facts: ExchangeFacts) => Promise) | undefined = undefined, + readonly headerPolicies: HeaderPolicies = {}, ) {} setSpan(span: Span, parentSpanContext?: SpanContext): void { @@ -209,6 +216,12 @@ export class OutboundAttemptObservation { const snapshot = Object.freeze({ url: request.url, method: request.method, + ...(this.headerPolicies.requestHeaders == null ? {} : { + headers: snapshotHeaders( + request.headers, + this.headerPolicies.requestHeaders, + ), + }), }); let response: Response; try { @@ -230,7 +243,15 @@ export class OutboundAttemptObservation { }); await this.#exchange(exchangeIndex, snapshot, { type: "response", - response: metadata, + response: Object.freeze({ + ...metadata, + ...(this.headerPolicies.responseHeaders == null ? {} : { + headers: snapshotHeaders( + response.headers, + this.headerPolicies.responseHeaders, + ), + }), + }), }); return response; } @@ -601,6 +622,7 @@ export class OutboundAttemptObserver { queue?: OutboundAttemptReport["queue"], readonly onHttpExchange: OutboundObservers["onHttpExchange"] = undefined, + readonly headerPolicies: HeaderPolicies = {}, ) { this.#queue = queue == null ? undefined : Object.freeze({ ...queue }); } @@ -611,6 +633,7 @@ export class OutboundAttemptObserver { message: OutboxMessage, provenance: OutboundProvenance, onHttpExchange?: OutboundObservers["onHttpExchange"], + headerPolicies: HeaderPolicies = {}, ): OutboundAttemptObserver | undefined { if (callback == null && onHttpExchange == null) return undefined; const identity = snapshotOutboxIdentity(message); @@ -624,6 +647,7 @@ export class OutboundAttemptObserver { queueAttempt: message.attempt, }, onHttpExchange, + headerPolicies, ); } @@ -658,6 +682,7 @@ export class OutboundAttemptObserver { }), "http-exchange", ), + this.onHttpExchange == null ? {} : this.headerPolicies, ); } } @@ -679,6 +704,7 @@ class OutboundDeliveryObservation { onAttempt: OutboundObservers["onAttempt"], onHttpExchange: OutboundObservers["onHttpExchange"], queue?: OutboundAttemptReport["queue"], + headerPolicies: HeaderPolicies = {}, ) { this.#reference = snapshotDeliveryIdentity(identity); this.#attempts = new OutboundAttemptObserver( @@ -688,6 +714,7 @@ class OutboundDeliveryObservation { this.provenance, queue, onHttpExchange, + headerPolicies, ); this.#plan = Object.freeze({ actorIds: Object.freeze(Array.from(destination.actorIds)), @@ -780,6 +807,7 @@ export class OutboundBroadcastObservation { readonly #onDelivery: OutboundObservers["onDelivery"]; readonly #onAttempt: OutboundObservers["onAttempt"]; readonly #onHttpExchange: OutboundObservers["onHttpExchange"]; + readonly #headerPolicies: HeaderPolicies; readonly #deliveries = new Map< string, OutboundDeliveryObservation @@ -801,6 +829,10 @@ export class OutboundBroadcastObservation { this.#onDelivery = observers.onDelivery; this.#onAttempt = observers.onAttempt; this.#onHttpExchange = observers.onHttpExchange; + this.#headerPolicies = { + requestHeaders: observers.requestHeaders, + responseHeaders: observers.responseHeaders, + }; } /** Continue an existing fanout; do not emit a new broadcast invocation. */ @@ -824,6 +856,8 @@ export class OutboundBroadcastObservation { onDelivery: observers.onDelivery, onAttempt: observers.onAttempt, onHttpExchange: observers.onHttpExchange, + requestHeaders: observers.requestHeaders, + responseHeaders: observers.responseHeaders, }, "send", getOutboundProvenance(context), @@ -870,6 +904,7 @@ export class OutboundBroadcastObservation { this.#onAttempt, this.#onHttpExchange, this.queue, + this.#headerPolicies, ); } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index ad8816466..28aec65e5 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -21,8 +21,9 @@ import { ok, rejects, strictEqual, + throws, } from "node:assert/strict"; -import { createFederationBuilder } from "./builder.ts"; +import { createFederationBuilder, FederationBuilderImpl } from "./builder.ts"; import { MemoryKvStore } from "./kv.ts"; import { ContextImpl, @@ -47,6 +48,7 @@ import { SendActivityError, } from "./send.ts"; import { OutboundBroadcastObservation } from "./outbound-observation.ts"; +import { snapshotHeaderPolicy, snapshotHeaders } from "./outbound-headers.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; const activity = new Create({ @@ -5238,3 +5240,378 @@ test("fanout worker actions preserve earlier direct entries across later setup f ); } }); + +test("outbound header policies validate atomically and copy registration", async () => { + const builder = createFederationBuilder(); + ok(builder instanceof FederationBuilderImpl); + const allow = ["X-Safe", "x-safe", "Content-Type"]; + const redact = ["X-Custom-Secret"]; + const policy = { allow, redact, maxBytes: 128 }; + const callback = () => {}; + builder.setOutboundObservers({ + requestHeaders: policy, + onHttpExchange: callback, + }); + const original = builder.outboundObservers; + deepStrictEqual(original.requestHeaders, { + allow: ["content-type", "x-safe"], + redact: ["x-custom-secret"], + maxBytes: 128, + }); + ok(Object.isFrozen(original.requestHeaders)); + ok(Object.isFrozen(original.requestHeaders?.allow)); + ok(Object.isFrozen(original.requestHeaders?.redact)); + allow.push("authorization"); + redact.length = 0; + policy.maxBytes = 1; + const invalid = [ + [{ allow: ["safe"], maxBytes: 0 }, RangeError], + [{ allow: ["safe"], maxBytes: -1 }, RangeError], + [{ allow: ["safe"], maxBytes: 1.5 }, RangeError], + [{ allow: ["safe"], maxBytes: Infinity }, RangeError], + [{ allow: ["safe"], maxBytes: NaN }, RangeError], + [{ allow: ["safe"], maxBytes: Number.MAX_SAFE_INTEGER + 1 }, RangeError], + [{ allow: ["safe"] }, TypeError], + [{ allow: ["safe"], maxBytes: "128" }, TypeError], + [{ allow: ["*"], maxBytes: 128 }, TypeError], + [{ allow: ["x-foo\nsecret"], maxBytes: 128 }, TypeError], + [{ allow: ["ümlaut"], maxBytes: 128 }, TypeError], + [{ allow: "x-safe", maxBytes: 128 }, TypeError], + [{ allow: [12], maxBytes: 128 }, TypeError], + [{ allow: [], redact: null, maxBytes: 128 }, TypeError], + [null, TypeError], + ] as const; + for (const [value, error] of invalid) { + throws(() => + builder.setOutboundObservers({ + requestHeaders: { allow: [], maxBytes: 1 }, + responseHeaders: value as never, + }), error); + strictEqual(builder.outboundObservers, original); + } + const federation = await builder.build(options); + ok(federation instanceof FederationImpl); + deepStrictEqual( + federation.outboundObservers.requestHeaders, + original.requestHeaders, + ); + builder.setOutboundObservers({}); + strictEqual(federation.outboundObservers.onHttpExchange, callback); + const registration = federation.outboundObservers; + throws( + () => + federation.setOutboundObservers({ + requestHeaders: { allow: [], maxBytes: 0 }, + }), + RangeError, + ); + strictEqual(federation.outboundObservers, registration); + federation.setOutboundObservers({}); + strictEqual(federation.outboundObservers.requestHeaders, undefined); +}); + +test("outbound header snapshots redact secrets and budget complete UTF-8 entries", () => { + const secrets = [ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "signature", + "signature-input", + "authentication-info", + "proxy-authentication-info", + "x-api-key", + "x-auth-token", + ]; + const headers = new Headers(secrets.map((name) => [name, "do-not-export"])); + headers.append("X-Visible", "first"); + headers.append("x-visible", "second"); + headers.set("X-Custom", "also-secret"); + const policy = snapshotHeaderPolicy({ + allow: [...secrets, "X-Visible", "x-custom", "x-missing"], + redact: ["X-Custom"], + maxBytes: 4096, + }); + ok(policy); + const snapshot = snapshotHeaders(headers, policy); + for (const [name, value] of snapshot.entries) { + strictEqual(value, name === "x-visible" ? "first, second" : "[redacted]"); + } + deepStrictEqual( + snapshot.entries.map(([name]) => name), + [...secrets, "x-visible", "x-custom"].sort(), + ); + strictEqual(JSON.stringify(snapshot).includes("do-not-export"), false); + strictEqual(JSON.stringify(snapshot).includes("also-secret"), false); + ok(Object.isFrozen(snapshot)); + ok(Object.isFrozen(snapshot.entries)); + for (const entry of snapshot.entries) ok(Object.isFrozen(entry)); + headers.set("x-visible", "changed"); + strictEqual( + snapshot.entries.find(([name]) => name === "x-visible")?.[1], + "first, second", + ); + const values = new Headers({ "a": "oversized value", "b": "é", "c": "z" }); + const capture = (maxBytes: number) => + snapshotHeaders( + values, + snapshotHeaderPolicy({ allow: ["c", "a", "b"], maxBytes })!, + ); + // `b: é\r\n` is seven UTF-8 bytes; skip a large entry and keep considering later ones. + deepStrictEqual(capture(7), { entries: [["b", "é"]], limitReached: true }); + deepStrictEqual(capture(6), { entries: [["c", "z"]], limitReached: true }); + deepStrictEqual( + snapshotHeaders( + new Headers({ b: "é" }), + snapshotHeaderPolicy({ allow: ["b"], maxBytes: 7 })!, + ), + { entries: [["b", "é"]], limitReached: false }, + ); + deepStrictEqual( + snapshotHeaders(values, snapshotHeaderPolicy({ allow: [], maxBytes: 1 })!), + { entries: [], limitReached: false }, + ); + deepStrictEqual( + snapshotHeaders( + new Headers({ authorization: "very long secret" }), + snapshotHeaderPolicy({ allow: ["authorization"], maxBytes: 27 })!, + ), + { entries: [["authorization", "[redacted]"]], limitReached: false }, + ); +}); + +test("outbound header policies retain selected snapshots across unsigned redirects", async () => { + const exchanges: OutboundHttpExchangeReport[] = []; + const allow = ["X-Trace", "Content-Type", "Authorization"]; + const builder = createFederationBuilder(); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + builder.setOutboundObservers({ + requestHeaders: { allow, maxBytes: 512 }, + responseHeaders: { + allow: ["Location", "X-Reply", "Set-Cookie"], + maxBytes: 512, + }, + async onHttpExchange(_ctx, report) { + exchanges.push(report); + entered(); + await gate; + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + builder.setOutboundObservers({}); + const message = { + ...continuationMessage(recipient.inboxId!.href, true), + headers: { + "X-Trace": "worker", + "Authorization": "secret", + "Content-Type": "application/activity+json", + }, + }; + const next = new URL("/next", message.inbox).href; + fetchMock.mockGlobal().post(message.inbox, { + status: 303, + headers: { + Location: next, + "X-Reply": "first", + "Set-Cookie": "secret-cookie", + }, + }).get(next, { status: 202, headers: { "X-Reply": "second" } }); + const processing = federation.processQueuedTask(undefined, message); + try { + await observing; + allow.length = 0; + federation.setOutboundObservers({ + onHttpExchange() { + throw new Error("replacement must not run"); + }, + }); + release(); + await processing; + strictEqual(exchanges.length, 2); + for (const report of exchanges) { + strictEqual( + report.request.headers?.entries.find(([name]) => + name === "authorization" + )?.[1], + "[redacted]", + ); + strictEqual( + report.request.headers?.entries.find(([name]) => name === "x-trace") + ?.[1], + "worker", + ); + ok(report.outcome.type === "response"); + ok(Object.isFrozen(report.outcome.response.headers)); + strictEqual(JSON.stringify(report).includes("secret-cookie"), false); + } + strictEqual(exchanges[1].request.method, "GET"); + strictEqual( + exchanges[1].request.headers?.entries.some(([name]) => + name === "content-type" + ), + false, + ); + ok( + exchanges[0].outcome.type === "response" && + exchanges[1].outcome.type === "response", + ); + deepStrictEqual(exchanges[0].outcome.response.headers?.entries, [ + ["location", next], + ["set-cookie", "[redacted]"], + ["x-reply", "first"], + ]); + deepStrictEqual(exchanges[1].outcome.response.headers?.entries, [[ + "x-reply", + "second", + ]]); + } finally { + release(); + await processing; + fetchMock.hardReset(); + } +}); + +test("outbound header snapshots follow signed challenge and redirect requests", async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + requestHeaders: snapshotHeaderPolicy({ + allow: ["content-digest", "signature", "signature-input"], + maxBytes: 1024, + }), + responseHeaders: snapshotHeaderPolicy({ + allow: ["accept-signature", "location"], + maxBytes: 1024, + }), + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + const next = new URL("/next", inbox).href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const responses: ResponseInit[] = [ + { + status: 401, + headers: { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created;nonce="observer-nonce"', + }, + }, + { status: 307, headers: { Location: next } }, + { status: 202 }, + ]; + const requests: Request[] = []; + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: { id: "https://example.com/activity" }, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { determineSpec: () => "rfc9421", rememberSpec() {} }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch(input) { + ok(input instanceof Request); + requests.push(input); + return Promise.resolve( + new Response(null, responses[requests.length - 1]), + ); + }, + close: () => Promise.resolve(), + }), + }); + strictEqual(exchanges.length, 3); + for (const [index, report] of exchanges.entries()) { + const expected = Array.from(requests[index].headers).filter(([name]) => + ["content-digest", "signature", "signature-input"].includes(name) + ).map(( + [name, value], + ) => [name, name.startsWith("signature") ? "[redacted]" : value]); + deepStrictEqual(report.request.headers?.entries, expected); + strictEqual(report.request.headers?.limitReached, false); + ok(requests[index].headers.has("signature")); + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, responses[index].status); + } + strictEqual(exchanges[2].request.url, next); + ok(exchanges[0].outcome.type === "response"); + strictEqual( + exchanges[0].outcome.response.headers?.entries[0][0], + "accept-signature", + ); +}); + +test("outbound header capture keeps each side independent and disabled by default", async (t) => { + for (const mode of ["disabled", "request", "response", "empty"] as const) { + await t.step(mode, async () => { + const reports: OutboundHttpExchangeReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + ...(mode === "request" || mode === "empty" + ? { + requestHeaders: { + allow: mode === "empty" ? [] : ["x-request"], + maxBytes: 128, + }, + } + : {}), + ...(mode === "response" + ? { responseHeaders: { allow: ["x-response"], maxBytes: 128 } } + : {}), + onHttpExchange(_ctx, report) { + reports.push(report); + }, + }); + const message = { + ...continuationMessage(recipient.inboxId!.href, false), + headers: { "X-Request": "request" }, + }; + fetchMock.mockGlobal().post(message.inbox, { + status: 202, + headers: { "X-Response": "response" }, + }); + try { + await federation.processQueuedTask(undefined, message); + strictEqual(reports.length, 1); + const report = reports[0]; + ok(report.outcome.type === "response"); + deepStrictEqual( + report.request.headers, + mode === "request" + ? { entries: [["x-request", "request"]], limitReached: false } + : mode === "empty" + ? { entries: [], limitReached: false } + : undefined, + ); + deepStrictEqual( + report.outcome.response.headers, + mode === "response" + ? { entries: [["x-response", "response"]], limitReached: false } + : undefined, + ); + } finally { + fetchMock.hardReset(); + } + }); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 730804e93..507d249a9 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -176,6 +176,24 @@ type OutboundAttemptFailure = readonly error: unknown; }; +/** Opt-in policy for one side of an actual HTTP exchange. @since 2.5.0 */ +export interface OutboundHeaderPolicy { + /** Literal HTTP names, normalized to lowercase and deduplicated at registration. */ + readonly allow: readonly string[]; + /** Additional names whose selected values are always `[redacted]`. */ + readonly redact?: readonly string[]; + /** Positive safe-integer UTF-8 budget for `name: value\r\n` entries. */ + readonly maxBytes: number; +} + +/** Immutable selected header values, using the runtime representation. @since 2.5.0 */ +export interface OutboundHeaderSnapshot { + /** Lowercase names in lexical order; credentials/signatures/cookies are redacted. */ + readonly entries: readonly (readonly [string, string])[]; + /** At least one selected entry was omitted because it exceeded the budget. */ + readonly limitReached: boolean; +} + interface OutboundHttpResponseMetadata { /** Actual HTTP status returned by fetch. */ readonly status: number; @@ -287,12 +305,20 @@ export type OutboundHttpExchangeReport = /** Zero-based submission order within this attempt. */ readonly exchangeIndex: number; /** Immutable facts of the request actually submitted. */ - readonly request: { readonly url: string; readonly method: string }; + readonly request: { + readonly url: string; + readonly method: string; + /** Selected header snapshot; absent when request header capture is disabled. */ + readonly headers?: OutboundHeaderSnapshot; + }; /** A received response, including non-2xx, or the original fetch rejection. */ readonly outcome: | { readonly type: "response"; - readonly response: OutboundHttpResponseMetadata; + readonly response: OutboundHttpResponseMetadata & { + /** Selected snapshot; absent when response header capture is disabled. */ + readonly headers?: OutboundHeaderSnapshot; + }; } | { readonly type: "failed"; readonly error: unknown }; }; @@ -429,6 +455,14 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * @since 2.5.0 */ export interface OutboundObservers { + /** + * Opt-in actual request headers for exchange reports; disabled by default. + * Protected credentials, signatures, and cookies remain redacted even if allowed. + */ + readonly requestHeaders?: OutboundHeaderPolicy; + /** Independent opt-in response header policy for exchange reports. */ + readonly responseHeaders?: OutboundHeaderPolicy; + /** * Observes terminal outgoing worker processing with current context/data. * Errors are isolated; results do not replace attempt outcomes or establish From 33ad055ecef4be3f35a0561debfe1d07ff793485 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 04:55:44 +0900 Subject: [PATCH 31/53] Bound response body diagnostics Limit diagnostic reads and their lifetime while preserving HTTP errors and partial previews when responses stall. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-body.test.ts | 138 ++++++++++++++++++ .../fedify/src/federation/outbound-body.ts | 80 ++++++++++ packages/fedify/src/federation/send.test.ts | 50 +++++++ packages/fedify/src/federation/send.ts | 26 +++- 4 files changed, 287 insertions(+), 7 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-body.test.ts create mode 100644 packages/fedify/src/federation/outbound-body.ts diff --git a/packages/fedify/src/federation/outbound-body.test.ts b/packages/fedify/src/federation/outbound-body.test.ts new file mode 100644 index 000000000..40d1afb4a --- /dev/null +++ b/packages/fedify/src/federation/outbound-body.test.ts @@ -0,0 +1,138 @@ +import { test } from "@fedify/fixture"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { ResponseBodyPump } from "./outbound-body.ts"; + +test("response body pump pulls once on demand and releases at EOF", async () => { + let pulls = 0; + let cancels = 0; + let controller!: ReadableStreamDefaultController; + let pulled!: () => void; + const pulling = new Promise((resolve) => { + pulled = resolve; + }); + const body = new ReadableStream({ + start(value) { + controller = value; + }, + pull() { + pulls++; + pulled(); + }, + cancel() { + cancels++; + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 1000); + try { + strictEqual(pulls, 0); + ok(body.locked); + const first = pump.read(); + const concurrent = pump.read(); + strictEqual(first, concurrent); + await pulling; + strictEqual(pulls, 1); + controller.enqueue(new Uint8Array([1, 2])); + const result = await first; + deepStrictEqual(result, { done: false, value: new Uint8Array([1, 2]) }); + strictEqual(await concurrent, result); + strictEqual(pulls, 1); + controller.close(); + deepStrictEqual(await pump.read(), { done: true, value: undefined }); + strictEqual(body.locked, false); + strictEqual(cancels, 0); + deepStrictEqual(await pump.read(), { done: true, value: undefined }); + } finally { + pump.close(); + } +}); + +test("response body pump deadline bounds stalled reads and cancellation cleanup", async (t) => { + for (const mode of ["no-pull", "stalled", "late-rejection"] as const) { + await t.step(mode, async () => { + let pulls = 0; + let cancels = 0; + let cancelled!: () => void; + const cancelling = new Promise((resolve) => { + cancelled = resolve; + }); + let rejectCancel!: (error: unknown) => void; + const cleanup = new Promise((_resolve, reject) => { + rejectCancel = reject; + }); + const body = new ReadableStream({ + pull() { + pulls++; + }, + cancel() { + cancels++; + cancelled(); + return cleanup; + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 25); + try { + if (mode === "no-pull") await cancelling; + else strictEqual(await pump.read(), undefined); + strictEqual(cancels, 1); + strictEqual(body.locked, false); + strictEqual(pulls, mode === "no-pull" ? 0 : 1); + strictEqual(await pump.read(), undefined); + pump.close(); + strictEqual(cancels, 1); + if (mode === "late-rejection") { + rejectCancel(new Error("late cleanup failure")); + // Flush rejection handlers; the late failure must remain observed. + await Promise.resolve(); + await Promise.resolve(); + } + } finally { + pump.close(); + } + }); + } +}); + +test("response body pump deadline also stops continuously ready empty chunks", async () => { + let pulls = 0; + let cancels = 0; + const body = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array()); + }, + cancel() { + cancels++; + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 25); + try { + while ( + await pump.read() != null + ) { /* Keep pulling a non-terminating source. */ } + ok(pulls > 0); + strictEqual(cancels, 1); + strictEqual(body.locked, false); + const stoppedAt = pulls; + strictEqual(await pump.read(), undefined); + strictEqual(pulls, stoppedAt); + } finally { + pump.close(); + } +}); + +test("response body pump preserves read failure and releases ownership", async () => { + const failure = new Error("broken body"); + const body = new ReadableStream({ + pull(controller) { + controller.error(failure); + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 1000); + try { + await rejects(pump.read(), (error: unknown) => error === failure); + strictEqual(body.locked, false); + deepStrictEqual(await pump.read(), { done: true, value: undefined }); + } finally { + pump.close(); + } +}); diff --git a/packages/fedify/src/federation/outbound-body.ts b/packages/fedify/src/federation/outbound-body.ts new file mode 100644 index 000000000..aa36d2fc4 --- /dev/null +++ b/packages/fedify/src/federation/outbound-body.ts @@ -0,0 +1,80 @@ +/** One pull-driven response reader with an absolute lifetime and no extra queue. */ +export class ResponseBodyPump { + readonly #reader: ReadableStreamDefaultReader; + readonly #expiresAt: number; + readonly #deadline: Promise; + readonly #timer: ReturnType; + #expireDeadline!: () => void; + #pending: + | Promise | undefined> + | undefined; + #timedOut = false; + #closed = false; + #ended = false; + + constructor(body: ReadableStream, timeoutMs: number) { + this.#reader = body.getReader(); + this.#expiresAt = performance.now() + timeoutMs; + this.#deadline = new Promise((resolve) => { + this.#expireDeadline = () => resolve(undefined); + }); + this.#timer = setTimeout(() => this.#expire(), timeoutMs); + } + + /** Undefined means the absolute lifetime expired, independently of EOF. */ + read(): Promise | undefined> { + if (this.#closed) { + return Promise.resolve( + this.#timedOut ? undefined : { done: true, value: undefined }, + ); + } + if (performance.now() >= this.#expiresAt) { + this.#expire(); + return Promise.resolve(undefined); + } + // Concurrent consumers share this read instead of issuing another one. + this.#pending ??= Promise.race([this.#reader.read(), this.#deadline]).then( + (result) => { + this.#pending = undefined; + // A continuously ready stream can keep timers from running. Check the + // same monotonic deadline on every read, including empty chunks. + if (this.#timedOut || performance.now() >= this.#expiresAt) { + this.#expire(); + return undefined; + } + if (this.#closed) return { done: true as const, value: undefined }; + if (result?.done) { + this.#ended = true; + this.close(); + } + return result; + }, + (error) => { + this.#pending = undefined; + this.close(); + throw error; + }, + ); + return this.#pending; + } + + /** Stop pulling and release ownership without awaiting arbitrary cancellation. */ + close(): void { + if (this.#closed) return; + this.#closed = true; + clearTimeout(this.#timer); + try { + if (!this.#ended) this.#reader.cancel().catch(() => {}); + } finally { + // Pending read rejection and late cancellation rejection are both observed. + this.#reader.releaseLock(); + } + } + + #expire(): void { + if (this.#timedOut) return; + this.#timedOut = true; + this.#expireDeadline(); + this.close(); + } +} diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index ad2efe2a7..ecb32dc67 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -1837,3 +1837,53 @@ function recordHops( }, }; } + +test("sendActivity() bounds a stalled diagnostic preview without changing HTTP failure", async () => { + const inbox = new URL("https://example.com/lifecycle/stalled-error"); + let reads = 0; + let cancels = 0; + let closed = 0; + let submissions = 0; + const body = new ReadableStream({ + pull(controller) { + if (reads++ === 0) { + controller.enqueue(new TextEncoder().encode("partial error")); + } + }, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const error = await assertRejects(() => + sendActivity({ + activity: lifecycleActivity, + keys: unsignedKeys, + inbox, + }, { + openHop: () => + Promise.resolve({ + url: inbox, + fetch: () => { + submissions++; + return Promise.resolve( + new Response(body, { + status: 503, + headers: { "Retry-After": "1" }, + }), + ); + }, + close: () => { + closed++; + assertFalse(body.locked); + return Promise.resolve(); + }, + }), + }), SendActivityError); + assertEquals(error.statusCode, 503); + assertEquals(error.responseBody, "partial error… (timed out)"); + assertEquals(error.responseHeaders.get("Retry-After"), "1"); + assertEquals(submissions, 1); + assertEquals(cancels, 1); + assertEquals(closed, 1); +}); diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 4ad9e44b0..8ec2b4bc8 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -1,3 +1,4 @@ +import { ResponseBodyPump } from "./outbound-body.ts"; import type { Recipient } from "@fedify/vocab"; import { FetchError, UrlError } from "@fedify/vocab-runtime"; import { @@ -388,6 +389,9 @@ export async function sendActivityThroughGateways( } const MAX_ERROR_RESPONSE_BODY_BYTES = 1024; +// Diagnostic previews must not indefinitely delay an already known HTTP failure. +// This budget is independent of opt-in application response-body capture. +const ERROR_RESPONSE_BODY_TIMEOUT_MS = 1000; function getActivityActorId(activity: unknown): string | undefined { if (!isRecord(activity)) return undefined; @@ -419,15 +423,25 @@ async function readLimitedResponseBody( if (response.body == null) { return ""; } - const reader = response.body.getReader(); + const pump = new ResponseBodyPump( + response.body, + ERROR_RESPONSE_BODY_TIMEOUT_MS, + ); const decoder = new TextDecoder(); const chunks: string[] = []; let totalBytes = 0; let truncated = false; + let timedOut = false; try { while (true) { - const { done, value } = await reader.read(); + const result = await pump.read(); + if (result == null) { + timedOut = true; + break; + } + const { done, value } = result; if (done) break; + if (value.length === 0) continue; if (totalBytes + value.length > maxBytes) { const remaining = maxBytes - totalBytes; if (remaining > 0) { @@ -442,15 +456,13 @@ async function readLimitedResponseBody( totalBytes += value.length; } } finally { - // Start cancelling the rest of a truncated body instead of leaving it - // unread. It is not awaited, since the cancellation of a tee branch - // (e.g., a body cloned by a fetch mock) may never settle: - if (truncated) reader.cancel().catch(() => {}); - reader.releaseLock(); + pump.close(); } let result = chunks.join(""); if (truncated) { result += "… (truncated)"; + } else if (timedOut) { + result += "… (timed out)"; } return result; } From c133d8f2f3b6bd737b0a3361b71956e7f2e22ccd Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 05:03:43 +0900 Subject: [PATCH 32/53] Capture bounded outgoing response bodies Allow opt-in capture with explicit limits and a revocable stream, keeping capture outcomes separate from HTTP delivery outcomes. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 2 + packages/fedify/src/federation/middleware.ts | 1 + .../fedify/src/federation/outbound-body.ts | 17 +- .../src/federation/outbound-capture.test.ts | 266 ++++++++++++++++++ .../fedify/src/federation/outbound-capture.ts | 261 +++++++++++++++++ .../src/federation/outbound-observation.ts | 114 +++++++- .../src/federation/outbound-report.test.ts | 244 ++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 67 ++++- packages/fedify/src/federation/send.ts | 3 + 9 files changed, 954 insertions(+), 21 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-capture.test.ts create mode 100644 packages/fedify/src/federation/outbound-capture.ts diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 113ae4402..ceb084e73 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -1,3 +1,4 @@ +import { snapshotBodyCapture } from "./outbound-capture.ts"; import { snapshotHeaderPolicy } from "./outbound-headers.ts"; import type { OutboundObservers } from "./outbound-report.ts"; import type { InboxRequestFinishedHandler } from "./inbox-report.ts"; @@ -666,6 +667,7 @@ export class FederationBuilderImpl setOutboundObservers(observers: OutboundObservers): void { this.outboundObservers = globalThis.Object.freeze({ + responseBodyCapture: snapshotBodyCapture(observers.responseBodyCapture), requestHeaders: snapshotHeaderPolicy(observers.requestHeaders), responseHeaders: snapshotHeaderPolicy(observers.responseHeaders), onBroadcast: observers.onBroadcast, diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 62cf65953..7be62cf1c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1629,6 +1629,7 @@ export class FederationImpl provenance, onHttpExchange, observers, + observers.responseBodyCapture, ); let parsedActorIds: URL[] | undefined; const getActorIds = () => { diff --git a/packages/fedify/src/federation/outbound-body.ts b/packages/fedify/src/federation/outbound-body.ts index aa36d2fc4..803c12eb4 100644 --- a/packages/fedify/src/federation/outbound-body.ts +++ b/packages/fedify/src/federation/outbound-body.ts @@ -1,9 +1,9 @@ /** One pull-driven response reader with an absolute lifetime and no extra queue. */ export class ResponseBodyPump { readonly #reader: ReadableStreamDefaultReader; - readonly #expiresAt: number; + #expiresAt: number; readonly #deadline: Promise; - readonly #timer: ReturnType; + #timer: ReturnType; #expireDeadline!: () => void; #pending: | Promise | undefined> @@ -58,6 +58,19 @@ export class ResponseBodyPump { return this.#pending; } + /** Shorten the remaining lifetime when ownership passes to diagnostic cleanup. */ + limitLifetime(timeoutMs: number): void { + if (this.#closed) return; + this.#expiresAt = Math.min( + this.#expiresAt, + performance.now() + Math.max(0, timeoutMs), + ); + clearTimeout(this.#timer); + if (performance.now() >= this.#expiresAt) this.#expire(); + else {this.#timer = setTimeout(() => + this.#expire(), this.#expiresAt - performance.now());} + } + /** Stop pulling and release ownership without awaiting arbitrary cancellation. */ close(): void { if (this.#closed) return; diff --git a/packages/fedify/src/federation/outbound-capture.test.ts b/packages/fedify/src/federation/outbound-capture.test.ts new file mode 100644 index 000000000..a738f2a58 --- /dev/null +++ b/packages/fedify/src/federation/outbound-capture.test.ts @@ -0,0 +1,266 @@ +import { test } from "@fedify/fixture"; +import { + deepStrictEqual, + ok, + rejects, + strictEqual, + throws, +} from "node:assert/strict"; +import { + captureResponseBody, + snapshotBodyCapture, + takeCapturedDiagnostic, +} from "./outbound-capture.ts"; +import { createFederationBuilder, FederationBuilderImpl } from "./builder.ts"; + +const limits = { maxBytes: 32, timeoutMs: 1000 }; +async function drain(body: ReadableStream): Promise { + const reader = body.getReader(); + const bytes: number[] = []; + try { + while (true) { + const value = await reader.read(); + if (value.done) return bytes; + bytes.push(...value.value); + } + } catch { + return bytes; + } finally { + reader.releaseLock(); + } +} + +test("response body capture validates explicit limits before replacing registration", () => { + const config = { maxBytes: 32, timeoutMs: 0.5, onBody() {} }; + const builder = createFederationBuilder(); + ok(builder instanceof FederationBuilderImpl); + builder.setOutboundObservers({ responseBodyCapture: config }); + const original = builder.outboundObservers; + deepStrictEqual(original.responseBodyCapture, { + ...config, + timeoutMs: 1, + responses: "all", + }); + ok(Object.isFrozen(original.responseBodyCapture)); + config.maxBytes = 1; + strictEqual(original.responseBodyCapture?.maxBytes, 32); + for ( + const [value, error] of [ + [null, TypeError], + [{ ...limits }, TypeError], + [{ onBody() {}, timeoutMs: 1 }, TypeError], + [{ onBody() {}, maxBytes: 1 }, TypeError], + [{ ...limits, onBody() {}, maxBytes: "1" }, TypeError], + [{ ...limits, onBody() {}, maxBytes: 0 }, RangeError], + [{ ...limits, onBody() {}, maxBytes: 1.5 }, RangeError], + [ + { ...limits, onBody() {}, maxBytes: Number.MAX_SAFE_INTEGER + 1 }, + RangeError, + ], + [{ ...limits, onBody() {}, timeoutMs: Infinity }, RangeError], + [{ ...limits, onBody() {}, timeoutMs: NaN }, RangeError], + [{ ...limits, onBody() {}, timeoutMs: 0 }, RangeError], + [{ ...limits, onBody() {}, timeoutMs: 2147483647.1 }, RangeError], + [{ ...limits, onBody() {}, responses: "failures" }, TypeError], + ] as const + ) { + throws( + () => + builder.setOutboundObservers({ responseBodyCapture: value as never }), + error, + ); + strictEqual(builder.outboundObservers, original); + } + strictEqual(snapshotBodyCapture(undefined), undefined); +}); + +test("response body capture distinguishes selection EOF and exact byte limits", async (t) => { + for ( + const mode of [ + "filtered", + "no-body", + "empty", + "short", + "exact", + "over", + ] as const + ) { + await t.step(mode, async () => { + const response = new Response( + mode === "no-body" + ? null + : mode === "empty" + ? "" + : mode === "over" + ? "abcd" + : "abc", + ); + let calls = 0; + let exposed: number[] = []; + let signal: AbortSignal | undefined; + const result = await captureResponseBody(response, { + maxBytes: mode === "exact" || mode === "over" ? 3 : 32, + timeoutMs: 1000, + ...(mode === "filtered" + ? { responses: "non-success-http-responses" as const } + : {}), + }, async (body, current) => { + calls++; + signal = current; + exposed = await drain(body); + }); + strictEqual( + result.type, + mode === "filtered" || mode === "no-body" + ? mode + : mode === "exact" || mode === "over" + ? "limit-reached" + : "complete", + ); + strictEqual( + result.bytesExposed, + mode === "filtered" || mode === "no-body" || mode === "empty" ? 0 : 3, + ); + strictEqual(calls, mode === "filtered" || mode === "no-body" ? 0 : 1); + if (calls) { + ok(signal?.aborted); + strictEqual(response.body?.locked, false); + } + if (result.bytesExposed) deepStrictEqual(exposed, [97, 98, 99]); + ok(Object.isFrozen(result)); + }); + } +}); + +test("response body capture revokes locked readers and bounds callback cleanup", async (t) => { + for ( + const mode of [ + "returned", + "cancelled", + "throws", + "hangs", + "pending-read", + "body-error", + ] as const + ) { + await t.step(mode, async () => { + const failure = new Error(mode); + let signal!: AbortSignal; + let retained!: ReadableStreamDefaultReader; + let pending: Promise | undefined; + let rejectLate!: (error: unknown) => void; + const late = new Promise((_resolve, reject) => { + rejectLate = reject; + }); + const body = new ReadableStream({ + pull(controller) { + if (mode === "body-error") controller.error(failure); + }, + cancel() { + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const result = await captureResponseBody(new Response(body), { + ...limits, + timeoutMs: 25, + }, async (stream, current) => { + signal = current; + retained = stream.getReader(); + if (mode === "throws") throw failure; + if (mode === "cancelled") await retained.cancel(failure); + if (mode === "hangs") await late; + if (mode === "pending-read") { + pending = retained.read().catch((error) => error); + } + if (mode === "body-error") await retained.read(); + }); + const expected = mode === "returned" || mode === "pending-read" + ? "callback-returned" + : mode === "cancelled" + ? "consumer-stopped" + : mode === "hangs" + ? "deadline" + : "failed"; + strictEqual(result.type, expected); + strictEqual(result.bytesExposed, 0); + ok(signal.aborted); + strictEqual(body.locked, false); + if (result.type === "failed") { + strictEqual(result.phase, mode === "throws" ? "observer" : "body"); + strictEqual(result.error, failure); + } + if (result.type === "consumer-stopped") { + strictEqual(result.reason, failure); + } + if (mode !== "cancelled") await rejects(retained.read()); + if (pending) ok(await pending instanceof Error); + retained.releaseLock(); + if (mode === "hangs") { + rejectLate(failure); + await Promise.resolve(); + await Promise.resolve(); + } + }); + } +}); + +test("response body capture shares one reader with bounded diagnostics", async () => { + const response = new Response("diagnostic failure", { status: 503 }); + ok(response.body); + const getReader = response.body.getReader.bind(response.body); + let acquisitions = 0; + Object.defineProperty(response.body, "getReader", { + value: () => { + acquisitions++; + return getReader(); + }, + }); + const summary = await captureResponseBody(response, { + ...limits, + maxBytes: 4, + }, async (body) => { + await drain(body); + }); + strictEqual(summary.type, "limit-reached"); + strictEqual(summary.bytesExposed, 4); + strictEqual(acquisitions, 1); + strictEqual(takeCapturedDiagnostic(response), "diagnostic failure"); + strictEqual(takeCapturedDiagnostic(response), undefined); +}); + +test("response body capture isolates unavailable readers and diagnostic deadlines", async (t) => { + await t.step("already locked source", async () => { + const response = new Response("locked"); + const reader = response.body!.getReader(); + try { + const result = await captureResponseBody(response, limits, () => { + throw new Error("must not invoke callback"); + }); + ok(result.type === "failed"); + strictEqual(result.phase, "body"); + strictEqual(result.bytesExposed, 0); + } finally { + await reader.cancel(); + reader.releaseLock(); + } + }); + await t.step("diagnostics retain their own deadline", async () => { + let cancels = 0; + const body = new ReadableStream({ + pull() {}, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const response = new Response(body, { status: 503 }); + const result = await captureResponseBody(response, { + ...limits, + timeoutMs: 10000, + }, () => {}); + deepStrictEqual(result, { type: "callback-returned", bytesExposed: 0 }); + strictEqual(takeCapturedDiagnostic(response), "… (timed out)"); + strictEqual(cancels, 1); + strictEqual(body.locked, false); + }); +}); diff --git a/packages/fedify/src/federation/outbound-capture.ts b/packages/fedify/src/federation/outbound-capture.ts new file mode 100644 index 000000000..a9daefd41 --- /dev/null +++ b/packages/fedify/src/federation/outbound-capture.ts @@ -0,0 +1,261 @@ +import { ResponseBodyPump } from "./outbound-body.ts"; +import type { + OutboundBodyCaptureSummary, + OutboundResponseBodyCapture, +} from "./outbound-report.ts"; + +const diagnosticTimeoutMs = 1000; +const diagnosticMaxBytes = 1024; +const diagnostics = new WeakMap(); + +/** Reuse a selected capture's diagnostic preview instead of acquiring a second reader. */ +export function takeCapturedDiagnostic(response: Response): string | undefined { + const value = diagnostics.get(response); + diagnostics.delete(response); + return value; +} + +/** Validate and copy before replacing the current observer registration. */ +export function snapshotBodyCapture( + value: OutboundResponseBodyCapture | undefined, +): OutboundResponseBodyCapture | undefined { + if (value === undefined) return undefined; + if ( + value == null || typeof value !== "object" || + typeof value.onBody !== "function" + ) { + throw new TypeError("Response body capture requires an onBody callback."); + } + const { maxBytes, timeoutMs, responses = "all", onBody } = value; + if (typeof maxBytes !== "number" || typeof timeoutMs !== "number") { + throw new TypeError( + "Response body capture requires numeric maxBytes and timeoutMs.", + ); + } + if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) { + throw new RangeError( + "Response body capture maxBytes must be a positive safe integer.", + ); + } + if ( + !Number.isFinite(timeoutMs) || timeoutMs <= 0 || + Math.ceil(timeoutMs) > 2147483647 + ) { + throw new RangeError( + "Response body capture timeoutMs is outside the timer range.", + ); + } + if (responses !== "all" && responses !== "non-success-http-responses") { + throw new TypeError("Invalid response body capture selector."); + } + return Object.freeze({ + maxBytes, + timeoutMs: Math.ceil(timeoutMs), + responses, + onBody, + }); +} + +type WithoutBytes = T extends unknown ? Omit : never; +type CaptureResult = WithoutBytes; + +/** One source reader feeds both the revocable application view and bounded diagnostics. */ +export async function captureResponseBody( + response: Response, + options: { + maxBytes: number; + timeoutMs: number; + responses?: "all" | "non-success-http-responses"; + }, + onBody: ( + body: ReadableStream, + signal: AbortSignal, + ) => void | Promise, +): Promise { + if (options.responses === "non-success-http-responses" && response.ok) { + return Object.freeze({ type: "filtered", bytesExposed: 0 }); + } + if (response.body == null) { + return Object.freeze({ type: "no-body", bytesExposed: 0 }); + } + const start = performance.now(); + const expires = start + options.timeoutMs; + let pump: ResponseBodyPump; + try { + pump = new ResponseBodyPump( + response.body, + Math.max(options.timeoutMs, response.ok ? 0 : diagnosticTimeoutMs), + ); + } catch (error) { + return Object.freeze({ + type: "failed", + phase: "body" as const, + error, + bytesExposed: 0, + }); + } + const signal = new AbortController(); + let controller!: ReadableStreamDefaultController; + let active = true; + let bytesExposed = 0; + let result: CaptureResult | undefined; + let eof = false; + let diagnosticBytes = 0; + let diagnosticTruncated = false; + let diagnosticFailed = false; + const preview: string[] = []; + const decoder = new TextDecoder(); + let pending: + | Promise | undefined> + | undefined; + const read = () => + pending ??= pump.read().then((value) => { + pending = undefined; + if (value?.done) eof = true; + if ( + !response.ok && value != null && !value.done && + performance.now() - start < diagnosticTimeoutMs + ) { + const remaining = diagnosticMaxBytes - diagnosticBytes; + const chunk = value.value; + if (chunk.length > remaining) diagnosticTruncated = true; + if (remaining > 0 && chunk.length > 0) { + const selected = chunk.subarray(0, remaining); + preview.push(decoder.decode(selected, { stream: true })); + diagnosticBytes += selected.length; + } + } + return value; + }, (error) => { + pending = undefined; + diagnosticFailed = true; + throw error; + }); + const revoke = (reason: unknown) => { + active = false; + if (!signal.signal.aborted) signal.abort(reason); + // Controller access works even while the application retains a locked reader. + try { + controller.error(reason); + } catch { /* The consumer may already have cancelled. */ } + }; + const body = new ReadableStream({ + start(value) { + controller = value; + }, + async pull() { + try { + while (active) { + if (performance.now() >= expires) { + result = { type: "deadline" }; + revoke( + new DOMException( + "Response body capture deadline exceeded.", + "TimeoutError", + ), + ); + return; + } + const value = await read(); + if (!active) return; + if (performance.now() >= expires || value == null) { + result = { type: "deadline" }; + revoke( + new DOMException( + "Response body capture deadline exceeded.", + "TimeoutError", + ), + ); + return; + } + if (value.done) { + result = { type: "complete" }; + active = false; + controller.close(); + return; + } + if (value.value.length === 0) continue; + const chunk = value.value.slice(0, options.maxBytes - bytesExposed); + bytesExposed += chunk.length; + controller.enqueue(chunk); + if (bytesExposed === options.maxBytes) { + result = { type: "limit-reached" }; + revoke( + new DOMException( + "Response body capture byte limit reached.", + "QuotaExceededError", + ), + ); + } + return; + } + } catch (error) { + if (active) { + result = { type: "failed", phase: "body", error }; + revoke(error); + } + } + }, + cancel(reason) { + result = { type: "consumer-stopped", reason }; + revoke(reason); + }, + }, { highWaterMark: 0 }); + let timer!: ReturnType; + const deadline = new Promise((resolve) => { + timer = setTimeout(() => { + result = { type: "deadline" }; + revoke( + new DOMException( + "Response body capture deadline exceeded.", + "TimeoutError", + ), + ); + resolve(); + }, Math.max(0, expires - performance.now())); + }); + const callback = Promise.resolve().then(() => onBody(body, signal.signal)) + .then( + () => {}, + (error) => { + if (result == null || result.type === "complete") { + result = { type: "failed", phase: "observer", error }; + } + }, + ); + try { + await Promise.race([callback, deadline]); + clearTimeout(timer); + if (performance.now() >= expires) result = { type: "deadline" }; + result ??= { type: "callback-returned" }; + revoke(new DOMException("Response body capture finished.", "AbortError")); + // Diagnostics may continue under their own budget, never exposing more app bytes. + if (!response.ok) { + pump.limitLifetime(diagnosticTimeoutMs - (performance.now() - start)); + try { + while ( + !eof && !diagnosticTruncated && + performance.now() - start < diagnosticTimeoutMs + ) { + if (await read() == null) break; + } + } catch { /* Diagnostic failure does not replace HTTP evidence. */ } + diagnostics.set( + response, + preview.join("") + + (diagnosticTruncated + ? "… (truncated)" + : !eof && !diagnosticFailed + ? "… (timed out)" + : ""), + ); + } + return Object.freeze({ + ...result, + bytesExposed, + }) as OutboundBodyCaptureSummary; + } finally { + clearTimeout(timer); + pump.close(); + } +} diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 6bbd03086..5d500300f 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,3 +1,4 @@ +import { captureResponseBody } from "./outbound-capture.ts"; import { snapshotHeaders } from "./outbound-headers.ts"; import { getOutboundProvenance, @@ -20,12 +21,14 @@ import type { } from "./queue.ts"; import type { OutboundAttemptReport, + OutboundBodyCaptureSummary, OutboundBroadcastEvent, OutboundDeliveryEvent, OutboundHttpExchangeReport, OutboundObservers, OutboundPreparationResult, OutboundProvenance, + OutboundResponseBodyEvent, OutboundWorkerReport, } from "./outbound-report.ts"; @@ -178,8 +181,27 @@ type ExchangeFacts = Pick< | "observedAt" | "request" | "outcome" + | "bodyCapture" >; +type BodyFacts = Pick< + OutboundResponseBodyEvent, + | "attemptId" + | "inboxUrl" + | "exchangeIndex" + | "observedAt" + | "request" + | "response" + | "body" + | "signal" +>; +type BodyCapture = { + maxBytes: number; + timeoutMs: number; + responses?: "all" | "non-success-http-responses"; + onBody: (event: BodyFacts) => void | Promise; +}; + type AttemptResponse = OutboundAttemptReport["outcome"]["response"]; type FinalResponse = Extract; type ObservedResponse = Pick; @@ -202,6 +224,7 @@ export class OutboundAttemptObservation { | ((facts: ExchangeFacts) => Promise) | undefined = undefined, readonly headerPolicies: HeaderPolicies = {}, + readonly bodyCapture?: BodyCapture, ) {} setSpan(span: Span, parentSpanContext?: SpanContext): void { @@ -241,18 +264,39 @@ export class OutboundAttemptObservation { type: "undetermined", lastObserved: observed, }); - await this.#exchange(exchangeIndex, snapshot, { - type: "response", - response: Object.freeze({ - ...metadata, - ...(this.headerPolicies.responseHeaders == null ? {} : { - headers: snapshotHeaders( - response.headers, - this.headerPolicies.responseHeaders, - ), - }), + const responseSnapshot = Object.freeze({ + ...metadata, + ...(this.headerPolicies.responseHeaders == null ? {} : { + headers: snapshotHeaders( + response.headers, + this.headerPolicies.responseHeaders, + ), }), }); + const bodyCapture = this.bodyCapture == null + ? undefined + : await captureResponseBody( + response, + this.bodyCapture, + (body, signal) => + this.bodyCapture!.onBody({ + attemptId: this.#attemptId, + inboxUrl: this.inboxUrl, + exchangeIndex, + observedAt: Temporal.Now.instant().epochMilliseconds, + request: snapshot, + response: responseSnapshot, + body, + signal, + }), + ); + if (bodyCapture?.type === "failed" && bodyCapture.phase === "observer") { + logObserverFailure("response-body", bodyCapture.error); + } + await this.#exchange(exchangeIndex, snapshot, { + type: "response", + response: responseSnapshot, + }, bodyCapture); return response; } @@ -260,6 +304,7 @@ export class OutboundAttemptObservation { exchangeIndex: number, request: OutboundHttpExchangeReport["request"], outcome: OutboundHttpExchangeReport["outcome"], + bodyCapture?: OutboundBodyCaptureSummary, ): Promise { return this.reportExchange?.({ attemptId: this.#attemptId, @@ -268,6 +313,7 @@ export class OutboundAttemptObservation { observedAt: Temporal.Now.instant().epochMilliseconds, request, outcome: Object.freeze(outcome), + ...(bodyCapture == null ? {} : { bodyCapture }), }) ?? Promise.resolve(); } @@ -623,6 +669,9 @@ export class OutboundAttemptObserver { readonly onHttpExchange: OutboundObservers["onHttpExchange"] = undefined, readonly headerPolicies: HeaderPolicies = {}, + readonly bodyCapture: OutboundObservers< + TContextData + >["responseBodyCapture"] = undefined, ) { this.#queue = queue == null ? undefined : Object.freeze({ ...queue }); } @@ -634,8 +683,11 @@ export class OutboundAttemptObserver { provenance: OutboundProvenance, onHttpExchange?: OutboundObservers["onHttpExchange"], headerPolicies: HeaderPolicies = {}, + bodyCapture?: OutboundObservers["responseBodyCapture"], ): OutboundAttemptObserver | undefined { - if (callback == null && onHttpExchange == null) return undefined; + if (callback == null && onHttpExchange == null && bodyCapture == null) { + return undefined; + } const identity = snapshotOutboxIdentity(message); return new OutboundAttemptObserver( context, @@ -648,11 +700,15 @@ export class OutboundAttemptObserver { }, onHttpExchange, headerPolicies, + bodyCapture, ); } startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { - if (this.callback == null && this.onHttpExchange == null) return undefined; + if ( + this.callback == null && this.onHttpExchange == null && + this.bodyCapture == null + ) return undefined; const context = this.context(); return new OutboundAttemptObservation( inboxUrl, @@ -682,7 +738,27 @@ export class OutboundAttemptObserver { }), "http-exchange", ), - this.onHttpExchange == null ? {} : this.headerPolicies, + this.onHttpExchange == null && this.bodyCapture == null + ? {} + : this.headerPolicies, + this.bodyCapture == null ? undefined : { + ...this.bodyCapture, + onBody: (event) => + withContext( + sourceLogContext(this.provenance), + () => + this.bodyCapture!.onBody( + context, + Object.freeze({ + ...this.identity, + ...event, + ...(this.#queue == null ? {} : { queue: this.#queue }), + provenance: this.provenance, + local: snapshotLocalContext(context), + }), + ), + ), + }, ); } } @@ -705,6 +781,7 @@ class OutboundDeliveryObservation { onHttpExchange: OutboundObservers["onHttpExchange"], queue?: OutboundAttemptReport["queue"], headerPolicies: HeaderPolicies = {}, + bodyCapture?: OutboundObservers["responseBodyCapture"], ) { this.#reference = snapshotDeliveryIdentity(identity); this.#attempts = new OutboundAttemptObserver( @@ -715,6 +792,7 @@ class OutboundDeliveryObservation { queue, onHttpExchange, headerPolicies, + bodyCapture, ); this.#plan = Object.freeze({ actorIds: Object.freeze(Array.from(destination.actorIds)), @@ -808,6 +886,7 @@ export class OutboundBroadcastObservation { readonly #onAttempt: OutboundObservers["onAttempt"]; readonly #onHttpExchange: OutboundObservers["onHttpExchange"]; readonly #headerPolicies: HeaderPolicies; + readonly #bodyCapture: OutboundObservers["responseBodyCapture"]; readonly #deliveries = new Map< string, OutboundDeliveryObservation @@ -829,6 +908,7 @@ export class OutboundBroadcastObservation { this.#onDelivery = observers.onDelivery; this.#onAttempt = observers.onAttempt; this.#onHttpExchange = observers.onHttpExchange; + this.#bodyCapture = observers.responseBodyCapture; this.#headerPolicies = { requestHeaders: observers.requestHeaders, responseHeaders: observers.responseHeaders, @@ -845,7 +925,8 @@ export class OutboundBroadcastObservation { ): Promise | undefined> { if ( observers.onDelivery == null && observers.onAttempt == null && - observers.onHttpExchange == null && observers.onWorker == null && + observers.onHttpExchange == null && + observers.responseBodyCapture == null && observers.onWorker == null && !Object.values(message.inboxes).some((entry) => entry.outbound != null) ) { return undefined; @@ -858,6 +939,7 @@ export class OutboundBroadcastObservation { onHttpExchange: observers.onHttpExchange, requestHeaders: observers.requestHeaders, responseHeaders: observers.responseHeaders, + responseBodyCapture: observers.responseBodyCapture, }, "send", getOutboundProvenance(context), @@ -905,6 +987,7 @@ export class OutboundBroadcastObservation { this.#onHttpExchange, this.queue, this.#headerPolicies, + this.#bodyCapture, ); } @@ -1109,7 +1192,8 @@ export function observeOutboundBroadcast( return withContext(sourceLogContext(provenance), () => { if ( observers.onBroadcast == null && observers.onDelivery == null && - observers.onAttempt == null && observers.onHttpExchange == null + observers.onAttempt == null && observers.onHttpExchange == null && + observers.responseBodyCapture == null ) { return execute(); } diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 28aec65e5..dd232efcb 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -40,6 +40,7 @@ import type { OutboundDeliveryEvent, OutboundHttpExchangeReport, OutboundObservers, + OutboundResponseBodyEvent, OutboundWorkerReport, } from "./mod.ts"; import type { FanoutMessage, OutboxMessage } from "./queue.ts"; @@ -5615,3 +5616,246 @@ test("outbound header capture keeps each side independent and disabled by defaul }); } }); + +async function readCapturedText( + body: ReadableStream, +): Promise { + const reader = body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const result = await reader.read(); + if (result.done) return text; + text += decoder.decode(result.value, { stream: true }); + } + } catch { + return text; + } finally { + reader.releaseLock(); + } +} + +test("outbound body-only observers retain worker context and selection across redirects", async () => { + const events: OutboundResponseBodyEvent[] = []; + const texts: string[] = []; + const builder = createFederationBuilder<{ label: string }>(); + const config = { + maxBytes: 128, + timeoutMs: 1000, + async onBody( + ctx: import("./context.ts").Context<{ label: string }>, + event: OutboundResponseBodyEvent, + ) { + strictEqual(ctx.data.label, "worker"); + events.push(event); + texts.push(await readCapturedText(event.body)); + federation.setOutboundObservers({}); + }, + }; + builder.setOutboundObservers({ responseBodyCapture: config }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + config.maxBytes = 1; + config.onBody = () => { + throw new Error("mutated callback must not be selected"); + }; + builder.setOutboundObservers({}); + const task = continuationMessage(recipient.inboxId!.href, false); + const next = new URL("/captured-next", task.inbox).href; + fetchMock.mockGlobal().post(task.inbox, { + status: 303, + headers: { Location: next }, + body: "first", + }).get(next, { status: 202, body: "second" }); + try { + await federation.processQueuedTask({ label: "worker" }, task); + deepStrictEqual(texts, ["first", "second"]); + strictEqual(events.length, 2); + for (const [index, event] of events.entries()) { + ok(Object.isFrozen(event)); + ok(event.signal.aborted); + strictEqual(event.exchangeIndex, index); + strictEqual(event.queue?.queueMessageId, task.id); + strictEqual(event.deliveryId, task.id); + strictEqual(event.response.status, index === 0 ? 303 : 202); + strictEqual(event.request.url, index === 0 ? task.inbox : next); + } + strictEqual(events[0].attemptId, events[1].attemptId); + } finally { + fetchMock.hardReset(); + } +}); + +test("outbound capture settles before exchanges without changing signed negotiation", async (t) => { + for (const fails of [false, true]) { + await t.step( + fails ? "intermediate observer failure" : "complete bodies", + async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const order: string[] = []; + const failure = new Error("capture observer failed"); + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 128, + timeoutMs: 1000, + async onBody(_ctx, event) { + order.push(`body ${event.exchangeIndex}`); + if (fails && event.exchangeIndex === 0) throw failure; + await readCapturedText(event.body); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + order.push(`exchange ${report.exchangeIndex}`); + }, + onAttempt(_ctx, report) { + attempts.push(report); + order.push("attempt"); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + const next = new URL("/capture-signed", inbox).href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const responses: ResponseInit[] = [ + { + status: 401, + headers: { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created;nonce="capture-nonce"', + }, + }, + { status: 307, headers: { Location: next } }, + { status: 202 }, + ]; + let submissions = 0; + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: { id: "https://example.com/capture" }, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { determineSpec: () => "rfc9421", rememberSpec() {} }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + const index = submissions++; + return Promise.resolve( + new Response(`body ${index}`, responses[index]), + ); + }, + close: () => Promise.resolve(), + }), + }); + strictEqual(submissions, 3); + deepStrictEqual(order, [ + "body 0", + "exchange 0", + "body 1", + "exchange 1", + "body 2", + "exchange 2", + "attempt", + ]); + for (const [index, report] of exchanges.entries()) { + if (fails && index === 0) { + deepStrictEqual(report.bodyCapture, { + type: "failed", + phase: "observer", + error: failure, + bytesExposed: 0, + }); + } else {deepStrictEqual(report.bodyCapture, { + type: "complete", + bytesExposed: 6, + });} + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, responses[index].status); + strictEqual("body" in report, false); + } + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(attempts[0].httpExchangeCount, 3); + const final = attempts[0].outcome.response; + ok(final.type === "final"); + strictEqual(final.metadata.status, 202); + }, + ); + } +}); + +test("outbound capture limits retain the original final HTTP diagnostic", async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 4, + timeoutMs: 1000, + async onBody(_ctx, event) { + await readCapturedText(event.body); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + let submissions = 0; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [], + inbox: new URL(inbox), + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return Promise.resolve( + new Response("full diagnostic error", { + status: 503, + headers: { "Retry-After": "1" }, + }), + ); + }, + close: () => Promise.resolve(), + }), + }), + (error: unknown) => { + ok(error instanceof SendActivityError); + strictEqual(error.statusCode, 503); + strictEqual(error.responseBody, "full diagnostic error"); + return true; + }, + ); + strictEqual(submissions, 1); + deepStrictEqual(exchanges[0].bodyCapture, { + type: "limit-reached", + bytesExposed: 4, + }); + ok(attempts[0].outcome.type === "failed"); + strictEqual(attempts[0].outcome.failure.kind, "http"); + ok(attempts[0].outcome.response.type === "final"); + strictEqual(attempts[0].outcome.response.metadata.status, 503); +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 507d249a9..8440f0601 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -176,6 +176,60 @@ type OutboundAttemptFailure = readonly error: unknown; }; +/** Result of an enabled response-body capture. Absence means disabled. @since 2.5.0 */ +export type OutboundBodyCaptureSummary = + & { + /** Bytes handed to the application, not bytes durably stored. */ + readonly bytesExposed: number; + } + & ( + | { + readonly type: + | "filtered" + | "no-body" + | "complete" + | "limit-reached" + | "deadline" + | "callback-returned"; + } + | { readonly type: "consumer-stopped"; readonly reason: unknown } + | { + readonly type: "failed"; + readonly phase: "body" | "observer"; + readonly error: unknown; + } + ); + +/** Bounded, revocable access to one selected response. @since 2.5.0 */ +export type OutboundResponseBodyEvent = + & Omit + & { + /** Immutable response metadata, independently of eventual attempt outcome. */ + readonly response: Extract< + OutboundHttpExchangeReport["outcome"], + { type: "response" } + >["response"]; + /** Pull-driven bytes; access ends when the callback settles or its limits apply. */ + readonly body: ReadableStream; + /** Capture-only lifetime signal; aborting it does not cancel transport. */ + readonly signal: AbortSignal; + }; + +/** Explicit opt-in response-body capture limits and consumer. @since 2.5.0 */ +export interface OutboundResponseBodyCapture { + /** Positive safe-integer byte limit. Reaching it without EOF proves no completeness. */ + readonly maxBytes: number; + /** Positive finite absolute lifetime in milliseconds, rounded up; at most 2^31-1. */ + readonly timeoutMs: number; + /** Actual response selector, defaulting to `all` only when enabled. */ + readonly responses?: "all" | "non-success-http-responses"; + /** Awaited within the bounded capture lifetime; errors cannot veto transport. */ + readonly onBody: ( + context: Context, + event: OutboundResponseBodyEvent, + ) => void | Promise; +} + /** Opt-in policy for one side of an actual HTTP exchange. @since 2.5.0 */ export interface OutboundHeaderPolicy { /** Literal HTTP names, normalized to lowercase and deduplicated at registration. */ @@ -285,8 +339,8 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { * * A redirect response remains an observed response even if a later hop fails. * This report does not establish the attempt's final response. Metadata is - * copied before callback awaiting; no live Request/Response/Headers or body - * capture is exposed. Exchange callbacks may run before the attempt settles. + * copied before callback awaiting; no live Request/Response/Headers are exposed. + * Enabled body capture uses a separate revocable callback; reports hold summaries. Exchange callbacks may run before the attempt settles. * @since 2.5.0 */ export type OutboundHttpExchangeReport = @@ -304,6 +358,8 @@ export type OutboundHttpExchangeReport = & { /** Zero-based submission order within this attempt. */ readonly exchangeIndex: number; + /** Enabled capture result; absent when disabled or no response was received. */ + readonly bodyCapture?: OutboundBodyCaptureSummary; /** Immutable facts of the request actually submitted. */ readonly request: { readonly url: string; @@ -444,10 +500,11 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * actions; they do not establish backend acknowledgment or future execution. * Known IDs and provenance survive queue continuations; callback context data * belongs to the current execution. Each callback is independently optional. - * Callbacks are awaited without a default timeout. + * Metadata callbacks are awaited without a default timeout. An enabled body + * consumer instead uses its explicit bounded capture lifetime. * Their errors, including errors logging observer failures, do not replace the * send's result. - * A callback that never settles can delay execution indefinitely. Detached + * A metadata callback that never settles can delay execution indefinitely. Detached * recording must handle its own rejection and shutdown; no persistence is * guaranteed. Existing portable forwarding deadlines may return while * background preparation, sending, and observer callbacks continue. @@ -455,6 +512,8 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * @since 2.5.0 */ export interface OutboundObservers { + /** Independent opt-in body consumer; explicit byte/time limits are required. */ + readonly responseBodyCapture?: OutboundResponseBodyCapture; /** * Opt-in actual request headers for exchange reports; disabled by default. * Protected credentials, signatures, and cookies remain redacted even if allowed. diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 8ec2b4bc8..540998345 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -1,3 +1,4 @@ +import { takeCapturedDiagnostic } from "./outbound-capture.ts"; import { ResponseBodyPump } from "./outbound-body.ts"; import type { Recipient } from "@fedify/vocab"; import { FetchError, UrlError } from "@fedify/vocab-runtime"; @@ -420,6 +421,8 @@ async function readLimitedResponseBody( response: Response, maxBytes: number, ): Promise { + const captured = takeCapturedDiagnostic(response); + if (captured !== undefined) return captured; if (response.body == null) { return ""; } From 576b3bedab259119d478f87cdbb322f439ad1a36 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 05:08:21 +0900 Subject: [PATCH 33/53] Test adverse response capture conditions Cover stalled and oversized streams, selector deadlines, and bookkeeping failures to verify capture remains bounded and useful. https://github.com/fedify-dev/fedify/issues/1316 Changelog: none Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-capture.test.ts | 136 ++++++++++++ .../src/federation/outbound-report.test.ts | 210 ++++++++++++++++++ 2 files changed, 346 insertions(+) diff --git a/packages/fedify/src/federation/outbound-capture.test.ts b/packages/fedify/src/federation/outbound-capture.test.ts index a738f2a58..cdb7254cb 100644 --- a/packages/fedify/src/federation/outbound-capture.test.ts +++ b/packages/fedify/src/federation/outbound-capture.test.ts @@ -264,3 +264,139 @@ test("response body capture isolates unavailable readers and diagnostic deadline strictEqual(body.locked, false); }); }); + +test("response body capture bounds oversized and nonterminating sources", async (t) => { + for (const mode of ["oversized", "infinite", "empty-infinite"] as const) { + await t.step(mode, async () => { + let pulls = 0; + let cancels = 0; + const source = new Uint8Array( + mode === "oversized" ? 1024 * 1024 : mode === "infinite" ? 1 : 0, + ).fill(255); + const original = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(source); + }, + cancel() { + cancels++; + }, + }, { highWaterMark: 0 }); + let exposed = 0; + const result = await captureResponseBody(new Response(original), { + maxBytes: 17, + timeoutMs: 50, + }, async (body) => { + const reader = body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + exposed += chunk.value.length; + ok(chunk.value.buffer.byteLength <= 17); + strictEqual(chunk.value[0], 255); + chunk.value.fill(0); + } + } catch { + /* Cap/deadline revokes the reader. */ + } finally { + reader.releaseLock(); + } + }); + strictEqual( + result.type, + mode === "empty-infinite" ? "deadline" : "limit-reached", + ); + strictEqual(result.bytesExposed, mode === "empty-infinite" ? 0 : 17); + strictEqual(exposed, result.bytesExposed); + strictEqual(cancels, 1); + strictEqual(original.locked, false); + if (mode !== "empty-infinite") { + strictEqual(pulls, mode === "oversized" ? 1 : 17); + strictEqual(source[0], 255); + } + }); + } +}); + +test("response body capture applies backpressure and revokes a paused consumer", async () => { + let pulls = 0; + let cancels = 0; + let retained!: ReadableStreamDefaultReader; + let exposed = 0; + const source = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array([1])); + }, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const result = await captureResponseBody(new Response(source), { + maxBytes: 32, + timeoutMs: 50, + }, async (body, signal) => { + retained = body.getReader(); + const first = await retained.read(); + ok(!first.done); + exposed += first.value.length; + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else signal.addEventListener("abort", () => resolve(), { once: true }); + }); + strictEqual(pulls, 1); + await rejects(retained.read()); + }); + deepStrictEqual(result, { type: "deadline", bytesExposed: 1 }); + strictEqual(exposed, 1); + strictEqual(pulls, 1); + strictEqual(cancels, 1); + strictEqual(source.locked, false); + await rejects(retained.read()); + retained.releaseLock(); +}); + +test("response body capture rejects late source data and observes late pull failure", async () => { + let controller!: ReadableStreamDefaultController; + let rejectPull!: (error: unknown) => void; + const pulling = new Promise((_resolve, reject) => { + rejectPull = reject; + }); + let cancels = 0; + const source = new ReadableStream({ + start(value) { + controller = value; + }, + pull() { + return pulling; + }, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + let retained!: ReadableStreamDefaultReader; + let exposed = 0; + const result = await captureResponseBody(new Response(source), { + maxBytes: 32, + timeoutMs: 25, + }, async (body) => { + retained = body.getReader(); + try { + const chunk = await retained.read(); + if (!chunk.done) exposed += chunk.value.length; + } catch { /* Revoked at deadline. */ } + }); + deepStrictEqual(result, { type: "deadline", bytesExposed: 0 }); + strictEqual(exposed, 0); + strictEqual(cancels, 1); + throws(() => controller.enqueue(new Uint8Array([1])), TypeError); + rejectPull(new Error("late upstream failure")); + await Promise.resolve(); + await Promise.resolve(); + await rejects(retained.read()); + retained.releaseLock(); + strictEqual(source.locked, false); +}); diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index dd232efcb..5a0e8a255 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -5859,3 +5859,213 @@ test("outbound capture limits retain the original final HTTP diagnostic", async ok(attempts[0].outcome.response.type === "final"); strictEqual(attempts[0].outcome.response.metadata.status, 503); }); + +test("outbound capture deadlines select each real exchange without changing HTTP flow", async (t) => { + const measurements: { + signed: boolean; + selector: string; + selected: number; + timeoutMs: number; + elapsedMs: number; + }[] = []; + for (const signed of [false, true]) { + for (const responses of ["all", "non-success-http-responses"] as const) { + await t.step( + `${signed ? "signed" : "unsigned"}: ${responses}`, + async () => { + const inbox = recipient.inboxId!.href; + const next = new URL("/capture-deadline", inbox).href; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const signals: AbortSignal[] = []; + const timeoutMs = 25; + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 32, + timeoutMs, + responses, + onBody(_ctx, event) { + signals.push(event.signal); + // The archive callback never pulls or settles; each exchange still proceeds. + return new Promise(() => {}); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const steps: ResponseInit[] = signed + ? [ + { + status: 401, + headers: { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created;nonce="deadline-nonce"', + }, + }, + { status: 307, headers: { Location: next } }, + { status: 202 }, + ] + : [{ status: 303, headers: { Location: next } }, { status: 202 }]; + let submissions = 0; + const start = performance.now(); + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: signed ? [sender] : [], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() {}, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + return Promise.resolve( + new Response("response", steps[submissions++]), + ); + }, + close: () => Promise.resolve(), + }), + }); + const elapsedMs = performance.now() - start; + const selected = steps.length - (responses === "all" ? 0 : 1); + strictEqual(submissions, steps.length); + strictEqual(signals.length, selected); + for (const signal of signals) ok(signal.aborted); + deepStrictEqual( + exchanges.map((event) => event.bodyCapture?.type), + steps.map((step) => + step.status === 202 && responses !== "all" + ? "filtered" + : "deadline" + ), + ); + for (const [index, exchange] of exchanges.entries()) { + strictEqual(exchange.bodyCapture?.bytesExposed, 0); + ok(exchange.outcome.type === "response"); + strictEqual(exchange.outcome.response.status, steps[index].status); + } + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(attempts[0].httpExchangeCount, steps.length); + const final = attempts[0].outcome.response; + ok(final.type === "final"); + strictEqual(final.metadata.status, 202); + measurements.push({ + signed, + selector: responses, + selected, + timeoutMs, + elapsedMs: Math.round(elapsedMs * 100) / 100, + }); + }, + ); + } + } + console.info( + "Selected-exchange capture latency (local test, not a benchmark):", + JSON.stringify(measurements), + ); +}); + +test("outbound capture evidence survives later final response bookkeeping failures", async (t) => { + for (const status of [202, 503]) { + await t.step(`${status}: cache and cleanup`, async () => { + const inbox = recipient.inboxId!.href; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 4, + timeoutMs: 1000, + async onBody(_ctx, event) { + await readCapturedText(event.body); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const cache = Symbol("cache after capture"); + const cleanup = Symbol("cleanup after capture"); + let submissions = 0; + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() { + throw cache; + }, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + return Promise.resolve( + new Response("response body", { + status: ++submissions === 1 ? 401 : status, + }), + ); + }, + close: () => Promise.reject(cleanup), + }), + }), + (error: unknown) => { + outward = error; + return error instanceof FetchError && error.cause === cache; + }, + ); + strictEqual(submissions, 2); + deepStrictEqual(exchanges.map((report) => report.bodyCapture), [{ + type: "limit-reached", + bytesExposed: 4, + }, { type: "limit-reached", bytesExposed: 4 }]); + const outcome = attempts[0].outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + ok(outcome.response.type === "final"); + strictEqual(outcome.response.metadata.status, status); + strictEqual(outcome.failure.kind, status === 503 ? "http" : "local"); + const failures = [outcome.failure, ...(outcome.diagnostics ?? [])]; + ok( + failures.some((failure) => + failure.phase === "post-response" && failure.error === cache + ), + ); + ok( + failures.some((failure) => + failure.phase === "cleanup" && failure.error === cleanup + ), + ); + strictEqual(attempts[0].httpExchangeCount, 2); + }); + } +}); From 0ba058042b9c263fa688ff6d61695f7f06abd558 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 05:20:46 +0900 Subject: [PATCH 34/53] Expose privacy-filtered sender payloads DrFed needs outgoing document evidence beyond the application Activity. Snapshot the concrete sender's existing input so immediate, forwarded, replicated, and queued attempts share one stage without extra serialization or JSON-LD processing. Blind recipients can occur inside nested and proof-bearing documents. Remove their fields from an independent immutable projection and report unavailable when context semantics cannot be established. Admit audited bundled contexts and exact generated extension definitions so ordinary Fedify payloads work without accepting arbitrary aliases or overrides. The sender-payload stage describes attempt input, not exact wire bytes. Redirects may drop the body, and a redacted copy does not retain a claim of signature or proof validity. Disabled export and failures before the snapshot boundary remain distinct from privacy-filter unavailability. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/builder.ts | 2 + .../src/federation/outbound-document.test.ts | 140 +++++++++ .../src/federation/outbound-document.ts | 267 ++++++++++++++++++ .../src/federation/outbound-observation.ts | 22 +- .../src/federation/outbound-report.test.ts | 228 +++++++++++++++ .../fedify/src/federation/outbound-report.ts | 32 +++ .../src/federation/portable-inbox.test.ts | 8 + packages/fedify/src/federation/send.ts | 1 + 8 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-document.test.ts create mode 100644 packages/fedify/src/federation/outbound-document.ts diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index ceb084e73..268368598 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -1,3 +1,4 @@ +import { snapshotSenderPayloadOption } from "./outbound-document.ts"; import { snapshotBodyCapture } from "./outbound-capture.ts"; import { snapshotHeaderPolicy } from "./outbound-headers.ts"; import type { OutboundObservers } from "./outbound-report.ts"; @@ -667,6 +668,7 @@ export class FederationBuilderImpl setOutboundObservers(observers: OutboundObservers): void { this.outboundObservers = globalThis.Object.freeze({ + senderPayload: snapshotSenderPayloadOption(observers.senderPayload), responseBodyCapture: snapshotBodyCapture(observers.responseBodyCapture), requestHeaders: snapshotHeaderPolicy(observers.requestHeaders), responseHeaders: snapshotHeaderPolicy(observers.responseHeaders), diff --git a/packages/fedify/src/federation/outbound-document.test.ts b/packages/fedify/src/federation/outbound-document.test.ts new file mode 100644 index 000000000..b5c1ee78e --- /dev/null +++ b/packages/fedify/src/federation/outbound-document.test.ts @@ -0,0 +1,140 @@ +import { test } from "@fedify/fixture"; +import { deepStrictEqual, ok, strictEqual, throws } from "node:assert/strict"; +import { projectSenderPayload } from "./outbound-document.ts"; +import { createFederationBuilder, FederationBuilderImpl } from "./builder.ts"; + +const as = "https://www.w3.org/ns/activitystreams"; +const secret = "https://private.example/recipient"; + +test("sender payload projects nested compact expanded and proof-bearing blind fields", () => { + const source = { + "@context": [as, "https://w3id.org/security/data-integrity/v2"], + type: "Create", + bto: [secret], + object: { + bcc: [secret], + children: [{ + "as:bto": secret, + [`${as}#bcc`]: secret, + content: "visible", + }], + proof: { + "@context": "https://w3id.org/identity/v1", + type: "DataIntegrityProof", + proofValue: "retained-proof", + "as:bcc": secret, + }, + }, + signature: { signatureValue: "retained-signature", [`${as}#bto`]: secret }, + }; + const original = structuredClone(source); + const result = projectSenderPayload(source); + ok(result.type === "available"); + strictEqual(result.representation, "privacy-projection"); + strictEqual(result.redacted, true); + strictEqual(JSON.stringify(result.value).includes(secret), false); + ok(JSON.stringify(result.value).includes("retained-proof")); + ok(JSON.stringify(result.value).includes("retained-signature")); + deepStrictEqual(source, original); + ok(Object.isFrozen(result)); + ok(Object.isFrozen(result.value)); + const value = result.value as Record; + ok(Object.isFrozen(value.object)); + source.object.children[0].content = "changed"; + ok(JSON.stringify(result.value).includes("visible")); + const expanded = projectSenderPayload([{ + "@type": [`${as}#Create`], + [`${as}#object`]: [{ + [`${as}#bcc`]: [{ "@id": secret }], + [`${as}#content`]: [{ "@value": "visible" }], + }], + }]); + ok(expanded.type === "available"); + strictEqual(expanded.redacted, true); + strictEqual(JSON.stringify(expanded.value).includes(secret), false); +}); + +test("sender payload fails closed without evaluating getters or JSON conversion", () => { + let getters = 0; + let conversions = 0; + const getter = { + "@context": as, + get object() { + getters++; + return "private"; + }, + }; + const conversion = { "@context": as }; + Object.defineProperty(conversion, "toJSON", { + value() { + conversions++; + return {}; + }, + }); + const context = [as]; + Object.defineProperty(context, "includes", { + get() { + getters++; + throw new Error("must not run"); + }, + enumerable: true, + }); + const cycle: Record = { "@context": as }; + cycle.object = cycle; + let deep: unknown = "leaf"; + for (let i = 0; i < 65; i++) deep = { object: deep }; + for ( + const source of [ + { "@context": [as, { private: `${as}#bcc` }], private: secret }, + { + "@context": as, + object: { + "@context": { as: "https://other.example/" }, + "as:bcc": secret, + }, + }, + { "@context": [as, "https://unknown.example/context"], private: secret }, + { "@context": [as, { sensitive: "as:bcc" }], sensitive: secret }, + { + "@context": [as, { toot: `${as}#`, Emoji: "toot:bcc" }], + Emoji: secret, + }, + { type: "Create", object: secret }, + getter, + conversion, + { "@context": context }, + cycle, + { "@context": as, object: deep }, + { "@context": as, object: NaN }, + { "@context": as, object: new Date() }, + { "@context": as, object: [undefined] }, + ] + ) { + deepStrictEqual(projectSenderPayload(source), { + stage: "sender-payload", + type: "unavailable", + reason: "privacy-filter-unavailable", + }); + } + strictEqual(getters, 0); + strictEqual(conversions, 0); +}); + +test("sender payload opt-in validates atomically and retains builder selection", () => { + const builder = createFederationBuilder(); + ok(builder instanceof FederationBuilderImpl); + const config = { senderPayload: true, onAttempt() {} }; + builder.setOutboundObservers(config); + const original = builder.outboundObservers; + config.senderPayload = false; + strictEqual(original.senderPayload, true); + for (const value of [null, 1, "true", {}]) { + throws( + () => builder.setOutboundObservers({ senderPayload: value as never }), + TypeError, + ); + strictEqual(builder.outboundObservers, original); + } + builder.setOutboundObservers({}); + strictEqual(builder.outboundObservers.senderPayload, false); +}); diff --git a/packages/fedify/src/federation/outbound-document.ts b/packages/fedify/src/federation/outbound-document.ts new file mode 100644 index 000000000..420bbd8fc --- /dev/null +++ b/packages/fedify/src/federation/outbound-document.ts @@ -0,0 +1,267 @@ +import { preloadedContexts } from "@fedify/vocab-runtime"; +import type { + OutboundJsonValue, + OutboundSenderPayload, +} from "./outbound-report.ts"; + +const asContext = "https://www.w3.org/ns/activitystreams"; +const asNamespace = `${asContext}#`; +const blind = new Set([ + "bto", + "bcc", + "as:bto", + "as:bcc", + `${asNamespace}bto`, + `${asNamespace}bcc`, +]); +const contextCandidates = [ + asContext, + "https://w3id.org/security/v1", + "https://w3id.org/identity/v1", + "https://gotosocial.org/ns", + "https://w3id.org/security/data-integrity/v1", + "https://w3id.org/security/data-integrity/v2", +]; + +// This audit is specific to blind addressing, unlike the attachment/Public +// normalization predicates. Unknown definitions never trigger context loading. +function safeContext(context: unknown): boolean { + if (Array.isArray(context)) return context.every(safeContext); + if (context == null || typeof context !== "object") return false; + for (const [name, definition] of Object.entries(context)) { + if (name === "@import") return false; + const mapping = typeof definition === "string" + ? definition + : definition != null && typeof definition === "object" + ? (definition as Record)["@id"] + : undefined; + if (name === "as" && mapping !== asNamespace) return false; + if ( + name !== "as" && typeof mapping === "string" && + mapping.startsWith(asNamespace) && + (mapping.endsWith("#") || mapping.endsWith("/") || + typeof definition === "object" && + (definition as Record)["@prefix"] === true) + ) return false; + if ( + typeof mapping === "string" && blind.has(mapping) && name !== "bto" && + name !== "bcc" + ) return false; + if ( + (name === "bto" || name === "bcc") && mapping !== `as:${name}` && + mapping !== `${asNamespace}${name}` + ) return false; + if ( + name === "@vocab" && typeof mapping === "string" && + mapping.startsWith(asNamespace) + ) return false; + if ( + definition != null && typeof definition === "object" && + "@context" in definition && + !safeContext((definition as Record)["@context"]) + ) return false; + } + return true; +} +const contexts = new Set(contextCandidates.filter((url) => { + const document = preloadedContexts[url]; + return document != null && typeof document === "object" && + safeContext((document as Record)["@context"]); +})); +// Exact generated extension definitions, not an interpreter for arbitrary inline +// contexts. In particular, callers cannot redefine prefixes or add blind aliases. +const generatedTerms: Record = { + toot: "http://joinmastodon.org/ns#", + misskey: "https://misskey-hub.net/ns#", + fedibird: "http://fedibird.com/ns#", + sensitive: "as:sensitive", + votersCount: { + "@id": "toot:votersCount", + "@type": "http://www.w3.org/2001/XMLSchema#nonNegativeInteger", + }, + Emoji: "toot:Emoji", + Hashtag: "as:Hashtag", + ChatMessage: "http://litepub.social/ns#ChatMessage", + quote: { "@id": "https://w3id.org/fep/044f#quote", "@type": "@id" }, + quoteUrl: "as:quoteUrl", + _misskey_quote: "misskey:_misskey_quote", + quoteUri: "fedibird:quoteUri", + QuoteAuthorization: "https://w3id.org/fep/044f#QuoteAuthorization", + QuoteRequest: "https://w3id.org/fep/044f#QuoteRequest", + quoteAuthorization: { + "@id": "https://w3id.org/fep/044f#quoteAuthorization", + "@type": "@id", + }, + emojiReactions: { "@id": "fedibird:emojiReactions", "@type": "@id" }, +}; +function sameDefinition(value: OutboundJsonValue, expected: unknown): boolean { + if (value === expected) return true; + if ( + value == null || expected == null || typeof value !== "object" || + typeof expected !== "object" || Array.isArray(value) + ) return false; + const actual = value as Record; + const reference = expected as Record; + return Object.keys(actual).length === Object.keys(reference).length && + Object.keys(actual).every((name) => + Object.hasOwn(reference, name) && + sameDefinition(actual[name], reference[name]) + ); +} +function generatedContext(value: OutboundJsonValue): boolean { + if (value == null || typeof value !== "object" || Array.isArray(value)) { + return false; + } + return Object.entries(value).every(([name, definition]) => + Object.hasOwn(generatedTerms, name) && + sameDefinition(definition, generatedTerms[name]) + ); +} +const keywords = new Set([ + "@id", + "@type", + "@value", + "@language", + "@direction", + "@index", + "@list", + "@set", + "@graph", + "@reverse", + "@included", + "@nest", +]); + +/** Validate the explicit opt-in before replacing observer registration. */ +export function snapshotSenderPayloadOption( + value: boolean | undefined, +): boolean { + if (value === undefined) return false; + if (typeof value !== "boolean") { + throw new TypeError("Outbound senderPayload must be a boolean."); + } + return value; +} + +/** Project existing plain JSON only; no getters, JSON-LD processing or serializers. */ +export function projectSenderPayload(input: unknown): OutboundSenderPayload { + try { + let compact = false; + let redacted = false; + const seen = new WeakSet(); + const copy = ( + value: unknown, + depth: number, + contextData = false, + ): OutboundJsonValue => { + if (depth >= 64) throw new TypeError("Unsafe document depth."); + if ( + value === null || typeof value === "string" || + typeof value === "boolean" + ) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "object" || value == null || seen.has(value)) { + throw new TypeError("Not independent JSON data."); + } + const array = Array.isArray(value); + const prototype = Object.getPrototypeOf(value); + if ( + array + ? prototype !== Array.prototype + : prototype !== Object.prototype && prototype !== null + ) throw new TypeError("Unsupported document object."); + seen.add(value); + try { + const descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.hasOwn(descriptors, "toJSON")) { + throw new TypeError("Custom JSON conversion is not projected."); + } + const names = Reflect.ownKeys(descriptors).filter((name) => + descriptors[name as string].enumerable + ); + if ( + names.some((name) => + typeof name !== "string" || !("value" in descriptors[name]) + ) + ) throw new TypeError("Not plain JSON properties."); + let contextValue: OutboundJsonValue | undefined; + if ( + !contextData && !array && + descriptors["@context"]?.enumerable + ) { + const context = copy(descriptors["@context"].value, depth + 1, true); + contextValue = context; + const urls = typeof context === "string" ? [context] : context; + if ( + !Array.isArray(urls) || urls.length === 0 || + (depth === 0 ? !urls.includes(asContext) : !compact) || + !urls.every((url) => + typeof url === "string" + ? contexts.has(url) + : depth === 0 && generatedContext(url) + ) + ) { + throw new TypeError("Unsupported document context."); + } + compact = true; + } + if (array) { + const length = descriptors.length.value as number; + if ( + names.length !== length || + names.some((name, index) => name !== String(index)) + ) throw new TypeError("Not a JSON array."); + return Object.freeze( + names.map((name) => + copy(descriptors[name as string].value, depth + 1, contextData) + ), + ); + } + const result: Record = Object.create(null); + for (const name of names as string[]) { + if (contextData) { + result[name] = copy(descriptors[name].value, depth + 1, true); + continue; + } + if (name === "@context") { + if (contextValue === undefined || !compact) { + throw new TypeError("Scoped context not supported."); + } + result[name] = contextValue!; + continue; + } + if ( + !compact && !keywords.has(name) && + !/^[A-Za-z][A-Za-z0-9+.-]*:/.test(name) + ) throw new TypeError("Unresolved document property."); + const next = copy(descriptors[name].value, depth + 1); + if (blind.has(name)) { + redacted = true; + continue; + } + result[name] = next; + } + return Object.freeze(result); + } finally { + seen.delete(value); + } + }; + if (input == null || typeof input !== "object") { + throw new TypeError("Not an outgoing document."); + } + const value = copy(input, 0); + return Object.freeze({ + stage: "sender-payload", + type: "available", + representation: "privacy-projection", + redacted, + value, + }); + } catch { + return Object.freeze({ + stage: "sender-payload", + type: "unavailable", + reason: "privacy-filter-unavailable", + }); + } +} diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 5d500300f..f60d3e5b8 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1,3 +1,4 @@ +import { projectSenderPayload } from "./outbound-document.ts"; import { captureResponseBody } from "./outbound-capture.ts"; import { snapshotHeaders } from "./outbound-headers.ts"; import { @@ -163,6 +164,7 @@ type AttemptFacts = Pick< | "attemptId" | "inboxUrl" | "httpExchangeCount" + | "senderPayload" | "observedAt" | "trace" | "outcome" @@ -170,7 +172,7 @@ type AttemptFacts = Pick< type HeaderPolicies = Pick< OutboundObservers, - "requestHeaders" | "responseHeaders" + "requestHeaders" | "responseHeaders" | "senderPayload" >; type ExchangeFacts = Pick< @@ -216,6 +218,7 @@ export class OutboundAttemptObservation { #trace: OutboundAttemptReport["trace"]; #finished = false; #exchangeCount = 0; + #senderPayload: OutboundAttemptReport["senderPayload"]; constructor( readonly inboxUrl: string, @@ -225,7 +228,18 @@ export class OutboundAttemptObservation { | undefined = undefined, readonly headerPolicies: HeaderPolicies = {}, readonly bodyCapture?: BodyCapture, - ) {} + readonly senderPayload = false, + ) { + this.#senderPayload = Object.freeze({ + stage: "sender-payload", + type: "unavailable", + reason: senderPayload ? "not-materialized" : "disabled", + }); + } + + snapshotSenderPayload(input: unknown): void { + if (this.senderPayload) this.#senderPayload = projectSenderPayload(input); + } setSpan(span: Span, parentSpanContext?: SpanContext): void { this.#trace = snapshotTrace(span, parentSpanContext); @@ -384,6 +398,7 @@ export class OutboundAttemptObservation { attemptId: this.#attemptId, inboxUrl: this.inboxUrl, httpExchangeCount: this.#exchangeCount, + senderPayload: this.#senderPayload, observedAt: Temporal.Now.instant().epochMilliseconds, ...(this.#trace == null ? {} : { trace: this.#trace }), outcome: Object.freeze(outcome), @@ -759,6 +774,7 @@ export class OutboundAttemptObserver { ), ), }, + this.callback != null && this.headerPolicies.senderPayload === true, ); } } @@ -912,6 +928,7 @@ export class OutboundBroadcastObservation { this.#headerPolicies = { requestHeaders: observers.requestHeaders, responseHeaders: observers.responseHeaders, + senderPayload: observers.senderPayload, }; } @@ -940,6 +957,7 @@ export class OutboundBroadcastObservation { requestHeaders: observers.requestHeaders, responseHeaders: observers.responseHeaders, responseBodyCapture: observers.responseBodyCapture, + senderPayload: observers.senderPayload, }, "send", getOutboundProvenance(context), diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 5a0e8a255..761bbc229 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -29,6 +29,7 @@ import { ContextImpl, createFederation, FederationImpl, + InboxContextImpl, OutboxContextImpl, } from "./middleware.ts"; import type { @@ -6069,3 +6070,230 @@ test("outbound capture evidence survives later final response bookkeeping failur }); } }); + +test("sender payload projections preserve actual send forward and queued inputs", async (t) => { + for ( + const mode of [ + "ordinary", + "outbox", + "queued", + "fanout", + "forward", + "forward-queued", + ] as const + ) { + await t.step(mode, async () => { + const tasks: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + tasks.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + ...(mode === "queued" || mode === "forward-queued" || mode === "fanout" + ? { + queue: { + outbox: queue, + ...(mode === "fanout" ? { fanout: queue } : {}), + }, + } + : {}), + }); + const reports: OutboundAttemptReport[] = []; + const contexts: string[] = []; + federation.setOutboundObservers({ + senderPayload: true, + onAttempt(ctx, report) { + reports.push(report); + contexts.push(ctx.data.label); + }, + }); + const secret = "https://private.example/recipient"; + const payload = { + ...forwardingPayload, + "@context": "https://www.w3.org/ns/activitystreams", + bto: [secret], + object: { type: "Note", bcc: [secret], content: "visible" }, + }; + const original = structuredClone(payload); + const outgoing = activity.clone({ + btos: [new URL(secret)], + bccs: [new URL(secret)], + }); + const ctx = mode.startsWith("forward") + ? forwardingContext(federation, "inbox", payload, { label: "producer" }) + : mode === "outbox" + ? forwardingContext(federation, "outbox", payload, { + label: "producer", + }) + : federation.createContext(new URL("https://example.com"), { + label: "producer", + }); + const bodies: unknown[] = []; + fetchMock.mockGlobal().post(recipient.inboxId!.href, (call) => { + bodies.push(JSON.parse(String(call.options.body))); + return 202; + }); + try { + if (mode.startsWith("forward")) { + ok(ctx instanceof InboxContextImpl); + await ctx.forwardActivity(sender, recipient, { + skipIfUnsigned: false, + immediate: mode === "forward", + }); + } else { + ok(ctx instanceof ContextImpl); + const senderContext: ContextImpl<{ label: string }> = ctx; + await senderContext.sendActivity(sender, recipient, outgoing, { + immediate: mode === "ordinary" || mode === "outbox", + fanout: mode === "fanout" ? "force" : "skip", + }); + } + while (tasks.length) { + await federation.processQueuedTask( + { label: "worker" }, + tasks.shift()!, + ); + } + strictEqual(reports.length, 1); + strictEqual(bodies.length, 1); + const document = reports[0].senderPayload; + ok( + document.type === "available", + JSON.stringify({ + document, + context: (bodies[0] as Record)["@context"], + }), + ); + strictEqual(document.stage, "sender-payload"); + strictEqual(document.redacted, true); + strictEqual(JSON.stringify(document.value).includes(secret), false); + ok(JSON.stringify(bodies[0]).includes(secret)); + deepStrictEqual(payload, original); + deepStrictEqual(contexts, [ + mode === "queued" || mode === "forward-queued" || mode === "fanout" + ? "worker" + : "producer", + ]); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("sender payload availability never changes serialization or redirected bodies", async (t) => { + for ( + const mode of [ + "disabled", + "preparation", + "conversion", + "getter", + "redirect", + ] as const + ) { + await t.step(mode, async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const reports: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + senderPayload: mode !== "disabled", + onAttempt(_ctx, report) { + reports.push(report); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const payload = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + bcc: ["https://private.example/recipient"], + object: { content: "visible" }, + }; + let conversions = 0; + let getters = 0; + const input = mode === "conversion" + ? Object.defineProperty({}, "toJSON", { + value() { + conversions++; + return payload; + }, + }) + : mode === "getter" + ? Object.defineProperty({ ...payload }, "object", { + enumerable: true, + get() { + getters++; + return payload.object; + }, + }) + : payload; + const requests: { method: string; body: string }[] = []; + const failure = new Error("session unavailable before serialization"); + const sending = sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: input, + keys: [], + inbox: new URL(inbox), + }, { + openHop: (url) => + mode === "preparation" ? Promise.reject(failure) : Promise.resolve({ + url: new URL(url), + async fetch(request) { + ok(request instanceof Request); + requests.push({ + method: request.method, + body: await request.text(), + }); + return new Response( + null, + mode === "redirect" && requests.length === 1 + ? { status: 303, headers: { Location: "/payload-next" } } + : { status: 202 }, + ); + }, + close: () => Promise.resolve(), + }), + }); + if (mode === "preparation") { + await rejects(sending, (error: unknown) => error === failure); + } else await sending; + strictEqual(reports.length, 1); + const document = reports[0].senderPayload; + if (mode === "redirect") { + ok(document.type === "available"); + strictEqual(document.redacted, true); + strictEqual( + JSON.stringify(document.value).includes("private.example"), + false, + ); + strictEqual(requests.length, 2); + strictEqual(requests[1].method, "GET"); + strictEqual(requests[1].body, ""); + ok(requests[0].body.includes("private.example")); + } else {deepStrictEqual(document, { + stage: "sender-payload", + type: "unavailable", + reason: mode === "disabled" + ? "disabled" + : mode === "preparation" + ? "not-materialized" + : "privacy-filter-unavailable", + });} + strictEqual(conversions, mode === "conversion" ? 1 : 0); + strictEqual(getters, mode === "getter" ? 1 : 0); + if (mode !== "preparation") { + strictEqual(requests[0].body, JSON.stringify(payload)); + } + }); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 8440f0601..cf0771503 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -176,6 +176,34 @@ type OutboundAttemptFailure = readonly error: unknown; }; +/** Immutable JSON data in a privacy projection. @since 2.5.0 */ +export type OutboundJsonValue = + | null + | boolean + | number + | string + | readonly OutboundJsonValue[] + | { readonly [key: string]: OutboundJsonValue }; + +/** Attempt input evidence, never proof of a particular exchange's wire body. @since 2.5.0 */ +export type OutboundSenderPayload = + & { readonly stage: "sender-payload" } + & ( + | { + readonly type: "available"; + readonly representation: "privacy-projection"; + readonly redacted: boolean; + readonly value: OutboundJsonValue; + } + | { + readonly type: "unavailable"; + readonly reason: + | "disabled" + | "not-materialized" + | "privacy-filter-unavailable"; + } + ); + /** Result of an enabled response-body capture. Absence means disabled. @since 2.5.0 */ export type OutboundBodyCaptureSummary = & { @@ -304,6 +332,8 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { readonly inboxUrl: string; /** Actual fetch submissions, including challenges and redirects; may be zero. */ readonly httpExchangeCount: number; + /** Opt-in independent sender input projection; unavailable states remain explicit. */ + readonly senderPayload: OutboundSenderPayload; /** Current queue execution; absent on immediate sends. */ readonly queue?: { readonly queueMessageId: string; @@ -512,6 +542,8 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * @since 2.5.0 */ export interface OutboundObservers { + /** Opt-in privacy-filtered sender input on attempt reports; disabled by default. */ + readonly senderPayload?: boolean; /** Independent opt-in body consumer; explicit byte/time limits are required. */ readonly responseBodyCapture?: OutboundResponseBodyCapture; /** diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index d4dde27dc..3cebaea30 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -2649,6 +2649,7 @@ test("portable replication attempts settle independently after the forwarding de }); let replacements = 0; federation.setOutboundObservers({ + senderPayload: true, onAttempt(_ctx, report) { attempts.push(report); if (report.outcome.type === "failed") failed(); @@ -2685,6 +2686,13 @@ test("portable replication attempts settle independently after the forwarding de assert(attempts[0].deliveryId !== attempts[1].deliveryId); assertEquals(attempts[0].broadcast, attempts[1].broadcast); for (const report of attempts) { + assertEquals(report.senderPayload.stage, "sender-payload"); + if (report.senderPayload.type === "unavailable") { + assertEquals(report.senderPayload.reason, "privacy-filter-unavailable"); + } else { + assertEquals(report.senderPayload.representation, "privacy-projection"); + assert(Object.isFrozen(report.senderPayload.value)); + } assertEquals(report.destinationInbox, report.inboxUrl); assertEquals(report.portableInbox, `ap+ef61://${did}/users/alice/inbox`); assertEquals( diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 540998345..220b366c4 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -583,6 +583,7 @@ async function sendActivityInternal( throw error; } try { + attempt?.snapshotSenderPayload(activity); body = JSON.stringify(activity); } catch (error) { attempt?.failure({ kind: "local", phase: "serialization", error }); From a4e1131b77606867c56c7f118660a3fbe6adcb3e Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 05:25:43 +0900 Subject: [PATCH 35/53] Document outgoing evidence capture Explain capture limits and privacy-filtered sender payloads so observers can interpret the evidence safely. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 7 +- changes.d/fedify/outbound-observers.md | 27 ++- claude-plugin/skills/fedify/SKILL.md | 64 +++++- docs/manual/mq.md | 6 + docs/manual/send.md | 204 +++++++++++++++++- packages/fedify/src/federation/federation.ts | 5 +- .../fedify/src/federation/outbound-report.ts | 56 ++++- packages/fedify/src/federation/send.ts | 6 +- 8 files changed, 341 insertions(+), 34 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 38367449e..36d429962 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -56,7 +56,12 @@ To be released. send results and exact HTTP exchange counts, including queued retries and gateway fallback, and `onHttpExchange` for each actual signed or unsigned fetch. Reports distinguish received responses from the selected final - response and retain HTTP evidence alongside later local failures. They + response and retain HTTP evidence alongside later local failures. Use + `onWorker` for fanout/outbox execution results, concrete send counts, and + actual child enqueue, retry, deferral, or abandonment actions. Worker + reports retain accepted children through partial failures and distinguish + unknown queue acceptance from confirmed scheduling; local completion does + not establish backend acknowledgment or remote success. Reports include observation timestamps and optional operation trace IDs. Callbacks receive application context and independent grouping IDs without requiring OpenTelemetry; their errors do not replace the original result. Known IDs diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 4bde643dd..d764cbdfc 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -15,13 +15,20 @@ links: actual child enqueue, retry, deferral, or abandonment actions. Worker reports retain accepted children through partial failures and distinguish unknown queue acceptance from confirmed scheduling; local completion does - not establish backend acknowledgment or remote success. Reports - include observation timestamps and optional operation trace IDs. Callbacks - receive application context and independent grouping IDs without requiring - OpenTelemetry; their errors do not replace the original result. Known IDs - and original inbox source survive queue continuations. Send/forward options - accept `dispatchId` for application correlation without deduplication, - limited to 1,024 UTF-8 bytes. Routed inbox reports include `inboxRequestId` - and informational `requestId`. Missing legacy relationships remain unknown. - Readiness and handoff observations do not establish remote delivery - success. [[#1316] by Jiwon Kwon] + not establish backend acknowledgment or remote success. Opt in to + independent redacted header snapshots, bounded streaming response-body + capture with explicit byte/time limits, and privacy-filtered sender-payload + projections. Capture summaries distinguish EOF, byte-limit uncertainty, + deadlines, and consumer/body failures without changing HTTP outcomes; + payload projections never claim exact wire bytes or proof validity. Error + diagnostics now have a separate one-second budget while retaining their 1 + KiB preview limit. Reports include observation timestamps and optional + operation trace IDs. Callbacks receive application context and independent + grouping IDs without requiring OpenTelemetry; their errors do not replace + the original result. Known IDs and original inbox source survive queue + continuations. Send/forward options accept `dispatchId` for application + correlation without deduplication, limited to 1,024 UTF-8 bytes. Routed + inbox reports include `inboxRequestId` and informational `requestId`. + Missing legacy relationships remain unknown. Readiness and handoff + observations do not establish remote delivery success. + [[#1316] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index c3d6ce973..ede06b0d2 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -273,7 +273,8 @@ delivery. `httpExchangeCount` counts actual fetches, including challenges and redirects; pre-fetch failures have zero. `onHttpExchange` emits immutable `OutboundHttpExchangeReport` snapshots of actual request URL/method and response status/text/available URL, or the original fetch error. `exchangeIndex` starts -at zero per attempt. No live transport objects, bodies, or headers are exposed. +at zero per attempt. No live transport objects are exposed; header snapshots +and a separate bounded body callback are opt-in. For attempt reports, narrow `report.outcome.response.type`: `final` has an `exchangeIndex` and @@ -309,15 +310,62 @@ settlement does not establish backend acknowledgment, a scheduled native retry, or remote success; later direct attempts may settle after a setup failure has already ended the worker. Do not infer scheduling from an attempt result. -Callbacks are awaited without a default timeout; observer/logging errors are -isolated. Nonsettling callbacks can delay execution indefinitely. Portable -forwarding can return at its existing deadline while background work and -observations continue. Detached recording must handle its own rejection, +Header values default to disabled. `requestHeaders` and `responseHeaders` +independently accept `allow`, positive safe-integer `maxBytes`, and optional +`redact`. Names are literal, lowercase/deduplicated; selected credentials, +signatures and cookies are always `[redacted]`. Exchange `headers.entries` are +immutable runtime name/value pairs. Budget UTF-8 `name + ": " + value + "\r\n"` +after redaction; omit whole entries and report `limitReached`. Invalid +configuration preserves prior registration; later caller mutation cannot change +selected policy. Header capture does not enable body capture. + +`responseBodyCapture` requires `onBody(ctx, event)`, positive safe-integer +`maxBytes`, and finite positive `timeoutMs` (rounded up, at most 2^31-1); no +numeric defaults. Enabled `responses` defaults to `all`, or select +`non-success-http-responses` (outside 200–299), independently for each actual +response including intermediate ones. The callback receives grouping metadata, +request/response snapshots, a controlled byte stream and capture-only signal, +before its terminal exchange report. It can be registered alone. Pull bytes +incrementally and pass the signal to application writes; do not accumulate a +whole body just to record it. + +Exchange `bodyCapture` is absent when disabled/no response, otherwise narrow +`type`: `filtered`, `no-body`, `complete`, `limit-reached`, `deadline`, +`callback-returned`, `consumer-stopped` (original `reason`), or `failed` +(`phase: "body" | "observer"`, original `error`). `bytesExposed` is not durable +storage. Exact cap without EOF proves neither completeness nor truncation. +Return/throw/cancel/cap/deadline revokes even locked readers; absolute lifetime +includes no-pull/slow consumers and bounded cleanup. Late rejections are +handled, but arbitrary JavaScript/storage cannot be forced to stop. One source +reader is shared with diagnostics, with no application clone/tee or extra +wrapper queue. Capture cannot abort transport or rewrite HTTP/retry evidence. +Diagnostics have a separate one-second budget and 1 KiB preview; timeout +appends `… (timed out)`. Selected exchanges accumulate waiting time; +upstream/runtime/application memory is outside Fedify's buffering claim. + +`senderPayload: true` adds an immutable `sender-payload` privacy projection to +attempt reports. Narrow `report.senderPayload.type`: `available` exposes +`value`, `representation: "privacy-projection"`, and `redacted`; `unavailable` +distinguishes `disabled`, `not-materialized`, and `privacy-filter-unavailable`. +Recursive compact/prefixed/expanded blind-recipient fields are removed without +changing transport/signing input. Only audited bundled contexts, exact +generated root extension definitions, supported nested context URLs, or +unambiguous expanded JSON qualify; arbitrary aliases/overrides and unsupported +data fail closed. No extra loading/serialization, raw fallback, exact-wire or +proof-validity claim is provided. A redirect can drop its body independently of +attempt input. Evidence is privileged: URLs/content can still contain private +data and require application access control/redaction. See the sending manual +for typed examples. + +Metadata callbacks are awaited without a default timeout; observer/logging +errors are isolated. Nonsettling callbacks can delay execution indefinitely. +Portable forwarding can return at its existing deadline while background work +and observations continue. Detached recording must handle its own rejection, shutdown, and persistence. Known IDs survive fanout expansion, retries, holds, and gateway changes. Fanout expansion continues existing delivery handoffs -without starting another broadcast. Legacy fanout can have an unknown broadcast; -legacy outbox reports use the current task UUID as `deliveryId` and omit an -unavailable original `destinationInbox`. Missing source facts remain +without starting another broadcast. Legacy fanout can have an unknown +broadcast; legacy outbox reports use the current task UUID as `deliveryId` and +omit an unavailable original `destinationInbox`. Missing source facts remain unavailable, and older consumers can discard new metadata. Attempt/exchange `queue` identifies the current message and available Fedify retry counter; workers receive current context data without observation-only loader calls. diff --git a/docs/manual/mq.md b/docs/manual/mq.md index c46788af5..a1aedb818 100644 --- a/docs/manual/mq.md +++ b/docs/manual/mq.md @@ -1319,6 +1319,12 @@ queue depth for messages still waiting. See [worker execution and actions](./send.md#worker-execution-and-actions) for report variants and a typed example. +Metadata callbacks are awaited without a default timeout. Optional +[response-body capture](./send.md#streaming-response-bodies) instead has +explicit byte/time limits; several selected exchanges can accumulate waiting +time within one worker execution. Applications own archival and must handle +recording work that outlives the capture signal or a backend deadline. + Queue depth reporting --------------------- diff --git a/docs/manual/send.md b/docs/manual/send.md index 598b6e76b..c4660c1a4 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -189,8 +189,10 @@ submission. `exchangeIndex` starts at zero within its `attemptId`. Its `request` copies the submitted URL and method, including those of signed requests. A `response` outcome records status, status text, and the response URL when fetch supplies it; a `failed` outcome retains the original fetch rejection. A non-2xx -response is still a received response. Neither live transport objects nor -response bodies or headers are exposed. +response is still a received response. Reports contain immutable metadata; +optional header snapshots and body-capture summaries are described below. +A separate bounded callback supplies selected response bytes, without exposing +live transport objects. ~~~~ typescript twoslash import type { Federation } from "@fedify/fedify"; @@ -341,6 +343,201 @@ if execution failed before constructing its usual HTTP exception. The separate `outcome.error` is the exact value that escaped the sender, including any existing wrapping. Observer errors never become attempt diagnostics. +### Capturing headers + +Header values are disabled by default. Enable `requestHeaders` and +`responseHeaders` independently to add immutable `headers` snapshots to actual +exchange requests and responses. Each policy requires an explicit `allow` list +of literal HTTP field names and a positive safe-integer `maxBytes`. An optional +`redact` list adds application-specific secret headers. Names are normalized to +lowercase and deduplicated; wildcards are not supported. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + requestHeaders: { allow: ["content-type", "signature"], maxBytes: 2048 }, + responseHeaders: { allow: ["content-type", "retry-after", "set-cookie"], maxBytes: 1024 }, + onHttpExchange(_ctx, report) { + console.log(report.request.headers?.entries); + if (report.outcome.type === "response") { + console.log(report.outcome.response.headers?.entries); + console.log(report.outcome.response.headers?.limitReached); + } + }, +}); +~~~~ + +Selected credentials, signatures, and cookies always become `[redacted]`, even +if allowlisted. The protected names are `authorization`, `proxy-authorization`, +`cookie`, `set-cookie`, `signature`, `signature-input`, `authentication-info`, +`proxy-authentication-info`, `x-api-key`, and `x-auth-token`. Application +redactions also take precedence over the allowlist. There is no raw-secret +option; add custom credential names to your redaction policy. + +`entries` contains readonly name/value pairs in lexical name order, using the +runtime's representation of repeated values. It does not promise raw wire +ordering or recover values already combined by the runtime. Each pair costs the +UTF-8 bytes of `name + ": " + value + "\r\n"`, after redaction. An oversized +entry is omitted whole while later entries remain eligible; `limitReached` +records any budget omission. An absent `headers` field means disabled, while an +enabled empty allowlist produces an empty snapshot. Signed/challenge/redirect +requests are captured at their actual submission, including changed headers. + +### Streaming response bodies + +Application body capture is disabled unless `responseBodyCapture` is supplied. +It is independent of header capture and can be registered without metadata +callbacks. It requires `onBody`, a positive safe-integer `maxBytes`, and a +finite positive `timeoutMs`; there are no numeric defaults. Fractional +milliseconds round up, and the rounded lifetime cannot exceed 2,147,483,647 ms. +Invalid configuration leaves the previous registration intact. + +`responses` defaults to `all` only when capture is enabled. The other selector, +`non-success-http-responses`, selects actual statuses outside 200–299, including +intermediate challenges and redirects. Selection does not predict the attempt +outcome. This example uses illustrative limits, not performance guarantees: + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +declare function saveChunk( + attemptId: string, + exchangeIndex: number, + chunk: Uint8Array, + signal: AbortSignal, +): Promise; +// ---cut-before--- +federation.setOutboundObservers({ + responseBodyCapture: { + maxBytes: 64 * 1024, + timeoutMs: 5000, + responses: "non-success-http-responses", + async onBody(_ctx, event) { + const reader = event.body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + await saveChunk(event.attemptId, event.exchangeIndex, chunk.value, event.signal); + } + } catch (error) { + if (!event.signal.aborted) throw error; + } finally { + reader.releaseLock(); + } + }, + }, + onHttpExchange(_ctx, report) { + console.log(report.attemptId, report.exchangeIndex, report.bodyCapture); + }, +}); +~~~~ + +`onBody` receives the current context/data, grouping metadata, an immutable +request/response snapshot, a controlled `ReadableStream`, and a +capture-only signal. The callback runs after headers and before the terminal +exchange report. Finished reports contain `bodyCapture` summaries, never +streams: + +| Summary type | Meaning | +| ------------------- | -------------------------------------------------------------------------------------------- | +| Absent | Application capture was disabled, or fetch produced no response. | +| `filtered` | The response was not selected; no callback ran. | +| `no-body` | A selected response had no body; distinct from an empty stream reaching EOF. | +| `complete` | EOF was observed and the callback settled successfully within its lifetime. | +| `limit-reached` | The byte cap was reached without observed EOF; completeness and truncation remain uncertain. | +| `deadline` | The absolute capture lifetime expired, including no-pull or slow-consumer time. | +| `callback-returned` | The callback returned before EOF or another stopping condition. | +| `consumer-stopped` | The consumer cancelled; `reason` retains its original value. | +| `failed` | `phase` identifies a `body` or `observer` failure and `error` retains its original value. | + +`bytesExposed` counts bytes handed to application code, not durable storage. +At an exact cap, another read does not establish EOF: access is revoked, and +`limit-reached` remains uncertain even if the remote body might end there. +Signal abort or stream rejection can also stop an outstanding application write; +the summary does not certify that exposed chunks were saved. + +Fedify uses one source reader shared with the bounded error diagnostic preview, +pull backpressure, no additional wrapper queue, and at most one pending source +read. It does not clone, tee, or accumulate the response body for application +capture. Its limits do not bound upstream chunk allocations, runtime buffering, +or memory/storage retained by the application. + +The absolute lifetime starts before callback invocation. Access is revoked on +callback return/throw, consumer cancellation, byte cap, or deadline, including +when an application retains a locked reader. Fedify bounds its own cleanup and +handles late callback/cancellation rejection without waiting indefinitely. +Signals cannot force arbitrary JavaScript or application storage to stop. Work +that ignores the signal can continue after its observation has ended. + +Capture failure or its signal does not abort the HTTP send, skip a challenge, +change a received status into a transport failure, or trigger another send. +The final error diagnostic can continue under its separate one-second budget +after application access ends. Error previews retain at most 1 KiB and append +`… (timed out)` when that diagnostic deadline expires, or `… (truncated)` when +additional body bytes establish truncation. Several selected exchanges can +accumulate waiting time; capture does not promise zero timing influence. + +### Inspecting the sender payload + +Enable `senderPayload: true` to include an independent privacy projection of the +concrete sender's existing JSON input in terminal attempt reports. It covers +immediate sends, worker/fanout execution, original forwarding, and portable +replication. It does not expose a producer's earlier Activity or perform extra +hydration, JSON-LD processing, context loading, or serialization. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + senderPayload: true, + onAttempt(_ctx, report) { + const document = report.senderPayload; + if (document.type === "available") { + console.log(document.stage, document.representation, document.redacted); + console.log(document.value); + } else { + console.log("Sender payload unavailable", document.reason); + } + }, +}); +~~~~ + +`stage` is `sender-payload`, and available values use +`representation: "privacy-projection"`. `redacted` means blind-recipient fields +were removed. The immutable copy removes compact, `as:`-prefixed, and full-IRI +`bto`/`bcc` fields throughout nested objects, arrays, and proof-bearing content. +It never changes the transport payload or signing input. A projection is not +exact wire evidence and does not establish signature/proof validity, including +when `redacted` is false. An unsigned redirect can drop its request body while +the attempt still retains this input projection. + +The supported subset includes ordinary plain JSON under audited ActivityStreams, +identity, security, data-integrity v1/v2, and GoToSocial context URLs; exact +Fedify-generated root extension definitions; and audited nested context URLs +under a supported compact root. Context-free expanded JSON-LD with unambiguous +IRI property names is also supported. Arbitrary inline aliases, context +overrides, unknown remote contexts, getters/custom JSON conversion, and +unsupported or cyclic/deep data produce an unavailable result. There is no +raw-data fallback. + +An unavailable payload has one of these reasons: + +| Reason | Meaning | +| ---------------------------- | ---------------------------------------------------------------- | +| `disabled` | Sender-payload export was not enabled. | +| `not-materialized` | The attempt failed before reaching the sender snapshot boundary. | +| `privacy-filter-unavailable` | A safe independent projection could not be established. | + +Header, body, and document evidence remain privileged application data. URLs and +content may contain other private values; blind-recipient filtering is not a +general personal-data sanitizer or permission to publish the evidence. Your +application owns access control, further redaction, archival, and saved original +dispatch documents. + ### Observation time and tracing Each event or report includes `observedAt`, Unix milliseconds captured at its @@ -466,7 +663,8 @@ attempt/exchange reports, `queue` contains the current `queueMessageId` and an available Fedify `queueAttempt` counter. Native redelivery can repeat that counter while creating fresh attempts. -Callbacks are awaited without a default timeout. Their return values and errors +Metadata callbacks are awaited without a default timeout; body consumers use +an explicit bounded capture lifetime. Callback return values and errors cannot veto sending, and callback or error-logging failures do not replace the send's original result. Slow callbacks delay processing; a callback that never settles can delay it indefinitely. Existing portable-forwarding deadlines can diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 80951a21a..78523ff85 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -87,7 +87,7 @@ export interface Federatable extends TaskRegistry { * Registers observers for outgoing activity processing. * * Each call replaces all previously registered outbound observers. - * Pass an empty object to clear them. Fedify copies the callbacks; + * Pass an empty object to clear them. Fedify copies callbacks and capture policies; * later changes to the supplied object do not affect registration. * Available on builders and directly created federations. Building copies * the callbacks, and an active send keeps the callback selected at entry. @@ -100,6 +100,9 @@ export interface Federatable extends TaskRegistry { * separately from attempts and backend acknowledgment. * Known IDs, original inbox source, and * application correlation survive queue work; workers use current context data. + * Header values, bounded response-body streaming and sender-payload projections + * are independently opt-in. Body consumers require explicit byte/time limits; + * other metadata callbacks have no default timeout. * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. * @param observers The optional callbacks to register. * @since 2.5.0 diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index cf0771503..1c31eb5c0 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -187,16 +187,25 @@ export type OutboundJsonValue = /** Attempt input evidence, never proof of a particular exchange's wire body. @since 2.5.0 */ export type OutboundSenderPayload = - & { readonly stage: "sender-payload" } + & { + /** Concrete sender input, not a producer history or per-exchange wire body. */ + readonly stage: "sender-payload"; + } & ( | { + /** A safe independent projection was established without extra processing. */ readonly type: "available"; + /** Always a projection; no signature/proof or exact-wire claim is retained. */ readonly representation: "privacy-projection"; + /** At least one blind-recipient field was removed from the copied value. */ readonly redacted: boolean; + /** Recursively immutable JSON data; transport/signing input is unchanged. */ readonly value: OutboundJsonValue; } | { + /** No document value is exposed, including when export is disabled. */ readonly type: "unavailable"; + /** Opt-out, pre-snapshot failure, or unsupported privacy/context semantics. */ readonly reason: | "disabled" | "not-materialized" @@ -212,6 +221,7 @@ export type OutboundBodyCaptureSummary = } & ( | { + /** Selection, verified EOF, byte-cap uncertainty, or local lifetime end. */ readonly type: | "filtered" | "no-body" @@ -220,10 +230,18 @@ export type OutboundBodyCaptureSummary = | "deadline" | "callback-returned"; } - | { readonly type: "consumer-stopped"; readonly reason: unknown } | { + /** Application cancellation ended capture. */ + readonly type: "consumer-stopped"; + /** Original consumer reason, including non-Error values. */ + readonly reason: unknown; + } + | { + /** Capture failed without changing received HTTP evidence. */ readonly type: "failed"; + /** Source reading versus application callback failure. */ readonly phase: "body" | "observer"; + /** Original capture error; it does not become the attempt's failure. */ readonly error: unknown; } ); @@ -245,13 +263,18 @@ export type OutboundResponseBodyEvent = /** Explicit opt-in response-body capture limits and consumer. @since 2.5.0 */ export interface OutboundResponseBodyCapture { - /** Positive safe-integer byte limit. Reaching it without EOF proves no completeness. */ + /** Positive safe-integer exposure limit; exact cap without EOF stays uncertain. */ readonly maxBytes: number; /** Positive finite absolute lifetime in milliseconds, rounded up; at most 2^31-1. */ readonly timeoutMs: number; /** Actual response selector, defaulting to `all` only when enabled. */ readonly responses?: "all" | "non-success-http-responses"; - /** Awaited within the bounded capture lifetime; errors cannot veto transport. */ + /** + * Invoked after headers, before the terminal exchange report, with current data. + * Awaited within the absolute lifetime, including no-pull/slow-consumer time. + * Return/throw/cancel/cap/deadline revokes access even for retained locked readers. + * Errors cannot veto transport; ignoring the signal cannot be forcibly stopped. + */ readonly onBody: ( context: Context, event: OutboundResponseBodyEvent, @@ -262,9 +285,15 @@ export interface OutboundResponseBodyCapture { export interface OutboundHeaderPolicy { /** Literal HTTP names, normalized to lowercase and deduplicated at registration. */ readonly allow: readonly string[]; - /** Additional names whose selected values are always `[redacted]`. */ + /** + * Additional names whose selected values are always `[redacted]`. + * Credentials, signatures and cookies remain protected even without this list. + */ readonly redact?: readonly string[]; - /** Positive safe-integer UTF-8 budget for `name: value\r\n` entries. */ + /** + * Positive safe-integer UTF-8 budget for `name: value\r\n` after redaction. + * Whole oversized entries are omitted; later entries remain eligible. + */ readonly maxBytes: number; } @@ -370,7 +399,8 @@ export type OutboundAttemptReport = OutboundSourceMetadata & { * A redirect response remains an observed response even if a later hop fails. * This report does not establish the attempt's final response. Metadata is * copied before callback awaiting; no live Request/Response/Headers are exposed. - * Enabled body capture uses a separate revocable callback; reports hold summaries. Exchange callbacks may run before the attempt settles. + * Enabled body capture uses a separate revocable callback; reports hold summaries. + * Exchange callbacks run after selected capture ends and may precede attempt settlement. * @since 2.5.0 */ export type OutboundHttpExchangeReport = @@ -542,9 +572,17 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * @since 2.5.0 */ export interface OutboundObservers { - /** Opt-in privacy-filtered sender input on attempt reports; disabled by default. */ + /** + * Opt-in immutable privacy projection of concrete sender input on attempt reports. + * Disabled by default. Unsupported context/data semantics yield no raw fallback; + * this is not exact wire evidence or proof that redirected requests kept a body. + */ readonly senderPayload?: boolean; - /** Independent opt-in body consumer; explicit byte/time limits are required. */ + /** + * Independent opt-in body consumer; explicit byte/time limits are required. + * Shares one source reader with separately bounded diagnostics, without cloning + * or whole-body accumulation for the application. Finished reports hold summaries. + */ readonly responseBodyCapture?: OutboundResponseBodyCapture; /** * Opt-in actual request headers for exchange reports; disabled by default. diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 220b366c4..400a073e1 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -746,7 +746,7 @@ function cancelResponseBody(response: Response, url: string): void { /** * Reads what is needed from the final response of a delivery while its * session is open: the body of a successful response is cancelled, and at - * most 1 KiB of an error response body is read. + * most 1 KiB of an error response body is read within its diagnostic budget. */ async function consumeSendActivityResponse( response: Response, @@ -895,7 +895,9 @@ export class SendActivityError extends Error { * The response body from the inbox, if any. Note that this may be * truncated to a maximum of 1 KiB to prevent excessive memory consumption * when remote servers return large error pages (e.g., Cloudflare error pages). - * If truncated, the string will end with `"… (truncated)"`. + * If truncated, the string will end with `"… (truncated)"`. Diagnostic reading + * has a separate one-second budget; a deadline retains any partial preview and + * appends `"… (timed out)"`. This is independent of application body capture. */ readonly responseBody: string; From 24255905174eacaee53b467f3b487f4905a695a3 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 05:36:43 +0900 Subject: [PATCH 36/53] Report incoming request cancellation Snapshot the local URL and native abort state and reason when building reports, keeping request cancellation separate from processing outcomes. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/inbox-observation.ts | 7 + .../src/federation/inbox-report.test.ts | 246 +++++++++++++++++- .../fedify/src/federation/inbox-report.ts | 22 ++ 3 files changed, 274 insertions(+), 1 deletion(-) diff --git a/packages/fedify/src/federation/inbox-observation.ts b/packages/fedify/src/federation/inbox-observation.ts index 8bd5ae243..2d495508b 100644 --- a/packages/fedify/src/federation/inbox-observation.ts +++ b/packages/fedify/src/federation/inbox-observation.ts @@ -127,11 +127,18 @@ export class InboxObservation { outcome: InboxRequestOutcome, ): Promise { if (handler == null) return; + const signal = context.request.signal; + const requestCancellation: InboxRequestReport["requestCancellation"] = + signal.aborted + ? Object.freeze({ type: "aborted", reason: signal.reason }) + : Object.freeze({ type: "not-aborted" }); const report: InboxRequestReport = { ...(this.source == null ? {} : { inboxRequestId: this.source.inboxRequestId, requestId: this.source.requestId, + local: Object.freeze({ ...this.source.context }), }), + requestCancellation, inbox: this.inbox, payload: this.payload, activity: this.activity, diff --git a/packages/fedify/src/federation/inbox-report.test.ts b/packages/fedify/src/federation/inbox-report.test.ts index 236762748..cc3fdf8bd 100644 --- a/packages/fedify/src/federation/inbox-report.test.ts +++ b/packages/fedify/src/federation/inbox-report.test.ts @@ -9,7 +9,8 @@ import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; import { createFederationBuilder } from "./builder.ts"; import type { InboxRequestReport } from "./inbox-report.ts"; import { MemoryKvStore } from "./kv.ts"; -import { createFederation } from "./middleware.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; +import type { Message } from "./queue.ts"; import { signRequest } from "../sig/http.ts"; import { signJsonLd } from "../sig/ld.ts"; import { signObject } from "../sig/proof.ts"; @@ -673,3 +674,246 @@ test("onRequestFinished observes the final response decoration result", async () }); } }); + +test("inbox request cancellation snapshots preserve exact reasons without stopping processing", async (t) => { + for ( + const [label, reason] of [ + ["live", undefined], + ["object", { kind: "client disconnect" }], + ["zero", 0], + ["nan", NaN], + ["symbol", Symbol("client")], + ] as const + ) { + await t.step(label, async () => { + const { federation, setters, reports } = setup(true); + let processed = 0; + setters.on(Create, () => { + processed++; + }); + const controller = new AbortController(); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + input.headers.set("X-Request-Id", "cancellation-case"); + if (label !== "live") controller.abort(reason); + const response = await federation.fetch(input, { + contextData: undefined, + }); + strictEqual(response.status, 202); + strictEqual(processed, 1); + strictEqual(reports.length, 1); + const report = reports[0]; + deepStrictEqual(report.outcome, { + type: "response", + status: 202, + disposition: "processed", + }); + deepStrictEqual(report.authentication, { status: "skipped" }); + strictEqual(report.payload.status, "parsed"); + strictEqual(report.requestId, "cancellation-case"); + ok(report.inboxRequestId); + deepStrictEqual(report.local, { + origin: "https://example.com", + host: "example.com", + hostname: "example.com", + canonicalOrigin: "https://example.com", + }); + ok(Object.isFrozen(report.local)); + ok(Object.isFrozen(report.requestCancellation)); + strictEqual( + report.requestCancellation.type, + label === "live" ? "not-aborted" : "aborted", + ); + if (report.requestCancellation.type === "aborted") { + ok(Object.is(report.requestCancellation.reason, reason)); + strictEqual(report.requestCancellation.reason, input.signal.reason); + } + }); + } +}); + +test("inbox request cancellation waits for processing and preserves original local facts", async () => { + const { federation, setters, reports } = setup(true); + let entered!: () => void; + const listening = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + setters.on(Create, async (ctx) => { + ok(ctx instanceof ContextImpl); + ctx.url.hostname = "changed.example"; + entered(); + await gate; + }); + const controller = new AbortController(); + const reason = Symbol("mid-processing disconnect"); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + const processing = federation.fetch(input, { contextData: undefined }); + try { + await listening; + controller.abort(reason); + strictEqual(reports.length, 0); + release(); + strictEqual((await processing).status, 202); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].outcome, { + type: "response", + status: 202, + disposition: "processed", + }); + deepStrictEqual(reports[0].requestCancellation, { + type: "aborted", + reason, + }); + strictEqual(reports[0].local?.hostname, "example.com"); + controller.abort("later reason"); + await Promise.resolve(); + strictEqual(reports.length, 1); + strictEqual(reports[0].local?.origin, "https://example.com"); + } finally { + release(); + await processing; + } +}); + +test("inbox request cancellation after report construction does not rewrite its snapshot", async () => { + const { federation, setters, reports } = setup(true); + setters.on(Create, () => {}); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + setters.onRequestFinished(async (_ctx, report) => { + reports.push(report); + entered(); + await gate; + throw new Error("recording failure"); + }); + const controller = new AbortController(); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + const processing = federation.fetch(input, { contextData: undefined }); + try { + await observing; + controller.abort({ kind: "late disconnect" }); + deepStrictEqual(reports[0].requestCancellation, { type: "not-aborted" }); + release(); + strictEqual((await processing).status, 202); + await Promise.resolve(); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].requestCancellation, { type: "not-aborted" }); + deepStrictEqual(reports[0].outcome, { + type: "response", + status: 202, + disposition: "processed", + }); + } finally { + release(); + await processing; + } +}); + +test("inbox request cancellation retains accepted queue work and its separate lifetime", async () => { + const controller = new AbortController(); + const reason = Symbol("disconnect after acceptance"); + const messages: Message[] = []; + const reports: InboxRequestReport[] = []; + let processed = 0; + let workerSignal: AbortSignal | undefined; + const federation = createFederation({ + kv: new MemoryKvStore(), + skipSignatureVerification: true, + manuallyStartQueue: true, + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + queue: { + inbox: { + enqueue(message) { + messages.push(message); + controller.abort(reason); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .on(Create, (ctx) => { + processed++; + workerSignal = Reflect.get(ctx, "signal"); + }) + .onRequestFinished((_ctx, report) => { + reports.push(report); + }); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + strictEqual( + (await federation.fetch(input, { contextData: undefined })).status, + 202, + ); + strictEqual(messages.length, 1); + strictEqual(processed, 0); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].outcome, { + type: "response", + status: 202, + disposition: "enqueued", + }); + deepStrictEqual(reports[0].requestCancellation, { type: "aborted", reason }); + strictEqual(Object.hasOwn(messages[0], "signal"), false); + strictEqual(Object.hasOwn(messages[0], "requestCancellation"), false); + await federation.processQueuedTask(undefined, messages[0]); + strictEqual(processed, 1); + strictEqual(workerSignal?.aborted === true, false); + strictEqual(reports.length, 1); +}); + +test("inbox request cancellation preserves early preparation failures and canonical source", async () => { + const error = new Error("initial loader factory failed"); + const reason = { kind: "client disconnected" }; + const controller = new AbortController(); + const reports: InboxRequestReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + origin: "https://canonical.example", + documentLoaderFactory: () => { + throw error; + }, + contextLoaderFactory: () => mockDocumentLoader, + }); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .onRequestFinished((_ctx, report) => { + reports.push(report); + }); + const input = new Request(request({}), { signal: controller.signal }); + controller.abort(reason); + await rejects( + federation.fetch(input, { contextData: undefined }), + (value: unknown) => value === error, + ); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].outcome, { + type: "exception", + stage: "prepare", + error, + }); + deepStrictEqual(reports[0].authentication, { status: "notDetermined" }); + deepStrictEqual(reports[0].payload, { status: "unavailable" }); + deepStrictEqual(reports[0].requestCancellation, { type: "aborted", reason }); + strictEqual(reports[0].local?.origin, "https://example.com"); + strictEqual(reports[0].local?.canonicalOrigin, "https://canonical.example"); +}); diff --git a/packages/fedify/src/federation/inbox-report.ts b/packages/fedify/src/federation/inbox-report.ts index 9fb9b31ea..f079f79d8 100644 --- a/packages/fedify/src/federation/inbox-report.ts +++ b/packages/fedify/src/federation/inbox-report.ts @@ -1,3 +1,4 @@ +import type { OutboundLocalContext } from "./outbound-report.ts"; import type { Activity } from "@fedify/vocab"; import type { RequestContext } from "./context.ts"; import type { @@ -26,6 +27,15 @@ export type InboxJsonValue = | readonly InboxJsonValue[] | { readonly [key: string]: InboxJsonValue }; +/** Request-signal state at inbox report construction, separate from processing outcome. @since 2.5.0 */ +export type InboxRequestCancellation = + | { readonly type: "not-aborted" } + | { + readonly type: "aborted"; + /** Exact Request.signal reason, including non-Error values. */ + readonly reason: unknown; + }; + /** * The final inbox authentication decision. * @since 2.4.0 @@ -145,6 +155,18 @@ export interface InboxRequestReport { * @since 2.5.0 */ readonly requestId?: string; + /** + * Original local URL facts from the routed request, independent of later contexts. + * Absent without a known routed source. + * @since 2.5.0 + */ + readonly local?: OutboundLocalContext; + /** + * Request.signal state immediately before report construction. An aborted request + * does not imply processing stopped or retract confirmed processing/enqueue outcomes. + * @since 2.5.0 + */ + readonly requestCancellation: InboxRequestCancellation; /** The delivery route, including a portable gateway route. */ readonly inbox: { readonly kind: "personal" | "shared" | "portable"; From d9924e0670375ed51d208b4b4ae711d74f72d92c Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 12:19:36 +0900 Subject: [PATCH 37/53] Document and verify outbound observers Clarify inbox request cancellation and observer guidance. Cover HTTP retry archival and Workers privacy and capture, and ensure declaration and test runtime compatibility. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 27 +-- changes.d/fedify/outbound-observers.md | 38 +--- claude-plugin/skills/fedify/SKILL.md | 13 +- docs/manual/context.md | 4 +- docs/manual/inbox.md | 16 ++ docs/manual/send.md | 143 +++++++-------- .../fedify/src/cfworkers/expected-tests.ts | 2 + .../src/federation/outbound-observation.ts | 2 +- .../src/federation/outbound-report.test.ts | 2 + .../src/federation/outbound-workers.test.ts | 58 ++++++ .../src/federation/outbound-workflow.test.ts | 170 ++++++++++++++++++ 11 files changed, 354 insertions(+), 121 deletions(-) create mode 100644 packages/fedify/src/federation/outbound-workers.test.ts create mode 100644 packages/fedify/src/federation/outbound-workflow.test.ts diff --git a/CHANGES.md b/CHANGES.md index 36d429962..9015044d0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -61,16 +61,23 @@ To be released. actual child enqueue, retry, deferral, or abandonment actions. Worker reports retain accepted children through partial failures and distinguish unknown queue acceptance from confirmed scheduling; local completion does - not establish backend acknowledgment or remote success. Reports - include observation timestamps and optional operation trace IDs. Callbacks - receive application context and independent grouping IDs without requiring - OpenTelemetry; their errors do not replace the original result. Known IDs - and original inbox source survive queue continuations. Send/forward options - accept `dispatchId` for application correlation without deduplication, - limited to 1,024 UTF-8 bytes. Routed inbox reports include `inboxRequestId` - and informational `requestId`. Missing legacy relationships remain unknown. - Readiness and handoff observations do not establish remote delivery - success. [[#1316] by Jiwon Kwon\] + not establish backend acknowledgment or remote success. Opt in to + independent redacted header snapshots, bounded streaming response-body + capture with explicit byte/time limits, and privacy-filtered sender-payload + projections. Capture summaries distinguish EOF, byte-limit uncertainty, + deadlines, and consumer/body failures without changing HTTP outcomes; + payload projections never claim exact wire bytes or proof validity. Error + diagnostics now have a separate one-second budget while retaining their 1 + KiB preview limit. Reports include observation timestamps and optional + operation trace IDs. Callbacks receive application context and independent + grouping IDs without requiring OpenTelemetry; their errors do not replace + the original result. Known IDs and original inbox source survive queue + continuations. Send/forward options accept `dispatchId` for application + correlation without deduplication, limited to 1,024 UTF-8 bytes. Routed + inbox reports include `inboxRequestId` and informational `requestId`. + Missing legacy relationships remain unknown. Readiness and handoff + observations do not establish remote delivery success. + [[#1316] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index d764cbdfc..2c852ae1d 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -2,33 +2,11 @@ links: '#1316': https://github.com/fedify-dev/fedify/issues/1316 --- - - Added `setOutboundObservers()` on federation builders and instances - for sends, original-payload forwarding, and automatic portable replication. - Register `onBroadcast` to record producer starts and preparation results, - and `onDelivery` for destination planning, direct handoffs, queue acceptance - or uncertainty, and local preparation failures. Use `onAttempt` for concrete - send results and exact HTTP exchange counts, including queued retries and - gateway fallback, and `onHttpExchange` for each actual signed or unsigned - fetch. Reports distinguish received responses from the selected final - response and retain HTTP evidence alongside later local failures. Use - `onWorker` for fanout/outbox execution results, concrete send counts, and - actual child enqueue, retry, deferral, or abandonment actions. Worker - reports retain accepted children through partial failures and distinguish - unknown queue acceptance from confirmed scheduling; local completion does - not establish backend acknowledgment or remote success. Opt in to - independent redacted header snapshots, bounded streaming response-body - capture with explicit byte/time limits, and privacy-filtered sender-payload - projections. Capture summaries distinguish EOF, byte-limit uncertainty, - deadlines, and consumer/body failures without changing HTTP outcomes; - payload projections never claim exact wire bytes or proof validity. Error - diagnostics now have a separate one-second budget while retaining their 1 - KiB preview limit. Reports include observation timestamps and optional - operation trace IDs. Callbacks receive application context and independent - grouping IDs without requiring OpenTelemetry; their errors do not replace - the original result. Known IDs and original inbox source survive queue - continuations. Send/forward options accept `dispatchId` for application - correlation without deduplication, limited to 1,024 UTF-8 bytes. Routed - inbox reports include `inboxRequestId` and informational `requestId`. - Missing legacy relationships remain unknown. Readiness and handoff - observations do not establish remote delivery success. - [[#1316] by Jiwon Kwon] + - Added application-facing outbound observers for sends, forwarding, + replication, and queued retries, independently of OpenTelemetry. Optional + callbacks report preparation, delivery handoffs, send/HTTP results, and + worker actions with correlation IDs. Applications can opt in to redacted + headers, bounded streaming response capture, and privacy-filtered sender + payloads. Inbox reports now preserve original request context and snapshot + request cancellation without changing processing outcomes. Error diagnostic + reads have a one-second deadline. [[#1316] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index ede06b0d2..36aa8093d 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -374,10 +374,15 @@ Events expose `provenance` for preserved source/app correlation and `local` for current URL facts. A known inbox source includes server-generated `inboxRequestId`, informational `requestId`, and the original URL snapshot. It survives context clones/conversions and queues; independent contexts do not -infer it from ambient execution. `observedAt` is current Unix milliseconds, -not a global ordering guarantee. Optional `trace` carries only valid IDs of the -operation's own span, never an ambient parent fallback; grouping works without -tracing. Entry events and exchanges omit trace when no own span exists. +infer it from ambient execution. Inbox completion `report.local` retains the +original URL facts. `report.requestCancellation` is `not-aborted` or `aborted` +with the exact native request-signal `reason`, captured before report creation. +It does not change authentication/processing outcomes, cancel accepted work, +inherit a signal into descendants, or emit a second report after a later abort. +`observedAt` is current Unix milliseconds, not a global ordering guarantee. +Optional `trace` carries only valid IDs of the operation's own span, never an +ambient parent fallback; grouping works without tracing. Entry events and +exchanges omit trace when no own span exists. Send/forward options accept `dispatchId`, preserved exactly in event provenance and queues even without producer observers. Empty strings are valid; the limit diff --git a/docs/manual/context.md b/docs/manual/context.md index 5d4bd8727..85fa1d672 100644 --- a/docs/manual/context.md +++ b/docs/manual/context.md @@ -86,7 +86,9 @@ For `RequestContext`, there is an additional property named Outgoing observations snapshot these values under `event.local`. For work caused by an inbox request, `event.provenance.inboxRequest.context` retains the -original URL facts through derived contexts and queues. See +original URL facts through derived contexts and queues. Inbox completion reports +preserve the original values under `report.local` and separately snapshot +request cancellation; neither changes the lifetime of accepted queue work. See [outgoing source and correlation](./send.md#original-source-and-current-context). [^1]: If no canonical origin is explicitly configured, it is the same as the diff --git a/docs/manual/inbox.md b/docs/manual/inbox.md index 81e9aab53..d13d069f7 100644 --- a/docs/manual/inbox.md +++ b/docs/manual/inbox.md @@ -206,6 +206,9 @@ federation .setInboxListeners("/users/{identifier}/inbox", "/inbox") .onRequestFinished(async (ctx, report) => { await saveReport(report); + if (report.requestCancellation.type === "aborted") { + console.log("Request aborted", report.requestCancellation.reason); + } }); ~~~~ @@ -221,6 +224,19 @@ server-generated ID for each HTTP delivery, even for the same Activity. The latter is an informational, externally supplied or generated logging label; it is not unique. These fields are absent when no routed source is known. +`report.local` preserves the original `origin`, `host`, `hostname`, and +`canonicalOrigin`, even when a later context's URL changes. It is absent when +no routed source is known. `report.requestCancellation` snapshots the native +request signal immediately before report construction: `not-aborted`, or +`aborted` with the exact `reason`, including non-Error values. These envelopes +are immutable; the reason object itself is not cloned or frozen. + +Request cancellation is separate from processing outcome. A processed or +enqueued `202` can coexist with an aborted signal. The callback still runs once +when processing settles; an abort after the snapshot does not change it or +trigger another report. Observation does not cancel HTTP processing or pass its +signal to accepted queue work or outgoing descendants. + Outgoing work using the handler's context retains this source through derived contexts and queues. See [outgoing source and correlation](./send.md#original-source-and-current-context). diff --git a/docs/manual/send.md b/docs/manual/send.md index c4660c1a4..dc737fb59 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -227,42 +227,30 @@ establish a worker's scheduling result. ### Worker execution and actions `onWorker` receives one `OutboundWorkerReport` after a fanout or outbox worker's -local execution settles, including its existing metrics and cleanup. It receives -the current worker context and application data. It also works when attempt and -exchange observers are disabled; inbox and custom-task workers are separate. - -`role` identifies `fanout` or `outbox`. `queue` identifies the current task and, -for outbox tasks, its Fedify retry counter. `attemptsStarted` counts actual -concrete sender entries: expanding fanout into child tasks counts zero, while -direct sends and alternate gateways each count when entered. An outbox report's -`delivery` identifies its obligation when available. Fanout has no single -`delivery`; its child actions carry their own delivery references. - -Narrow `result.type` to read the execution result: - -| Type | Meaning | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| `completed` | The local worker path returned normally, including handled send failures and abandonment. | -| `failed` | Processing escaped with the original `error`, even if an earlier attempt succeeded. | -| `preparation-failed` | Key import, activity parsing, context creation, or request preparation failed; `failure.phase` identifies the operation. | - -`actions` records actual local decisions and enqueue settlements independently -of the result. An empty array means no such action was observed in that -execution. Narrow an action's `type`, then its `action` when present: - -| Type | Evidence | -| --------------- | --------------------------------------------------------------------------------------------------------------- | -| `scheduled` | An `enqueue`, `retry`, or `defer` call resolved. `queue` identifies the submitted task and counter. | -| `action-failed` | An enqueue rejected, with `acceptance: "unknown"` and the original `error`, or an available `drop` step failed. | -| `handoff` | A fanout worker entered a concrete sender (`action: "direct"`); this does not establish HTTP success. | -| `abandoned` | The local path stopped work for the listed routes, with a `reason` such as `permanent-failure`. | - -Each action's `inboxUrls` contains its affected routes. Gateway abandonment can -remove one route while another succeeds or is rescheduled. Retry/defer actions -include the actual ISO 8601 `delay` passed to the queue and any per-route -`notBefore` constraints retained in the continuation. Child `enqueue` actions -and direct handoffs include their known `delivery` references. Constructing a -child or deciding to retry does not itself establish an enqueue. +local execution and instrumentation settle. `role` and `queue` identify the +current worker/task. `attemptsStarted` counts concrete sender entries, including +fallback gateways; child enqueues count zero. Outbox `delivery` identifies its +obligation. Fanout has no single delivery; child actions carry their references. + +| `result.type` | Meaning | +| -------------------- | -------------------------------------------------------------------------- | +| `completed` | Local normal return, including handled send failures and abandonment. | +| `failed` | The original error escaped, even if an earlier attempt succeeded. | +| `preparation-failed` | `failure.phase` identifies the preparation operation producing that error. | + +Read `actions` independently of `result` and attempt outcomes: + +| `type` | Evidence | +| --------------- | ---------------------------------------------------------------------------------------------- | +| `scheduled` | An `enqueue`, `retry`, or `defer` call resolved; `queue` identifies its actual task/counter. | +| `action-failed` | Enqueue rejection with unknown acceptance and original error, or a failed `drop` step. | +| `handoff` | A fanout worker entered a concrete sender (`action: "direct"`), without claiming HTTP success. | +| `abandoned` | Work stopped locally for the listed routes, with a reason such as `permanent-failure`. | + +`inboxUrls` lists affected routes; one gateway can be abandoned while another +continues. Retry/defer includes the actual ISO `delay` and any per-route +`notBefore` constraints. Child enqueues/direct handoffs retain delivery +references. An empty action list means no action was observed by settlement. ~~~~ typescript twoslash import type { Federation } from "@fedify/fedify"; @@ -270,43 +258,23 @@ declare const federation: Federation; // ---cut-before--- federation.setOutboundObservers({ onWorker(_ctx, report) { - console.log(report.role, report.queue.queueMessageId, report.attemptsStarted); - if (report.result.type === "preparation-failed") { - console.error(report.result.failure.phase, report.result.error); - } + console.log(report.role, report.result.type, report.attemptsStarted); for (const action of report.actions) { if (action.type === "scheduled") { console.log(action.action, action.queue.queueMessageId, action.inboxUrls); - if (action.action === "enqueue") { - console.log(action.delivery?.deliveryId); - } else { - console.log(action.delay, action.notBefore); - } - } else if (action.type === "action-failed") { - console.error(action.action, action.error); - if (action.action !== "drop") { - console.error("Queue acceptance is unknown", action.queue.queueMessageId); - } + if (action.action !== "enqueue") console.log(action.delay); } } }, }); ~~~~ -Accepted child tasks remain in the report when another child fails or later -bookkeeping throws. A rejected batch leaves acceptance unknown for every -submitted child, even if the backend accepted some of them. Fanout replay keeps -known delivery IDs while creating fresh child task IDs, without another -broadcast or a change to the producer's preparation result. A later destination -setup failure can leave earlier direct sends running; the worker report captures -entries observed by its settlement, while those attempt callbacks may arrive -later. - -Worker settlement does not establish backend acknowledgment, a future native -retry, or remote application effects. A native rethrow records failure without -claiming a scheduled continuation; native redelivery can reuse the task and -Fedify retry counter. See [message queues](./mq.md#observing-outgoing-workers) -for how this relates to queue processing. +Accepted actions survive sibling or later bookkeeping failures. A rejected batch +leaves every submitted child's acceptance unknown. Later destination setup +failure can leave earlier direct attempts running after worker settlement. +Neither local completion nor a native rethrow establishes backend +acknowledgment, a future replay, or remote application effects. See +[message queues](./mq.md#observing-outgoing-workers). ### Response versus final response @@ -459,18 +427,14 @@ At an exact cap, another read does not establish EOF: access is revoked, and Signal abort or stream rejection can also stop an outstanding application write; the summary does not certify that exposed chunks were saved. -Fedify uses one source reader shared with the bounded error diagnostic preview, -pull backpressure, no additional wrapper queue, and at most one pending source -read. It does not clone, tee, or accumulate the response body for application -capture. Its limits do not bound upstream chunk allocations, runtime buffering, -or memory/storage retained by the application. +Fedify uses one source reader shared with diagnostics and pull backpressure, +without cloning, teeing, or accumulating application response bodies. Its bounds +exclude upstream chunk allocations, runtime buffering, and application storage. -The absolute lifetime starts before callback invocation. Access is revoked on -callback return/throw, consumer cancellation, byte cap, or deadline, including -when an application retains a locked reader. Fedify bounds its own cleanup and -handles late callback/cancellation rejection without waiting indefinitely. -Signals cannot force arbitrary JavaScript or application storage to stop. Work -that ignores the signal can continue after its observation has ended. +The absolute lifetime starts before callback invocation and covers no-pull/slow +consumption and cleanup. Return/throw/cancel/cap/deadline revokes even retained +locked readers. Late rejections are handled without waiting indefinitely; +arbitrary JavaScript or storage that ignores the signal can still continue. Capture failure or its signal does not abort the HTTP send, skip a challenge, change a received status into a transport failure, or trigger another send. @@ -575,8 +539,17 @@ inbox request -> handler -> send/forward -> planned delivery +-> fanout task -> outbox tasks Queue continuations preserve known IDs and the original source. + +Portable fallback: delivery D -> gateway A -> gateway B (stop on success) +Portable replication: broadcast B -> delivery D1 -> gateway A + -> delivery D2 -> gateway B + +HTTP abort -> inbox report.requestCancellation +Accepted queue work -> worker (the HTTP abort does not cancel it) ~~~~ +Diagram generated with Codex. + IDs provide grouping, not ordering or deduplication. Replayed work can produce repeated observations. Treat IDs as opaque values. @@ -588,6 +561,21 @@ An explicit canonical `portableInbox` can still be known. Missing source and application correlation remain unavailable. Older consumers can discard these fields, so mixed-version processing can lose correlation. +### Retry history + +Persist attempt reports by `deliveryId` and each fresh `attemptId`; correlate +scheduled retries with worker action task IDs. For example: + +~~~~ text +delivery D -> attempt A1: failed, final HTTP 503 + -> worker: retry scheduled as task Q, queueAttempt 1 + -> attempt A2: succeeded, final HTTP 202 +~~~~ + +A native redelivery can reuse a task/counter but still creates a new attempt. +The application owns this history and any later manual retry; Fedify emits no +distributed delivery-completed result. + ### Original source and current context Outgoing events distinguish preserved source facts from current URL facts: @@ -671,6 +659,11 @@ settles can delay it indefinitely. Existing portable-forwarding deadlines can return control while background preparation, sending, and observation continue; a deadline alone does not create a failed preparation result or attempt outcome. +Inbox request cancellation is reported separately from processing. The queue +listener's stop signal controls intake; this observer API does not yet report +per-execution queue cancellation. A backend ceasing to wait is not evidence that +its worker terminated. + If you detach recording work, handle its rejections and shutdown yourself; Fedify does not guarantee its persistence. Incoming C2S `setOutboxListeners()` registration is separate. Mock federations accept outbound observer diff --git a/packages/fedify/src/cfworkers/expected-tests.ts b/packages/fedify/src/cfworkers/expected-tests.ts index 04c51f943..ac9ee715e 100644 --- a/packages/fedify/src/cfworkers/expected-tests.ts +++ b/packages/fedify/src/cfworkers/expected-tests.ts @@ -1,5 +1,7 @@ // The Workers suite is deliberately bounded to tests validated on workerd. export const expectedTestNames = [ + "outbound sender payload privacy on Workers", + "outbound body capture revocation on Workers", "authentication preserves the queued principal wire format", "proof principals adapt internal signer kinds at the public boundary", "ignored nested test steps return false", diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index f60d3e5b8..c243157c4 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -227,7 +227,7 @@ export class OutboundAttemptObservation { | ((facts: ExchangeFacts) => Promise) | undefined = undefined, readonly headerPolicies: HeaderPolicies = {}, - readonly bodyCapture?: BodyCapture, + readonly bodyCapture: BodyCapture | undefined = undefined, readonly senderPayload = false, ) { this.#senderPayload = Object.freeze({ diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 761bbc229..c596ca653 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -6223,6 +6223,8 @@ test("sender payload availability never changes serialization or redirected bodi let getters = 0; const input = mode === "conversion" ? Object.defineProperty({}, "toJSON", { + // Bun ignores a non-enumerable converter; use a shared runtime fixture. + enumerable: true, value() { conversions++; return payload; diff --git a/packages/fedify/src/federation/outbound-workers.test.ts b/packages/fedify/src/federation/outbound-workers.test.ts new file mode 100644 index 000000000..00374654a --- /dev/null +++ b/packages/fedify/src/federation/outbound-workers.test.ts @@ -0,0 +1,58 @@ +import { test } from "@fedify/fixture"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { captureResponseBody } from "./outbound-capture.ts"; +import { projectSenderPayload } from "./outbound-document.ts"; + +test("outbound sender payload privacy on Workers", () => { + const secret = "https://private.example/recipient"; + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + bcc: [secret], + object: { "as:bto": [secret], content: "visible" }, + }; + const result = projectSenderPayload(input); + ok(result.type === "available"); + strictEqual(result.redacted, true); + strictEqual(JSON.stringify(result.value).includes(secret), false); + ok(Object.isFrozen(result.value)); + strictEqual(input.bcc[0], secret); + deepStrictEqual( + projectSenderPayload({ + ...input, + "@context": [input["@context"], { private: "as:bcc" }], + }), + { + stage: "sender-payload", + type: "unavailable", + reason: "privacy-filter-unavailable", + }, + ); +}); + +test("outbound body capture revocation on Workers", async () => { + let cancelled = 0; + let reader!: ReadableStreamDefaultReader; + let signal!: AbortSignal; + const source = new ReadableStream({ + pull() {}, + cancel() { + cancelled++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const result = await captureResponseBody(new Response(source), { + maxBytes: 16, + timeoutMs: 10, + }, async (body, current) => { + signal = current; + reader = body.getReader(); + await rejects(reader.read()); + }); + deepStrictEqual(result, { type: "deadline", bytesExposed: 0 }); + ok(signal.aborted); + strictEqual(cancelled, 1); + strictEqual(source.locked, false); + await rejects(reader.read()); + reader.releaseLock(); +}); diff --git a/packages/fedify/src/federation/outbound-workflow.test.ts b/packages/fedify/src/federation/outbound-workflow.test.ts new file mode 100644 index 000000000..8a2ce1d60 --- /dev/null +++ b/packages/fedify/src/federation/outbound-workflow.test.ts @@ -0,0 +1,170 @@ +import { test } from "@fedify/fixture"; +import { Create, Person } from "@fedify/vocab"; +import { deepStrictEqual, ok, strictEqual } from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; +import { createFederation } from "./middleware.ts"; +import { MemoryKvStore } from "./kv.ts"; +import type { Message } from "./queue.ts"; +import type { + OutboundAttemptReport, + OutboundHttpExchangeReport, + OutboundWorkerReport, +} from "./outbound-report.ts"; + +test({ + name: "outbound recorder archives real HTTP exchanges across a queued retry", + // The package Deno suite is offline; focused --allow-all and Node/Bun run it. + ignore: "Deno" in globalThis && + (await Deno.permissions.query({ name: "net" })).state !== "granted", + async fn() { + let accepting = false; + const received: { status: number; activity: unknown }[] = []; + const receiverErrors: unknown[] = []; + const server = createServer((request, response) => { + void (async () => { + let body = ""; + for await (const chunk of request) body += chunk.toString(); + const status = accepting ? 202 : 503; + received.push({ status, activity: JSON.parse(body) }); + response.writeHead(status, { + "Content-Type": "text/plain", + "Connection": "close", + }); + response.end(accepting ? "accepted" : "retry later"); + })().catch((error) => { + receiverErrors.push(error); + response.writeHead(500); + response.end(); + }); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const inbox = new URL( + `http://127.0.0.1:${(server.address() as AddressInfo).port}/inbox`, + ); + const queued: Message[] = []; + const attempts: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const workers: OutboundWorkerReport[] = []; + const archive = new Map(); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: false, + outboxRetryPolicy: () => Temporal.Duration.from({ milliseconds: 0 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + senderPayload: true, + responseBodyCapture: { + maxBytes: 4096, + timeoutMs: 1000, + async onBody(_ctx, event) { + const key = `${event.attemptId}:${event.exchangeIndex}`; + const chunks: Uint8Array[] = []; + archive.set(key, chunks); + const reader = event.body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + chunks.push(chunk.value.slice()); + } + } finally { + reader.releaseLock(); + } + }, + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onWorker(_ctx, report) { + workers.push(report); + }, + }); + try { + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }, + new Person({ id: new URL(inbox.origin + "/actor"), inbox }), + new Create({ + id: new URL("https://example.com/workflow/activity"), + actor: new URL("https://example.com/person2"), + }), + { fanout: "skip", dispatchId: "saved-workflow-dispatch" }, + ); + strictEqual(received.length, 0); + strictEqual(queued.length, 1); + await federation.processQueuedTask(undefined, queued.shift()!); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "failed"); + ok(attempts[0].outcome.response.type === "final"); + strictEqual(attempts[0].outcome.response.metadata.status, 503); + strictEqual(queued.length, 1); + const retry = workers[0].actions.find((action) => + action.type === "scheduled" && action.action === "retry" + ); + ok(retry?.type === "scheduled"); + strictEqual(retry.queue.queueAttempt, 1); + accepting = true; + await federation.processQueuedTask(undefined, queued.shift()!); + strictEqual(queued.length, 0); + strictEqual(attempts.length, 2); + strictEqual(attempts[1].outcome.type, "succeeded"); + strictEqual(attempts[0].deliveryId, attempts[1].deliveryId); + ok(attempts[0].attemptId !== attempts[1].attemptId); + deepStrictEqual(attempts[0].broadcast, attempts[1].broadcast); + strictEqual(attempts[1].queue?.queueAttempt, 1); + for (const report of attempts) { + strictEqual(report.provenance.dispatchId, "saved-workflow-dispatch"); + strictEqual(report.senderPayload.type, "available"); + strictEqual( + report.httpExchangeCount, + exchanges.filter((exchange) => + exchange.attemptId === report.attemptId + ) + .length, + ); + } + strictEqual(received.length, exchanges.length); + strictEqual(archive.size, exchanges.length); + for (const [index, exchange] of exchanges.entries()) { + deepStrictEqual(exchange.bodyCapture, { + type: "complete", + bytesExposed: received[index].status === 503 ? 11 : 8, + }); + const chunks = archive.get( + `${exchange.attemptId}:${exchange.exchangeIndex}`, + )!; + const decoder = new TextDecoder(); + strictEqual( + chunks.map((chunk) => decoder.decode(chunk)).join(""), + received[index].status === 503 ? "retry later" : "accepted", + ); + } + deepStrictEqual(workers.map((worker) => worker.attemptsStarted), [1, 1]); + deepStrictEqual(receiverErrors, []); + } finally { + const closed = new Promise((resolve, reject) => + server.close((error) => error ? reject(error) : resolve()) + ); + server.closeAllConnections(); + await closed; + } + }, +}); From 2bbff1bf07814dbfb635b08e1c90314ef7315076 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 12:38:34 +0900 Subject: [PATCH 38/53] Fix outbound observer type checking Preserve tuple header inference and the literal failure discriminant. Access Deno permissions through an optional typed global for Node checks. https://github.com/fedify-dev/fedify/issues/1316 Changelog: none Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 38 ++++--------------- .../fedify/src/federation/outbound-capture.ts | 2 +- .../src/federation/outbound-report.test.ts | 4 +- .../src/federation/outbound-workflow.test.ts | 12 +++++- 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9015044d0..8fd84bb03 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,36 +48,14 @@ To be released. `idempotency` strategy, as direct processing already did, instead of always deduplicating them per origin. Activities recorded by workers of older versions are still recognized. [[#1290], [#1299]] - - Added `setOutboundObservers()` on federation builders and instances - for sends, original-payload forwarding, and automatic portable replication. - Register `onBroadcast` to record producer starts and preparation results, - and `onDelivery` for destination planning, direct handoffs, queue acceptance - or uncertainty, and local preparation failures. Use `onAttempt` for concrete - send results and exact HTTP exchange counts, including queued retries and - gateway fallback, and `onHttpExchange` for each actual signed or unsigned - fetch. Reports distinguish received responses from the selected final - response and retain HTTP evidence alongside later local failures. Use - `onWorker` for fanout/outbox execution results, concrete send counts, and - actual child enqueue, retry, deferral, or abandonment actions. Worker - reports retain accepted children through partial failures and distinguish - unknown queue acceptance from confirmed scheduling; local completion does - not establish backend acknowledgment or remote success. Opt in to - independent redacted header snapshots, bounded streaming response-body - capture with explicit byte/time limits, and privacy-filtered sender-payload - projections. Capture summaries distinguish EOF, byte-limit uncertainty, - deadlines, and consumer/body failures without changing HTTP outcomes; - payload projections never claim exact wire bytes or proof validity. Error - diagnostics now have a separate one-second budget while retaining their 1 - KiB preview limit. Reports include observation timestamps and optional - operation trace IDs. Callbacks receive application context and independent - grouping IDs without requiring OpenTelemetry; their errors do not replace - the original result. Known IDs and original inbox source survive queue - continuations. Send/forward options accept `dispatchId` for application - correlation without deduplication, limited to 1,024 UTF-8 bytes. Routed - inbox reports include `inboxRequestId` and informational `requestId`. - Missing legacy relationships remain unknown. Readiness and handoff - observations do not establish remote delivery success. - [[#1316] by Jiwon Kwon\] + - Added application-facing outbound observers for sends, forwarding, + replication, and queued retries, independently of OpenTelemetry. Optional + callbacks report preparation, delivery handoffs, send/HTTP results, and + worker actions with correlation IDs. Applications can opt in to redacted + headers, bounded streaming response capture, and privacy-filtered sender + payloads. Inbox reports now preserve original request context and snapshot + request cancellation without changing processing outcomes. Error diagnostic + reads have a one-second deadline. [[#1316] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin diff --git a/packages/fedify/src/federation/outbound-capture.ts b/packages/fedify/src/federation/outbound-capture.ts index a9daefd41..523ce5538 100644 --- a/packages/fedify/src/federation/outbound-capture.ts +++ b/packages/fedify/src/federation/outbound-capture.ts @@ -88,7 +88,7 @@ export async function captureResponseBody( ); } catch (error) { return Object.freeze({ - type: "failed", + type: "failed" as const, phase: "body" as const, error, bytesExposed: 0, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index c596ca653..62b32f3ed 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -5325,7 +5325,9 @@ test("outbound header snapshots redact secrets and budget complete UTF-8 entries "x-api-key", "x-auth-token", ]; - const headers = new Headers(secrets.map((name) => [name, "do-not-export"])); + const headers = new Headers( + secrets.map((name): [string, string] => [name, "do-not-export"]), + ); headers.append("X-Visible", "first"); headers.append("x-visible", "second"); headers.set("X-Custom", "also-secret"); diff --git a/packages/fedify/src/federation/outbound-workflow.test.ts b/packages/fedify/src/federation/outbound-workflow.test.ts index 8a2ce1d60..f4ee42445 100644 --- a/packages/fedify/src/federation/outbound-workflow.test.ts +++ b/packages/fedify/src/federation/outbound-workflow.test.ts @@ -14,11 +14,19 @@ import type { OutboundWorkerReport, } from "./outbound-report.ts"; +const { Deno: deno } = globalThis as typeof globalThis & { + Deno?: { + permissions: { + query(options: { name: "net" }): Promise<{ state: string }>; + }; + }; +}; + test({ name: "outbound recorder archives real HTTP exchanges across a queued retry", // The package Deno suite is offline; focused --allow-all and Node/Bun run it. - ignore: "Deno" in globalThis && - (await Deno.permissions.query({ name: "net" })).state !== "granted", + ignore: deno != null && + (await deno.permissions.query({ name: "net" })).state !== "granted", async fn() { let accepting = false; const received: { status: number; activity: unknown }[] = []; From 5beb7a22cd343c9bc07476d850358db4ea38ba48 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 13:55:04 +0900 Subject: [PATCH 39/53] Add PR number in changelogs --- CHANGES.md | 3 ++- changes.d/fedify/outbound-observers.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8fd84bb03..2d48c6b9d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -55,7 +55,7 @@ To be released. headers, bounded streaming response capture, and privacy-filtered sender payloads. Inbox reports now preserve original request context and snapshot request cancellation without changing processing outcomes. Error diagnostic - reads have a one-second deadline. [[#1316] by Jiwon Kwon\] + reads have a one-second deadline. [[#1316], [#1359] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin @@ -99,6 +99,7 @@ To be released. [#1318]: https://github.com/fedify-dev/fedify/issues/1318 [#1328]: https://github.com/fedify-dev/fedify/pull/1328 [#1353]: https://github.com/fedify-dev/fedify/pull/1353 +[#1359]: https://github.com/fedify-dev/fedify/pull/1359 ### @fedify/cli diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 2c852ae1d..8e14886cc 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -1,6 +1,7 @@ --- links: '#1316': https://github.com/fedify-dev/fedify/issues/1316 + '#1359': https://github.com/fedify-dev/fedify/pull/1359 --- - Added application-facing outbound observers for sends, forwarding, replication, and queued retries, independently of OpenTelemetry. Optional @@ -9,4 +10,4 @@ links: headers, bounded streaming response capture, and privacy-filtered sender payloads. Inbox reports now preserve original request context and snapshot request cancellation without changing processing outcomes. Error diagnostic - reads have a one-second deadline. [[#1316] by Jiwon Kwon] + reads have a one-second deadline. [[#1316], [#1359] by Jiwon Kwon] From b5ca4e04573a1567c7ee06db42432927670f0dac Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 14:51:28 +0900 Subject: [PATCH 40/53] Add async keyword to processQueuedTask() Fix https://github.com/fedify-dev/fedify/pull/1359#discussion_r4451187984 --- docs/manual/send.md | 2 -- packages/fedify/src/federation/middleware.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/manual/send.md b/docs/manual/send.md index dc737fb59..fc7a95b2d 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -548,8 +548,6 @@ HTTP abort -> inbox report.requestCancellation Accepted queue work -> worker (the HTTP abort does not cancel it) ~~~~ -Diagram generated with Codex. - IDs provide grouping, not ordering or deduplication. Replayed work can produce repeated observations. Treat IDs as opaque values. diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 7be62cf1c..5824df881 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1229,7 +1229,7 @@ export class FederationImpl await Promise.all(promises); } - processQueuedTask( + async processQueuedTask( contextData: TContextData, message: Message, ): Promise { @@ -1243,7 +1243,7 @@ export class FederationImpl if (message.type !== "task" && message.provenance !== undefined) { message = { ...message, provenance }; } - return withContext({ + return await withContext({ messageId: message.id, inboxRequestId: undefined, ...sourceLogContext(provenance), From 93695894ed4cd61ef18b0eeded8ab8971e1586a4 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 15:18:01 +0900 Subject: [PATCH 41/53] Simplify outgoing observation guidance Place the guide after queueing and group report concepts by usage while preserving all TypeScript examples. https://github.com/fedify-dev/fedify/pull/1359 Changelog: none Assisted-by: Codex:gpt-6.1-sol --- docs/manual/send.md | 1028 ++++++++++++++++++------------------------- 1 file changed, 419 insertions(+), 609 deletions(-) diff --git a/docs/manual/send.md b/docs/manual/send.md index fc7a95b2d..c4ac0b25d 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -59,615 +59,6 @@ async function sendFollow( > the *Context* section. -Observing outgoing activities ------------------------------ - -*This API is available since Fedify 2.5.0.* - -Use `Federatable.setOutboundObservers()` on a federation builder or a directly -created federation to observe sends, original-payload forwarding, and automatic -portable-inbox replication. Each callback is optional. This example records -producer preparation and delivery handoffs: - -~~~~ typescript twoslash -import type { - Federation, - OutboundBroadcastEvent, - OutboundDeliveryEvent, -} from "@fedify/fedify"; -declare const federation: Federation<{ requestId: string }>; -declare function recordBroadcast( - requestId: string, - event: OutboundBroadcastEvent, -): Promise; -declare function recordDelivery( - requestId: string, - event: OutboundDeliveryEvent, -): Promise; -// ---cut-before--- -federation.setOutboundObservers({ - async onBroadcast(ctx, event) { - await recordBroadcast(ctx.data.requestId, event); - if (event.stage === "preparation-finished" && event.result.type === "failed") { - console.error(event.result.failure.phase, event.result.failure.error); - } - }, - async onDelivery(ctx, event) { - await recordDelivery(ctx.data.requestId, event); - if (event.stage === "handoff" && event.result.type === "enqueue-failed") { - console.error("Queue acceptance is unknown", event.result.error); - } - }, -}); -~~~~ - -Here, `recordBroadcast()` and `recordDelivery()` are your application's -recording functions. Callbacks receive context and application data without -loading additional data for observation. Queued work uses the current worker's -context and data; the original HTTP cause is recorded separately in -`provenance`. The report fields contain immutable snapshots of known routing -and source facts; original thrown values remain available as `unknown` values. -The context, application data, Activity objects, and exceptions are not -deep-copied by this API. - -### Broadcast lifecycle - -Each invocation has a fresh `broadcastId`, shared by its `started` and -`preparation-finished` events. The `operation` is `send`, `forward`, or -`portable-replication`. Broadcast and delivery identities work without -OpenTelemetry, including when no tracer provider is configured or sampling is -disabled. - -`started` precedes failure-prone preparation. The `preparation-finished` event -contains an `OutboundPreparationResult`: - - - `prepared` means the current producer reached readiness for fanout or - delivery. `destinationCount` counts deduplicated destinations or confirmed - replication targets, not successful deliveries. Fanout workers still need - to prepare deliveries. - - `skipped` records an existing no-send decision: `no-destinations`, - `unsigned` when forwarding skips an unsigned payload, - `forwarding-disabled`, `forwarding-unavailable` when replication cannot - claim targets, or `no-forwarding-targets` when none remain claimed. - - `failed` reports the preparation phase, the original thrown value (which - need not be an `Error`), and the extent of destination planning. An - incomplete plan's count covers confirmed obligations so far. - -The `unsigned` reason reflects the existing `skipIfUnsigned` signature/proof -shape check; it does not establish authentication. - -Preparation readiness does not confirm queue acceptance or successful delivery. -Immediate sends can already be running when preparation finishes, and a later -send or enqueue failure does not replace the preparation result. A crash or -nonsettling operation can leave a start without a later result. - -### Destination observations - -`onDelivery` receives an `OutboundDeliveryEvent` for each established delivery -obligation. Its `broadcast` reference is `{ type: "known", id }` when the -invocation is known, or `{ type: "unknown" }` for legacy fanout work. Its -`deliveryId` groups events for one deduplicated destination or confirmed -replication target. Different inboxes on the same host remain distinct. - -| Stage | Meaning | -| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `planned` | The destination was established. `actorIds` lists actors already known to share it; `sharedInbox` describes the selected route. | -| `handoff`, result `direct` | Control entered the direct-send path. This does not confirm an HTTP request or remote success. | -| `handoff`, result `enqueued` | The enqueue resolved. `queue` identifies `fanout` or `outbox`, and `queueMessageId` identifies the accepted task. | -| `handoff`, result `enqueue-failed` | Enqueue rejected. `acceptance` remains `unknown`, because a backend effect may already have happened; `error` retains the original failure. | -| `preparation-failed` | Destination-local setup failed before concrete send entry. This creates no synthetic attempt. | - -`destinationInbox` retains the planned destination. For a portable inbox, -`portableInbox` contains its canonical portable ID. The planned event's `mode` -is `ordinary`, `portable-fallback`, or `portable-replication`. Alternate -fallback gateways share one obligation; automatic replication establishes a -separate obligation for each successfully claimed target. - -Several deliveries can share one accepted fanout task and therefore the same -`queueMessageId`; this task ID is separate from their delivery IDs. Acceptance -is reported before later bookkeeping, so a metrics or logging failure does not -turn an accepted enqueue into an enqueue failure. - -Recipients filtered out before planning create no delivery. Common producer -failures are broadcast failures; a destination-local failure is also reported -for that destination. Earlier handoffs remain valid, and later unvisited -destinations can remain planned without a handoff. Parallel work can interleave -callbacks, so IDs provide grouping rather than a total event order. - -### Attempts and HTTP exchanges - -`onAttempt` receives an `OutboundAttemptReport` once a concrete send settles, -including immediate sends, forwarding, portable replication, and queue workers. -Each entry into the sender gets a fresh `attemptId` under its `deliveryId`. -Alternate gateways and retries create new attempts; signature challenges and -redirects can create several fetches within one attempt. `inboxUrl` is the -inbox selected for that attempt, while `request.url` on an exchange identifies -the actual URL fetched at that step. - -`onHttpExchange` receives an `OutboundHttpExchangeReport` for each actual fetch -submission. `exchangeIndex` starts at zero within its `attemptId`. Its `request` -copies the submitted URL and method, including those of signed requests. A -`response` outcome records status, status text, and the response URL when fetch -supplies it; a `failed` outcome retains the original fetch rejection. A non-2xx -response is still a received response. Reports contain immutable metadata; -optional header snapshots and body-capture summaries are described below. -A separate bounded callback supplies selected response bytes, without exposing -live transport objects. - -~~~~ typescript twoslash -import type { Federation } from "@fedify/fedify"; -declare const federation: Federation; -// ---cut-before--- -federation.setOutboundObservers({ - onHttpExchange(_ctx, report) { - console.log(report.attemptId, report.exchangeIndex, report.request.url); - if (report.outcome.type === "response") { - console.log("HTTP response", report.outcome.response.status); - } - }, - onAttempt(_ctx, report) { - console.log(report.attemptId, report.httpExchangeCount); - const response = report.outcome.response; - if (response.type === "final") { - console.log("Final response", response.exchangeIndex, response.metadata.status); - } - if (report.outcome.type === "failed") { - console.error(report.outcome.failure.kind, report.outcome.failure.error); - } - }, -}); -~~~~ - -An attempt can fail before fetch, for example during URL validation or signing. -It still has an attempt ID, with `httpExchangeCount: 0` and no exchange report. -Failures before sender entry, such as queue key import or fanout parsing, and -worker holds, drops, or deferrals do not create attempts. Attempt reports do not -establish a worker's scheduling result. - -### Worker execution and actions - -`onWorker` receives one `OutboundWorkerReport` after a fanout or outbox worker's -local execution and instrumentation settle. `role` and `queue` identify the -current worker/task. `attemptsStarted` counts concrete sender entries, including -fallback gateways; child enqueues count zero. Outbox `delivery` identifies its -obligation. Fanout has no single delivery; child actions carry their references. - -| `result.type` | Meaning | -| -------------------- | -------------------------------------------------------------------------- | -| `completed` | Local normal return, including handled send failures and abandonment. | -| `failed` | The original error escaped, even if an earlier attempt succeeded. | -| `preparation-failed` | `failure.phase` identifies the preparation operation producing that error. | - -Read `actions` independently of `result` and attempt outcomes: - -| `type` | Evidence | -| --------------- | ---------------------------------------------------------------------------------------------- | -| `scheduled` | An `enqueue`, `retry`, or `defer` call resolved; `queue` identifies its actual task/counter. | -| `action-failed` | Enqueue rejection with unknown acceptance and original error, or a failed `drop` step. | -| `handoff` | A fanout worker entered a concrete sender (`action: "direct"`), without claiming HTTP success. | -| `abandoned` | Work stopped locally for the listed routes, with a reason such as `permanent-failure`. | - -`inboxUrls` lists affected routes; one gateway can be abandoned while another -continues. Retry/defer includes the actual ISO `delay` and any per-route -`notBefore` constraints. Child enqueues/direct handoffs retain delivery -references. An empty action list means no action was observed by settlement. - -~~~~ typescript twoslash -import type { Federation } from "@fedify/fedify"; -declare const federation: Federation; -// ---cut-before--- -federation.setOutboundObservers({ - onWorker(_ctx, report) { - console.log(report.role, report.result.type, report.attemptsStarted); - for (const action of report.actions) { - if (action.type === "scheduled") { - console.log(action.action, action.queue.queueMessageId, action.inboxUrls); - if (action.action !== "enqueue") console.log(action.delay); - } - } - }, -}); -~~~~ - -Accepted actions survive sibling or later bookkeeping failures. A rejected batch -leaves every submitted child's acceptance unknown. Later destination setup -failure can leave earlier direct attempts running after worker settlement. -Neither local completion nor a native rethrow establishes backend -acknowledgment, a future replay, or remote application effects. See -[message queues](./mq.md#observing-outgoing-workers). - -### Response versus final response - -A response means one fetch received HTTP headers. A final response is the -response that the HTTP flow selected as the attempt's result. For example: - -~~~~ text -401 challenge -> retry -> 202 accepted -exchange 0 exchange 1 (final) -~~~~ - -If signing the retry fails, the 401 stays intermediate; being the last response -does not make it final. An attempt's `outcome.response` makes that distinction -explicit: - -| Type | Evidence | -| ------------------- | -------------------------------------------------------------------------------------------------- | -| `final` | The flow selected this response. `exchangeIndex` and immutable `metadata` identify it. | -| `not-observed` | No response headers were received, including pre-fetch failures and rejected fetches. | -| `intermediate-only` | The flow chose to continue past the received response. `lastObserved` retains its index/metadata. | -| `undetermined` | Execution failed before finality was established. `lastObserved` retains the known response facts. | - -`httpExchangeCount` counts actual fetch submissions, including challenge, -redirect, and transport retries. It does not count signing or preparation work. -A `succeeded` attempt has a successful final HTTP response after the send -execution settles; it does not establish subsequent remote application work. - -A selected final response survives later cache or cleanup failures. With a -final non-OK response, `failure.kind` remains `http` and `diagnostics` retains -additional local failures. With a final 2xx response and a local failure, the -attempt is `failed` with the 2xx evidence retained. `failure.error` preserves -original operation evidence when available; an HTTP failure can lack an error -if execution failed before constructing its usual HTTP exception. The separate -`outcome.error` is the exact value that escaped the sender, including any -existing wrapping. Observer errors never become attempt diagnostics. - -### Capturing headers - -Header values are disabled by default. Enable `requestHeaders` and -`responseHeaders` independently to add immutable `headers` snapshots to actual -exchange requests and responses. Each policy requires an explicit `allow` list -of literal HTTP field names and a positive safe-integer `maxBytes`. An optional -`redact` list adds application-specific secret headers. Names are normalized to -lowercase and deduplicated; wildcards are not supported. - -~~~~ typescript twoslash -import type { Federation } from "@fedify/fedify"; -declare const federation: Federation; -// ---cut-before--- -federation.setOutboundObservers({ - requestHeaders: { allow: ["content-type", "signature"], maxBytes: 2048 }, - responseHeaders: { allow: ["content-type", "retry-after", "set-cookie"], maxBytes: 1024 }, - onHttpExchange(_ctx, report) { - console.log(report.request.headers?.entries); - if (report.outcome.type === "response") { - console.log(report.outcome.response.headers?.entries); - console.log(report.outcome.response.headers?.limitReached); - } - }, -}); -~~~~ - -Selected credentials, signatures, and cookies always become `[redacted]`, even -if allowlisted. The protected names are `authorization`, `proxy-authorization`, -`cookie`, `set-cookie`, `signature`, `signature-input`, `authentication-info`, -`proxy-authentication-info`, `x-api-key`, and `x-auth-token`. Application -redactions also take precedence over the allowlist. There is no raw-secret -option; add custom credential names to your redaction policy. - -`entries` contains readonly name/value pairs in lexical name order, using the -runtime's representation of repeated values. It does not promise raw wire -ordering or recover values already combined by the runtime. Each pair costs the -UTF-8 bytes of `name + ": " + value + "\r\n"`, after redaction. An oversized -entry is omitted whole while later entries remain eligible; `limitReached` -records any budget omission. An absent `headers` field means disabled, while an -enabled empty allowlist produces an empty snapshot. Signed/challenge/redirect -requests are captured at their actual submission, including changed headers. - -### Streaming response bodies - -Application body capture is disabled unless `responseBodyCapture` is supplied. -It is independent of header capture and can be registered without metadata -callbacks. It requires `onBody`, a positive safe-integer `maxBytes`, and a -finite positive `timeoutMs`; there are no numeric defaults. Fractional -milliseconds round up, and the rounded lifetime cannot exceed 2,147,483,647 ms. -Invalid configuration leaves the previous registration intact. - -`responses` defaults to `all` only when capture is enabled. The other selector, -`non-success-http-responses`, selects actual statuses outside 200–299, including -intermediate challenges and redirects. Selection does not predict the attempt -outcome. This example uses illustrative limits, not performance guarantees: - -~~~~ typescript twoslash -import type { Federation } from "@fedify/fedify"; -declare const federation: Federation; -declare function saveChunk( - attemptId: string, - exchangeIndex: number, - chunk: Uint8Array, - signal: AbortSignal, -): Promise; -// ---cut-before--- -federation.setOutboundObservers({ - responseBodyCapture: { - maxBytes: 64 * 1024, - timeoutMs: 5000, - responses: "non-success-http-responses", - async onBody(_ctx, event) { - const reader = event.body.getReader(); - try { - while (true) { - const chunk = await reader.read(); - if (chunk.done) break; - await saveChunk(event.attemptId, event.exchangeIndex, chunk.value, event.signal); - } - } catch (error) { - if (!event.signal.aborted) throw error; - } finally { - reader.releaseLock(); - } - }, - }, - onHttpExchange(_ctx, report) { - console.log(report.attemptId, report.exchangeIndex, report.bodyCapture); - }, -}); -~~~~ - -`onBody` receives the current context/data, grouping metadata, an immutable -request/response snapshot, a controlled `ReadableStream`, and a -capture-only signal. The callback runs after headers and before the terminal -exchange report. Finished reports contain `bodyCapture` summaries, never -streams: - -| Summary type | Meaning | -| ------------------- | -------------------------------------------------------------------------------------------- | -| Absent | Application capture was disabled, or fetch produced no response. | -| `filtered` | The response was not selected; no callback ran. | -| `no-body` | A selected response had no body; distinct from an empty stream reaching EOF. | -| `complete` | EOF was observed and the callback settled successfully within its lifetime. | -| `limit-reached` | The byte cap was reached without observed EOF; completeness and truncation remain uncertain. | -| `deadline` | The absolute capture lifetime expired, including no-pull or slow-consumer time. | -| `callback-returned` | The callback returned before EOF or another stopping condition. | -| `consumer-stopped` | The consumer cancelled; `reason` retains its original value. | -| `failed` | `phase` identifies a `body` or `observer` failure and `error` retains its original value. | - -`bytesExposed` counts bytes handed to application code, not durable storage. -At an exact cap, another read does not establish EOF: access is revoked, and -`limit-reached` remains uncertain even if the remote body might end there. -Signal abort or stream rejection can also stop an outstanding application write; -the summary does not certify that exposed chunks were saved. - -Fedify uses one source reader shared with diagnostics and pull backpressure, -without cloning, teeing, or accumulating application response bodies. Its bounds -exclude upstream chunk allocations, runtime buffering, and application storage. - -The absolute lifetime starts before callback invocation and covers no-pull/slow -consumption and cleanup. Return/throw/cancel/cap/deadline revokes even retained -locked readers. Late rejections are handled without waiting indefinitely; -arbitrary JavaScript or storage that ignores the signal can still continue. - -Capture failure or its signal does not abort the HTTP send, skip a challenge, -change a received status into a transport failure, or trigger another send. -The final error diagnostic can continue under its separate one-second budget -after application access ends. Error previews retain at most 1 KiB and append -`… (timed out)` when that diagnostic deadline expires, or `… (truncated)` when -additional body bytes establish truncation. Several selected exchanges can -accumulate waiting time; capture does not promise zero timing influence. - -### Inspecting the sender payload - -Enable `senderPayload: true` to include an independent privacy projection of the -concrete sender's existing JSON input in terminal attempt reports. It covers -immediate sends, worker/fanout execution, original forwarding, and portable -replication. It does not expose a producer's earlier Activity or perform extra -hydration, JSON-LD processing, context loading, or serialization. - -~~~~ typescript twoslash -import type { Federation } from "@fedify/fedify"; -declare const federation: Federation; -// ---cut-before--- -federation.setOutboundObservers({ - senderPayload: true, - onAttempt(_ctx, report) { - const document = report.senderPayload; - if (document.type === "available") { - console.log(document.stage, document.representation, document.redacted); - console.log(document.value); - } else { - console.log("Sender payload unavailable", document.reason); - } - }, -}); -~~~~ - -`stage` is `sender-payload`, and available values use -`representation: "privacy-projection"`. `redacted` means blind-recipient fields -were removed. The immutable copy removes compact, `as:`-prefixed, and full-IRI -`bto`/`bcc` fields throughout nested objects, arrays, and proof-bearing content. -It never changes the transport payload or signing input. A projection is not -exact wire evidence and does not establish signature/proof validity, including -when `redacted` is false. An unsigned redirect can drop its request body while -the attempt still retains this input projection. - -The supported subset includes ordinary plain JSON under audited ActivityStreams, -identity, security, data-integrity v1/v2, and GoToSocial context URLs; exact -Fedify-generated root extension definitions; and audited nested context URLs -under a supported compact root. Context-free expanded JSON-LD with unambiguous -IRI property names is also supported. Arbitrary inline aliases, context -overrides, unknown remote contexts, getters/custom JSON conversion, and -unsupported or cyclic/deep data produce an unavailable result. There is no -raw-data fallback. - -An unavailable payload has one of these reasons: - -| Reason | Meaning | -| ---------------------------- | ---------------------------------------------------------------- | -| `disabled` | Sender-payload export was not enabled. | -| `not-materialized` | The attempt failed before reaching the sender snapshot boundary. | -| `privacy-filter-unavailable` | A safe independent projection could not be established. | - -Header, body, and document evidence remain privileged application data. URLs and -content may contain other private values; blind-recipient filtering is not a -general personal-data sanitizer or permission to publish the evidence. Your -application owns access control, further redaction, archival, and saved original -dispatch documents. - -### Observation time and tracing - -Each event or report includes `observedAt`, Unix milliseconds captured at its -observation boundary before awaiting the callback. Replay gets a new observation -time while retaining known grouping and historical source facts. Wall clocks do -not establish a global order across workers. - -Optional `trace` contains valid `traceId` and `spanId` from the observed -operation's own span, including unsampled spans. Fedify omits it when no valid -own span exists. Producer entry precedes span creation, and exchanges have no -dedicated span; their reports omit `trace`. Ambient parent/server spans are not -substituted. Grouping IDs and observations work independently of tracing or -exporter storage. - -### Identities across queues - -Observed broadcasts and deliveries retain their IDs through fanout expansion, -retries, holds, and gateway changes. Fanout expansion can create new outbox task -IDs without starting another broadcast. Source metadata and `dispatchId` are -serialized even when producer observers are disabled; broadcast/delivery -metadata is allocated when producer observation is enabled. - -| Boundary | Identity behavior | -| ----------------------------------- | --------------------------------------------------------------------------------------------------- | -| Observed send or forward invocation | Fresh `broadcastId`; one `deliveryId` per deduplicated destination or confirmed replication target. | -| Fanout expansion | Existing broadcast/delivery IDs, with new child outbox task IDs. | -| Retry, hold, or deferral | Same delivery identity and provenance. | -| Portable fallback | One delivery across alternate gateways; the planned destination remains unchanged. | -| Portable replication | A separate delivery for each confirmed target claim. | - -~~~~ text -inbox request -> handler -> send/forward -> planned delivery - +-> direct handoff - +-> outbox task -> retry/hold - +-> fanout task -> outbox tasks - -Queue continuations preserve known IDs and the original source. - -Portable fallback: delivery D -> gateway A -> gateway B (stop on success) -Portable replication: broadcast B -> delivery D1 -> gateway A - -> delivery D2 -> gateway B - -HTTP abort -> inbox report.requestCancellation -Accepted queue work -> worker (the HTTP abort does not cancel it) -~~~~ - -IDs provide grouping, not ordering or deduplication. Replayed work can produce -repeated observations. Treat IDs as opaque values. - -Older messages remain processable. Fanout tasks without identity metadata get -stable delivery IDs with `broadcast: { type: "unknown" }`. Legacy outbox attempt -reports use the current task UUID as `deliveryId` and unknown broadcast -membership; `destinationInbox` is absent when the original plan is unavailable. -An explicit canonical `portableInbox` can still be known. Missing source and -application correlation remain unavailable. Older consumers can discard these -fields, so mixed-version processing can lose correlation. - -### Retry history - -Persist attempt reports by `deliveryId` and each fresh `attemptId`; correlate -scheduled retries with worker action task IDs. For example: - -~~~~ text -delivery D -> attempt A1: failed, final HTTP 503 - -> worker: retry scheduled as task Q, queueAttempt 1 - -> attempt A2: succeeded, final HTTP 202 -~~~~ - -A native redelivery can reuse a task/counter but still creates a new attempt. -The application owns this history and any later manual retry; Fedify emits no -distributed delivery-completed result. - -### Original source and current context - -Outgoing events distinguish preserved source facts from current URL facts: - - - `provenance.inboxRequest`, when known, contains the original - `inboxRequestId`, informational `requestId`, and a `context` snapshot. - - `local` describes the context invoking this callback now. - - `provenance.originatingRequestId` retains the logging label for - HTTP-derived work without an inbox cause. - -Both URL snapshots contain `origin`, `host`, `hostname`, and `canonicalOrigin`. -`host` includes an optional port; `hostname` does not. `canonicalOrigin` uses -the configured web origin or falls back to `origin`. - -The original source survives derived sends, forwarding, context clones and -conversions, and queues. Its snapshot stays unchanged if a derived context's -URL changes. Independent contexts do not acquire an inbox cause from ambient -execution. See [observing inbox requests](./inbox.md#observing-inbox-requests) -for the difference between the server-generated `inboxRequestId` and the -informational logging `requestId`. - -Worker logs use `sourceInboxRequestId` and `sourceRequestId` for historical -correlation, preserving an unrelated current `requestId`. - -### Application dispatch correlation - -Send and forward options accept an application-owned `dispatchId`. Fedify -preserves it exactly in `event.provenance.dispatchId` and outgoing queue -continuations. Empty strings are valid; the limit is 1,024 UTF-8 bytes. A -non-string value raises `TypeError`, and an oversized string raises `RangeError` -before preparation. Queue consumers enforce the same limit. - -This example adds application correlation to an inbox-triggered acceptance: - -~~~~ typescript twoslash -import type { Federation } from "@fedify/fedify"; -declare const federation: Federation; -declare function saveAcceptance(follow: import("@fedify/vocab").Follow): Promise; -// ---cut-before--- -import { Accept, Follow } from "@fedify/vocab"; - -federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") - .on(Follow, async (ctx, follow) => { - if (follow.id == null || follow.objectId == null) return; - const actor = ctx.parseUri(follow.objectId); - if (actor?.type !== "actor") return; - const follower = await follow.getActor(ctx); - if (follower == null) return; - const dispatchId = await saveAcceptance(follow); - await ctx.sendActivity( - { identifier: actor.identifier }, - follower, - new Accept({ actor: follow.objectId, object: follow.id }), - { dispatchId }, - ); - }); -~~~~ - -`saveAcceptance()` represents your application's record creation; its ID must -satisfy the byte limit. The new Activity retains the incoming request's source. -Reusing a `dispatchId` sends again with a fresh broadcast ID. A new public call -that omits it does not inherit a previous app ID; queued continuations of the -existing call retain it. - -### Registration and callback behavior - -Each registration call replaces all previous outbound observers; pass `{}` to -clear them. Fedify copies the supplied callbacks, and building a federation -copies the builder's registration. An active invocation or worker execution -keeps its selected callbacks even if registration changes while it is running. -You can register any callback independently, including `onWorker` alone. On -attempt/exchange reports, `queue` contains the current `queueMessageId` and an -available Fedify `queueAttempt` counter. Native redelivery can repeat that -counter while creating fresh attempts. - -Metadata callbacks are awaited without a default timeout; body consumers use -an explicit bounded capture lifetime. Callback return values and errors -cannot veto sending, and callback or error-logging failures do not replace the -send's original result. Slow callbacks delay processing; a callback that never -settles can delay it indefinitely. Existing portable-forwarding deadlines can -return control while background preparation, sending, and observation continue; -a deadline alone does not create a failed preparation result or attempt outcome. - -Inbox request cancellation is reported separately from processing. The queue -listener's stop signal controls intake; this observer API does not yet report -per-execution queue cancellation. A backend ceasing to wait is not evidence that -its worker terminated. - -If you detach recording work, handle its rejections and shutdown yourself; -Fedify does not guarantee its persistence. Incoming C2S `setOutboxListeners()` -registration is separate. Mock federations accept outbound observer -registration but do not execute these observers. - - Specifying a sender ------------------- @@ -1028,6 +419,425 @@ an error and does not retry the delivery. [`@fedify/mysql`]: https://github.com/fedify-dev/fedify/tree/main/packages/mysql +Observing outgoing activities +----------------------------- + +*This API is available since Fedify 2.5.0.* + +Register callbacks with `Federatable.setOutboundObservers()` to record outgoing +sends, original-payload forwarding, and automatic portable-inbox replication. +The method is available on federation builders and directly created federations. +Each callback receives the current `Context` and an event or report, and is +awaited at its observation boundary: + +~~~~ typescript twoslash +import type { + Federation, + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "@fedify/fedify"; +declare const federation: Federation<{ requestId: string }>; +declare function recordBroadcast( + requestId: string, + event: OutboundBroadcastEvent, +): Promise; +declare function recordDelivery( + requestId: string, + event: OutboundDeliveryEvent, +): Promise; +// ---cut-before--- +federation.setOutboundObservers({ + async onBroadcast(ctx, event) { + await recordBroadcast(ctx.data.requestId, event); + if (event.stage === "preparation-finished" && event.result.type === "failed") { + console.error(event.result.failure.phase, event.result.failure.error); + } + }, + async onDelivery(ctx, event) { + await recordDelivery(ctx.data.requestId, event); + if (event.stage === "handoff" && event.result.type === "enqueue-failed") { + console.error("Queue acceptance is unknown", event.result.error); + } + }, +}); +~~~~ + +`recordBroadcast()` and `recordDelivery()` represent your application's +recording functions. Queued work uses the current worker's context and data. +The original HTTP source, when known, is recorded separately in `provenance`. +Observation does not load additional data or serialize an activity again. +Routing and source metadata are immutable snapshots; the context, application +data, Activity objects, and original exceptions are not deep-copied. + +`onBroadcast` reports one send or forward invocation. Its `started` event runs +before preparation; `preparation-finished` describes whether the producer +prepared work for delivery or fanout, skipped sending, or failed. Fanout-ready +work still needs worker-side delivery preparation. A prepared result does not +confirm queue acceptance or HTTP delivery, and later send failures do not +rewrite it. Skips such as `no-destinations` or `forwarding-disabled` describe +existing decisions; an `unsigned` skip reflects the forwarding signature/proof +shape check, not authentication. + +`onDelivery` reports each deduplicated destination or claimed replication +target. Different inboxes on one host remain separate deliveries. A `planned` +event identifies the destination; a `handoff` records direct sender entry or +the result of enqueueing work. An `enqueued` result confirms producer +acceptance. An `enqueue-failed` result leaves backend acceptance unknown, +because the backend may have accepted work before rejecting the call. +Destination-local preparation failures are reported without inventing an +attempt. Earlier handoffs remain valid if preparation for a later destination +fails. + +`onAttempt` reports the result when a concrete send settles. Each sender entry +gets a fresh `attemptId`; retries and alternative gateways create new attempts. +`onHttpExchange` reports each actual fetch within an attempt, including signed +requests, redirects, and signature challenges. The pair of `attemptId` and +`exchangeIndex` identifies an exchange. A fetch that receives non-2xx headers +still has a `response` outcome; a rejected fetch has a `failed` outcome. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + onHttpExchange(_ctx, report) { + console.log(report.attemptId, report.exchangeIndex, report.request.url); + if (report.outcome.type === "response") { + console.log("HTTP response", report.outcome.response.status); + } + }, + onAttempt(_ctx, report) { + console.log(report.attemptId, report.httpExchangeCount); + const response = report.outcome.response; + if (response.type === "final") { + console.log("Final response", response.exchangeIndex, response.metadata.status); + } + if (report.outcome.type === "failed") { + console.error(report.outcome.failure.kind, report.outcome.failure.error); + } + }, +}); +~~~~ + +An attempt can fail during URL validation or signing before any fetch, leaving +`httpExchangeCount` at zero. Failures before sender entry, such as queue key +import or fanout parsing, and worker holds or drops create no attempt. + +An exchange response and an attempt's final response are different. A 401 +challenge followed by a successful retry produces two exchanges; the later +response is final. If signing the retry fails, the 401 remains intermediate. +`outcome.response` distinguishes `final`, `intermediate-only`, `not-observed`, +and `undetermined` evidence. A final response is retained even when later cache +or cleanup work fails. For a final non-OK response, HTTP failure remains primary +and additional local errors appear in `diagnostics`; a local failure after a +final 2xx still makes the attempt fail. `outcome.error` retains the value that +escaped the sender, while `failure.error` retains operation evidence when +available. Neither queue acceptance nor remote HTTP acceptance proves that the +remote application applied the activity. + +### Worker execution and actions + +`onWorker` receives one `OutboundWorkerReport` when a fanout or outbox worker's +local execution and instrumentation settle. `attemptsStarted` counts concrete +sender entries, including fallback gateways; enqueueing a child task does not +increase it. Outbox reports identify one delivery, while fanout reports carry +delivery references on their child actions. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + onWorker(_ctx, report) { + console.log(report.role, report.result.type, report.attemptsStarted); + for (const action of report.actions) { + if (action.type === "scheduled") { + console.log(action.action, action.queue.queueMessageId, action.inboxUrls); + if (action.action !== "enqueue") console.log(action.delay); + } + } + }, +}); +~~~~ + +Read `actions` separately from `result`. A `completed` result means the worker +returned normally, including handled failures or abandonment. Its actions record +accepted enqueues, retries and deferrals, direct handoffs, abandoned routes, and +failed scheduling or drop operations. Retry and deferral actions include their +actual delay and any per-route constraints. One gateway can be abandoned while +another continues. An empty action list means none was observed by settlement. + +Accepted actions survive later failures; a rejected batch leaves every submitted +child's acceptance unknown. Earlier direct attempts can continue after a later +preparation failure settles the worker. A local result does not establish +backend acknowledgment or future native redelivery. See +[observing outgoing workers](./mq.md#observing-outgoing-workers). + +### Identities across queues + +`broadcastId` groups an invocation and its continuations. `deliveryId` +identifies one destination obligation and survives fanout replay, retries, +holds, and route changes. Alternative portable gateways share one delivery; +automatic replication creates independent deliveries. Each concrete attempt +gets a new `attemptId`. Queue task IDs and `queueAttempt` describe scheduling, +not the number of send executions. Native redelivery can reuse both while +creating a fresh attempt. + +The identity relationships are: + +| Unit | Identity and lifetime | +| ------------- | ------------------------------------------------------------------------------------------------------------------ | +| Broadcast | Fresh `broadcastId` for an observed invocation; retained through queue continuations. | +| Delivery | One `deliveryId` per deduplicated destination or claimed replication target; retained across retries and fallback. | +| Attempt | Fresh `attemptId` at each concrete sender entry. | +| HTTP exchange | `attemptId` plus `exchangeIndex`, starting at zero within the attempt. | +| Queue task | Current `queueMessageId` and, when available, `queueAttempt`; fanout replay can create new task IDs. | + +IDs provide grouping, not ordering, deduplication, or remote idempotency. +Callbacks can interleave or repeat, and a crash or operation that never settles +can leave an invocation without a terminal report. Applications own persistence +and aggregate delivery state. + +A failed attempt receiving HTTP 503 and a later successful retry receiving HTTP +202 share the same `deliveryId`, but have different `attemptId` values. The +worker report records the scheduled retry and its queue task ID. + +### Original source and current context + +`local` describes the context invoking the callback. For inbox-derived work, +`provenance.inboxRequest` preserves the original `inboxRequestId`, informational +logging `requestId`, and URL snapshot through context clones, conversions, new +sends, forwarding, and queues. Both URL snapshots contain `origin`, `host`, +`hostname`, and `canonicalOrigin`. Independent contexts do not acquire an inbox +cause from ambient execution. Worker logs use `sourceInboxRequestId` and +`sourceRequestId` without replacing an unrelated current request's label. +See [observing inbox requests](./inbox.md#observing-inbox-requests). + +Send and forward options accept an application-owned `dispatchId` for grouping +related calls, including manual retries. It is preserved exactly through +outgoing queue continuations and never suppresses a send. For example, an inbox +listener can associate an acceptance with its application record: + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +declare function saveAcceptance(follow: import("@fedify/vocab").Follow): Promise; +// ---cut-before--- +import { Accept, Follow } from "@fedify/vocab"; + +federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .on(Follow, async (ctx, follow) => { + if (follow.id == null || follow.objectId == null) return; + const actor = ctx.parseUri(follow.objectId); + if (actor?.type !== "actor") return; + const follower = await follow.getActor(ctx); + if (follower == null) return; + const dispatchId = await saveAcceptance(follow); + await ctx.sendActivity( + { identifier: actor.identifier }, + follower, + new Accept({ actor: follow.objectId, object: follow.id }), + { dispatchId }, + ); + }); +~~~~ + +`saveAcceptance()` represents your application's record creation. Dispatch IDs +may be empty but must fit within 1,024 UTF-8 bytes. Invalid types raise +`TypeError` and oversized strings raise `RangeError` before preparation; queue +consumers apply the same validation. A new call that omits the ID does not +inherit it from a previous invocation. + +Older messages remain processable, with missing relationships explicitly +unknown. Legacy fanout can recover stable delivery grouping; legacy outbox +reports use the current task UUID as the delivery ID and cannot recover +broadcast membership. Unavailable original destinations and source facts remain +absent. Older consumers can discard new metadata, so mixed-version processing +can lose correlation. Source metadata and dispatch IDs propagate even without +producer observers; broadcast and delivery metadata is allocated when producer +observation is enabled. + +### Capturing headers + +Header values are disabled by default. Configure `requestHeaders` and +`responseHeaders` independently with literal names in `allow` and a positive +safe-integer `maxBytes`. Optional `redact` names add application-specific +secrets. Names are normalized to lowercase; wildcards are not supported. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + requestHeaders: { allow: ["content-type", "signature"], maxBytes: 2048 }, + responseHeaders: { allow: ["content-type", "retry-after", "set-cookie"], maxBytes: 1024 }, + onHttpExchange(_ctx, report) { + console.log(report.request.headers?.entries); + if (report.outcome.type === "response") { + console.log(report.outcome.response.headers?.entries); + console.log(report.outcome.response.headers?.limitReached); + } + }, +}); +~~~~ + +Credentials, signatures, and cookies in the built-in protected set are always +redacted, even when allowlisted. Add custom credential headers to `redact`. +Snapshots contain immutable name/value pairs in lexical name order, using the +runtime's representation of repeated headers. The budget counts UTF-8 bytes of +`name + ": " + value + "\r\n"` after redaction. Oversized entries are omitted +whole, and `limitReached` records omissions. An absent snapshot means capture +was disabled; an enabled empty allowlist produces an empty snapshot. Each +signed request, challenge retry, and redirect captures its actual submitted +headers. + +### Streaming response bodies + +Body capture is disabled unless `responseBodyCapture` is supplied. It is +independent of header capture and can be used without metadata callbacks. +Provide `onBody`, a positive safe-integer `maxBytes`, and a finite positive +`timeoutMs`. Fractional milliseconds round up; the rounded lifetime cannot +exceed 2,147,483,647 ms. Invalid configuration preserves the previous +registration. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +declare function saveChunk( + attemptId: string, + exchangeIndex: number, + chunk: Uint8Array, + signal: AbortSignal, +): Promise; +// ---cut-before--- +federation.setOutboundObservers({ + responseBodyCapture: { + maxBytes: 64 * 1024, + timeoutMs: 5000, + responses: "non-success-http-responses", + async onBody(_ctx, event) { + const reader = event.body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + await saveChunk(event.attemptId, event.exchangeIndex, chunk.value, event.signal); + } + } catch (error) { + if (!event.signal.aborted) throw error; + } finally { + reader.releaseLock(); + } + }, + }, + onHttpExchange(_ctx, report) { + console.log(report.attemptId, report.exchangeIndex, report.bodyCapture); + }, +}); +~~~~ + +`responses` defaults to `all` when capture is enabled. The example selects +`non-success-http-responses`, including intermediate challenges and redirects; +selection does not predict the attempt's final result. Its limits are +illustrative. + +`onBody` receives the current context, grouping metadata, immutable HTTP +metadata, a controlled stream, and a capture-only signal. It runs after +response headers and before the exchange report. Fedify uses one source reader +for capture and diagnostics, with backpressure and no clone, tee, or whole-body +accumulation. Runtime buffering, upstream chunk allocation, and application +storage remain outside these bounds. + +The report's `bodyCapture` summary distinguishes disabled or filtered capture, +a missing body, EOF, a byte limit, a deadline, early callback return, consumer +cancellation, and body or observer failure. `bytesExposed` counts bytes handed +to application code, not bytes durably saved. Reaching the exact cap does not +establish EOF, so `limit-reached` leaves completeness uncertain. + +The lifetime includes waiting for a consumer that is slow or never reads, as +well as cleanup. Callback settlement, consumer cancellation, the cap, or the +deadline revokes access even through retained locked readers. Late promise +rejections are handled, but application storage that ignores the signal can +continue. Applications must handle interrupted writes and archive cleanup +themselves. + +Capture failures retain observed HTTP facts and do not independently trigger a +retry or skip a challenge. Error diagnostics have a separate one-second budget +and retain at most 1 KiB, appending `… (timed out)` or `… (truncated)` as +appropriate. Waiting for capture on multiple exchanges can add to total send +time. + +### Inspecting the sender payload + +Enable `senderPayload: true` to include a privacy-filtered copy of the concrete +sender's existing JSON input in attempt reports. It covers immediate and queued +sends, forwarding, and portable replication without additional serialization or +context loading: + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + senderPayload: true, + onAttempt(_ctx, report) { + const document = report.senderPayload; + if (document.type === "available") { + console.log(document.stage, document.representation, document.redacted); + console.log(document.value); + } else { + console.log("Sender payload unavailable", document.reason); + } + }, +}); +~~~~ + +An available payload is an immutable `privacy-projection`. It removes compact, +`as:`-prefixed, and full-IRI `bto`/`bcc` fields throughout nested content, +including proof-bearing objects. `redacted` indicates that fields were removed. +It does not modify transport or signing input, establish proof validity, or +represent exact wire bytes. Redirects can change the actual request body +independently. + +Unsupported contexts, aliases, getters, custom JSON conversion, cyclic data, or +excessive nesting fail closed with `privacy-filter-unavailable`; there is no raw +fallback. `disabled` means export was not enabled, and `not-materialized` means +the sender did not reach the snapshot boundary. Supported context forms and +field definitions are described by `OutboundSenderPayload`. + +Headers, bodies, and payloads remain privileged application data. URLs and +content can contain private values beyond blind recipients. Your application +owns access control, additional redaction, archival, and saved original +dispatch documents. + +### Registration and callback behavior + +Registering again replaces the complete observer set; pass `{}` to clear it. +Builders copy registrations into built federations, and active invocations and +workers retain their selected callbacks. Any callback can be registered alone. +Mock federations accept registration but do not execute these observers. + +Observer errors, including logging errors, are isolated from the original result +and retry ownership. Metadata callbacks are awaited without a default timeout; +a callback that never settles can delay processing indefinitely. Detached +recording must handle its own rejections and shutdown. Neither awaited recording +nor detached work makes persistence atomic with network effects or queue +acceptance. Portable-forwarding deadlines can return control while sends and +observation continue; the deadline alone does not create a failed attempt. + +Each event or report has an `observedAt` Unix timestamp in milliseconds. It does +not establish ordering across workers. Observation works without OpenTelemetry +or sampling. Optional `trace` references the operation's own valid span, +including unsampled spans; broadcast starts and HTTP exchanges omit it because +they have no span of their own at those boundaries. + +Inbox request cancellation is observed separately from processing. Queue intake +shutdown and per-execution cancellation are also distinct. Execution +cancellation is not currently reported by this API. Incoming C2S +`setOutboxListeners()` registration serves a separate purpose from outbound +observation. + + Optimizing activity delivery for large audiences ------------------------------------------------ From c495ca5f2b8a85bf74d60fcafb2ff2208b5a8a2f Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 16:18:51 +0900 Subject: [PATCH 42/53] Classify worker preparation failures Preserve the operation phase and original error when shared fanout preparation or pre-send URL and trace setup fails. This reports the failure without changing retries or inventing delivery attempts. https://github.com/fedify-dev/fedify/pull/1359 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 192 +++++++----- .../src/federation/outbound-report.test.ts | 275 ++++++++++++++++++ .../fedify/src/federation/outbound-report.ts | 5 + 3 files changed, 398 insertions(+), 74 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 5824df881..f690f1cc0 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1771,7 +1771,10 @@ export class FederationImpl return; } try { - const inbox = new URL(message.inbox); + const parseInbox = () => new URL(message.inbox); + const inbox = worker == null + ? parseInbox() + : await worker.prepare("validation", parseInbox); const circuit = this.outboxQueue == null ? undefined : this.circuitBreaker; @@ -2251,12 +2254,18 @@ export class FederationImpl Temporal.Now.instant(), ) <= 0 ) { - const inbox = new URL(message.gatewayInboxes[0]); + const parseInbox = () => new URL(message.gatewayInboxes[0]); + const inbox = worker == null + ? parseInbox() + : await worker.prepare("validation", parseInbox); await drop(circuit, getRemoteHost(inbox), inbox, heldSince); return; } for (const inboxHref of message.gatewayInboxes) { - const inbox = new URL(inboxHref); + const parseInbox = () => new URL(inboxHref); + const inbox = worker == null + ? parseInbox() + : await worker.prepare("validation", parseInbox); const remoteHost = getRemoteHost(inbox); const now = Temporal.Now.instant(); const notBefore = parseInstant(message.gatewayNotBefore?.[inboxHref]); @@ -3459,72 +3468,95 @@ export class FederationImpl provenance = getOutboundProvenance(ctx), } = options; observation?.setPhase("validation"); - if (activity.id == null) { - throw new TypeError("The activity to send must have an id."); - } - if (activity.actorId == null) { - throw new TypeError( - "The activity to send must have at least one actor property.", - ); - } else if (keys.length < 1) { - throw new TypeError("The keys must not be empty."); - } + const validateActivity = () => { + if (activity.id == null) { + throw new TypeError("The activity to send must have an id."); + } + if (activity.actorId == null) { + throw new TypeError( + "The activity to send must have at least one actor property.", + ); + } else if (keys.length < 1) { + throw new TypeError("The keys must not be empty."); + } + return activity.id.href; + }; + const activityId = worker == null + ? validateActivity() + : await worker.prepare("validation", validateActivity); observation?.setPhase("serialization"); - const contextLoader = this.contextLoaderFactory( - this.#getLoaderOptions(ctx.origin), - ); - const activityId = activity.id.href; - let rsaKey: { keyId: URL; privateKey: CryptoKey } | null = null; + const createContextLoader = () => + this.contextLoaderFactory(this.#getLoaderOptions(ctx.origin)); + const contextLoader = worker == null + ? createContextLoader() + : await worker.prepare("serialization", createContextLoader); // A Linked Data Signature attests authorship, which neither a gateway key // nor any key other than the DID's can do for a portable actor's // activity; its Object Integrity Proof does that instead: - const ldSignable = !hasPortableActor(activity); - observation?.setPhase("key-resolution"); - for (const { keyId, privateKey } of keys) { - validateCryptoKey(privateKey, "private"); - if ( - ldSignable && rsaKey == null && - privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && - !isPortableKeyId(keyId) - ) { - rsaKey = { keyId, privateKey }; + const resolveRsaKey = () => { + const ldSignable = !hasPortableActor(activity); + let rsaKey: { keyId: URL; privateKey: CryptoKey } | null = null; + for (const { keyId, privateKey } of keys) { + validateCryptoKey(privateKey, "private"); + if ( + ldSignable && rsaKey == null && + privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && + !isPortableKeyId(keyId) + ) { + rsaKey = { keyId, privateKey }; + } } - } + return { rsaKey, ldSignable }; + }; + observation?.setPhase("key-resolution"); + const { rsaKey, ldSignable } = worker == null + ? resolveRsaKey() + : await worker.prepare("key-resolution", resolveRsaKey); // If Object Integrity Proofs were already created before fanout (e.g., in // sendActivityInternal()), skip signing to avoid duplicates. observation?.setPhase("signing"); - const signed = await signOutgoingActivity( - activity, - keys.map(({ keyId, privateKey }) => ({ - verificationMethod: keyId, - privateKey, - })), - { - contextLoader, - tracerProvider: this.tracerProvider, - appendToExistingProofs: false, - }, - ); + const signActivity = () => + signOutgoingActivity( + activity, + keys.map(({ keyId, privateKey }) => ({ + verificationMethod: keyId, + privateKey, + })), + { + contextLoader, + tracerProvider: this.tracerProvider, + appendToExistingProofs: false, + }, + ); + const signed = worker == null + ? await signActivity() + : await worker.prepare("signing", signActivity); activity = signed.activity; const { hasProof, proofCreated } = signed; observation?.setPhase("serialization"); - let jsonLd = !proofCreated && options.activityJsonLd != null - ? options.activityJsonLd - : await activity.toJsonLd({ - format: "compact", - contextLoader, - }); - // Existing proofs are preserved by default because they may have been - // created over the compact JSON-LD bytes exactly as supplied. Fedify can - // safely normalize unsigned activities, proofs it just created, or - // locally pre-signed activities when callers opt in. - if (proofCreated || !hasProof || options.normalizeExistingProofs) { - jsonLd = await normalizeOutgoingActivityJsonLd(jsonLd, contextLoader, { - preserveNestedSecuredDocuments: true, - }); - } + const serializeActivity = async () => { + let result = !proofCreated && options.activityJsonLd != null + ? options.activityJsonLd + : await activity.toJsonLd({ format: "compact", contextLoader }); + // Existing proofs are preserved by default because they may have been + // created over the compact JSON-LD bytes exactly as supplied. Fedify can + // safely normalize unsigned activities, proofs it just created, or + // locally pre-signed activities when callers opt in. + if (proofCreated || !hasProof || options.normalizeExistingProofs) { + result = await normalizeOutgoingActivityJsonLd(result, contextLoader, { + preserveNestedSecuredDocuments: true, + }); + } + return result; + }; + let jsonLd = worker == null + ? await serializeActivity() + : await worker.prepare("serialization", serializeActivity); observation?.setPhase("validation"); - assertSupportedCompoundProofShape(jsonLd, activityId); + const validateProofShape = () => + assertSupportedCompoundProofShape(jsonLd, activityId); + if (worker == null) validateProofShape(); + else await worker.prepare("validation", validateProofShape); if (rsaKey == null && !ldSignable) { logger.debug( "The activity {activityId} is performed by a portable actor, so it " + @@ -3546,21 +3578,27 @@ export class FederationImpl }, ); } else { - try { - observation?.setPhase("signing"); - jsonLd = await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, { - contextLoader, - tracerProvider: this.tracerProvider, - }); - } catch (error) { - if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error; - logger.warn( - "Failed to create a Linked Data signature for the activity " + - "{activityId}. The activity will be sent without a Linked " + - "Data signature.", - { activityId, error }, - ); - } + const signLinkedData = async () => { + try { + observation?.setPhase("signing"); + return await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, { + contextLoader, + tracerProvider: this.tracerProvider, + }); + } catch (error) { + if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error; + logger.warn( + "Failed to create a Linked Data signature for the activity " + + "{activityId}. The activity will be sent without a Linked " + + "Data signature.", + { activityId, error }, + ); + return jsonLd; + } + }; + jsonLd = worker == null + ? await signLinkedData() + : await worker.prepare("signing", signLinkedData); } if (!hasProof) { logger.warn( @@ -3661,13 +3699,19 @@ export class FederationImpl observation?.setPhase("key-resolution"); const keyJwkPairs: SenderKeyJwkPair[] = []; for (const { keyId, privateKey } of keys) { - const privateKeyJwk = await exportJwk(privateKey); + const exportPrivateKey = () => exportJwk(privateKey); + const privateKeyJwk = worker == null + ? await exportPrivateKey() + : await worker.prepare("key-export", exportPrivateKey); keyJwkPairs.push({ keyId: keyId.href, privateKey: privateKeyJwk }); } if (!this.manuallyStartQueue) this._startQueueInternal(ctx.data); observation?.setPhase("request-preparation"); const carrier: Record = {}; - propagation.inject(context.active(), carrier); + const injectTraceContext = () => + propagation.inject(context.active(), carrier); + if (worker == null) injectTraceContext(); + else await worker.prepare("request-preparation", injectTraceContext); const messages: { message: OutboxMessage; orderingKey?: string }[] = []; for (const inbox in inboxes) { try { diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 62b32f3ed..2134c84db 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -9,6 +9,7 @@ import { FetchError, UrlError } from "@fedify/vocab-runtime"; import { configure, reset } from "@logtape/logtape"; import { isSpanContextValid, + propagation, SpanKind, SpanStatusCode, trace, @@ -52,6 +53,7 @@ import { import { OutboundBroadcastObservation } from "./outbound-observation.ts"; import { snapshotHeaderPolicy, snapshotHeaders } from "./outbound-headers.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; +import { exportJwk } from "../sig/key.ts"; const activity = new Create({ id: new URL("https://example.com/create"), @@ -4105,6 +4107,279 @@ test("outbound worker observers report explicit key and fanout parse failures wi } }); +test("outbound worker classifies inbox URL parsing failures as validation", async (t) => { + for (const mode of ["ordinary", "gateway", "held-gateway"] as const) { + await t.step(mode, async () => { + const reports: OutboundWorkerReport[] = []; + const queued: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: mode === "held-gateway" + ? { heldActivityTtl: { seconds: 1 } } + : false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + nativeRetrial: true, + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const original = continuationMessage(recipient.inboxId!.href, true); + const malformed = "not a URL"; + const message: OutboxMessage = mode === "ordinary" + ? { ...original, inbox: malformed } + : { + ...original, + gatewayInboxes: [malformed, "https://gateway.example/inbox"], + ...(mode === "held-gateway" + ? { circuitHeld: true, circuitHeldSince: "2000-01-01T00:00:00Z" } + : {}), + }; + let outward: unknown; + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => { + outward = error; + return error instanceof TypeError; + }, + ); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 0); + deepStrictEqual(reports[0].result, { + type: "preparation-failed", + failure: { kind: "local", phase: "validation", error: outward }, + error: outward, + }); + strictEqual(queued.length, 0); + }); + } +}); + +test("fanout worker classifies sendActivity validation, signing, serialization, and normalization failures as preparation", async (t) => { + for ( + const phase of [ + "validation", + "signing", + "serialization", + "normalization", + "key-export", + ] as const + ) { + await t.step(phase, async () => { + const failure = Symbol(phase); + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + ...options, + ...(phase === "key-export" + ? { + queue: { + outbox: { + enqueue: () => Promise.resolve(), + listen: () => Promise.resolve(), + }, + }, + } + : {}), + }) as FederationImpl; + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const originalSendActivity = federation.sendActivity; + Object.defineProperty(federation, "sendActivity", { + value: async (...args: Parameters) => { + const [keys, inboxes, outgoing, sendOptions] = args; + if (phase === "validation") { + Object.defineProperty(outgoing, "actorId", { + get() { + throw failure; + }, + }); + } else if (phase === "signing") { + Object.defineProperty(outgoing, "getProofs", { + value() { + throw failure; + }, + }); + } else if (phase === "serialization") { + Object.defineProperty(outgoing, "toJsonLd", { + value() { + throw failure; + }, + }); + } else if (phase === "normalization") { + Object.defineProperty(outgoing, "toJsonLd", { + value: () => + new Proxy({}, { + ownKeys() { + throw failure; + }, + }), + }); + } + const options = phase === "serialization" || phase === "normalization" + ? { ...sendOptions, activityJsonLd: undefined } + : sendOptions; + return await originalSendActivity.call( + federation, + keys, + inboxes, + outgoing, + options, + ); + }, + }); + const activityJsonLd = { + "@context": "https://www.w3.org/ns/activitystreams", + id: activity.id!.href, + type: "Create", + actor: activity.actorId!.href, + }; + const message: FanoutMessage = { + type: "fanout", + id: crypto.randomUUID(), + baseUrl: "https://example.com", + keys: [{ + keyId: sender.keyId.href, + privateKey: await exportJwk(sender.privateKey), + }], + inboxes: { + [recipient.inboxId!.href]: { + actorIds: [], + sharedInbox: false, + }, + }, + activity: activityJsonLd, + activityId: activity.id!.href, + activityType: Create.typeId.href, + traceContext: {}, + }; + let outward: unknown; + const exportKeyDescriptor = Object.getOwnPropertyDescriptor( + crypto.subtle, + "exportKey", + ); + try { + if (phase === "key-export") { + Object.defineProperty(crypto.subtle, "exportKey", { + configurable: true, + value: () => Promise.reject(failure), + }); + } + await rejects( + federation.processQueuedTask(undefined, message), + (error) => { + outward = error; + return error === failure; + }, + ); + } finally { + if (phase === "key-export") { + if (exportKeyDescriptor == null) { + Reflect.deleteProperty(crypto.subtle, "exportKey"); + } else { + Object.defineProperty( + crypto.subtle, + "exportKey", + exportKeyDescriptor, + ); + } + } + } + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].result, { + type: "preparation-failed", + failure: { + kind: "local", + phase: phase === "normalization" ? "serialization" : phase, + error: outward, + }, + error: outward, + }); + strictEqual(reports[0].attemptsStarted, 0); + }); + } +}); + +test("fanout worker preserves a trace propagator failure as request preparation", async () => { + const failure = Symbol("trace injection"); + const reports: OutboundWorkerReport[] = []; + const fanouts: Message[] = []; + const outboxMessages: Message[] = []; + const federation = createFederation({ + ...options, + manuallyStartQueue: true, + queue: { + fanout: { + enqueue(message) { + fanouts.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + outbox: { + enqueue(message) { + outboxMessages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, [recipient], activity, { fanout: "force" }); + const message = fanouts[0]; + ok(message?.type === "fanout"); + try { + strictEqual( + propagation.setGlobalPropagator({ + inject() { + throw failure; + }, + extract(ctx) { + return ctx; + }, + fields() { + return []; + }, + }), + true, + ); + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + } finally { + propagation.disable(); + } + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].result, { + type: "preparation-failed", + failure: { kind: "local", phase: "request-preparation", error: failure }, + error: failure, + }); + strictEqual(reports[0].attemptsStarted, 0); + strictEqual(outboxMessages.length, 0); +}); + test("outbound worker reports distinguish fanout child tasks from concrete sends", async (t) => { for (const queued of [false, true]) { await t.step(queued ? "child outbox tasks" : "direct sends", async () => { diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 1c31eb5c0..09f6658d5 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -541,6 +541,11 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { | "key-import" | "activity-parsing" | "context-preparation" + | "validation" + | "key-resolution" + | "signing" + | "serialization" + | "key-export" | "request-preparation"; readonly error: unknown; }; From e721280a1044e18ef719ca9bfb3f935b7e4988ab Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 16:40:18 +0900 Subject: [PATCH 43/53] Retain handled worker preparation failures Caught route preparation failures survive successful gateway fallback or native retry. Scheduling actions retain causes separately from enqueue errors while preserving settlement and retry behavior. https://github.com/fedify-dev/fedify/pull/1359 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 11 +- changes.d/fedify/outbound-observers.md | 11 +- claude-plugin/skills/fedify/SKILL.md | 21 +- docs/manual/mq.md | 13 +- docs/manual/send.md | 8 + packages/fedify/src/federation/middleware.ts | 67 +++++- .../src/federation/outbound-observation.ts | 34 ++++ .../src/federation/outbound-report.test.ts | 190 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 23 +++ 9 files changed, 343 insertions(+), 35 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2d48c6b9d..d654172be 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -51,11 +51,12 @@ To be released. - Added application-facing outbound observers for sends, forwarding, replication, and queued retries, independently of OpenTelemetry. Optional callbacks report preparation, delivery handoffs, send/HTTP results, and - worker actions with correlation IDs. Applications can opt in to redacted - headers, bounded streaming response capture, and privacy-filtered sender - payloads. Inbox reports now preserve original request context and snapshot - request cancellation without changing processing outcomes. Error diagnostic - reads have a one-second deadline. [[#1316], [#1359] by Jiwon Kwon\] + worker actions with correlation IDs and preparation failure details. + Applications can opt in to redacted headers, bounded streaming response + capture, and privacy-filtered sender payloads. Inbox reports now preserve + original request context and snapshot request cancellation without changing + processing outcomes. Error diagnostic reads have a one-second deadline. + [[#1316], [#1359] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 8e14886cc..0e74dec98 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -6,8 +6,9 @@ links: - Added application-facing outbound observers for sends, forwarding, replication, and queued retries, independently of OpenTelemetry. Optional callbacks report preparation, delivery handoffs, send/HTTP results, and - worker actions with correlation IDs. Applications can opt in to redacted - headers, bounded streaming response capture, and privacy-filtered sender - payloads. Inbox reports now preserve original request context and snapshot - request cancellation without changing processing outcomes. Error diagnostic - reads have a one-second deadline. [[#1316], [#1359] by Jiwon Kwon] + worker actions with correlation IDs and preparation failure details. + Applications can opt in to redacted headers, bounded streaming response + capture, and privacy-filtered sender payloads. Inbox reports now preserve + original request context and snapshot request cancellation without changing + processing outcomes. Error diagnostic reads have a one-second deadline. + [[#1316], [#1359] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index 36aa8093d..05f36e520 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -302,13 +302,20 @@ Retry/defer includes ISO `delay` and optional per-route `notBefore` constraints. `action-failed` retains enqueue rejection with `acceptance: "unknown"`, or a failed `drop` step. `handoff`/`direct` records concrete fanout sender entry; `abandoned` records a local reason. Each action lists affected `inboxUrls`, so -gateway outcomes can differ by route. Child enqueues and direct handoffs carry -known delivery references. Accepted children survive sibling/bookkeeping -failure; a rejected batch leaves all submitted children uncertain. Replay keeps -delivery IDs with fresh child task IDs, without another broadcast. Worker -settlement does not establish backend acknowledgment, a scheduled native retry, -or remote success; later direct attempts may settle after a setup failure has -already ended the worker. Do not infer scheduling from an attempt result. +gateway outcomes can differ by route. Retry, deferral, and abandonment actions +retain contributing local preparation failures in `preparationFailures` entries +with `inboxUrl` and `failure` (phase/original error), even on a `completed` +worker. Worker-level `preparationFailures` retains all caught route preparation +failures even when a later gateway succeeds or native retry rethrows without a +scheduling action. An enqueue rejection remains the action's separate `error`. +These diagnostics are not serialized into retry messages. Child enqueues and +direct handoffs carry known delivery references. Accepted children survive +sibling/bookkeeping failure; a rejected batch leaves all submitted children +uncertain. Replay keeps delivery IDs with fresh child task IDs, without another +broadcast. Worker settlement does not establish backend acknowledgment, a +scheduled native retry, or remote success; later direct attempts may settle +after a setup failure has already ended the worker. Do not infer scheduling +from an attempt result. Header values default to disabled. `requestHeaders` and `responseHeaders` independently accept `allow`, positive safe-integer `maxBytes`, and optional diff --git a/docs/manual/mq.md b/docs/manual/mq.md index a1aedb818..bc103d0d4 100644 --- a/docs/manual/mq.md +++ b/docs/manual/mq.md @@ -1311,11 +1311,14 @@ known delivery identities and source metadata restored from the queued message. Inbox and custom-task processing use separate mechanisms. A completed worker report means the local path returned normally; it can include -a handled send failure or abandonment. An accepted enqueue can survive a later -worker failure, and a rejected enqueue or batch does not prove the backend -accepted nothing. Reports do not establish backend acknowledgment or future -native retry execution. Use attempt observers for concrete send results and -queue depth for messages still waiting. See +a handled send failure or abandonment. Reports retain caught preparation errors +under `preparationFailures`, even when another gateway succeeds. Actions also +retain their contributing preparation failures, with affected routes and +operation phases. An accepted enqueue can survive a later worker failure, and a +rejected enqueue or batch does not prove the backend accepted nothing. Reports +do not establish backend acknowledgment or future native retry execution. Use +attempt observers for concrete send results and queue depth for messages still +waiting. See [worker execution and actions](./send.md#worker-execution-and-actions) for report variants and a typed example. diff --git a/docs/manual/send.md b/docs/manual/send.md index c4ac0b25d..7c213a1d3 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -567,6 +567,14 @@ failed scheduling or drop operations. Retry and deferral actions include their actual delay and any per-route constraints. One gateway can be abandoned while another continues. An empty action list means none was observed by settlement. +`report.preparationFailures` retains caught preparation failures with their +`inboxUrl`, phase, and original error, even if another gateway later succeeds or +native retry handling rethrows a later error without scheduling an action. +When a failure contributes to a retry, deferral, or abandonment, the action's +`preparationFailures` also identifies that cause. The worker can still be +`completed` after handling a failure. If enqueueing fails, the action's `error` +describes the enqueue failure separately. + Accepted actions survive later failures; a rejected batch leaves every submitted child's acceptance unknown. Earlier direct attempts can continue after a later preparation failure settles the worker. A local result does not establish diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index f690f1cc0..1d63bd62a 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -11,6 +11,7 @@ import type { OutboundInboxRequest, OutboundObservers, OutboundProvenance, + OutboundWorkerPreparationDiagnostic, } from "./outbound-report.ts"; import { observeOutboundBroadcast, @@ -1665,6 +1666,7 @@ export class FederationImpl const enqueueHeldOutboxMessage = async ( delay: Temporal.Duration, heldSince: Temporal.Instant, + preparationFailures?: readonly OutboundWorkerPreparationDiagnostic[], ) => { const { outboxQueue } = this; if (outboxQueue == null) return; @@ -1684,6 +1686,7 @@ export class FederationImpl heldMessage, enqueueOptions.delay, () => outboxQueue.enqueue(heldMessage, enqueueOptions), + preparationFailures, )); this.metrics.recordQueueTaskEnqueued( { @@ -1858,6 +1861,10 @@ export class FederationImpl } } } catch (error) { + const preparationFailures = worker?.recordPreparationFailure( + error, + message.inbox, + ) ?? []; span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); const remoteHost = (() => { if (error instanceof SendActivityError) { @@ -1988,7 +1995,11 @@ export class FederationImpl } if (circuitDrop != null) { - worker?.abandon("held-ttl-expired", [circuitDrop.inbox.href]); + worker?.abandon( + "held-ttl-expired", + [circuitDrop.inbox.href], + preparationFailures, + ); const drop = () => dropHeldOutboxMessage( circuitDrop.circuit, @@ -2007,7 +2018,11 @@ export class FederationImpl if ( isPermanentFailure ) { - worker?.abandon("permanent-failure", [message.inbox]); + worker?.abandon( + "permanent-failure", + [message.inbox], + preparationFailures, + ); this.metrics.recordPermanentFailure( error.inbox, error.statusCode, @@ -2079,6 +2094,7 @@ export class FederationImpl await enqueueHeldOutboxMessage( holdDelay, circuitHold.heldSince, + preparationFailures, ); return; } @@ -2117,6 +2133,7 @@ export class FederationImpl retryMessage, enqueueOptions.delay, () => outboxQueue.enqueue(retryMessage, enqueueOptions), + preparationFailures, )); this.metrics.recordQueueTaskEnqueued( { @@ -2133,7 +2150,11 @@ export class FederationImpl ); } } else { - worker?.abandon("retry-policy-declined", [message.inbox]); + worker?.abandon( + "retry-policy-declined", + [message.inbox], + preparationFailures, + ); logger.error( "Failed to send activity {activityId} to {inbox} after {attempt} " + "attempts; giving up:\n{error}", @@ -2219,6 +2240,10 @@ export class FederationImpl readonly retryAfter?: Temporal.Duration; } const survivors: Survivor[] = []; + const preparationFailuresByRoute = new Map< + string, + readonly OutboundWorkerPreparationDiagnostic[] + >(); const permanentRoutes = new Set(); let lastError: unknown; let lastPermanent: { error: SendActivityError; inbox: URL } | undefined; @@ -2233,7 +2258,11 @@ export class FederationImpl const routes = message.gatewayInboxes.filter((route) => !permanentRoutes.has(route) ); - worker?.abandon("held-ttl-expired", routes); + worker?.abandon( + "held-ttl-expired", + routes, + routes.flatMap((route) => preparationFailuresByRoute.get(route) ?? []), + ); const execute = async () => dropHeldOutboxMessage( circuit, @@ -2330,6 +2359,13 @@ export class FederationImpl tracerProvider: this.tracerProvider, }); } catch (error) { + const preparationFailures = worker?.recordPreparationFailure( + error, + inboxHref, + ) ?? []; + if (preparationFailures.length > 0) { + preparationFailuresByRoute.set(inboxHref, preparationFailures); + } lastError = error; const isPermanentFailure = error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode); @@ -2398,7 +2434,11 @@ export class FederationImpl } if (isPermanentFailure) { permanentRoutes.add(inboxHref); - worker?.abandon("permanent-failure", [inboxHref]); + worker?.abandon( + "permanent-failure", + [inboxHref], + preparationFailures, + ); logger.warn( "Permanent delivery failure for activity {activityId} to " + "{inbox} ({status}); not trying the gateway again.", @@ -2575,6 +2615,9 @@ export class FederationImpl heldMessage, enqueueOptions.delay, () => outboxQueue.enqueue(heldMessage, enqueueOptions), + survivors.flatMap((route) => + preparationFailuresByRoute.get(route.inbox) ?? [] + ), )); this.metrics.recordQueueTaskEnqueued( { @@ -2605,6 +2648,9 @@ export class FederationImpl worker?.abandon( outboxQueue == null ? "queue-unavailable" : "retry-policy-declined", survivors.map((route) => route.inbox), + survivors.flatMap((route) => + preparationFailuresByRoute.get(route.inbox) ?? [] + ), ); logger.error( "Failed to send activity {activityId} to every gateway of " + @@ -2638,8 +2684,15 @@ export class FederationImpl }; await (worker == null ? outboxQueue.enqueue(retryMessage, enqueueOptions) - : worker.enqueue("retry", retryMessage, enqueueOptions.delay, () => - outboxQueue.enqueue(retryMessage, enqueueOptions))); + : worker.enqueue( + "retry", + retryMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(retryMessage, enqueueOptions), + survivors.flatMap((route) => + preparationFailuresByRoute.get(route.inbox) ?? [] + ), + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index c243157c4..f9096b665 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -30,6 +30,7 @@ import type { OutboundPreparationResult, OutboundProvenance, OutboundResponseBodyEvent, + OutboundWorkerPreparationDiagnostic, OutboundWorkerReport, } from "./outbound-report.ts"; @@ -430,6 +431,8 @@ type WorkerPreparationFailure = Extract< type WorkerAction = NonNullable[number]; type WorkerAbandonment = Extract; +type WorkerPreparationDiagnostic = + readonly OutboundWorkerPreparationDiagnostic[]; /** Per-execution state, independent of concrete attempt/exchange callbacks. */ export class OutboundWorkerObservation { @@ -437,6 +440,7 @@ export class OutboundWorkerObservation { #actions: WorkerAction[] | undefined; #trace: OutboundWorkerReport["trace"]; #preparationFailure: WorkerPreparationFailure | undefined; + #preparationFailures: OutboundWorkerPreparationDiagnostic[] = []; readonly #queue: OutboundWorkerReport["queue"]; readonly #delivery: AttemptIdentity | undefined; readonly #role: OutboundWorkerReport["role"]; @@ -529,6 +533,7 @@ export class OutboundWorkerObservation { message: OutboxMessage, delay: Temporal.Duration, enqueue: () => Promise, + preparationFailures?: WorkerPreparationDiagnostic, ): Promise { if (this.#actions == null) return await enqueue(); const facts = { @@ -543,6 +548,13 @@ export class OutboundWorkerObservation { : [message.inbox], ), delay: delay.toString(), + ...(preparationFailures == null || preparationFailures.length === 0 + ? {} + : { + preparationFailures: Object.freeze( + [...preparationFailures], + ), + }), ...(message.gatewayNotBefore == null ? {} : { notBefore: Object.freeze({ ...message.gatewayNotBefore }), }), @@ -566,12 +578,20 @@ export class OutboundWorkerObservation { abandon( reason: WorkerAbandonment["reason"], inboxUrls: readonly string[], + preparationFailures?: WorkerPreparationDiagnostic, ): void { this.#actions?.push( Object.freeze({ type: "abandoned", reason, inboxUrls: Object.freeze([...inboxUrls]), + ...(preparationFailures == null || preparationFailures.length === 0 + ? {} + : { + preparationFailures: Object.freeze( + [...preparationFailures], + ), + }), }), ); } @@ -625,6 +645,17 @@ export class OutboundWorkerObservation { } } + recordPreparationFailure( + error: unknown, + inboxUrl: string, + ): WorkerPreparationDiagnostic { + const failure = this.#preparationFailure; + if (failure == null || !Object.is(failure.error, error)) return []; + const diagnostic = Object.freeze({ inboxUrl, failure }); + this.#preparationFailures.push(diagnostic); + return [diagnostic]; + } + async run(operation: () => Promise): Promise { try { await operation(); @@ -657,6 +688,9 @@ export class OutboundWorkerObservation { ...(this.#delivery == null ? {} : { delivery: this.#delivery }), attemptsStarted: this.#attemptsStarted, actions: Object.freeze([...(this.#actions ?? [])]), + ...(this.#preparationFailures.length === 0 ? {} : { + preparationFailures: Object.freeze([...this.#preparationFailures]), + }), observedAt, ...(this.#trace == null ? {} : { trace: this.#trace }), provenance: this.provenance, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 2134c84db..58224b08a 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -3310,6 +3310,61 @@ function continuationMessage( }; } +test("gateway retry preparation diagnostics include only the failed route", async () => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + ]; + const queued: Message[] = []; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + gatewayNotBefore: { [gateways[0]]: "2999-01-01T00:00:00Z" }, + headers: { "bad\nname": "value" }, + }; + try { + await federation.processQueuedTask(undefined, message); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 0); + strictEqual(queued.length, 1); + const action = reports[0].actions?.[0]; + ok(action?.type === "scheduled"); + strictEqual(action.action, "retry"); + deepStrictEqual(action.inboxUrls, gateways); + deepStrictEqual(action.preparationFailures?.map((item) => item.inboxUrl), [ + gateways[1], + ]); + strictEqual( + action.preparationFailures?.[0].failure.phase, + "request-preparation", + ); + } finally { + fetchMock.hardReset(); + } +}); + test("gateway worker actions retain route abandonment without claiming an unqueued deferral", async () => { const gateways = [ "https://gw1.example/inbox", @@ -3712,6 +3767,92 @@ test("ordinary outbox worker actions record actual retry and hold enqueues", asy } }); +test("handled header preparation failures stay attached to actual retry actions", async (t) => { + for (const mode of ["accepted", "enqueue-failed", "declined"] as const) { + await t.step(mode, async () => { + const reports: OutboundWorkerReport[] = []; + const enqueueError = new Error("queue unavailable"); + const preparationErrors: unknown[] = []; + let enqueueCalls = 0; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => + mode === "declined" + ? null + : Temporal.Duration.from({ milliseconds: 250 }), + onOutboxError(error) { + preparationErrors.push(error); + }, + queue: { + outbox: { + enqueue() { + enqueueCalls++; + return mode === "enqueue-failed" + ? Promise.reject(enqueueError) + : Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = { + ...continuationMessage(recipient.inboxId!.href, true), + headers: { "bad\nname": "value" }, + }; + try { + if (mode === "enqueue-failed") { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === enqueueError, + ); + } else await federation.processQueuedTask(undefined, message); + const report = reports[0]; + if (mode === "enqueue-failed") { + ok(report.result.type === "failed"); + strictEqual(report.result.error, enqueueError); + } else deepStrictEqual(report.result, { type: "completed" }); + strictEqual(report.attemptsStarted, 0); + strictEqual(enqueueCalls, mode === "declined" ? 0 : 1); + const action = report.actions?.[0]; + if (mode === "declined") { + ok(action?.type === "abandoned"); + strictEqual(action.reason, "retry-policy-declined"); + } else { + ok( + action?.type === + (mode === "accepted" ? "scheduled" : "action-failed"), + ); + strictEqual(action.action, "retry"); + } + deepStrictEqual(action.inboxUrls, [message.inbox]); + const [diagnostic] = action.preparationFailures!; + const [workerDiagnostic] = report.preparationFailures!; + strictEqual(workerDiagnostic, diagnostic); + strictEqual(diagnostic.inboxUrl, message.inbox); + strictEqual(diagnostic.failure.kind, "local"); + strictEqual(diagnostic.failure.phase, "request-preparation"); + strictEqual(diagnostic.failure.error, preparationErrors[0]); + if (mode === "enqueue-failed") { + ok(action.type === "action-failed"); + strictEqual(action.error, enqueueError); + strictEqual(diagnostic.failure.error === enqueueError, false); + } else { + strictEqual(Object.isFrozen(action.preparationFailures), true); + } + } finally { + fetchMock.hardReset(); + } + }); + } +}); + test("ordinary outbox worker actions preserve enqueue uncertainty and accepted bookkeeping failures", async (t) => { for (const mode of ["rejected", "accepted"] as const) { await t.step(mode, async () => { @@ -3911,6 +4052,7 @@ test("outbound worker-only observers count real sends and report local settlemen "native", "gateway", "recovered-preparation", + "recovered-preparation-success", ] as const ) { await t.step(mode, async () => { @@ -3951,18 +4093,26 @@ test("outbound worker-only observers count real sends and report local settlemen }, }); const message = continuationMessage(recipient.inboxId!.href, true); - if (mode === "gateway" || mode === "recovered-preparation") { + if ( + mode === "gateway" || mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { Object.assign(message, { gatewayInboxes: [message.inbox, "https://gateway.example/inbox"], }); } - if (mode === "recovered-preparation") { + let preparationError: Error | undefined; + if ( + mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { let reads = 0; + preparationError = new Error("first gateway header preparation"); Object.assign(message, { headers: new Proxy({}, { get(target, property, receiver) { if (property === Symbol.iterator && ++reads === 1) { - throw new Error("first gateway header preparation"); + throw preparationError; } return Reflect.get(target, property, receiver); }, @@ -3974,10 +4124,15 @@ test("outbound worker-only observers count real sends and report local settlemen message.inbox, mode === "success" ? 202 : 503, ); - if (mode === "gateway" || mode === "recovered-preparation") { + if ( + mode === "gateway" || mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { fetchMock.post( "https://gateway.example/inbox", - mode === "gateway" ? 202 : 503, + mode === "gateway" || mode === "recovered-preparation-success" + ? 202 + : 503, ); } let outward: unknown; @@ -3999,7 +4154,30 @@ test("outbound worker-only observers count real sends and report local settlemen } else if (mode === "handled") { ok(report.actions?.[0].type === "scheduled"); } else deepStrictEqual(report.actions, []); - strictEqual(report.attemptsStarted, mode === "gateway" ? 2 : 1); + strictEqual( + report.attemptsStarted, + mode === "gateway" ? 2 : 1, + ); + if ( + mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { + deepStrictEqual( + report.preparationFailures?.map((item) => item.inboxUrl), + [ + message.inbox, + ], + ); + strictEqual(report.preparationFailures?.length, 1); + strictEqual(Object.isFrozen(report.preparationFailures), true); + const [diagnostic] = report.preparationFailures!; + strictEqual(Object.isFrozen(diagnostic), true); + strictEqual(diagnostic.failure.error, preparationError); + strictEqual( + report.result.type, + mode === "recovered-preparation-success" ? "completed" : "failed", + ); + } else strictEqual("preparationFailures" in report, false); deepStrictEqual(report.queue, { queueMessageId: message.id, queueAttempt: 0, diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 09f6658d5..9d2809b93 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -449,6 +449,21 @@ type OutboundWorkerSchedule = { readonly delay: string; /** Per-route ISO instants retained by the scheduled continuation, if any. */ readonly notBefore?: Readonly>; + /** Local preparation failures that contributed to this actual action. */ + readonly preparationFailures?: readonly OutboundWorkerPreparationDiagnostic[]; +}; + +/** + * One local preparation failure associated with a worker route. + * @since 2.5.0 + */ +export type OutboundWorkerPreparationDiagnostic = { + /** Route whose local setup produced the failure. */ + readonly inboxUrl: string; + /** Existing worker preparation classification and original thrown value. */ + readonly failure: Extract["failure"]; }; type OutboundWorkerChild = { @@ -487,6 +502,9 @@ type OutboundWorkerAction = | "queue-unavailable"; /** Routes locally abandoned; other routes of the delivery may continue. */ readonly inboxUrls: readonly string[]; + /** Local preparation failures that contributed to this abandonment. */ + readonly preparationFailures?: + readonly OutboundWorkerPreparationDiagnostic[]; } | { readonly type: "action-failed"; @@ -518,6 +536,11 @@ export type OutboundWorkerReport = OutboundSourceMetadata & { * submitted child uncertain. Empty when no action was observed by settlement. */ readonly actions: readonly OutboundWorkerAction[]; + /** + * Caught preparation failures retained when later routes succeed or when + * the worker exits without scheduling a continuation. Omitted when none. + */ + readonly preparationFailures?: readonly OutboundWorkerPreparationDiagnostic[]; /** * Outbox obligation, with unavailable legacy relationships omitted. * Fanout has no single delivery; its child actions carry their own references. From dc369e64199fd0937df66b6a4f16b79f905f72c1 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 16:46:05 +0900 Subject: [PATCH 44/53] Record mock send dispatch correlation Preserve optional dispatchId values in context and federation send records, including repeated and empty IDs. Document compatible mock observer registration. https://github.com/fedify-dev/fedify/pull/1359 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 5 ++++ changes.d/testing/outbound-observers.md | 10 +++++++ claude-plugin/skills/fedify/SKILL.md | 6 ++-- packages/testing/README.md | 6 ++++ packages/testing/src/mock.test.ts | 38 +++++++++++++++++++++++++ packages/testing/src/mock.ts | 11 +++++++ 6 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 changes.d/testing/outbound-observers.md diff --git a/CHANGES.md b/CHANGES.md index d654172be..be07869dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -323,6 +323,11 @@ To be released. ### @fedify/testing + - Added outbound observer registration support to mock federations and + dispatch IDs to recorded sends. Tests can inspect `dispatchId` through + `context.getSentActivities()` and `federation.sentActivities` to assert + application correlation. Mock sends do not invoke outbound observer + callbacks. [[#1316], [#1359] by Jiwon Kwon\] - Mock inbox contexts made by `createInboxContext()` now have the `authentication` property, which can be given as an option and defaults to evidence that nothing was authenticated. [[#1290], [#1299]] diff --git a/changes.d/testing/outbound-observers.md b/changes.d/testing/outbound-observers.md new file mode 100644 index 000000000..bab74ef07 --- /dev/null +++ b/changes.d/testing/outbound-observers.md @@ -0,0 +1,10 @@ +--- +links: + '#1316': https://github.com/fedify-dev/fedify/issues/1316 + '#1359': https://github.com/fedify-dev/fedify/pull/1359 +--- + - Added outbound observer registration support to mock federations and + dispatch IDs to recorded sends. Tests can inspect `dispatchId` through + `context.getSentActivities()` and `federation.sentActivities` to assert + application correlation. Mock sends do not invoke outbound observer + callbacks. [[#1316], [#1359] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index 05f36e520..37192bb5d 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -398,8 +398,10 @@ is 1,024 UTF-8 bytes. Invalid types raise `TypeError` and oversized values broadcast ID. New public calls supply their own app ID; queued continuations retain it. -Mock federations accept registration without executing observers. Incoming C2S -listeners use the separate `setOutboxListeners()` API. +Mock federations accept registration without executing observers. Both +`context.getSentActivities()` and `federation.sentActivities` preserve optional +`dispatchId` values, including repeated IDs without suppressing sends. Incoming +C2S listeners use the separate `setOutboxListeners()` API. See . diff --git a/packages/testing/README.md b/packages/testing/README.md index a352a5479..6b32b3431 100644 --- a/packages/testing/README.md +++ b/packages/testing/README.md @@ -87,6 +87,12 @@ const sent = context.getSentActivities(); console.log(sent[0].activity); ~~~~ +Both `context.getSentActivities()` and `federation.sentActivities` retain the +optional `dispatchId` supplied to `sendActivity()`. Reusing a dispatch ID +records another send, so tests can check correlation across application-managed +retries. Mock federations also accept `setOutboundObservers()` registration, +but mock sends do not invoke those callbacks. + ### Helper functions The package also exports helper functions for creating various context types: diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index ca1916bbb..4ceff407c 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -177,6 +177,44 @@ test("getSentActivities returns sent activities", async () => { deepStrictEqual(mockFederation.sentActivities[0].sentOrder, 1); }); +test("sent activity records preserve dispatch IDs", async () => { + const federation = createFederation(); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sender = { identifier: "alice" }; + const recipient = new Person({ + id: new URL("https://example.com/users/bob"), + }); + const activity = new Create({ + id: new URL("https://example.com/activities/dispatch-id"), + actor: new URL("https://example.com/users/alice"), + }); + + await context.sendActivity(sender, recipient, activity, { + dispatchId: "same-id", + }); + await context.sendActivity(sender, recipient, activity, { + dispatchId: "same-id", + }); + await context.sendActivity(sender, recipient, activity); + await context.sendActivity(sender, recipient, activity, { dispatchId: "" }); + + const contextRecords = context.getSentActivities(); + const federationRecords = federation.sentActivities; + deepStrictEqual(contextRecords.length, 4); + deepStrictEqual(federationRecords.length, 4); + strictEqual(contextRecords[0].dispatchId, "same-id"); + strictEqual(contextRecords[1].dispatchId, "same-id"); + strictEqual(federationRecords[0].dispatchId, "same-id"); + strictEqual(federationRecords[1].dispatchId, "same-id"); + strictEqual(contextRecords[3].dispatchId, ""); + strictEqual(federationRecords[3].dispatchId, ""); + strictEqual(Object.hasOwn(contextRecords[2], "dispatchId"), false); + strictEqual(Object.hasOwn(federationRecords[2], "dispatchId"), false); +}); + test("reset clears sent activities", async () => { const mockFederation = createFederation(); const context = mockFederation.createContext( diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index 09f218346..dbafdd6f4 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -275,6 +275,8 @@ interface SentActivity { activity: Activity; /** The raw forwarded payload, if preserved by the caller. */ rawActivity?: unknown; + /** Application-provided correlation identifier, if present. */ + dispatchId?: string; /** The order in which the activity was sent (auto-incrementing counter). */ sentOrder: number; } @@ -309,6 +311,7 @@ interface TestContext recipients: any; activity: Activity; rawActivity?: unknown; + dispatchId?: string; }>; reset(): void; } @@ -1210,6 +1213,7 @@ class MockContext implements Context { recipients: any; activity: Activity; rawActivity?: unknown; + dispatchId?: string; }> = []; constructor( @@ -1733,6 +1737,9 @@ class MockContext implements Context { recipients, activity, rawActivity: options?.rawActivity, + ...(options?.dispatchId === undefined + ? {} + : { dispatchId: options.dispatchId }), }); // If this is a MockFederation, also record it there @@ -1743,6 +1750,9 @@ class MockContext implements Context { queue: queued ? "outbox" : undefined, activity, rawActivity: options?.rawActivity, + ...(options?.dispatchId === undefined + ? {} + : { dispatchId: options.dispatchId }), sentOrder: ++this.federation.sentCounter, }); } @@ -1770,6 +1780,7 @@ class MockContext implements Context { recipients: any; activity: Activity; rawActivity?: unknown; + dispatchId?: string; }> { return [...this.sentActivities]; } From 19cf65f6baaceefb15c14c962df4fff0262a13b5 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 16:50:39 +0900 Subject: [PATCH 45/53] Export outbound report constituent types Expose the report's component types so consumers can use named imports when typing observer report values. https://github.com/fedify-dev/fedify/pull/1359 Assisted-by: Codex:gpt-6.1-sol --- .../fedify/src/federation/outbound-report.ts | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 9d2809b93..811a67db1 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -143,7 +143,11 @@ export type OutboundDeliveryEvent = } ); -type OutboundAttemptFailure = +/** + * The failure classification and available error evidence for a send attempt. + * @since 2.5.0 + */ +export type OutboundAttemptFailure = | { readonly kind: "http"; readonly phase: "response"; @@ -305,7 +309,11 @@ export interface OutboundHeaderSnapshot { readonly limitReached: boolean; } -interface OutboundHttpResponseMetadata { +/** + * Immutable HTTP response facts captured from a received response. + * @since 2.5.0 + */ +export interface OutboundHttpResponseMetadata { /** Actual HTTP status returned by fetch. */ readonly status: number; /** Copied HTTP status text. */ @@ -314,7 +322,11 @@ interface OutboundHttpResponseMetadata { readonly url?: string; } -type OutboundAttemptResponse = +/** + * Response evidence and finality established during a send attempt. + * @since 2.5.0 + */ +export type OutboundAttemptResponse = | { /** The HTTP flow selected this response before follow-up work. */ readonly type: "final"; @@ -439,7 +451,11 @@ export type OutboundHttpExchangeReport = | { readonly type: "failed"; readonly error: unknown }; }; -type OutboundWorkerSchedule = { +/** + * A worker retry or deferral submission, separate from its enqueue outcome. + * @since 2.5.0 + */ +export type OutboundWorkerSchedule = { readonly action: "retry" | "defer"; /** Actual continuation task and Fedify retry counter. */ readonly queue: NonNullable; @@ -466,7 +482,11 @@ export type OutboundWorkerPreparationDiagnostic = { }>["failure"]; }; -type OutboundWorkerChild = { +/** + * A child outbox task submitted by a fanout worker. + * @since 2.5.0 + */ +export type OutboundWorkerChild = { readonly action: "enqueue"; /** Actual child task identity and initial Fedify retry counter. */ readonly queue: NonNullable; @@ -476,7 +496,11 @@ type OutboundWorkerChild = { readonly delivery?: OutboundWorkerReport["delivery"]; }; -type OutboundWorkerAction = +/** + * An observed worker handoff, scheduling outcome, or local abandonment. + * @since 2.5.0 + */ +export type OutboundWorkerAction = | ((OutboundWorkerSchedule | OutboundWorkerChild) & { readonly type: "scheduled"; }) @@ -710,7 +734,11 @@ export interface OutboundProvenance { readonly originatingRequestId?: string; } -interface OutboundSourceMetadata { +/** + * Observation time, optional tracing, original provenance, and current context. + * @since 2.5.0 + */ +export interface OutboundSourceMetadata { /** Unix milliseconds at observation; wall clocks do not ensure global ordering. */ readonly observedAt: number; /** Valid IDs of this operation's own span, when one is available. */ From 9de8328d3a2edf3cc0186534921ce749cd76eb68 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 17:00:10 +0900 Subject: [PATCH 46/53] Clarify observer usage and compatibility Explain how forwarding preserves the received payload and carries correlation IDs, and document the member required of custom Federation implementations. https://github.com/fedify-dev/fedify/pull/1359 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 3 ++ changes.d/fedify/outbound-observers.md | 3 ++ docs/manual/inbox.md | 3 ++ docs/manual/send.md | 46 +++++++++++++++++++++----- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index be07869dd..8ecedb2fc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -56,6 +56,9 @@ To be released. capture, and privacy-filtered sender payloads. Inbox reports now preserve original request context and snapshot request cancellation without changing processing outcomes. Error diagnostic reads have a one-second deadline. + The public `Federatable` interface now requires `setOutboundObservers()`, + breaking source compatibility for custom implementations. Update custom + mocks, decorators, and wrappers to implement this method. [[#1316], [#1359] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index 0e74dec98..d01eb1e93 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -11,4 +11,7 @@ links: capture, and privacy-filtered sender payloads. Inbox reports now preserve original request context and snapshot request cancellation without changing processing outcomes. Error diagnostic reads have a one-second deadline. + The public `Federatable` interface now requires `setOutboundObservers()`, + breaking source compatibility for custom implementations. Update custom + mocks, decorators, and wrappers to implement this method. [[#1316], [#1359] by Jiwon Kwon] diff --git a/docs/manual/inbox.md b/docs/manual/inbox.md index d13d069f7..f0820266f 100644 --- a/docs/manual/inbox.md +++ b/docs/manual/inbox.md @@ -989,6 +989,9 @@ federation.setInboxListeners("/{identifier}/inbox", "/inbox") }) ~~~~ +To record forwarding preparation, delivery attempts, and HTTP results, see +[observing outgoing activities](./send.md#observing-outgoing-activities). + > [!NOTE] > The `~InboxContext.forwardActivity()` method does not guarantee that the > forwarded activity is successfully delivered to the recipient, since diff --git a/docs/manual/send.md b/docs/manual/send.md index 7c213a1d3..1c0c078f4 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -591,15 +591,21 @@ gets a new `attemptId`. Queue task IDs and `queueAttempt` describe scheduling, not the number of send executions. Native redelivery can reuse both while creating a fresh attempt. -The identity relationships are: - -| Unit | Identity and lifetime | -| ------------- | ------------------------------------------------------------------------------------------------------------------ | -| Broadcast | Fresh `broadcastId` for an observed invocation; retained through queue continuations. | -| Delivery | One `deliveryId` per deduplicated destination or claimed replication target; retained across retries and fallback. | -| Attempt | Fresh `attemptId` at each concrete sender entry. | -| HTTP exchange | `attemptId` plus `exchangeIndex`, starting at zero within the attempt. | -| Queue task | Current `queueMessageId` and, when available, `queueAttempt`; fanout replay can create new task IDs. | +Identity allocation and propagation follow these rules: + +| Field | Allocation or source | Propagation and replacement | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `inboxRequestId` | Fresh server-generated ID for each routed HTTP inbox request, before preparation. | Preserved through inbox retries, derived contexts, outgoing sends, fanout, outbox retries, and replication. Repeated deliveries of the same Activity have different IDs. | +| Original logging `requestId` | The existing logging label selected at HTTP ingress; externally supplied values are informational and nonunique. | Preserved as source metadata through queues. It does not replace an unrelated worker's current logging label. | +| Original URL snapshot | The incoming request context's `origin`, `host`, `hostname`, and `canonicalOrigin`. | Preserved unchanged through derived contexts and queues, independently of current worker configuration. | +| `dispatchId` | Optional application-owned send or forward option. | Preserved through that invocation's descendants. New calls can reuse it without suppressing sends; a call that omits it does not inherit a previous invocation's value. | +| `broadcastId` | Fresh ID for each observed public send/forward invocation or internal portable-replication invocation, before preparation. | Preserved through fanout replay and retries. Another public invocation creates a new broadcast, even for the same Activity. | +| `deliveryId` | Allocated for each deduplicated destination or claimed replication target, before its handoff. | Preserved through fanout replay, native redelivery, retries, holds, fallback, and gateway-list changes. Alternative routes share a delivery; replication targets have separate deliveries. | +| `attemptId` | Fresh ID at each concrete sender entry. | Never reused on retries or native redelivery, and never derived from a queue round or trace ID. | +| `exchangeIndex` | Starts at zero and increments for actual fetch submissions within an attempt. | Identifies an exchange together with `attemptId`; redirects and signature challenges remain within their attempt. | +| `queueMessageId` | The current queue task's actual UUID. | Fanout expansion or replay can create new task IDs without creating new logical deliveries. | +| `queueAttempt` | The current message's Fedify retry counter, when available. | Native redelivery can leave it unchanged. One round can include multiple gateway attempts; it is not an execution ordinal. | +| Trace/span IDs | The observed operation's own valid OpenTelemetry span, when available. | Optional supplementary context. Worker and attempt reports use their own spans; trace IDs never replace grouping identities. | IDs provide grouping, not ordering, deduplication, or remote idempotency. Callbacks can interleave or repeat, and a crash or operation that never settles @@ -656,6 +662,28 @@ may be empty but must fit within 1,024 UTF-8 bytes. Invalid types raise consumers apply the same validation. A new call that omits the ID does not inherit it from a previous invocation. +To forward the received payload instead of constructing a new Activity, use the +inbox handler's `forwardActivity()` method with the same application correlation +option: + +~~~~ typescript twoslash +import type { InboxContext } from "@fedify/fedify"; +declare const ctx: InboxContext; +declare const dispatchId: string; +// ---cut-before--- +await ctx.forwardActivity( + { identifier: "alice" }, + "followers", + { dispatchId, skipIfUnsigned: true }, +); +~~~~ + +This creates a `forward` broadcast, preserves the received payload, and carries +the original inbox provenance and `dispatchId` into outgoing observations and +queue continuations. The example skips unsigned activities. See +[forwarding activities](./inbox.md#forwarding-activities-to-another-server) for +delivery and signature considerations. + Older messages remain processable, with missing relationships explicitly unknown. Legacy fanout can recover stable delivery grouping; legacy outbox reports use the current task UUID as the delivery ID and cannot recover From 1db180e87d4370895042e411d7cdcc1504e343fa Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 17:09:46 +0900 Subject: [PATCH 47/53] Harden outbound observation invariants Invalid optional consumer identities now warn and fall back to legacy grouping without blocking delivery. Producer validation remains strict, and successful reports require an observed final response. https://github.com/fedify-dev/fedify/pull/1359 Assisted-by: Codex:gpt-6.1-sol --- .../src/federation/outbound-observation.ts | 74 +++++++---- .../src/federation/outbound-report.test.ts | 124 ++++++++++++++++-- 2 files changed, 166 insertions(+), 32 deletions(-) diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index f9096b665..e48da780f 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -139,6 +139,17 @@ function logObserverFailure(unit: string, error: unknown): void { } } +function warnInvalidDeliveryIdentity(messageId: string, error: unknown): void { + try { + getLogger(["fedify", "federation", "outbox"]).warn( + "Ignoring invalid outbound delivery identity in queue message {messageId}: {error}", + { messageId, error }, + ); + } catch { + // Logging must not prevent a queued delivery either. + } +} + function snapshotTrace( span: Span, parentSpanContext?: SpanContext, @@ -361,7 +372,13 @@ export class OutboundAttemptObservation { } succeeded(): Promise { - return this.#finish({ type: "succeeded", response: this.#finalResponse! }); + if (this.#finished) return Promise.resolve(); + if (this.#finalResponse == null) { + throw new Error( + "A successful outbound attempt must have an observed final response.", + ); + } + return this.#finish({ type: "succeeded", response: this.#finalResponse }); } failed(error: unknown): Promise { @@ -413,15 +430,20 @@ type AttemptIdentity = Pick< >; function snapshotOutboxIdentity(message: OutboxMessage): AttemptIdentity { - return message.outbound == null - ? Object.freeze({ - broadcast: Object.freeze({ type: "unknown" as const }), - deliveryId: message.id, - ...(message.portableInbox == null - ? {} - : { portableInbox: message.portableInbox }), - }) - : snapshotDeliveryIdentity(message.outbound); + if (message.outbound != null) { + try { + return snapshotDeliveryIdentity(message.outbound); + } catch (error) { + warnInvalidDeliveryIdentity(message.id, error); + } + } + return Object.freeze({ + broadcast: Object.freeze({ type: "unknown" as const }), + deliveryId: message.id, + ...(message.portableInbox == null + ? {} + : { portableInbox: message.portableInbox }), + }); } type WorkerPreparationFailure = Extract< @@ -458,12 +480,7 @@ export class OutboundWorkerObservation { ...(message.type === "outbox" ? { queueAttempt: message.attempt } : {}), }); if (message.type === "outbox") { - try { - this.#delivery = snapshotOutboxIdentity(message); - } catch { - // Invalid optional grouping cannot change the worker's own execution. - this.#delivery = undefined; - } + this.#delivery = snapshotOutboxIdentity(message); } } @@ -1002,12 +1019,25 @@ export class OutboundBroadcastObservation { observation.#finished = true; observation.planned(Object.keys(message.inboxes).length); for (const [inbox, destination] of Object.entries(message.inboxes)) { - const identity = destination.outbound ?? - await legacyFanoutIdentity( - message.id, - inbox, - destination.portableInbox, - ); + let identity: SerializedOutboundDelivery | undefined; + if (destination.outbound != null) { + try { + const snapshot = snapshotDeliveryIdentity(destination.outbound); + if (snapshot.destinationInbox !== inbox) { + throw new TypeError( + "Outbound delivery identity does not match its fanout destination.", + ); + } + identity = snapshot; + } catch (error) { + warnInvalidDeliveryIdentity(message.id, error); + } + } + identity ??= await legacyFanoutIdentity( + message.id, + inbox, + destination.portableInbox, + ); observation.#deliveries.set( inbox, observation.#createDelivery(inbox, destination, identity), diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 58224b08a..14838d4b0 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -50,7 +50,10 @@ import { sendActivity as sendConcreteActivity, SendActivityError, } from "./send.ts"; -import { OutboundBroadcastObservation } from "./outbound-observation.ts"; +import { + OutboundAttemptObservation, + OutboundBroadcastObservation, +} from "./outbound-observation.ts"; import { snapshotHeaderPolicy, snapshotHeaders } from "./outbound-headers.ts"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; import { exportJwk } from "../sig/key.ts"; @@ -66,6 +69,40 @@ const options = { contextLoaderFactory: () => mockDocumentLoader, }; +test("successful outbound attempts require an observed final response", async () => { + const reports: unknown[] = []; + const observation = new OutboundAttemptObservation( + "https://receiver.example/inbox", + async (facts) => { + await Promise.resolve(); + reports.push(facts); + }, + ); + throws(() => observation.succeeded(), { + message: + "A successful outbound attempt must have an observed final response.", + }); + observation.response(new Response(null, { status: 202 }), "final"); + throws(() => observation.succeeded(), { + message: + "A successful outbound attempt must have an observed final response.", + }); + strictEqual(reports.length, 0); + + const response = new Response(null, { status: 202 }); + await observation.fetch( + new Request("https://receiver.example/inbox", { method: "POST" }), + async () => { + await Promise.resolve(); + return response; + }, + ); + observation.response(response, "final"); + await observation.succeeded(); + await observation.succeeded(); + strictEqual(reports.length, 1); +}); + test("outbound observation timestamps snapshot each boundary and refresh on fanout replay", async () => { const federation = createFederation(options); const ctx = federation.createContext( @@ -3192,15 +3229,33 @@ test("outbound queue identities survive fanout replay and legacy messages", asyn strictEqual(identities.size, 2); strictEqual(events.length, legacy ? 4 : 2); if (!legacy) { - const invalid = structuredClone(original); - const first = Object.values(invalid.inboxes)[0]; - ok(first.outbound); - Object.assign(first.outbound, { deliveryId: "" }); - await rejects( - federation.processQueuedTask(undefined, invalid), - TypeError, - ); - strictEqual(outboxes.length, 4); + for (const mismatch of [false, true]) { + const invalid = structuredClone(original); + const first = Object.values(invalid.inboxes)[0]; + ok(first.outbound); + Object.assign( + first.outbound, + mismatch + ? { destinationInbox: "https://other.example/inbox" } + : { deliveryId: "" }, + ); + await federation.processQueuedTask(undefined, invalid); + const fallback = outboxes.at(-2); + ok(fallback?.type === "outbox" && fallback.outbound); + deepStrictEqual(fallback.outbound.broadcast, { type: "unknown" }); + strictEqual(fallback.outbound.destinationInbox, fallback.inbox); + strictEqual( + fallback.outbound.deliveryId.startsWith("legacy-fanout:"), + true, + ); + const retained = outboxes.at(-1); + ok(retained?.type === "outbox"); + deepStrictEqual( + retained.outbound, + Object.values(original.inboxes)[1].outbound, + ); + } + strictEqual(outboxes.length, 8); } }, ); @@ -3258,6 +3313,55 @@ test("outbound observers preserve processing of legacy outbox messages", async ( } }); +test("invalid outbox grouping does not block delivery or observers", async () => { + const task = continuationMessage(recipient.inboxId!.href, true); + task.outbound && Object.assign(task.outbound, { deliveryId: "" }); + const attempts: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const workers: OutboundWorkerReport[] = []; + const bodies: OutboundResponseBodyEvent[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onWorker(_ctx, report) { + workers.push(report); + }, + responseBodyCapture: { + maxBytes: 128, + timeoutMs: 1000, + onBody(_ctx, event) { + bodies.push(event); + }, + }, + }); + fetchMock.mockGlobal().post(task.inbox, { status: 202, body: "accepted" }); + try { + await federation.processQueuedTask(undefined, task); + strictEqual(fetchMock.callHistory.calls().length, 1); + strictEqual(attempts.length, 1); + strictEqual(exchanges.length, 1); + strictEqual(workers.length, 1); + strictEqual(bodies.length, 1); + for (const report of [...attempts, ...exchanges, ...bodies]) { + strictEqual(report.deliveryId, task.id); + deepStrictEqual(report.broadcast, { type: "unknown" }); + strictEqual("destinationInbox" in report, false); + } + strictEqual(workers[0].delivery?.deliveryId, task.id); + deepStrictEqual(workers[0].delivery?.broadcast, { type: "unknown" }); + } finally { + fetchMock.hardReset(); + } +}); + function continuationMessage( inbox: string, observed: boolean, From 3f2a9c22fca38fb347c2c99737a317785e4890db Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 18:30:56 +0900 Subject: [PATCH 48/53] Add activity references to broadcast events Capture immutable activity ID and type snapshots using existing producer reads, and drop stale transformer facts without changing queue schemas. Remove the unused source wrapper while preserving dispatch behavior. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- claude-plugin/skills/fedify/SKILL.md | 7 +- docs/manual/send.md | 28 +++++--- packages/fedify/src/federation/middleware.ts | 72 +++++++++++++++---- .../src/federation/outbound-dispatch.test.ts | 12 ++++ .../src/federation/outbound-observation.ts | 18 +++++ .../src/federation/outbound-provenance.ts | 7 -- .../src/federation/outbound-report.test.ts | 71 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 17 +++++ .../src/federation/portable-inbox.test.ts | 4 ++ 9 files changed, 206 insertions(+), 30 deletions(-) diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index 37192bb5d..d454dacbe 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -253,7 +253,12 @@ portable replication. Add `onAttempt(ctx, report)` and callbacks are optional. Registration replaces the whole set; `{}` clears it. Registration and builds copy callbacks, and active executions retain their selection. Callbacks receive existing context/data without extra loading and -work without OpenTelemetry. +work without OpenTelemetry. Broadcast `event.activity` snapshots optional known +`id` and `type` strings, with `type` containing the Activity type IRI. The +reference contains no payload and does not establish authentication. Early +events can have an empty reference. Transformers can change it; missing new +facts must not be inferred from the old Activity. Activity IDs do not replace +broadcast or application dispatch identities. `onBroadcast` groups start and preparation results with an independent `broadcastId`; `operation` distinguishes `send`, `forward`, and diff --git a/docs/manual/send.md b/docs/manual/send.md index 1c0c078f4..f730b2c5e 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -469,14 +469,26 @@ Observation does not load additional data or serialize an activity again. Routing and source metadata are immutable snapshots; the context, application data, Activity objects, and original exceptions are not deep-copied. -`onBroadcast` reports one send or forward invocation. Its `started` event runs -before preparation; `preparation-finished` describes whether the producer -prepared work for delivery or fanout, skipped sending, or failed. Fanout-ready -work still needs worker-side delivery preparation. A prepared result does not -confirm queue acceptance or HTTP delivery, and later send failures do not -rewrite it. Skips such as `no-destinations` or `forwarding-disabled` describe -existing decisions; an `unsigned` skip reflects the forwarding signature/proof -shape check, not authentication. +Broadcast events include an immutable `activity` reference with optional `id` +and `type` strings; `type` is the Activity's type IRI. These are facts already +obtained by the send path, not a copy of the payload. The `started` event can +have an empty reference, and later events can contain more information. After +transformation, unavailable facts are omitted rather than copied from the old +Activity. Observation does not read additional Activity properties, parse +payloads, or load contexts to fill the reference. Activity IDs can repeat +across broadcasts; use `broadcastId` for invocation identity and `dispatchId` +for application correlation. A reported Activity ID does not establish +authentication. + +`onBroadcast` reports one send, forward, or automatic portable-replication +invocation. Its `started` event runs before preparation; `preparation-finished` +describes whether the producer prepared work for delivery or fanout, skipped +sending, or failed. Fanout-ready work still needs worker-side delivery +preparation. A prepared result does not confirm queue acceptance or HTTP +delivery, and later send failures do not rewrite it. Skips such as +`no-destinations` or `forwarding-disabled` describe existing decisions; an +`unsigned` skip reflects the forwarding signature/proof shape check, not +authentication. `onDelivery` reports each deduplicated destination or claimed replication target. Different inboxes on one host remain separate deliveries. A `planned` diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 1d63bd62a..04023dd74 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -3696,6 +3696,12 @@ export class FederationImpl inboxes[inbox].actorIds, ), }); + const sendActivityId = activity.id?.href; + const activityType = getTypeId(activity).href; + observation?.setActivity({ + ...(sendActivityId == null ? {} : { id: sendActivityId }), + type: activityType, + }); sending = sendActivityToInbox( inbox, inboxes[inbox].gatewayInboxes, @@ -3711,8 +3717,8 @@ export class FederationImpl allowPrivateAddress: this.allowPrivateAddress, keys, activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, + activityId: sendActivityId, + activityType, sharedInbox: inboxes[inbox].sharedInbox, headers: createHeaders == null ? undefined @@ -3773,6 +3779,12 @@ export class FederationImpl const messageOrderingKey = orderingKey == null ? undefined : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; + const messageActivityId = activity.id?.href; + const messageActivityType = getTypeId(activity).href; + observation?.setActivity({ + ...(messageActivityId == null ? {} : { id: messageActivityId }), + type: messageActivityType, + }); const message: OutboxMessage = { type: "outbox", ...serializeProvenance(provenance), @@ -3780,8 +3792,8 @@ export class FederationImpl baseUrl: ctx.origin, keys: keyJwkPairs, activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, + activityId: messageActivityId, + activityType: messageActivityType, inbox, ...observation?.continuation(inbox), sharedInbox: inboxes[inbox].sharedInbox, @@ -5044,6 +5056,10 @@ export class FederationImpl context, this.outboundObservers, async (observation) => { + observation?.setActivity({ + ...(activityId == null ? {} : { id: activityId.href }), + type: activityType, + }); if (this.portableInboxForwarding.maxTargets < 1) { if (observation != null) { await observation.skipped("forwarding-disabled"); @@ -6229,6 +6245,8 @@ export class ContextImpl implements Context { metadata.version, ); const parentSpanContext = trace.getSpanContext(context.active()); + const activityType = getTypeId(activity).href; + observation?.setActivity({ type: activityType }); return tracer.startActiveSpan( this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" @@ -6238,7 +6256,7 @@ export class ContextImpl implements Context { ? SpanKind.CLIENT : SpanKind.PRODUCER, attributes: { - "activitypub.activity.type": getTypeId(activity).href, + "activitypub.activity.type": activityType, "activitypub.activity.to": activity.toIds.map((to) => to.href), "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), "activitypub.activity.bto": activity.btoIds.map((bto) => @@ -6252,9 +6270,14 @@ export class ContextImpl implements Context { async (span) => { observation?.setSpan(span, parentSpanContext); try { - if (activity.id != null) { - span.setAttribute("activitypub.activity.id", activity.id.href); + const activityId = activity.id?.href; + if (activityId != null) { + span.setAttribute("activitypub.activity.id", activityId); } + observation?.setActivity({ + type: activityType, + ...(activityId == null ? {} : { id: activityId }), + }); await this.sendActivityInternal( sender, recipients, @@ -6393,10 +6416,16 @@ export class ContextImpl implements Context { }; span.setAttribute("activitypub.inboxes", expandedRecipients.length); observation?.setPhase("transformation"); + if (this.federation.activityTransformers.length > 0) { + // A transformer can replace the Activity; do not retain its old ID/type. + observation?.setActivity({}); + } for (const activityTransformer of this.federation.activityTransformers) { activity = activityTransformer(activity, this); } - span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? ""); + const transformedActivityId = activity?.id?.href; + span?.setAttribute("activitypub.activity.id", transformedActivityId ?? ""); + observation?.setActivityId(transformedActivityId); observation?.setPhase("validation"); if (activity.actorId == null) { logger.error( @@ -6511,6 +6540,12 @@ export class ContextImpl implements Context { observation?.setPhase("request-preparation"); const carrier: Record = {}; propagation.inject(context.active(), carrier); + const activityId = activity.id?.href; + const activityType = getTypeId(activity).href; + observation?.setActivity({ + ...(activityId == null ? {} : { id: activityId }), + type: activityType, + }); const message: FanoutMessage = { type: "fanout", ...serializeProvenance(provenance), @@ -6528,8 +6563,8 @@ export class ContextImpl implements Context { }]), ), activity: activityJsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, + activityId, + activityType, collectionSync: opts.collectionSync, orderingKey: options.orderingKey, normalizeExistingProofs: proofCreated || options.normalizeExistingProofs, @@ -7360,6 +7395,10 @@ function forwardActivity( ctx, ctx.federation.outboundObservers, (observation, provenance) => { + observation?.setActivity({ + ...(ctx.activityId == null ? {} : { id: ctx.activityId }), + type: ctx.activityType, + }); const tracer = ctx.tracerProvider.getTracer( metadata.name, metadata.version, @@ -7823,6 +7862,8 @@ export class OutboxContextImpl extends ContextImpl metadata.version, ); const parentSpanContext = trace.getSpanContext(context.active()); + const activityType = getTypeId(activity).href; + observation?.setActivity({ type: activityType }); return tracer.startActiveSpan( this.federation.outboxQueue == null || options.immediate ? "activitypub.outbox" @@ -7832,7 +7873,7 @@ export class OutboxContextImpl extends ContextImpl ? SpanKind.CLIENT : SpanKind.PRODUCER, attributes: { - "activitypub.activity.type": getTypeId(activity).href, + "activitypub.activity.type": activityType, "activitypub.activity.to": activity.toIds.map((to) => to.href), "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), "activitypub.activity.bto": activity.btoIds.map((bto) => @@ -7846,9 +7887,14 @@ export class OutboxContextImpl extends ContextImpl async (span) => { observation?.setSpan(span, parentSpanContext); try { - if (activity.id != null) { - span.setAttribute("activitypub.activity.id", activity.id.href); + const activityId = activity.id?.href; + if (activityId != null) { + span.setAttribute("activitypub.activity.id", activityId); } + observation?.setActivity({ + type: activityType, + ...(activityId == null ? {} : { id: activityId }), + }); const delivered = await this.sendActivityInternal( sender, recipients, diff --git a/packages/fedify/src/federation/outbound-dispatch.test.ts b/packages/fedify/src/federation/outbound-dispatch.test.ts index cb5db15ae..b59571130 100644 --- a/packages/fedify/src/federation/outbound-dispatch.test.ts +++ b/packages/fedify/src/federation/outbound-dispatch.test.ts @@ -236,6 +236,18 @@ test("dispatch IDs correlate send and forward entries without deduplication or c const starts = broadcasts.filter((event) => event.stage === "started"); strictEqual(starts.length, 3); strictEqual(new Set(starts.map((event) => event.broadcastId)).size, 3); + deepStrictEqual(starts.map((event) => event.activity), [{}, {}, {}]); + const finishes = broadcasts.filter((event) => + event.stage === "preparation-finished" + ); + strictEqual(finishes.length, 3); + for (const event of finishes) { + deepStrictEqual(event.activity, { + id: activity.id!.href, + type: Create.typeId.href, + }); + ok(Object.isFrozen(event.activity)); + } for (const ctx of contexts) strictEqual(ctx, context); for (let index = 0; index < 3; index++) { const message = messages[index]; diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index e48da780f..603d6097e 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -21,6 +21,7 @@ import type { SerializedOutboundDelivery, } from "./queue.ts"; import type { + OutboundActivityReference, OutboundAttemptReport, OutboundBodyCaptureSummary, OutboundBroadcastEvent, @@ -947,6 +948,7 @@ export class OutboundBroadcastObservation { #planning: PreparationFailure["planning"] = "not-started"; #destinationCount = 0; #trace: OutboundBroadcastEvent["trace"]; + #activity: OutboundActivityReference = Object.freeze({}); readonly #onBroadcast: OutboundObservers["onBroadcast"]; readonly #onDelivery: OutboundObservers["onDelivery"]; @@ -1144,6 +1146,20 @@ export class OutboundBroadcastObservation { if (phase === "destination-planning") this.#planning = "incomplete"; } + /** Replace the Activity facts already obtained by the producer path. */ + setActivity(reference: OutboundActivityReference): void { + this.#activity = Object.freeze({ ...reference }); + } + + /** Refresh the identifier from a producer read without retaining stale IDs. */ + setActivityId(id: string | undefined): void { + const reference: { id?: string; type?: string } = { + ...(id == null ? {} : { id }), + ...(this.#activity.type == null ? {} : { type: this.#activity.type }), + }; + this.#activity = Object.freeze(reference); + } + planned(destinationCount: number): void { this.#planning = "complete"; this.#destinationCount = destinationCount; @@ -1188,6 +1204,7 @@ export class OutboundBroadcastObservation { broadcastId: this.#broadcast.id, operation: this.operation, stage: "started", + activity: this.#activity, })); } try { @@ -1221,6 +1238,7 @@ export class OutboundBroadcastObservation { operation: this.operation, stage: "preparation-finished", result, + activity: this.#activity, })); } } diff --git a/packages/fedify/src/federation/outbound-provenance.ts b/packages/fedify/src/federation/outbound-provenance.ts index 0007e0f65..d035739f8 100644 --- a/packages/fedify/src/federation/outbound-provenance.ts +++ b/packages/fedify/src/federation/outbound-provenance.ts @@ -1,4 +1,3 @@ -import { withContext } from "@logtape/logtape"; import type { Context } from "./context.ts"; import type { OutboundInboxRequest, @@ -48,12 +47,6 @@ export function createInboxRequestSource( }); } -export function withOutboundSource(context: object, operation: () => T): T { - const source = getOutboundProvenance(context); - // Explicit absence masks unrelated ambient source labels as well. - return withContext(sourceLogContext(source), operation); -} - /** The same exact-string budget applies to public input and queued metadata. */ export function validateDispatchId( value: unknown, diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 14838d4b0..4093ff5ba 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -4,7 +4,7 @@ import { mockDocumentLoader, test, } from "@fedify/fixture"; -import { Create, Person } from "@fedify/vocab"; +import { Create, Follow, Person } from "@fedify/vocab"; import { FetchError, UrlError } from "@fedify/vocab-runtime"; import { configure, reset } from "@logtape/logtape"; import { @@ -1838,6 +1838,68 @@ test("outbound preparation preserves failure provenance and plan completeness", } }); +test("broadcast activity references are immutable and follow transformed activities", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const transformedId = "https://example.com/transformed"; + const federation = createFederation({ + ...options, + queue: { fanout: queue }, + manuallyStartQueue: true, + activityTransformers: [() => + new Follow({ + id: new URL(transformedId), + actor: new URL("https://example.com/alice"), + object: new URL("https://example.com/bob"), + })], + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await ctx.sendActivity(sender, recipient, activity, { fanout: "force" }); + await ctx.sendActivity(sender, recipient, activity, { fanout: "force" }); + await ctx.sendActivity( + sender, + new Person({ + id: new URL("https://receiver.example/no-inbox"), + }), + activity, + ); + + strictEqual(messages.length, 2); + const starts = events.filter((event) => event.stage === "started"); + const finishes = events.filter((event) => + event.stage === "preparation-finished" + ); + strictEqual(starts.length, 3); + strictEqual(finishes.length, 3); + deepStrictEqual(starts.map((event) => event.activity), [{}, {}, {}]); + strictEqual(new Set(starts.map((event) => event.broadcastId)).size, 3); + for (const event of finishes.slice(0, 2)) { + deepStrictEqual(event.activity, { + id: transformedId, + type: Follow.typeId.href, + }); + ok(Object.isFrozen(event.activity)); + } + deepStrictEqual(finishes[2].activity, { id: transformedId }); + ok(!("type" in finishes[2].activity)); + ok(Object.isFrozen(starts[0].activity)); +}); + test("outbound preparation precedes queue acceptance and is not rewritten by enqueue failure", async (t) => { for (const role of ["outbox", "fanout"] as const) { for (const rejected of [false, true]) { @@ -2678,6 +2740,13 @@ test("forwarding observers await readiness before enqueue while preserving raw p ok(messages[0].type === "outbox"); strictEqual(messages[0].activity, forwardingPayload); ok(!("broadcastId" in messages[0])); + deepStrictEqual(events[0].activity, {}); + deepStrictEqual(events[1].activity, { + id: forwardingPayload.id, + type: Create.typeId.href, + }); + ok(Object.isFrozen(events[0].activity)); + ok(Object.isFrozen(events[1].activity)); for (const event of events) strictEqual(event.operation, "forward"); if (ctx instanceof OutboxContextImpl) { strictEqual(ctx.hasDeliveredActivity(), !rejected); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 811a67db1..693c6306a 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -52,6 +52,21 @@ export type OutboundPreparationResult = readonly plannedDestinationCount: number; }; +/** + * Known Activity ID and type at a broadcast observation boundary. + * + * Fields are included only after the send path has already obtained them. + * A `started` event may have an empty reference, and later events may reflect + * Activity transformations that happened after that event. + * @since 2.5.0 + */ +export type OutboundActivityReference = { + /** Activity identifier, when already available in the send path. */ + readonly id?: string; + /** Activity type IRI, when already available in the send path. */ + readonly type?: string; +}; + /** * A send, original-payload forwarding, or portable replication producer lifecycle. * @@ -65,6 +80,8 @@ export type OutboundBroadcastEvent = readonly broadcastId: string; /** The outgoing operation being observed. */ readonly operation: "send" | "forward" | "portable-replication"; + /** Immutable Activity identity facts already obtained by the send path. */ + readonly activity: OutboundActivityReference; } & ( | { diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index 3cebaea30..28b35ddd2 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -2361,6 +2361,10 @@ test("portable replication observers distinguish disabled, unavailable, and uncl ? "forwarding-unavailable" : "no-forwarding-targets", }); + assertEquals(events[0].activity, {}); + assertEquals(events[1].activity.type, Follow.typeId.href); + assert(events[1].activity.id?.startsWith("ap+ef61://")); + assert(Object.isFrozen(events[1].activity)); assertStrictEquals(contexts[0], contexts[1]); assertEquals(deliveries, []); assertEquals(queue.outbox.length, 0); From 2bd3774f3c1a009a5841157b30f9b33ffbf9e768 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 18:35:56 +0900 Subject: [PATCH 49/53] Report unavailable outbox retry queues Previously, ordinary outbox retries without a queue were abandoned without reporting their worker action. Report the unavailable queue while preserving the preparation failures and existing retry behavior. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- packages/fedify/src/federation/middleware.ts | 6 ++ .../src/federation/outbound-report.test.ts | 66 +++++++++++++------ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 04023dd74..bd183a22b 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -2148,6 +2148,12 @@ export class FederationImpl "retried", retryMessage.activityType, ); + } else { + worker?.abandon( + "queue-unavailable", + [message.inbox], + preparationFailures, + ); } } else { worker?.abandon( diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 4093ff5ba..6748660aa 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -3941,7 +3941,14 @@ test("ordinary outbox worker actions record actual retry and hold enqueues", asy }); test("handled header preparation failures stay attached to actual retry actions", async (t) => { - for (const mode of ["accepted", "enqueue-failed", "declined"] as const) { + for ( + const mode of [ + "accepted", + "enqueue-failed", + "declined", + "queue-unavailable", + ] as const + ) { await t.step(mode, async () => { const reports: OutboundWorkerReport[] = []; const enqueueError = new Error("queue unavailable"); @@ -3958,17 +3965,19 @@ test("handled header preparation failures stay attached to actual retry actions" onOutboxError(error) { preparationErrors.push(error); }, - queue: { - outbox: { - enqueue() { - enqueueCalls++; - return mode === "enqueue-failed" - ? Promise.reject(enqueueError) - : Promise.resolve(); + ...(mode === "queue-unavailable" ? {} : { + queue: { + outbox: { + enqueue() { + enqueueCalls++; + return mode === "enqueue-failed" + ? Promise.reject(enqueueError) + : Promise.resolve(); + }, + listen: () => Promise.resolve(), }, - listen: () => Promise.resolve(), }, - }, + }), }); federation.setOutboundObservers({ onWorker(_ctx, report) { @@ -3992,11 +4001,18 @@ test("handled header preparation failures stay attached to actual retry actions" strictEqual(report.result.error, enqueueError); } else deepStrictEqual(report.result, { type: "completed" }); strictEqual(report.attemptsStarted, 0); - strictEqual(enqueueCalls, mode === "declined" ? 0 : 1); + strictEqual( + enqueueCalls, + mode === "declined" || mode === "queue-unavailable" ? 0 : 1, + ); + strictEqual(report.actions?.length, 1); const action = report.actions?.[0]; - if (mode === "declined") { + if (mode === "declined" || mode === "queue-unavailable") { ok(action?.type === "abandoned"); - strictEqual(action.reason, "retry-policy-declined"); + strictEqual( + action.reason, + mode === "declined" ? "retry-policy-declined" : "queue-unavailable", + ); } else { ok( action?.type === @@ -4118,6 +4134,7 @@ test("ordinary outbox worker actions retain local abandonment and failed drop st const mode of [ "permanent", "declined", + "queue-unavailable", "ttl", "ttl-parse", "ttl-handler", @@ -4133,17 +4150,22 @@ test("ordinary outbox worker actions retain local abandonment and failed drop st kv: new MemoryKvStore(), allowPrivateAddress: true, manuallyStartQueue: true, - outboxRetryPolicy: () => null, + outboxRetryPolicy: () => + mode === "queue-unavailable" + ? Temporal.Duration.from({ seconds: 1 }) + : null, circuitBreaker: ttl ? { heldActivityTtl: { seconds: 1 } } : false, - queue: { - outbox: { - enqueue(message) { - queued.push(message); - return Promise.resolve(); + ...(mode === "queue-unavailable" ? {} : { + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), }, - listen: () => Promise.resolve(), }, - }, + }), }); federation.setOutboundObservers({ onWorker(_ctx, report) { @@ -4199,6 +4221,8 @@ test("ordinary outbox worker actions retain local abandonment and failed drop st ? "held-ttl-expired" : mode === "declined" ? "retry-policy-declined" + : mode === "queue-unavailable" + ? "queue-unavailable" : "permanent-failure", ); deepStrictEqual(abandoned.inboxUrls, [message.inbox]); From 25d4c61d0cb458d61b6da0a1fe6d5ef62bd03abe Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 18:41:35 +0900 Subject: [PATCH 50/53] Validate mock send dispatch IDs Reject invalid dispatch IDs before recording sends, matching production validation behavior. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 5 ++- changes.d/testing/outbound-observers.md | 5 ++- packages/testing/src/mock.test.ts | 57 +++++++++++++++++++++++++ packages/testing/src/mock.ts | 21 ++++++--- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8ecedb2fc..050b2a7e8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -329,8 +329,9 @@ To be released. - Added outbound observer registration support to mock federations and dispatch IDs to recorded sends. Tests can inspect `dispatchId` through `context.getSentActivities()` and `federation.sentActivities` to assert - application correlation. Mock sends do not invoke outbound observer - callbacks. [[#1316], [#1359] by Jiwon Kwon\] + application correlation. Mock sends reject invalid dispatch IDs before + recording, using production's 1,024-byte UTF-8 limit. Mock sends do not + invoke outbound observer callbacks. [[#1316], [#1359] by Jiwon Kwon\] - Mock inbox contexts made by `createInboxContext()` now have the `authentication` property, which can be given as an option and defaults to evidence that nothing was authenticated. [[#1290], [#1299]] diff --git a/changes.d/testing/outbound-observers.md b/changes.d/testing/outbound-observers.md index bab74ef07..10f282b74 100644 --- a/changes.d/testing/outbound-observers.md +++ b/changes.d/testing/outbound-observers.md @@ -6,5 +6,6 @@ links: - Added outbound observer registration support to mock federations and dispatch IDs to recorded sends. Tests can inspect `dispatchId` through `context.getSentActivities()` and `federation.sentActivities` to assert - application correlation. Mock sends do not invoke outbound observer - callbacks. [[#1316], [#1359] by Jiwon Kwon] + application correlation. Mock sends reject invalid dispatch IDs before + recording, using production's 1,024-byte UTF-8 limit. Mock sends do not + invoke outbound observer callbacks. [[#1316], [#1359] by Jiwon Kwon] diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index 4ceff407c..fde5aadc6 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -215,6 +215,63 @@ test("sent activity records preserve dispatch IDs", async () => { strictEqual(Object.hasOwn(federationRecords[2], "dispatchId"), false); }); +test("sendActivity validates dispatch IDs before recording", async () => { + const federation = createFederation(); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sender = { identifier: "alice" }; + const recipient = new Person({ + id: new URL("https://example.com/users/bob"), + }); + const activity = new Create({ + id: new URL("https://example.com/activities/invalid-dispatch-id"), + actor: new URL("https://example.com/users/alice"), + }); + + for (const dispatchId of [null, 1, {}]) { + await rejects( + context.sendActivity(sender, recipient, activity, { + dispatchId, + } as never), + { + name: "TypeError", + message: "dispatchId must be a string.", + }, + ); + } + for (const dispatchId of ["x".repeat(1025), "한".repeat(342)]) { + await rejects( + context.sendActivity(sender, recipient, activity, { dispatchId }), + { + name: "RangeError", + message: "dispatchId must not exceed 1,024 UTF-8 bytes.", + }, + ); + } + + strictEqual(context.getSentActivities().length, 0); + strictEqual(federation.sentActivities.length, 0); + + for (const dispatchId of ["x".repeat(1024), "한".repeat(341) + "a"]) { + await context.sendActivity(sender, recipient, activity, { dispatchId }); + } + deepStrictEqual( + federation.sentActivities.map((record) => record.sentOrder), + [1, 2], + ); + const expectedDispatchIds = ["x".repeat(1024), "한".repeat(341) + "a"]; + deepStrictEqual( + context.getSentActivities().map((record) => record.dispatchId), + expectedDispatchIds, + ); + deepStrictEqual( + federation.sentActivities.map((record) => record.dispatchId), + expectedDispatchIds, + ); +}); + test("reset clears sent activities", async () => { const mockFederation = createFederation(); const context = mockFederation.createContext( diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index dbafdd6f4..b1161c54c 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -1732,14 +1732,25 @@ class MockContext implements Context { activity: Activity, options?: any, ): Promise { + const dispatchId: unknown = options?.dispatchId; + if (dispatchId !== undefined && typeof dispatchId !== "string") { + return Promise.reject(new TypeError("dispatchId must be a string.")); + } + if ( + dispatchId !== undefined && + (dispatchId.length > 1024 || + new TextEncoder().encode(dispatchId).byteLength > 1024) + ) { + return Promise.reject( + new RangeError("dispatchId must not exceed 1,024 UTF-8 bytes."), + ); + } this.sentActivities.push({ sender, recipients, activity, rawActivity: options?.rawActivity, - ...(options?.dispatchId === undefined - ? {} - : { dispatchId: options.dispatchId }), + ...(dispatchId === undefined ? {} : { dispatchId }), }); // If this is a MockFederation, also record it there @@ -1750,9 +1761,7 @@ class MockContext implements Context { queue: queued ? "outbox" : undefined, activity, rawActivity: options?.rawActivity, - ...(options?.dispatchId === undefined - ? {} - : { dispatchId: options.dispatchId }), + ...(dispatchId === undefined ? {} : { dispatchId }), sentOrder: ++this.federation.sentCounter, }); } From b725e9a5b5703ca43914a864425e90a1bc977540 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 18:42:22 +0900 Subject: [PATCH 51/53] Simplify Fedify skill guidance Condense outbound observer guidance into a callback summary and direct readers to the sending manual for report variants and capture limits. Tighten wording across the related Claude plugin skills. Changelog: none Assisted-by: Codex:gpt-6.1-sol --- claude-plugin/skills/actor/SKILL.md | 2 +- claude-plugin/skills/docs/SKILL.md | 2 +- claude-plugin/skills/fedify/SKILL.md | 224 +++++++----------------- claude-plugin/skills/fep/SKILL.md | 4 +- claude-plugin/skills/migration/SKILL.md | 3 +- 5 files changed, 65 insertions(+), 170 deletions(-) diff --git a/claude-plugin/skills/actor/SKILL.md b/claude-plugin/skills/actor/SKILL.md index 32d65f5ab..cfdd1d48b 100644 --- a/claude-plugin/skills/actor/SKILL.md +++ b/claude-plugin/skills/actor/SKILL.md @@ -20,7 +20,7 @@ Walk through: 3. Chaining `.setKeyPairsDispatcher()` to supply RSA and Ed25519 keys. 4. Optionally configuring `mapActorAlias()` for handle-based or fixed-path aliases. -5. Making the actor discoverable via WebFinger. Fedify handles +5. Making the actor discoverable via WebFinger. Fedify handles `/.well-known/webfinger` automatically once the actor dispatcher is registered. diff --git a/claude-plugin/skills/docs/SKILL.md b/claude-plugin/skills/docs/SKILL.md index d7c9a9be1..56d6404e8 100644 --- a/claude-plugin/skills/docs/SKILL.md +++ b/claude-plugin/skills/docs/SKILL.md @@ -19,7 +19,7 @@ Fetch and explain Fedify documentation about “$ARGUMENTS”. The documentation index is at . -2. Summarise the key points with runnable TypeScript examples. +2. Summarize the key points with runnable TypeScript examples. 3. Mention related pages the user might also want to read. diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index d454dacbe..fb9bf14cd 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -244,171 +244,65 @@ deliveries have no retry. > those must be distinct activities. Use a fresh UUID or counter in the > fragment. -Use -`federation.setOutboundObservers({ onBroadcast(ctx, event) { ... }, onDelivery(ctx, event) { ... } })` -(also available on builders) for producer observations of -ordinary/outbox-context sends, original-payload forwarding, and automatic -portable replication. Add `onAttempt(ctx, report)` and -`onHttpExchange(ctx, report)` for concrete send and actual fetch results. All -callbacks are optional. Registration replaces the whole set; `{}` clears it. -Registration and builds copy callbacks, and active executions retain their -selection. Callbacks receive existing context/data without extra loading and -work without OpenTelemetry. Broadcast `event.activity` snapshots optional known -`id` and `type` strings, with `type` containing the Activity type IRI. The -reference contains no payload and does not establish authentication. Early -events can have an empty reference. Transformers can change it; missing new -facts must not be inferred from the old Activity. Activity IDs do not replace -broadcast or application dispatch identities. - -`onBroadcast` groups start and preparation results with an independent -`broadcastId`; `operation` distinguishes `send`, `forward`, and -`portable-replication`. Narrow `event.stage` before reading `event.result`, then -narrow its `type` (`prepared`, `skipped`, or `failed`). `onDelivery` reports -`planned`, `handoff`, or destination-local `preparation-failed` events with a -`deliveryId` and a `broadcast` reference (`known` with an `id`, or `unknown` -for legacy fanout work). Handoffs distinguish `direct`, -`enqueued` (with actual queue/task ID), and `enqueue-failed` (acceptance -unknown). Readiness, queue acceptance, and remote effects are distinct. -Portable fallback keeps one delivery across alternate gateways; replication -claims independent targets. Filtered recipients create no planned delivery. - -`onAttempt` emits one terminal `OutboundAttemptReport` per concrete send, -including workers and each gateway/retry, with a fresh `attemptId` under its -delivery. `httpExchangeCount` counts actual fetches, including challenges and -redirects; pre-fetch failures have zero. `onHttpExchange` emits immutable -`OutboundHttpExchangeReport` snapshots of actual request URL/method and response -status/text/available URL, or the original fetch error. `exchangeIndex` starts -at zero per attempt. No live transport objects are exposed; header snapshots -and a separate bounded body callback are opt-in. - -For attempt reports, narrow `report.outcome.response.type`: `final` has an -`exchangeIndex` and -`metadata`; `intermediate-only` and `undetermined` have `lastObserved`; -`not-observed` has no response. The last received challenge or redirect is not -necessarily final. A selected non-OK final response stays primary `http` failure -when cache/cleanup also fails; local errors appear in `diagnostics`. A final 2xx -plus a local error is still a failed attempt. Preserve `failure.error` -separately from the outward `outcome.error`; HTTP failure evidence may lack an -exception. Holds, drops, and failures before sender entry produce no attempt, -and attempt reports do not establish worker scheduling results. - -`onWorker(ctx, report)` reports each settled fanout/outbox execution, including -its instrumentation, with current worker context/data. It works independently -of attempt/exchange callbacks. `role` and `queue` identify the current worker; -`attemptsStarted` counts concrete sender entries, never child enqueues. Narrow -`result.type`: `completed` is local normal return (including handled failures -and abandonment), `failed` retains the escaping error, and `preparation-failed` -identifies the actual preparation phase. Outbox `delivery` identifies its -obligation; fanout has no single delivery. - -Read `actions` separately from the worker result. `scheduled` records a resolved -`enqueue`, `retry`, or `defer`; its `queue` identifies the actual task/counter. -Retry/defer includes ISO `delay` and optional per-route `notBefore` constraints. -`action-failed` retains enqueue rejection with `acceptance: "unknown"`, or a -failed `drop` step. `handoff`/`direct` records concrete fanout sender entry; -`abandoned` records a local reason. Each action lists affected `inboxUrls`, so -gateway outcomes can differ by route. Retry, deferral, and abandonment actions -retain contributing local preparation failures in `preparationFailures` entries -with `inboxUrl` and `failure` (phase/original error), even on a `completed` -worker. Worker-level `preparationFailures` retains all caught route preparation -failures even when a later gateway succeeds or native retry rethrows without a -scheduling action. An enqueue rejection remains the action's separate `error`. -These diagnostics are not serialized into retry messages. Child enqueues and -direct handoffs carry known delivery references. Accepted children survive -sibling/bookkeeping failure; a rejected batch leaves all submitted children -uncertain. Replay keeps delivery IDs with fresh child task IDs, without another -broadcast. Worker settlement does not establish backend acknowledgment, a -scheduled native retry, or remote success; later direct attempts may settle -after a setup failure has already ended the worker. Do not infer scheduling -from an attempt result. - -Header values default to disabled. `requestHeaders` and `responseHeaders` -independently accept `allow`, positive safe-integer `maxBytes`, and optional -`redact`. Names are literal, lowercase/deduplicated; selected credentials, -signatures and cookies are always `[redacted]`. Exchange `headers.entries` are -immutable runtime name/value pairs. Budget UTF-8 `name + ": " + value + "\r\n"` -after redaction; omit whole entries and report `limitReached`. Invalid -configuration preserves prior registration; later caller mutation cannot change -selected policy. Header capture does not enable body capture. - -`responseBodyCapture` requires `onBody(ctx, event)`, positive safe-integer -`maxBytes`, and finite positive `timeoutMs` (rounded up, at most 2^31-1); no -numeric defaults. Enabled `responses` defaults to `all`, or select -`non-success-http-responses` (outside 200–299), independently for each actual -response including intermediate ones. The callback receives grouping metadata, -request/response snapshots, a controlled byte stream and capture-only signal, -before its terminal exchange report. It can be registered alone. Pull bytes -incrementally and pass the signal to application writes; do not accumulate a -whole body just to record it. - -Exchange `bodyCapture` is absent when disabled/no response, otherwise narrow -`type`: `filtered`, `no-body`, `complete`, `limit-reached`, `deadline`, -`callback-returned`, `consumer-stopped` (original `reason`), or `failed` -(`phase: "body" | "observer"`, original `error`). `bytesExposed` is not durable -storage. Exact cap without EOF proves neither completeness nor truncation. -Return/throw/cancel/cap/deadline revokes even locked readers; absolute lifetime -includes no-pull/slow consumers and bounded cleanup. Late rejections are -handled, but arbitrary JavaScript/storage cannot be forced to stop. One source -reader is shared with diagnostics, with no application clone/tee or extra -wrapper queue. Capture cannot abort transport or rewrite HTTP/retry evidence. -Diagnostics have a separate one-second budget and 1 KiB preview; timeout -appends `… (timed out)`. Selected exchanges accumulate waiting time; -upstream/runtime/application memory is outside Fedify's buffering claim. - -`senderPayload: true` adds an immutable `sender-payload` privacy projection to -attempt reports. Narrow `report.senderPayload.type`: `available` exposes -`value`, `representation: "privacy-projection"`, and `redacted`; `unavailable` -distinguishes `disabled`, `not-materialized`, and `privacy-filter-unavailable`. -Recursive compact/prefixed/expanded blind-recipient fields are removed without -changing transport/signing input. Only audited bundled contexts, exact -generated root extension definitions, supported nested context URLs, or -unambiguous expanded JSON qualify; arbitrary aliases/overrides and unsupported -data fail closed. No extra loading/serialization, raw fallback, exact-wire or -proof-validity claim is provided. A redirect can drop its body independently of -attempt input. Evidence is privileged: URLs/content can still contain private -data and require application access control/redaction. See the sending manual -for typed examples. - -Metadata callbacks are awaited without a default timeout; observer/logging -errors are isolated. Nonsettling callbacks can delay execution indefinitely. -Portable forwarding can return at its existing deadline while background work -and observations continue. Detached recording must handle its own rejection, -shutdown, and persistence. Known IDs survive fanout expansion, retries, holds, -and gateway changes. Fanout expansion continues existing delivery handoffs -without starting another broadcast. Legacy fanout can have an unknown -broadcast; legacy outbox reports use the current task UUID as `deliveryId` and -omit an unavailable original `destinationInbox`. Missing source facts remain -unavailable, and older consumers can discard new metadata. Attempt/exchange -`queue` identifies the current message and available Fedify retry counter; -workers receive current context data without observation-only loader calls. - -Events expose `provenance` for preserved source/app correlation and `local` for -current URL facts. A known inbox source includes server-generated -`inboxRequestId`, informational `requestId`, and the original URL snapshot. -It survives context clones/conversions and queues; independent contexts do not -infer it from ambient execution. Inbox completion `report.local` retains the -original URL facts. `report.requestCancellation` is `not-aborted` or `aborted` -with the exact native request-signal `reason`, captured before report creation. -It does not change authentication/processing outcomes, cancel accepted work, -inherit a signal into descendants, or emit a second report after a later abort. -`observedAt` is current Unix milliseconds, not a global ordering guarantee. -Optional `trace` carries only valid IDs of the operation's own span, never an -ambient parent fallback; grouping works without tracing. Entry events and -exchanges omit trace when no own span exists. - -Send/forward options accept `dispatchId`, preserved exactly in event provenance -and queues even without producer observers. Empty strings are valid; the limit -is 1,024 UTF-8 bytes. Invalid types raise `TypeError` and oversized values -`RangeError` before preparation. Reusing an ID sends again with a fresh -broadcast ID. New public calls supply their own app ID; queued continuations -retain it. - -Mock federations accept registration without executing observers. Both -`context.getSentActivities()` and `federation.sentActivities` preserve optional -`dispatchId` values, including repeated IDs without suppressing sends. Incoming -C2S listeners use the separate `setOutboxListeners()` API. - -See . +### Observing outbound delivery + +Register callbacks with `federation.setOutboundObservers()` or the builder's +matching method. Registration replaces the whole set; `{}` clears it. Active +sends keep the callbacks selected when they began. Observers use the existing +context and work without OpenTelemetry. + +Choose callbacks by the question you need to answer: + +| Callback | Reports | +| ---------------- | -------------------------------------------------------------------------- | +| `onBroadcast` | One send or forward invocation, including preparation and its result. | +| `onDelivery` | Each destination's plan, handoff, or preparation failure. | +| `onAttempt` | The outcome of one concrete send, including each retry or gateway attempt. | +| `onHttpExchange` | Each actual HTTP request and its response or fetch error. | +| `onWorker` | The local result and scheduling actions of a fanout or outbox worker. | + +`onBroadcast` and `onDelivery` also cover automatic portable replication. +Broadcasts have a `broadcastId`; destinations have a `deliveryId`. An +`onDelivery` event can refer to an unknown broadcast when it comes from legacy +fanout work. These IDs group reports; they do not prove remote receipt. +The optional `event.activity` is a snapshot of known `id` and type IRI strings, +not the activity payload or an authentication result. A transformer may change +those facts during preparation. + +For broadcasts, narrow `event.stage` before reading `event.result.type`. +For deliveries, narrow `event.stage` and, for handoffs, `event.result.type`. +Attempt reports have a fresh `attemptId` and an `outcome`; their +`httpExchangeCount` counts actual fetches, including redirects and signature +challenges. A failure before the first fetch has a count of zero. Worker +reports distinguish normal local completion from an escaping error. Their +`actions` record enqueues, retries, deferrals, direct handoffs, abandonment, +and failed drops. A completed worker does not imply remote success or queue +acknowledgment. + +Header capture is off by default. Set `requestHeaders` or `responseHeaders` +with an allowlist and byte limit to add redacted header snapshots to exchange +reports. `responseBodyCapture` separately invokes `onBody` with a bounded +stream and deadline; read it incrementally and honor its cancellation signal. +Use `senderPayload: true` only when the attempt report needs a privacy-filtered +projection of the outgoing activity. These options do not change the bytes +sent or signed. Treat all captured evidence as sensitive application data. + +Callbacks are awaited, and callback or logging errors do not replace delivery +results. A callback that never settles can still delay the send. If you +record events in the background, handle rejection, shutdown, and persistence +in the application. + +Use the `dispatchId` send or forward option to carry an application correlation +ID through queues. Event `provenance` preserves known source facts; `local` +describes the current execution. Optional `trace` contains the operation's +own span IDs when tracing is enabled. Older queue messages can lack these +facts. Invalid `dispatchId` values raise an error before preparation; the +limit is 1,024 UTF-8 bytes. Reusing an ID does not deduplicate sends. + +For typed examples, report variants, and capture limits, see the +[sending manual]. + +[sending manual]: https://fedify.dev/manual/send.md Vocabulary imports diff --git a/claude-plugin/skills/fep/SKILL.md b/claude-plugin/skills/fep/SKILL.md index 88fc4ef48..d051f37ca 100644 --- a/claude-plugin/skills/fep/SKILL.md +++ b/claude-plugin/skills/fep/SKILL.md @@ -15,7 +15,7 @@ explain how to implement it with Fedify. Normalization ------------- -Normalise the identifier first: strip any leading `FEP-` or `fep-` prefix +Normalize the identifier first: strip any leading `FEP-` or `fep-` prefix and lowercase the result to get the bare four-character hex id (e.g. `8fcf`). Call that `$ID` in the steps below. If the result does not match `^[0-9a-f]{4}$`, stop and ask the user to provide a valid FEP identifier. @@ -39,7 +39,7 @@ cat "$FEP_DIR/fep/$ID/fep-$ID.md" Summary ------- -Summarise: status, what problem it solves, and what extensions it defines +Summarize: status, what problem it solves, and what extensions it defines (new JSON-LD terms, HTTP endpoints, or activity shapes). diff --git a/claude-plugin/skills/migration/SKILL.md b/claude-plugin/skills/migration/SKILL.md index 09e1082ee..4d8b120db 100644 --- a/claude-plugin/skills/migration/SKILL.md +++ b/claude-plugin/skills/migration/SKILL.md @@ -16,7 +16,8 @@ Migration workflow 1. Fetch *CHANGES.md* from the repo to identify breaking changes between the versions in question: `https://raw.githubusercontent.com/fedify-dev/fedify/main/CHANGES.md` -2. List every breaking change that affects the user's code range. +2. List the breaking changes between those versions that affect the user's + code. 3. For each breaking change, show the old API, the new API, and a concrete before/after code snippet. 4. Search the user's codebase for usages of deprecated symbols and suggest From ea81b8ba2f1b4af38eebade5ef1d6539db4d0ea5 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 21:36:04 +0900 Subject: [PATCH 52/53] Ignore malformed queued provenance Queued provenance is observation metadata. Warn and fall back to empty source facts when a consumer reads an invalid value, so delivery and retries continue. Keep producer validation strict. Update the sending manual and unreleased notes. https://github.com/fedify-dev/fedify/pull/1359#discussion_r4454098673 Assisted-by: Codex:gpt-6.1-sol --- CHANGES.md | 4 +- changes.d/fedify/outbound-observers.md | 4 +- claude-plugin/skills/fedify/SKILL.md | 6 +- docs/manual/send.md | 15 +-- packages/fedify/src/federation/middleware.ts | 18 +++- .../src/federation/outbound-dispatch.test.ts | 94 +++++++++++++++++-- .../federation/outbound-provenance.test.ts | 20 ---- 7 files changed, 117 insertions(+), 44 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 050b2a7e8..0a038110a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -55,7 +55,9 @@ To be released. Applications can opt in to redacted headers, bounded streaming response capture, and privacy-filtered sender payloads. Inbox reports now preserve original request context and snapshot request cancellation without changing - processing outcomes. Error diagnostic reads have a one-second deadline. + processing outcomes. Malformed optional queue metadata no longer prevents + delivery; consumers warn and continue without its correlation facts. Error + diagnostic reads have a one-second deadline. The public `Federatable` interface now requires `setOutboundObservers()`, breaking source compatibility for custom implementations. Update custom mocks, decorators, and wrappers to implement this method. diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md index d01eb1e93..13405d13b 100644 --- a/changes.d/fedify/outbound-observers.md +++ b/changes.d/fedify/outbound-observers.md @@ -10,7 +10,9 @@ links: Applications can opt in to redacted headers, bounded streaming response capture, and privacy-filtered sender payloads. Inbox reports now preserve original request context and snapshot request cancellation without changing - processing outcomes. Error diagnostic reads have a one-second deadline. + processing outcomes. Malformed optional queue metadata no longer prevents + delivery; consumers warn and continue without its correlation facts. Error + diagnostic reads have a one-second deadline. The public `Federatable` interface now requires `setOutboundObservers()`, breaking source compatibility for custom implementations. Update custom mocks, decorators, and wrappers to implement this method. diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index fb9bf14cd..f1e67bd7b 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -296,8 +296,10 @@ Use the `dispatchId` send or forward option to carry an application correlation ID through queues. Event `provenance` preserves known source facts; `local` describes the current execution. Optional `trace` contains the operation's own span IDs when tracing is enabled. Older queue messages can lack these -facts. Invalid `dispatchId` values raise an error before preparation; the -limit is 1,024 UTF-8 bytes. Reusing an ID does not deduplicate sends. +facts. Consumers warn and omit malformed queued provenance so delivery can +continue. Invalid `dispatchId` values on new sends raise an error before +preparation; the limit is 1,024 UTF-8 bytes. Reusing an ID does not +deduplicate sends. For typed examples, report variants, and capture limits, see the [sending manual]. diff --git a/docs/manual/send.md b/docs/manual/send.md index f730b2c5e..f5ba87624 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -670,9 +670,8 @@ federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") `saveAcceptance()` represents your application's record creation. Dispatch IDs may be empty but must fit within 1,024 UTF-8 bytes. Invalid types raise -`TypeError` and oversized strings raise `RangeError` before preparation; queue -consumers apply the same validation. A new call that omits the ID does not -inherit it from a previous invocation. +`TypeError` and oversized strings raise `RangeError` before preparation. A new +call that omits the ID does not inherit it from a previous invocation. To forward the received payload instead of constructing a new Activity, use the inbox handler's `forwardActivity()` method with the same application correlation @@ -700,10 +699,12 @@ Older messages remain processable, with missing relationships explicitly unknown. Legacy fanout can recover stable delivery grouping; legacy outbox reports use the current task UUID as the delivery ID and cannot recover broadcast membership. Unavailable original destinations and source facts remain -absent. Older consumers can discard new metadata, so mixed-version processing -can lose correlation. Source metadata and dispatch IDs propagate even without -producer observers; broadcast and delivery metadata is allocated when producer -observation is enabled. +absent. If queued source metadata is malformed, the consumer warns and continues +without its provenance; a retry carries the empty provenance rather than the +invalid value. Older consumers can discard new metadata, so mixed-version +processing can lose correlation. Source metadata and dispatch IDs propagate even +without producer observers; broadcast and delivery metadata is allocated when +producer observation is enabled. ### Capturing headers diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index bd183a22b..76d6bde8b 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1238,9 +1238,21 @@ export class FederationImpl context.active(), message.traceContext, ); - const provenance = message.type === "task" - ? emptyProvenance - : restoreProvenance(message.provenance); + let provenance: OutboundProvenance = emptyProvenance; + if (message.type !== "task") { + try { + provenance = restoreProvenance(message.provenance); + } catch (error) { + try { + getLogger(["fedify", "federation", "queue"]).warn( + "Ignoring invalid provenance in queue message {messageId}: {error}", + { messageId: message.id, error }, + ); + } catch { + // A logging sink must not prevent processing a queued task. + } + } + } if (message.type !== "task" && message.provenance !== undefined) { message = { ...message, provenance }; } diff --git a/packages/fedify/src/federation/outbound-dispatch.test.ts b/packages/fedify/src/federation/outbound-dispatch.test.ts index b59571130..b1acc5f96 100644 --- a/packages/fedify/src/federation/outbound-dispatch.test.ts +++ b/packages/fedify/src/federation/outbound-dispatch.test.ts @@ -285,7 +285,7 @@ test("dispatch IDs correlate send and forward entries without deduplication or c } }); -test("dispatch ID byte limits and queue validation preserve exact values", async () => { +test("dispatch ID byte limits and queue fallback preserve exact values", async () => { const messages: Message[] = []; const queue: MessageQueue = { enqueue(message) { @@ -312,6 +312,10 @@ test("dispatch ID byte limits and queue validation preserve exact values", async category: ["test", "dispatch"], sinks: ["buffer"], lowestLevel: "info", + }, { + category: ["fedify", "federation", "queue"], + sinks: ["buffer"], + lowestLevel: "warning", }, { category: ["logtape", "meta"], sinks: [] }], contextLocalStorage: new AsyncLocalStorage(), }); @@ -336,16 +340,41 @@ test("dispatch ID byte limits and queue validation preserve exact values", async try { const queued = messages[0]; ok(queued.type === "outbox"); - for (const dispatchId of [null, "x".repeat(1025), "😀".repeat(257)]) { - await rejects( - async () => - await consumer.processQueuedTask(undefined, { + const malformed = [ + { dispatchId: null }, + { dispatchId: "x".repeat(1025) }, + { dispatchId: "😀".repeat(257) }, + { + inboxRequest: { + inboxRequestId: "", + requestId: "bad", + context: { + origin: "x", + host: "x", + hostname: "x", + canonicalOrigin: "x", + }, + }, + }, + ]; + for (const provenance of malformed) { + await withContext( + { requestId: "current-request", dispatchId: "ambient-dispatch" }, + () => + consumer.processQueuedTask(undefined, { ...queued, - provenance: { dispatchId: dispatchId as string }, + provenance: provenance as NonNullable, }), - dispatchId === null ? TypeError : RangeError, ); - strictEqual(fetchMock.callHistory.calls().length, 0); + } + ok(fetchMock.callHistory.calls().length >= malformed.length); + const warnings = records.filter((record) => + record.rawMessage === + "Ignoring invalid provenance in queue message {messageId}: {error}" + ); + strictEqual(warnings.length, malformed.length); + for (const warning of warnings) { + strictEqual(warning.properties.messageId, queued.id); } for (let index = 0; index < messages.length; index++) { const message = messages[index]; @@ -363,11 +392,14 @@ test("dispatch ID byte limits and queue validation preserve exact values", async const logs = records.filter((record) => record.rawMessage === "original dispatch" ); - deepStrictEqual(logs.map((record) => record.properties.dispatchId), valid); + deepStrictEqual( + logs.map((record) => record.properties.dispatchId), + [...malformed.map(() => undefined), ...valid], + ); for (const record of logs) { strictEqual(record.properties.requestId, "current-request"); } - strictEqual(descendants.length, 6); + strictEqual(descendants.length, 2 * (malformed.length + valid.length)); for (const event of descendants) { strictEqual(event.provenance.dispatchId, undefined); } @@ -376,3 +408,45 @@ test("dispatch ID byte limits and queue validation preserve exact values", async await reset(); } }); + +test("malformed queued provenance does not poison a retry", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + queue: { outbox: queue }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, target, activity, { fanout: "skip" }); + const original = messages[0]; + ok(original.type === "outbox"); + const malformed = { + ...original, + provenance: { inboxRequest: { inboxRequestId: "" } }, + } as Message; + fetchMock.mockGlobal().post(target.inboxId!.href, 503); + try { + await federation.processQueuedTask(undefined, malformed); + strictEqual(messages.length, 2); + const retry = messages[1]; + ok(retry.type === "outbox"); + deepStrictEqual(retry.provenance, {}); + strictEqual(retry.attempt, 1); + await federation.processQueuedTask(undefined, retry); + strictEqual(messages.length, 3); + const next = messages[2]; + ok(next.type === "outbox"); + deepStrictEqual(next.provenance, {}); + } finally { + fetchMock.hardReset(); + } +}); diff --git a/packages/fedify/src/federation/outbound-provenance.test.ts b/packages/fedify/src/federation/outbound-provenance.test.ts index ad2f3d5fb..1d1faa211 100644 --- a/packages/fedify/src/federation/outbound-provenance.test.ts +++ b/packages/fedify/src/federation/outbound-provenance.test.ts @@ -608,26 +608,6 @@ test("queued provenance survives inbox retries, fanout, and outbox worker recons sourceInboxRequestId: "ambient-source", sourceRequestId: "ambient-label", }, () => federation.processQueuedTask(undefined, message)); - if (!legacy && !native) { - await rejects(async () => - await execute({ - ...task, - provenance: { - inboxRequest: { - inboxRequestId: "", - requestId: "bad", - context: { - origin: "x", - host: "x", - hostname: "x", - canonicalOrigin: "x", - }, - }, - }, - }), TypeError); - strictEqual(calls, 0); - strictEqual(events.length, 0); - } messages.length = 0; const logStart = records.length; if (native) { From 1e6f49396baba70d89757ea785e5496f807aa3ea Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 3 Nov 2026 22:01:20 +0900 Subject: [PATCH 53/53] Preserve transformed activity references Record both available object ID and type when validating transformed activities, including skipped reports and accessor failures. Preserve the original ID accessor error and remove the obsolete ID-only setter. https://github.com/fedify-dev/fedify/issues/1316 Assisted-by: Codex:gpt-6.1-sol --- claude-plugin/skills/fedify/SKILL.md | 5 +- docs/manual/send.md | 12 +- packages/fedify/src/federation/middleware.ts | 21 ++- .../src/federation/outbound-observation.ts | 11 +- .../src/federation/outbound-report.test.ts | 128 +++++++++++++++++- .../fedify/src/federation/outbound-report.ts | 9 +- 6 files changed, 160 insertions(+), 26 deletions(-) diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index f1e67bd7b..a2ed71474 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -265,9 +265,8 @@ Choose callbacks by the question you need to answer: Broadcasts have a `broadcastId`; destinations have a `deliveryId`. An `onDelivery` event can refer to an unknown broadcast when it comes from legacy fanout work. These IDs group reports; they do not prove remote receipt. -The optional `event.activity` is a snapshot of known `id` and type IRI strings, -not the activity payload or an authentication result. A transformer may change -those facts during preparation. +The event.activity reference is a snapshot of known id and type IRI strings; it +may be empty on started and can gain facts later. For broadcasts, narrow `event.stage` before reading `event.result.type`. For deliveries, narrow `event.stage` and, for handoffs, `event.result.type`. diff --git a/docs/manual/send.md b/docs/manual/send.md index f5ba87624..e8e6b7912 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -470,12 +470,14 @@ Routing and source metadata are immutable snapshots; the context, application data, Activity objects, and original exceptions are not deep-copied. Broadcast events include an immutable `activity` reference with optional `id` -and `type` strings; `type` is the Activity's type IRI. These are facts already -obtained by the send path, not a copy of the payload. The `started` event can +and `type` strings; `type` is the Activity class's type IRI. The reference +contains identity facts, not a copy of the payload. The `started` event can have an empty reference, and later events can contain more information. After -transformation, unavailable facts are omitted rather than copied from the old -Activity. Observation does not read additional Activity properties, parse -payloads, or load contexts to fill the reference. Activity IDs can repeat +successful transformation, both fields are refreshed from the resulting +Activity, including for skipped or early-failed sends. Unavailable facts are +omitted rather than copied from the old Activity. Reading the class type for +observation cannot replace a send error. Observation does not parse payloads or +load contexts to fill the reference. Activity IDs can repeat across broadcasts; use `broadcastId` for invocation identity and `dispatchId` for application correlation. A reported Activity ID does not establish authentication. diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 76d6bde8b..04b369f7c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -6441,9 +6441,26 @@ export class ContextImpl implements Context { for (const activityTransformer of this.federation.activityTransformers) { activity = activityTransformer(activity, this); } - const transformedActivityId = activity?.id?.href; + let transformedActivityId: string | undefined; + try { + transformedActivityId = activity?.id?.href; + } finally { + if (observation != null) { + let activityType: string | undefined; + try { + activityType = getTypeId(activity)?.href; + } catch { + // Observation must not replace an error from reading the Activity. + } + observation.setActivity({ + ...(transformedActivityId == null + ? {} + : { id: transformedActivityId }), + ...(activityType == null ? {} : { type: activityType }), + }); + } + } span?.setAttribute("activitypub.activity.id", transformedActivityId ?? ""); - observation?.setActivityId(transformedActivityId); observation?.setPhase("validation"); if (activity.actorId == null) { logger.error( diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts index 603d6097e..7be7949d8 100644 --- a/packages/fedify/src/federation/outbound-observation.ts +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -1146,20 +1146,11 @@ export class OutboundBroadcastObservation { if (phase === "destination-planning") this.#planning = "incomplete"; } - /** Replace the Activity facts already obtained by the producer path. */ + /** Replace the Activity facts captured by the producer path. */ setActivity(reference: OutboundActivityReference): void { this.#activity = Object.freeze({ ...reference }); } - /** Refresh the identifier from a producer read without retaining stale IDs. */ - setActivityId(id: string | undefined): void { - const reference: { id?: string; type?: string } = { - ...(id == null ? {} : { id }), - ...(this.#activity.type == null ? {} : { type: this.#activity.type }), - }; - this.#activity = Object.freeze(reference); - } - planned(destinationCount: number): void { this.#planning = "complete"; this.#destinationCount = destinationCount; diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts index 6748660aa..3028d1aab 100644 --- a/packages/fedify/src/federation/outbound-report.test.ts +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -1895,11 +1895,135 @@ test("broadcast activity references are immutable and follow transformed activit }); ok(Object.isFrozen(event.activity)); } - deepStrictEqual(finishes[2].activity, { id: transformedId }); - ok(!("type" in finishes[2].activity)); + deepStrictEqual(finishes[2].activity, { + id: transformedId, + type: Follow.typeId.href, + }); ok(Object.isFrozen(starts[0].activity)); }); +test("transformed activity type capture does not mask validation failures", async () => { + const validationMessage = + "The activity to send must have at least one actor property."; + class UnreadableTypeFollow extends Follow { + static override get typeId(): URL { + throw new Error("type must not be read"); + } + } + for (const unreadableType of [false, true]) { + const transformedId = unreadableType + ? "https://example.com/unreadable" + : "https://example.com/missing-actor"; + const federation = createFederation({ + ...options, + activityTransformers: [() => { + const ActivityClass = unreadableType ? UnreadableTypeFollow : Follow; + return new ActivityClass({ id: new URL(transformedId) } as never); + }], + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + + let rejected: unknown; + await rejects( + ctx.sendActivity(sender, recipient, activity), + (error: unknown) => { + rejected = error; + return error instanceof TypeError && + error.message === validationMessage; + }, + ); + const finished = events.find((event) => + event.stage === "preparation-finished" + ); + ok(finished); + strictEqual(finished.activity.id, transformedId); + if (unreadableType) { + ok(!("type" in finished.activity)); + } else { + strictEqual(finished.activity.type, Follow.typeId.href); + } + strictEqual(finished.result.type, "failed"); + if (finished.result.type === "failed") { + strictEqual(finished.result.failure.phase, "validation"); + strictEqual(finished.result.failure.error, rejected); + } + ok(Object.isFrozen(finished.activity)); + } +}); + +test("transformed activity ID getter failures retain the readable type", async () => { + const failure = new Error("transformed ID is unreadable"); + for (const unreadableType of [false, true]) { + let idReads = 0; + const transformed = new Follow({ + id: new URL("https://example.com/transformed"), + actor: new URL("https://example.com/alice"), + object: new URL("https://example.com/bob"), + }); + Object.defineProperty(transformed, "id", { + get() { + idReads++; + throw failure; + }, + }); + if (unreadableType) { + Object.defineProperty(transformed, "constructor", { + get() { + throw new Error("transformed type is unreadable"); + }, + }); + } + const federation = createFederation({ + ...options, + activityTransformers: [() => transformed], + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + + await rejects(ctx.sendActivity(sender, recipient, activity), (error) => { + strictEqual(error, failure); + return true; + }); + + strictEqual(idReads, 1); + const started = events.find((event) => event.stage === "started"); + ok(started); + deepStrictEqual(started.activity, {}); + ok(Object.isFrozen(started.activity)); + const finished = events.find((event) => + event.stage === "preparation-finished" + ); + ok(finished); + deepStrictEqual( + finished.activity, + unreadableType ? {} : { type: Follow.typeId.href }, + ); + ok(Object.isFrozen(finished.activity)); + strictEqual(finished.result.type, "failed"); + if (finished.result.type === "failed") { + strictEqual(finished.result.failure.phase, "transformation"); + strictEqual(finished.result.failure.error, failure); + } + } +}); + test("outbound preparation precedes queue acceptance and is not rewritten by enqueue failure", async (t) => { for (const role of ["outbox", "fanout"] as const) { for (const rejected of [false, true]) { diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts index 693c6306a..25155b8b9 100644 --- a/packages/fedify/src/federation/outbound-report.ts +++ b/packages/fedify/src/federation/outbound-report.ts @@ -55,15 +55,16 @@ export type OutboundPreparationResult = /** * Known Activity ID and type at a broadcast observation boundary. * - * Fields are included only after the send path has already obtained them. + * Fields are captured without serializing the Activity or loading contexts. + * Successful transformation refreshes both the ID and the Activity class type. * A `started` event may have an empty reference, and later events may reflect * Activity transformations that happened after that event. * @since 2.5.0 */ export type OutboundActivityReference = { - /** Activity identifier, when already available in the send path. */ + /** Activity identifier, when available at the observation boundary. */ readonly id?: string; - /** Activity type IRI, when already available in the send path. */ + /** Activity class type IRI, when available at the observation boundary. */ readonly type?: string; }; @@ -80,7 +81,7 @@ export type OutboundBroadcastEvent = readonly broadcastId: string; /** The outgoing operation being observed. */ readonly operation: "send" | "forward" | "portable-replication"; - /** Immutable Activity identity facts already obtained by the send path. */ + /** Immutable Activity identity facts captured by the send path. */ readonly activity: OutboundActivityReference; } & (