diff --git a/CHANGES.md b/CHANGES.md index 823ec2b1a..0a038110a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,6 +48,20 @@ To be released. `idempotency` strategy, as direct processing already did, instead of always deduplicating them per origin. Activities recorded by workers of older versions are still recognized. [[#1290], [#1299]] + - Added application-facing outbound observers for sends, forwarding, + replication, and queued retries, independently of OpenTelemetry. Optional + callbacks report preparation, delivery handoffs, send/HTTP results, and + worker actions with correlation IDs and preparation failure details. + Applications can opt in to redacted headers, bounded streaming response + capture, and privacy-filtered sender payloads. Inbox reports now preserve + original request context and snapshot request cancellation without changing + processing outcomes. Malformed optional queue metadata no longer prevents + delivery; consumers warn and continue without its correlation facts. Error + diagnostic reads have a one-second deadline. + The public `Federatable` interface now requires `setOutboundObservers()`, + breaking source compatibility for custom implementations. Update custom + mocks, decorators, and wrappers to implement this method. + [[#1316], [#1359] by Jiwon Kwon\] - Added the `activity-origin-mismatch` reason to rejected `InboxAuthentication` decisions in `onRequestFinished` reports. Inbox deliveries rejected because the activity's ID does not have the same origin @@ -87,9 +101,11 @@ To be released. [#1289]: https://github.com/fedify-dev/fedify/issues/1289 [#1290]: https://github.com/fedify-dev/fedify/issues/1290 [#1299]: https://github.com/fedify-dev/fedify/pull/1299 +[#1316]: https://github.com/fedify-dev/fedify/issues/1316 [#1318]: https://github.com/fedify-dev/fedify/issues/1318 [#1328]: https://github.com/fedify-dev/fedify/pull/1328 [#1353]: https://github.com/fedify-dev/fedify/pull/1353 +[#1359]: https://github.com/fedify-dev/fedify/pull/1359 ### @fedify/cli @@ -312,6 +328,12 @@ To be released. ### @fedify/testing + - Added outbound observer registration support to mock federations and + dispatch IDs to recorded sends. Tests can inspect `dispatchId` through + `context.getSentActivities()` and `federation.sentActivities` to assert + application correlation. Mock sends reject invalid dispatch IDs before + recording, using production's 1,024-byte UTF-8 limit. Mock sends do not + invoke outbound observer callbacks. [[#1316], [#1359] by Jiwon Kwon\] - Mock inbox contexts made by `createInboxContext()` now have the `authentication` property, which can be given as an option and defaults to evidence that nothing was authenticated. [[#1290], [#1299]] diff --git a/changes.d/fedify/outbound-observers.md b/changes.d/fedify/outbound-observers.md new file mode 100644 index 000000000..13405d13b --- /dev/null +++ b/changes.d/fedify/outbound-observers.md @@ -0,0 +1,19 @@ +--- +links: + '#1316': https://github.com/fedify-dev/fedify/issues/1316 + '#1359': https://github.com/fedify-dev/fedify/pull/1359 +--- + - Added application-facing outbound observers for sends, forwarding, + replication, and queued retries, independently of OpenTelemetry. Optional + callbacks report preparation, delivery handoffs, send/HTTP results, and + worker actions with correlation IDs and preparation failure details. + Applications can opt in to redacted headers, bounded streaming response + capture, and privacy-filtered sender payloads. Inbox reports now preserve + original request context and snapshot request cancellation without changing + processing outcomes. Malformed optional queue metadata no longer prevents + delivery; consumers warn and continue without its correlation facts. Error + diagnostic reads have a one-second deadline. + The public `Federatable` interface now requires `setOutboundObservers()`, + breaking source compatibility for custom implementations. Update custom + mocks, decorators, and wrappers to implement this method. + [[#1316], [#1359] by Jiwon Kwon] diff --git a/changes.d/testing/outbound-observers.md b/changes.d/testing/outbound-observers.md new file mode 100644 index 000000000..10f282b74 --- /dev/null +++ b/changes.d/testing/outbound-observers.md @@ -0,0 +1,11 @@ +--- +links: + '#1316': https://github.com/fedify-dev/fedify/issues/1316 + '#1359': https://github.com/fedify-dev/fedify/pull/1359 +--- + - Added outbound observer registration support to mock federations and + dispatch IDs to recorded sends. Tests can inspect `dispatchId` through + `context.getSentActivities()` and `federation.sentActivities` to assert + application correlation. Mock sends reject invalid dispatch IDs before + recording, using production's 1,024-byte UTF-8 limit. Mock sends do not + invoke outbound observer callbacks. [[#1316], [#1359] by Jiwon Kwon] diff --git a/claude-plugin/skills/actor/SKILL.md b/claude-plugin/skills/actor/SKILL.md index 32d65f5ab..cfdd1d48b 100644 --- a/claude-plugin/skills/actor/SKILL.md +++ b/claude-plugin/skills/actor/SKILL.md @@ -20,7 +20,7 @@ Walk through: 3. Chaining `.setKeyPairsDispatcher()` to supply RSA and Ed25519 keys. 4. Optionally configuring `mapActorAlias()` for handle-based or fixed-path aliases. -5. Making the actor discoverable via WebFinger. Fedify handles +5. Making the actor discoverable via WebFinger. Fedify handles `/.well-known/webfinger` automatically once the actor dispatcher is registered. diff --git a/claude-plugin/skills/docs/SKILL.md b/claude-plugin/skills/docs/SKILL.md index d7c9a9be1..56d6404e8 100644 --- a/claude-plugin/skills/docs/SKILL.md +++ b/claude-plugin/skills/docs/SKILL.md @@ -19,7 +19,7 @@ Fetch and explain Fedify documentation about “$ARGUMENTS”. The documentation index is at . -2. Summarise the key points with runnable TypeScript examples. +2. Summarize the key points with runnable TypeScript examples. 3. Mention related pages the user might also want to read. diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index c6385e05b..a2ed71474 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -244,7 +244,66 @@ deliveries have no retry. > those must be distinct activities. Use a fresh UUID or counter in the > fragment. -See . +### Observing outbound delivery + +Register callbacks with `federation.setOutboundObservers()` or the builder's +matching method. Registration replaces the whole set; `{}` clears it. Active +sends keep the callbacks selected when they began. Observers use the existing +context and work without OpenTelemetry. + +Choose callbacks by the question you need to answer: + +| Callback | Reports | +| ---------------- | -------------------------------------------------------------------------- | +| `onBroadcast` | One send or forward invocation, including preparation and its result. | +| `onDelivery` | Each destination's plan, handoff, or preparation failure. | +| `onAttempt` | The outcome of one concrete send, including each retry or gateway attempt. | +| `onHttpExchange` | Each actual HTTP request and its response or fetch error. | +| `onWorker` | The local result and scheduling actions of a fanout or outbox worker. | + +`onBroadcast` and `onDelivery` also cover automatic portable replication. +Broadcasts have a `broadcastId`; destinations have a `deliveryId`. An +`onDelivery` event can refer to an unknown broadcast when it comes from legacy +fanout work. These IDs group reports; they do not prove remote receipt. +The event.activity reference is a snapshot of known id and type IRI strings; it +may be empty on started and can gain facts later. + +For broadcasts, narrow `event.stage` before reading `event.result.type`. +For deliveries, narrow `event.stage` and, for handoffs, `event.result.type`. +Attempt reports have a fresh `attemptId` and an `outcome`; their +`httpExchangeCount` counts actual fetches, including redirects and signature +challenges. A failure before the first fetch has a count of zero. Worker +reports distinguish normal local completion from an escaping error. Their +`actions` record enqueues, retries, deferrals, direct handoffs, abandonment, +and failed drops. A completed worker does not imply remote success or queue +acknowledgment. + +Header capture is off by default. Set `requestHeaders` or `responseHeaders` +with an allowlist and byte limit to add redacted header snapshots to exchange +reports. `responseBodyCapture` separately invokes `onBody` with a bounded +stream and deadline; read it incrementally and honor its cancellation signal. +Use `senderPayload: true` only when the attempt report needs a privacy-filtered +projection of the outgoing activity. These options do not change the bytes +sent or signed. Treat all captured evidence as sensitive application data. + +Callbacks are awaited, and callback or logging errors do not replace delivery +results. A callback that never settles can still delay the send. If you +record events in the background, handle rejection, shutdown, and persistence +in the application. + +Use the `dispatchId` send or forward option to carry an application correlation +ID through queues. Event `provenance` preserves known source facts; `local` +describes the current execution. Optional `trace` contains the operation's +own span IDs when tracing is enabled. Older queue messages can lack these +facts. Consumers warn and omit malformed queued provenance so delivery can +continue. Invalid `dispatchId` values on new sends raise an error before +preparation; the limit is 1,024 UTF-8 bytes. Reusing an ID does not +deduplicate sends. + +For typed examples, report variants, and capture limits, see the +[sending manual]. + +[sending manual]: https://fedify.dev/manual/send.md Vocabulary imports diff --git a/claude-plugin/skills/fep/SKILL.md b/claude-plugin/skills/fep/SKILL.md index 88fc4ef48..d051f37ca 100644 --- a/claude-plugin/skills/fep/SKILL.md +++ b/claude-plugin/skills/fep/SKILL.md @@ -15,7 +15,7 @@ explain how to implement it with Fedify. Normalization ------------- -Normalise the identifier first: strip any leading `FEP-` or `fep-` prefix +Normalize the identifier first: strip any leading `FEP-` or `fep-` prefix and lowercase the result to get the bare four-character hex id (e.g. `8fcf`). Call that `$ID` in the steps below. If the result does not match `^[0-9a-f]{4}$`, stop and ask the user to provide a valid FEP identifier. @@ -39,7 +39,7 @@ cat "$FEP_DIR/fep/$ID/fep-$ID.md" Summary ------- -Summarise: status, what problem it solves, and what extensions it defines +Summarize: status, what problem it solves, and what extensions it defines (new JSON-LD terms, HTTP endpoints, or activity shapes). diff --git a/claude-plugin/skills/migration/SKILL.md b/claude-plugin/skills/migration/SKILL.md index 09e1082ee..4d8b120db 100644 --- a/claude-plugin/skills/migration/SKILL.md +++ b/claude-plugin/skills/migration/SKILL.md @@ -16,7 +16,8 @@ Migration workflow 1. Fetch *CHANGES.md* from the repo to identify breaking changes between the versions in question: `https://raw.githubusercontent.com/fedify-dev/fedify/main/CHANGES.md` -2. List every breaking change that affects the user's code range. +2. List the breaking changes between those versions that affect the user's + code. 3. For each breaking change, show the old API, the new API, and a concrete before/after code snippet. 4. Search the user's codebase for usages of deprecated symbols and suggest diff --git a/docs/manual/context.md b/docs/manual/context.md index 3f6561d1c..85fa1d672 100644 --- a/docs/manual/context.md +++ b/docs/manual/context.md @@ -84,6 +84,13 @@ The `Context` object has properties to get the base URL of the current request: For `RequestContext`, there is an additional property named `~RequestContext.url` that contains the full URL of the current request. +Outgoing observations snapshot these values under `event.local`. For work +caused by an inbox request, `event.provenance.inboxRequest.context` retains the +original URL facts through derived contexts and queues. Inbox completion reports +preserve the original values under `report.local` and separately snapshot +request cancellation; neither changes the lifetime of accepted queue work. See +[outgoing source and correlation](./send.md#original-source-and-current-context). + [^1]: If no canonical origin is explicitly configured, it is the same as the `Context.origin`. See also the [*Explicitly setting the canonical origin* section](./federation.md#explicitly-setting-the-canonical-origin) in the diff --git a/docs/manual/inbox.md b/docs/manual/inbox.md index 962323d92..f0820266f 100644 --- a/docs/manual/inbox.md +++ b/docs/manual/inbox.md @@ -206,6 +206,9 @@ federation .setInboxListeners("/users/{identifier}/inbox", "/inbox") .onRequestFinished(async (ctx, report) => { await saveReport(report); + if (report.requestCancellation.type === "aborted") { + console.log("Request aborted", report.requestCancellation.reason); + } }); ~~~~ @@ -215,6 +218,29 @@ covers failures while preparing document loaders or resolving a portable recipient. Inbox collection requests, unmatched routes, programmatic `routeActivity()` calls, and queue workers do not invoke it. +Since Fedify 2.5.0, routed HTTP inbox reports include `report.inboxRequestId` +and `report.requestId`, including preparation failures. The former is a fresh +server-generated ID for each HTTP delivery, even for the same Activity. The +latter is an informational, externally supplied or generated logging label; +it is not unique. These fields are absent when no routed source is known. + +`report.local` preserves the original `origin`, `host`, `hostname`, and +`canonicalOrigin`, even when a later context's URL changes. It is absent when +no routed source is known. `report.requestCancellation` snapshots the native +request signal immediately before report construction: `not-aborted`, or +`aborted` with the exact `reason`, including non-Error values. These envelopes +are immutable; the reason object itself is not cloned or frozen. + +Request cancellation is separate from processing outcome. A processed or +enqueued `202` can coexist with an aborted signal. The callback still runs once +when processing settles; an abort after the snapshot does not change it or +trigger another report. Observation does not cancel HTTP processing or pass its +signal to accepted queue work or outgoing descendants. + +Outgoing work using the handler's context retains this source through derived +contexts and queues. See +[outgoing source and correlation](./send.md#original-source-and-current-context). + `report.payload` is either `unavailable` or `parsed`, whose `value` contains the original JSON. A parsed JSON `null` is distinct from an unavailable body. `report.activity` contains the `Activity` obtained by the existing processing @@ -963,6 +989,9 @@ federation.setInboxListeners("/{identifier}/inbox", "/inbox") }) ~~~~ +To record forwarding preparation, delivery attempts, and HTTP results, see +[observing outgoing activities](./send.md#observing-outgoing-activities). + > [!NOTE] > The `~InboxContext.forwardActivity()` method does not guarantee that the > forwarded activity is successfully delivered to the recipient, since diff --git a/docs/manual/mq.md b/docs/manual/mq.md index 2cb99c78f..bc103d0d4 100644 --- a/docs/manual/mq.md +++ b/docs/manual/mq.md @@ -1297,6 +1297,38 @@ Optimized performance : Backend-specific optimizations for retry logic. +Observing outgoing workers +-------------------------- + +*This API is available since Fedify 2.5.0.* + +Register `onWorker` through `Federatable.setOutboundObservers()` to observe +local fanout and outbox execution, including calls to +`Federation.processQueuedTask()`. Reports include the current task, concrete +send count, local result, and actual child enqueue, retry, deferral, or +abandonment actions. They receive the current worker context and data, with +known delivery identities and source metadata restored from the queued message. +Inbox and custom-task processing use separate mechanisms. + +A completed worker report means the local path returned normally; it can include +a handled send failure or abandonment. Reports retain caught preparation errors +under `preparationFailures`, even when another gateway succeeds. Actions also +retain their contributing preparation failures, with affected routes and +operation phases. An accepted enqueue can survive a later worker failure, and a +rejected enqueue or batch does not prove the backend accepted nothing. Reports +do not establish backend acknowledgment or future native retry execution. Use +attempt observers for concrete send results and queue depth for messages still +waiting. See +[worker execution and actions](./send.md#worker-execution-and-actions) for +report variants and a typed example. + +Metadata callbacks are awaited without a default timeout. Optional +[response-body capture](./send.md#streaming-response-bodies) instead has +explicit byte/time limits; several selected exchanges can accumulate waiting +time within one worker execution. Applications own archival and must handle +recording work that outlives the capture signal or a backend deadline. + + Queue depth reporting --------------------- diff --git a/docs/manual/send.md b/docs/manual/send.md index 1f442a27d..e8e6b7912 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -419,6 +419,476 @@ an error and does not retry the delivery. [`@fedify/mysql`]: https://github.com/fedify-dev/fedify/tree/main/packages/mysql +Observing outgoing activities +----------------------------- + +*This API is available since Fedify 2.5.0.* + +Register callbacks with `Federatable.setOutboundObservers()` to record outgoing +sends, original-payload forwarding, and automatic portable-inbox replication. +The method is available on federation builders and directly created federations. +Each callback receives the current `Context` and an event or report, and is +awaited at its observation boundary: + +~~~~ typescript twoslash +import type { + Federation, + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "@fedify/fedify"; +declare const federation: Federation<{ requestId: string }>; +declare function recordBroadcast( + requestId: string, + event: OutboundBroadcastEvent, +): Promise; +declare function recordDelivery( + requestId: string, + event: OutboundDeliveryEvent, +): Promise; +// ---cut-before--- +federation.setOutboundObservers({ + async onBroadcast(ctx, event) { + await recordBroadcast(ctx.data.requestId, event); + if (event.stage === "preparation-finished" && event.result.type === "failed") { + console.error(event.result.failure.phase, event.result.failure.error); + } + }, + async onDelivery(ctx, event) { + await recordDelivery(ctx.data.requestId, event); + if (event.stage === "handoff" && event.result.type === "enqueue-failed") { + console.error("Queue acceptance is unknown", event.result.error); + } + }, +}); +~~~~ + +`recordBroadcast()` and `recordDelivery()` represent your application's +recording functions. Queued work uses the current worker's context and data. +The original HTTP source, when known, is recorded separately in `provenance`. +Observation does not load additional data or serialize an activity again. +Routing and source metadata are immutable snapshots; the context, application +data, Activity objects, and original exceptions are not deep-copied. + +Broadcast events include an immutable `activity` reference with optional `id` +and `type` strings; `type` is the Activity class's type IRI. The reference +contains identity facts, not a copy of the payload. The `started` event can +have an empty reference, and later events can contain more information. After +successful transformation, both fields are refreshed from the resulting +Activity, including for skipped or early-failed sends. Unavailable facts are +omitted rather than copied from the old Activity. Reading the class type for +observation cannot replace a send error. Observation does not parse payloads or +load contexts to fill the reference. Activity IDs can repeat +across broadcasts; use `broadcastId` for invocation identity and `dispatchId` +for application correlation. A reported Activity ID does not establish +authentication. + +`onBroadcast` reports one send, forward, or automatic portable-replication +invocation. Its `started` event runs before preparation; `preparation-finished` +describes whether the producer prepared work for delivery or fanout, skipped +sending, or failed. Fanout-ready work still needs worker-side delivery +preparation. A prepared result does not confirm queue acceptance or HTTP +delivery, and later send failures do not rewrite it. Skips such as +`no-destinations` or `forwarding-disabled` describe existing decisions; an +`unsigned` skip reflects the forwarding signature/proof shape check, not +authentication. + +`onDelivery` reports each deduplicated destination or claimed replication +target. Different inboxes on one host remain separate deliveries. A `planned` +event identifies the destination; a `handoff` records direct sender entry or +the result of enqueueing work. An `enqueued` result confirms producer +acceptance. An `enqueue-failed` result leaves backend acceptance unknown, +because the backend may have accepted work before rejecting the call. +Destination-local preparation failures are reported without inventing an +attempt. Earlier handoffs remain valid if preparation for a later destination +fails. + +`onAttempt` reports the result when a concrete send settles. Each sender entry +gets a fresh `attemptId`; retries and alternative gateways create new attempts. +`onHttpExchange` reports each actual fetch within an attempt, including signed +requests, redirects, and signature challenges. The pair of `attemptId` and +`exchangeIndex` identifies an exchange. A fetch that receives non-2xx headers +still has a `response` outcome; a rejected fetch has a `failed` outcome. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + onHttpExchange(_ctx, report) { + console.log(report.attemptId, report.exchangeIndex, report.request.url); + if (report.outcome.type === "response") { + console.log("HTTP response", report.outcome.response.status); + } + }, + onAttempt(_ctx, report) { + console.log(report.attemptId, report.httpExchangeCount); + const response = report.outcome.response; + if (response.type === "final") { + console.log("Final response", response.exchangeIndex, response.metadata.status); + } + if (report.outcome.type === "failed") { + console.error(report.outcome.failure.kind, report.outcome.failure.error); + } + }, +}); +~~~~ + +An attempt can fail during URL validation or signing before any fetch, leaving +`httpExchangeCount` at zero. Failures before sender entry, such as queue key +import or fanout parsing, and worker holds or drops create no attempt. + +An exchange response and an attempt's final response are different. A 401 +challenge followed by a successful retry produces two exchanges; the later +response is final. If signing the retry fails, the 401 remains intermediate. +`outcome.response` distinguishes `final`, `intermediate-only`, `not-observed`, +and `undetermined` evidence. A final response is retained even when later cache +or cleanup work fails. For a final non-OK response, HTTP failure remains primary +and additional local errors appear in `diagnostics`; a local failure after a +final 2xx still makes the attempt fail. `outcome.error` retains the value that +escaped the sender, while `failure.error` retains operation evidence when +available. Neither queue acceptance nor remote HTTP acceptance proves that the +remote application applied the activity. + +### Worker execution and actions + +`onWorker` receives one `OutboundWorkerReport` when a fanout or outbox worker's +local execution and instrumentation settle. `attemptsStarted` counts concrete +sender entries, including fallback gateways; enqueueing a child task does not +increase it. Outbox reports identify one delivery, while fanout reports carry +delivery references on their child actions. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + onWorker(_ctx, report) { + console.log(report.role, report.result.type, report.attemptsStarted); + for (const action of report.actions) { + if (action.type === "scheduled") { + console.log(action.action, action.queue.queueMessageId, action.inboxUrls); + if (action.action !== "enqueue") console.log(action.delay); + } + } + }, +}); +~~~~ + +Read `actions` separately from `result`. A `completed` result means the worker +returned normally, including handled failures or abandonment. Its actions record +accepted enqueues, retries and deferrals, direct handoffs, abandoned routes, and +failed scheduling or drop operations. Retry and deferral actions include their +actual delay and any per-route constraints. One gateway can be abandoned while +another continues. An empty action list means none was observed by settlement. + +`report.preparationFailures` retains caught preparation failures with their +`inboxUrl`, phase, and original error, even if another gateway later succeeds or +native retry handling rethrows a later error without scheduling an action. +When a failure contributes to a retry, deferral, or abandonment, the action's +`preparationFailures` also identifies that cause. The worker can still be +`completed` after handling a failure. If enqueueing fails, the action's `error` +describes the enqueue failure separately. + +Accepted actions survive later failures; a rejected batch leaves every submitted +child's acceptance unknown. Earlier direct attempts can continue after a later +preparation failure settles the worker. A local result does not establish +backend acknowledgment or future native redelivery. See +[observing outgoing workers](./mq.md#observing-outgoing-workers). + +### Identities across queues + +`broadcastId` groups an invocation and its continuations. `deliveryId` +identifies one destination obligation and survives fanout replay, retries, +holds, and route changes. Alternative portable gateways share one delivery; +automatic replication creates independent deliveries. Each concrete attempt +gets a new `attemptId`. Queue task IDs and `queueAttempt` describe scheduling, +not the number of send executions. Native redelivery can reuse both while +creating a fresh attempt. + +Identity allocation and propagation follow these rules: + +| Field | Allocation or source | Propagation and replacement | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `inboxRequestId` | Fresh server-generated ID for each routed HTTP inbox request, before preparation. | Preserved through inbox retries, derived contexts, outgoing sends, fanout, outbox retries, and replication. Repeated deliveries of the same Activity have different IDs. | +| Original logging `requestId` | The existing logging label selected at HTTP ingress; externally supplied values are informational and nonunique. | Preserved as source metadata through queues. It does not replace an unrelated worker's current logging label. | +| Original URL snapshot | The incoming request context's `origin`, `host`, `hostname`, and `canonicalOrigin`. | Preserved unchanged through derived contexts and queues, independently of current worker configuration. | +| `dispatchId` | Optional application-owned send or forward option. | Preserved through that invocation's descendants. New calls can reuse it without suppressing sends; a call that omits it does not inherit a previous invocation's value. | +| `broadcastId` | Fresh ID for each observed public send/forward invocation or internal portable-replication invocation, before preparation. | Preserved through fanout replay and retries. Another public invocation creates a new broadcast, even for the same Activity. | +| `deliveryId` | Allocated for each deduplicated destination or claimed replication target, before its handoff. | Preserved through fanout replay, native redelivery, retries, holds, fallback, and gateway-list changes. Alternative routes share a delivery; replication targets have separate deliveries. | +| `attemptId` | Fresh ID at each concrete sender entry. | Never reused on retries or native redelivery, and never derived from a queue round or trace ID. | +| `exchangeIndex` | Starts at zero and increments for actual fetch submissions within an attempt. | Identifies an exchange together with `attemptId`; redirects and signature challenges remain within their attempt. | +| `queueMessageId` | The current queue task's actual UUID. | Fanout expansion or replay can create new task IDs without creating new logical deliveries. | +| `queueAttempt` | The current message's Fedify retry counter, when available. | Native redelivery can leave it unchanged. One round can include multiple gateway attempts; it is not an execution ordinal. | +| Trace/span IDs | The observed operation's own valid OpenTelemetry span, when available. | Optional supplementary context. Worker and attempt reports use their own spans; trace IDs never replace grouping identities. | + +IDs provide grouping, not ordering, deduplication, or remote idempotency. +Callbacks can interleave or repeat, and a crash or operation that never settles +can leave an invocation without a terminal report. Applications own persistence +and aggregate delivery state. + +A failed attempt receiving HTTP 503 and a later successful retry receiving HTTP +202 share the same `deliveryId`, but have different `attemptId` values. The +worker report records the scheduled retry and its queue task ID. + +### Original source and current context + +`local` describes the context invoking the callback. For inbox-derived work, +`provenance.inboxRequest` preserves the original `inboxRequestId`, informational +logging `requestId`, and URL snapshot through context clones, conversions, new +sends, forwarding, and queues. Both URL snapshots contain `origin`, `host`, +`hostname`, and `canonicalOrigin`. Independent contexts do not acquire an inbox +cause from ambient execution. Worker logs use `sourceInboxRequestId` and +`sourceRequestId` without replacing an unrelated current request's label. +See [observing inbox requests](./inbox.md#observing-inbox-requests). + +Send and forward options accept an application-owned `dispatchId` for grouping +related calls, including manual retries. It is preserved exactly through +outgoing queue continuations and never suppresses a send. For example, an inbox +listener can associate an acceptance with its application record: + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +declare function saveAcceptance(follow: import("@fedify/vocab").Follow): Promise; +// ---cut-before--- +import { Accept, Follow } from "@fedify/vocab"; + +federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .on(Follow, async (ctx, follow) => { + if (follow.id == null || follow.objectId == null) return; + const actor = ctx.parseUri(follow.objectId); + if (actor?.type !== "actor") return; + const follower = await follow.getActor(ctx); + if (follower == null) return; + const dispatchId = await saveAcceptance(follow); + await ctx.sendActivity( + { identifier: actor.identifier }, + follower, + new Accept({ actor: follow.objectId, object: follow.id }), + { dispatchId }, + ); + }); +~~~~ + +`saveAcceptance()` represents your application's record creation. Dispatch IDs +may be empty but must fit within 1,024 UTF-8 bytes. Invalid types raise +`TypeError` and oversized strings raise `RangeError` before preparation. A new +call that omits the ID does not inherit it from a previous invocation. + +To forward the received payload instead of constructing a new Activity, use the +inbox handler's `forwardActivity()` method with the same application correlation +option: + +~~~~ typescript twoslash +import type { InboxContext } from "@fedify/fedify"; +declare const ctx: InboxContext; +declare const dispatchId: string; +// ---cut-before--- +await ctx.forwardActivity( + { identifier: "alice" }, + "followers", + { dispatchId, skipIfUnsigned: true }, +); +~~~~ + +This creates a `forward` broadcast, preserves the received payload, and carries +the original inbox provenance and `dispatchId` into outgoing observations and +queue continuations. The example skips unsigned activities. See +[forwarding activities](./inbox.md#forwarding-activities-to-another-server) for +delivery and signature considerations. + +Older messages remain processable, with missing relationships explicitly +unknown. Legacy fanout can recover stable delivery grouping; legacy outbox +reports use the current task UUID as the delivery ID and cannot recover +broadcast membership. Unavailable original destinations and source facts remain +absent. If queued source metadata is malformed, the consumer warns and continues +without its provenance; a retry carries the empty provenance rather than the +invalid value. Older consumers can discard new metadata, so mixed-version +processing can lose correlation. Source metadata and dispatch IDs propagate even +without producer observers; broadcast and delivery metadata is allocated when +producer observation is enabled. + +### Capturing headers + +Header values are disabled by default. Configure `requestHeaders` and +`responseHeaders` independently with literal names in `allow` and a positive +safe-integer `maxBytes`. Optional `redact` names add application-specific +secrets. Names are normalized to lowercase; wildcards are not supported. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + requestHeaders: { allow: ["content-type", "signature"], maxBytes: 2048 }, + responseHeaders: { allow: ["content-type", "retry-after", "set-cookie"], maxBytes: 1024 }, + onHttpExchange(_ctx, report) { + console.log(report.request.headers?.entries); + if (report.outcome.type === "response") { + console.log(report.outcome.response.headers?.entries); + console.log(report.outcome.response.headers?.limitReached); + } + }, +}); +~~~~ + +Credentials, signatures, and cookies in the built-in protected set are always +redacted, even when allowlisted. Add custom credential headers to `redact`. +Snapshots contain immutable name/value pairs in lexical name order, using the +runtime's representation of repeated headers. The budget counts UTF-8 bytes of +`name + ": " + value + "\r\n"` after redaction. Oversized entries are omitted +whole, and `limitReached` records omissions. An absent snapshot means capture +was disabled; an enabled empty allowlist produces an empty snapshot. Each +signed request, challenge retry, and redirect captures its actual submitted +headers. + +### Streaming response bodies + +Body capture is disabled unless `responseBodyCapture` is supplied. It is +independent of header capture and can be used without metadata callbacks. +Provide `onBody`, a positive safe-integer `maxBytes`, and a finite positive +`timeoutMs`. Fractional milliseconds round up; the rounded lifetime cannot +exceed 2,147,483,647 ms. Invalid configuration preserves the previous +registration. + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +declare function saveChunk( + attemptId: string, + exchangeIndex: number, + chunk: Uint8Array, + signal: AbortSignal, +): Promise; +// ---cut-before--- +federation.setOutboundObservers({ + responseBodyCapture: { + maxBytes: 64 * 1024, + timeoutMs: 5000, + responses: "non-success-http-responses", + async onBody(_ctx, event) { + const reader = event.body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + await saveChunk(event.attemptId, event.exchangeIndex, chunk.value, event.signal); + } + } catch (error) { + if (!event.signal.aborted) throw error; + } finally { + reader.releaseLock(); + } + }, + }, + onHttpExchange(_ctx, report) { + console.log(report.attemptId, report.exchangeIndex, report.bodyCapture); + }, +}); +~~~~ + +`responses` defaults to `all` when capture is enabled. The example selects +`non-success-http-responses`, including intermediate challenges and redirects; +selection does not predict the attempt's final result. Its limits are +illustrative. + +`onBody` receives the current context, grouping metadata, immutable HTTP +metadata, a controlled stream, and a capture-only signal. It runs after +response headers and before the exchange report. Fedify uses one source reader +for capture and diagnostics, with backpressure and no clone, tee, or whole-body +accumulation. Runtime buffering, upstream chunk allocation, and application +storage remain outside these bounds. + +The report's `bodyCapture` summary distinguishes disabled or filtered capture, +a missing body, EOF, a byte limit, a deadline, early callback return, consumer +cancellation, and body or observer failure. `bytesExposed` counts bytes handed +to application code, not bytes durably saved. Reaching the exact cap does not +establish EOF, so `limit-reached` leaves completeness uncertain. + +The lifetime includes waiting for a consumer that is slow or never reads, as +well as cleanup. Callback settlement, consumer cancellation, the cap, or the +deadline revokes access even through retained locked readers. Late promise +rejections are handled, but application storage that ignores the signal can +continue. Applications must handle interrupted writes and archive cleanup +themselves. + +Capture failures retain observed HTTP facts and do not independently trigger a +retry or skip a challenge. Error diagnostics have a separate one-second budget +and retain at most 1 KiB, appending `… (timed out)` or `… (truncated)` as +appropriate. Waiting for capture on multiple exchanges can add to total send +time. + +### Inspecting the sender payload + +Enable `senderPayload: true` to include a privacy-filtered copy of the concrete +sender's existing JSON input in attempt reports. It covers immediate and queued +sends, forwarding, and portable replication without additional serialization or +context loading: + +~~~~ typescript twoslash +import type { Federation } from "@fedify/fedify"; +declare const federation: Federation; +// ---cut-before--- +federation.setOutboundObservers({ + senderPayload: true, + onAttempt(_ctx, report) { + const document = report.senderPayload; + if (document.type === "available") { + console.log(document.stage, document.representation, document.redacted); + console.log(document.value); + } else { + console.log("Sender payload unavailable", document.reason); + } + }, +}); +~~~~ + +An available payload is an immutable `privacy-projection`. It removes compact, +`as:`-prefixed, and full-IRI `bto`/`bcc` fields throughout nested content, +including proof-bearing objects. `redacted` indicates that fields were removed. +It does not modify transport or signing input, establish proof validity, or +represent exact wire bytes. Redirects can change the actual request body +independently. + +Unsupported contexts, aliases, getters, custom JSON conversion, cyclic data, or +excessive nesting fail closed with `privacy-filter-unavailable`; there is no raw +fallback. `disabled` means export was not enabled, and `not-materialized` means +the sender did not reach the snapshot boundary. Supported context forms and +field definitions are described by `OutboundSenderPayload`. + +Headers, bodies, and payloads remain privileged application data. URLs and +content can contain private values beyond blind recipients. Your application +owns access control, additional redaction, archival, and saved original +dispatch documents. + +### Registration and callback behavior + +Registering again replaces the complete observer set; pass `{}` to clear it. +Builders copy registrations into built federations, and active invocations and +workers retain their selected callbacks. Any callback can be registered alone. +Mock federations accept registration but do not execute these observers. + +Observer errors, including logging errors, are isolated from the original result +and retry ownership. Metadata callbacks are awaited without a default timeout; +a callback that never settles can delay processing indefinitely. Detached +recording must handle its own rejections and shutdown. Neither awaited recording +nor detached work makes persistence atomic with network effects or queue +acceptance. Portable-forwarding deadlines can return control while sends and +observation continue; the deadline alone does not create a failed attempt. + +Each event or report has an `observedAt` Unix timestamp in milliseconds. It does +not establish ordering across workers. Observation works without OpenTelemetry +or sampling. Optional `trace` references the operation's own valid span, +including unsampled spans; broadcast starts and HTTP exchanges omit it because +they have no span of their own at those boundaries. + +Inbox request cancellation is observed separately from processing. Queue intake +shutdown and per-execution cancellation are also distinct. Execution +cancellation is not currently reported by this API. Incoming C2S +`setOutboxListeners()` registration serves a separate purpose from outbound +observation. + + Optimizing activity delivery for large audiences ------------------------------------------------ diff --git a/packages/fedify/src/cfworkers/expected-tests.ts b/packages/fedify/src/cfworkers/expected-tests.ts index 04c51f943..ac9ee715e 100644 --- a/packages/fedify/src/cfworkers/expected-tests.ts +++ b/packages/fedify/src/cfworkers/expected-tests.ts @@ -1,5 +1,7 @@ // The Workers suite is deliberately bounded to tests validated on workerd. export const expectedTestNames = [ + "outbound sender payload privacy on Workers", + "outbound body capture revocation on Workers", "authentication preserves the queued principal wire format", "proof principals adapt internal signer kinds at the public boundary", "ignored nested test steps return false", diff --git a/packages/fedify/src/federation/builder.ts b/packages/fedify/src/federation/builder.ts index 4b660ba0d..268368598 100644 --- a/packages/fedify/src/federation/builder.ts +++ b/packages/fedify/src/federation/builder.ts @@ -1,3 +1,7 @@ +import { snapshotSenderPayloadOption } from "./outbound-document.ts"; +import { snapshotBodyCapture } from "./outbound-capture.ts"; +import { snapshotHeaderPolicy } from "./outbound-headers.ts"; +import type { OutboundObservers } from "./outbound-report.ts"; import type { InboxRequestFinishedHandler } from "./inbox-report.ts"; import { assertPath, @@ -183,6 +187,9 @@ export class FederationBuilderImpl inboxListeners?: ActivityListenerSet>; outboxListeners?: ActivityListenerSet>; inboxRequestFinishedHandler?: InboxRequestFinishedHandler; + outboundObservers: OutboundObservers = globalThis.Object.freeze( + {}, + ); inboxErrorHandler?: InboxErrorHandler; outboxListenerErrorHandler?: OutboxListenerErrorHandler; @@ -288,6 +295,7 @@ export class FederationBuilderImpl f.inboxListeners = this.inboxListeners?.clone(); f.outboxListeners = this.outboxListeners?.clone(); f.inboxRequestFinishedHandler = this.inboxRequestFinishedHandler; + f.setOutboundObservers(this.outboundObservers); f.inboxErrorHandler = this.inboxErrorHandler; f.outboxListenerErrorHandler = this.outboxListenerErrorHandler; f.outboxAuthorizePredicate = this.outboxAuthorizePredicate; @@ -658,6 +666,20 @@ export class FederationBuilderImpl return setters; } + setOutboundObservers(observers: OutboundObservers): void { + this.outboundObservers = globalThis.Object.freeze({ + senderPayload: snapshotSenderPayloadOption(observers.senderPayload), + responseBodyCapture: snapshotBodyCapture(observers.responseBodyCapture), + requestHeaders: snapshotHeaderPolicy(observers.requestHeaders), + responseHeaders: snapshotHeaderPolicy(observers.responseHeaders), + onBroadcast: observers.onBroadcast, + onDelivery: observers.onDelivery, + onAttempt: observers.onAttempt, + onHttpExchange: observers.onHttpExchange, + onWorker: observers.onWorker, + }); + } + setNodeInfoDispatcher( path: string, dispatcher: NodeInfoDispatcher, diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index ed1ebd19f..50689d9f1 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -1867,6 +1867,15 @@ export type ParseUriResult = * Options for {@link Context.sendActivity} method. */ export interface SendActivityOptions { + /** + * Application-owned correlation for this invocation, never a deduplication + * key. Preserved exactly, including an empty string. At most 1,024 UTF-8 + * bytes; invalid types and oversized values are rejected before preparation. + * Omission means no application dispatch ID for this new invocation. + * @since 2.5.0 + */ + readonly dispatchId?: string; + /** * Whether to prefer the shared inbox for the recipients. */ diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index 286d9c830..78523ff85 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -1,3 +1,4 @@ +import type { OutboundObservers } from "./outbound-report.ts"; import type { InboxListenerOptions } from "./activity-listener.ts"; import type { InboxRequestFinishedHandler } from "./inbox-report.ts"; import type { @@ -82,6 +83,32 @@ export interface FederationStartQueueOptions { * @since 1.6.0 */ export interface Federatable extends TaskRegistry { + /** + * Registers observers for outgoing activity processing. + * + * Each call replaces all previously registered outbound observers. + * Pass an empty object to clear them. Fedify copies callbacks and capture policies; + * later changes to the supplied object do not affect registration. + * Available on builders and directly created federations. Building copies + * the callbacks, and an active send keeps the callback selected at entry. + * Observes send, original-payload forwarding, and automatic portable + * replication preparation, destination planning, and handoffs, including + * outbox-context operations and fanout continuation handoffs. Separate + * callbacks report concrete send attempts and actual HTTP exchanges, including + * queued retries and gateway fallback. Worker reports retain local execution + * results and actual enqueue, retry, deferral, and abandonment actions, + * separately from attempts and backend acknowledgment. + * Known IDs, original inbox source, and + * application correlation survive queue work; workers use current context data. + * Header values, bounded response-body streaming and sender-payload projections + * are independently opt-in. Body consumers require explicit byte/time limits; + * other metadata callbacks have no default timeout. + * This is separate from incoming C2S {@link Federatable.setOutboxListeners}. + * @param observers The optional callbacks to register. + * @since 2.5.0 + */ + setOutboundObservers(observers: OutboundObservers): void; + /** * Registers a NodeInfo dispatcher. * @param path The URI path pattern for the NodeInfo dispatcher. The syntax diff --git a/packages/fedify/src/federation/inbox-observation.ts b/packages/fedify/src/federation/inbox-observation.ts index 576c1fe6c..2d495508b 100644 --- a/packages/fedify/src/federation/inbox-observation.ts +++ b/packages/fedify/src/federation/inbox-observation.ts @@ -1,6 +1,8 @@ import { Activity } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; import type { Span } from "@opentelemetry/api"; +import type { OutboundInboxRequest } from "./outbound-report.ts"; +import { withContext } from "@logtape/logtape"; import type { RequestContext } from "./context.ts"; import type { InboxAuthentication, @@ -47,7 +49,10 @@ export class InboxObservation { } }, }; - constructor(readonly inbox: InboxRequestReport["inbox"]) {} + constructor( + readonly inbox: InboxRequestReport["inbox"], + readonly source: OutboundInboxRequest | undefined = undefined, + ) {} /** Run an ingress operation and project telemetry; fetch owns completion. */ async run( @@ -122,7 +127,18 @@ export class InboxObservation { outcome: InboxRequestOutcome, ): Promise { if (handler == null) return; + const signal = context.request.signal; + const requestCancellation: InboxRequestReport["requestCancellation"] = + signal.aborted + ? Object.freeze({ type: "aborted", reason: signal.reason }) + : Object.freeze({ type: "not-aborted" }); const report: InboxRequestReport = { + ...(this.source == null ? {} : { + inboxRequestId: this.source.inboxRequestId, + requestId: this.source.requestId, + local: Object.freeze({ ...this.source.context }), + }), + requestCancellation, inbox: this.inbox, payload: this.payload, activity: this.activity, @@ -131,7 +147,12 @@ export class InboxObservation { outcome, }; try { - await handler(context, report); + await withContext( + this.source == null + ? {} + : { inboxRequestId: this.source.inboxRequestId }, + () => handler(context, report), + ); } catch (error) { try { getLogger(["fedify", "federation", "inbox"]).error( diff --git a/packages/fedify/src/federation/inbox-report.test.ts b/packages/fedify/src/federation/inbox-report.test.ts index 236762748..cc3fdf8bd 100644 --- a/packages/fedify/src/federation/inbox-report.test.ts +++ b/packages/fedify/src/federation/inbox-report.test.ts @@ -9,7 +9,8 @@ import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; import { createFederationBuilder } from "./builder.ts"; import type { InboxRequestReport } from "./inbox-report.ts"; import { MemoryKvStore } from "./kv.ts"; -import { createFederation } from "./middleware.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; +import type { Message } from "./queue.ts"; import { signRequest } from "../sig/http.ts"; import { signJsonLd } from "../sig/ld.ts"; import { signObject } from "../sig/proof.ts"; @@ -673,3 +674,246 @@ test("onRequestFinished observes the final response decoration result", async () }); } }); + +test("inbox request cancellation snapshots preserve exact reasons without stopping processing", async (t) => { + for ( + const [label, reason] of [ + ["live", undefined], + ["object", { kind: "client disconnect" }], + ["zero", 0], + ["nan", NaN], + ["symbol", Symbol("client")], + ] as const + ) { + await t.step(label, async () => { + const { federation, setters, reports } = setup(true); + let processed = 0; + setters.on(Create, () => { + processed++; + }); + const controller = new AbortController(); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + input.headers.set("X-Request-Id", "cancellation-case"); + if (label !== "live") controller.abort(reason); + const response = await federation.fetch(input, { + contextData: undefined, + }); + strictEqual(response.status, 202); + strictEqual(processed, 1); + strictEqual(reports.length, 1); + const report = reports[0]; + deepStrictEqual(report.outcome, { + type: "response", + status: 202, + disposition: "processed", + }); + deepStrictEqual(report.authentication, { status: "skipped" }); + strictEqual(report.payload.status, "parsed"); + strictEqual(report.requestId, "cancellation-case"); + ok(report.inboxRequestId); + deepStrictEqual(report.local, { + origin: "https://example.com", + host: "example.com", + hostname: "example.com", + canonicalOrigin: "https://example.com", + }); + ok(Object.isFrozen(report.local)); + ok(Object.isFrozen(report.requestCancellation)); + strictEqual( + report.requestCancellation.type, + label === "live" ? "not-aborted" : "aborted", + ); + if (report.requestCancellation.type === "aborted") { + ok(Object.is(report.requestCancellation.reason, reason)); + strictEqual(report.requestCancellation.reason, input.signal.reason); + } + }); + } +}); + +test("inbox request cancellation waits for processing and preserves original local facts", async () => { + const { federation, setters, reports } = setup(true); + let entered!: () => void; + const listening = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + setters.on(Create, async (ctx) => { + ok(ctx instanceof ContextImpl); + ctx.url.hostname = "changed.example"; + entered(); + await gate; + }); + const controller = new AbortController(); + const reason = Symbol("mid-processing disconnect"); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + const processing = federation.fetch(input, { contextData: undefined }); + try { + await listening; + controller.abort(reason); + strictEqual(reports.length, 0); + release(); + strictEqual((await processing).status, 202); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].outcome, { + type: "response", + status: 202, + disposition: "processed", + }); + deepStrictEqual(reports[0].requestCancellation, { + type: "aborted", + reason, + }); + strictEqual(reports[0].local?.hostname, "example.com"); + controller.abort("later reason"); + await Promise.resolve(); + strictEqual(reports.length, 1); + strictEqual(reports[0].local?.origin, "https://example.com"); + } finally { + release(); + await processing; + } +}); + +test("inbox request cancellation after report construction does not rewrite its snapshot", async () => { + const { federation, setters, reports } = setup(true); + setters.on(Create, () => {}); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + setters.onRequestFinished(async (_ctx, report) => { + reports.push(report); + entered(); + await gate; + throw new Error("recording failure"); + }); + const controller = new AbortController(); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + const processing = federation.fetch(input, { contextData: undefined }); + try { + await observing; + controller.abort({ kind: "late disconnect" }); + deepStrictEqual(reports[0].requestCancellation, { type: "not-aborted" }); + release(); + strictEqual((await processing).status, 202); + await Promise.resolve(); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].requestCancellation, { type: "not-aborted" }); + deepStrictEqual(reports[0].outcome, { + type: "response", + status: 202, + disposition: "processed", + }); + } finally { + release(); + await processing; + } +}); + +test("inbox request cancellation retains accepted queue work and its separate lifetime", async () => { + const controller = new AbortController(); + const reason = Symbol("disconnect after acceptance"); + const messages: Message[] = []; + const reports: InboxRequestReport[] = []; + let processed = 0; + let workerSignal: AbortSignal | undefined; + const federation = createFederation({ + kv: new MemoryKvStore(), + skipSignatureVerification: true, + manuallyStartQueue: true, + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + queue: { + inbox: { + enqueue(message) { + messages.push(message); + controller.abort(reason); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .on(Create, (ctx) => { + processed++; + workerSignal = Reflect.get(ctx, "signal"); + }) + .onRequestFinished((_ctx, report) => { + reports.push(report); + }); + const input = new Request(request(await payload()), { + signal: controller.signal, + }); + strictEqual( + (await federation.fetch(input, { contextData: undefined })).status, + 202, + ); + strictEqual(messages.length, 1); + strictEqual(processed, 0); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].outcome, { + type: "response", + status: 202, + disposition: "enqueued", + }); + deepStrictEqual(reports[0].requestCancellation, { type: "aborted", reason }); + strictEqual(Object.hasOwn(messages[0], "signal"), false); + strictEqual(Object.hasOwn(messages[0], "requestCancellation"), false); + await federation.processQueuedTask(undefined, messages[0]); + strictEqual(processed, 1); + strictEqual(workerSignal?.aborted === true, false); + strictEqual(reports.length, 1); +}); + +test("inbox request cancellation preserves early preparation failures and canonical source", async () => { + const error = new Error("initial loader factory failed"); + const reason = { kind: "client disconnected" }; + const controller = new AbortController(); + const reports: InboxRequestReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + origin: "https://canonical.example", + documentLoaderFactory: () => { + throw error; + }, + contextLoaderFactory: () => mockDocumentLoader, + }); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .onRequestFinished((_ctx, report) => { + reports.push(report); + }); + const input = new Request(request({}), { signal: controller.signal }); + controller.abort(reason); + await rejects( + federation.fetch(input, { contextData: undefined }), + (value: unknown) => value === error, + ); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].outcome, { + type: "exception", + stage: "prepare", + error, + }); + deepStrictEqual(reports[0].authentication, { status: "notDetermined" }); + deepStrictEqual(reports[0].payload, { status: "unavailable" }); + deepStrictEqual(reports[0].requestCancellation, { type: "aborted", reason }); + strictEqual(reports[0].local?.origin, "https://example.com"); + strictEqual(reports[0].local?.canonicalOrigin, "https://canonical.example"); +}); diff --git a/packages/fedify/src/federation/inbox-report.ts b/packages/fedify/src/federation/inbox-report.ts index 0d987f090..f079f79d8 100644 --- a/packages/fedify/src/federation/inbox-report.ts +++ b/packages/fedify/src/federation/inbox-report.ts @@ -1,3 +1,4 @@ +import type { OutboundLocalContext } from "./outbound-report.ts"; import type { Activity } from "@fedify/vocab"; import type { RequestContext } from "./context.ts"; import type { @@ -26,6 +27,15 @@ export type InboxJsonValue = | readonly InboxJsonValue[] | { readonly [key: string]: InboxJsonValue }; +/** Request-signal state at inbox report construction, separate from processing outcome. @since 2.5.0 */ +export type InboxRequestCancellation = + | { readonly type: "not-aborted" } + | { + readonly type: "aborted"; + /** Exact Request.signal reason, including non-Error values. */ + readonly reason: unknown; + }; + /** * The final inbox authentication decision. * @since 2.4.0 @@ -132,6 +142,31 @@ export type InboxRequestOutcome = * @since 2.4.0 */ export interface InboxRequestReport { + /** + * Fresh server-generated identity for one routed HTTP inbox request, + * including preparation failures and duplicate deliveries. Preserved on + * outgoing causal descendants; absent without a known routed source. + * @since 2.5.0 + */ + readonly inboxRequestId?: string; + /** + * Original informational logging label; externally controlled and nonunique, + * independent of inboxRequestId. Absent without a known routed source. + * @since 2.5.0 + */ + readonly requestId?: string; + /** + * Original local URL facts from the routed request, independent of later contexts. + * Absent without a known routed source. + * @since 2.5.0 + */ + readonly local?: OutboundLocalContext; + /** + * Request.signal state immediately before report construction. An aborted request + * does not imply processing stopped or retract confirmed processing/enqueue outcomes. + * @since 2.5.0 + */ + readonly requestCancellation: InboxRequestCancellation; /** The delivery route, including a portable gateway route. */ readonly inbox: { readonly kind: "personal" | "shared" | "portable"; diff --git a/packages/fedify/src/federation/inbox.ts b/packages/fedify/src/federation/inbox.ts index e07be771d..f3a814a86 100644 --- a/packages/fedify/src/federation/inbox.ts +++ b/packages/fedify/src/federation/inbox.ts @@ -1,3 +1,7 @@ +import { + getOutboundProvenance, + serializeProvenance, +} from "./outbound-provenance.ts"; import { getTypeId } from "@fedify/vocab"; import type { Activity } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; @@ -425,6 +429,7 @@ export async function routeActivity( propagation.inject(context.active(), carrier); message = { type: "inbox", + ...serializeProvenance(getOutboundProvenance(ctx)), id: crypto.randomUUID(), baseUrl: ctx.origin, activity: originalJson ?? json, diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 5889daa7a..04b369f7c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1,3 +1,24 @@ +import { + createInboxRequestSource, + emptyProvenance, + getOutboundProvenance, + outboundProvenance, + restoreProvenance, + serializeProvenance, + sourceLogContext, +} from "./outbound-provenance.ts"; +import type { + OutboundInboxRequest, + OutboundObservers, + OutboundProvenance, + OutboundWorkerPreparationDiagnostic, +} from "./outbound-report.ts"; +import { + observeOutboundBroadcast, + OutboundAttemptObserver, + OutboundBroadcastObservation, + OutboundWorkerObservation, +} from "./outbound-observation.ts"; import { InboxObservation } from "./inbox-observation.ts"; import { type Path, RouterError } from "@fedify/uri-template"; import type { @@ -46,6 +67,7 @@ import { metrics, propagation, type Span, + type SpanContext, SpanKind, SpanStatusCode, trace, @@ -1208,7 +1230,7 @@ export class FederationImpl await Promise.all(promises); } - processQueuedTask( + async processQueuedTask( contextData: TContextData, message: Message, ): Promise { @@ -1216,46 +1238,109 @@ export class FederationImpl context.active(), message.traceContext, ); - return withContext({ messageId: message.id }, async () => { + let provenance: OutboundProvenance = emptyProvenance; + if (message.type !== "task") { + try { + provenance = restoreProvenance(message.provenance); + } catch (error) { + try { + getLogger(["fedify", "federation", "queue"]).warn( + "Ignoring invalid provenance in queue message {messageId}: {error}", + { messageId: message.id, error }, + ); + } catch { + // A logging sink must not prevent processing a queued task. + } + } + } + if (message.type !== "task" && message.provenance !== undefined) { + message = { ...message, provenance }; + } + return await withContext({ + messageId: message.id, + inboxRequestId: undefined, + ...sourceLogContext(provenance), + }, async () => { if (message.type === "fanout") { + const observers = this.outboundObservers; + const worker = this.#createOutboundWorkerObservation( + contextData, + message, + provenance, + observers.onWorker, + ); const common: QueueTaskCommonAttributes = { role: "fanout", queue: this.fanoutQueue, activityType: message.activityType, }; - await this.#runWorkerSpan( - "activitypub.fanout", - { "activitypub.activity.type": message.activityType }, - extractedContext, - this.#instrumentWorkerBody(common, async (span) => { - if (message.activityId != null) { - span.setAttribute("activitypub.activity.id", message.activityId); - } - await this.#listenFanoutMessage(contextData, message); - return { outcome: "completed" }; - }, classifyAbortableError), - ); + const execute = () => + this.#runWorkerSpan( + "activitypub.fanout", + { "activitypub.activity.type": message.activityType }, + extractedContext, + this.#instrumentWorkerBody(common, async (span) => { + if (message.activityId != null) { + span.setAttribute( + "activitypub.activity.id", + message.activityId, + ); + } + await this.#listenFanoutMessage( + contextData, + message, + provenance, + span, + trace.getSpanContext(extractedContext), + observers, + worker, + ); + return { outcome: "completed" }; + }, classifyAbortableError), + worker, + ); + await (worker == null ? execute() : worker.run(execute)); } else if (message.type === "outbox") { + const observers = this.outboundObservers; + const worker = this.#createOutboundWorkerObservation( + contextData, + message, + provenance, + observers.onWorker, + ); const common: QueueTaskCommonAttributes = { role: "outbox", queue: this.outboxQueue, activityType: message.activityType, }; - await this.#runWorkerSpan( - "activitypub.outbox", - { - "activitypub.activity.type": message.activityType, - "activitypub.activity.retries": message.attempt, - }, - extractedContext, - this.#instrumentWorkerBody(common, async (span) => { - if (message.activityId != null) { - span.setAttribute("activitypub.activity.id", message.activityId); - } - await this.#listenOutboxMessage(contextData, message, span); - return { outcome: "completed" }; - }, classifyAbortableError), - ); + const execute = () => + this.#runWorkerSpan( + "activitypub.outbox", + { + "activitypub.activity.type": message.activityType, + "activitypub.activity.retries": message.attempt, + }, + extractedContext, + this.#instrumentWorkerBody(common, async (span) => { + if (message.activityId != null) { + span.setAttribute( + "activitypub.activity.id", + message.activityId, + ); + } + await this.#listenOutboxMessage( + contextData, + message, + span, + provenance, + observers, + worker, + ); + return { outcome: "completed" }; + }, classifyAbortableError), + worker, + ); + await (worker == null ? execute() : worker.run(execute)); } else if (message.type === "inbox") { const common: QueueTaskCommonAttributes = { role: "inbox", @@ -1270,6 +1355,7 @@ export class FederationImpl contextData, message, span, + provenance, (activityType) => { common.activityType = activityType; }, @@ -1301,6 +1387,39 @@ export class FederationImpl }); } + #createOutboundWorkerObservation( + contextData: TContextData, + message: FanoutMessage | OutboxMessage, + provenance: OutboundProvenance, + callback: OutboundObservers["onWorker"], + ): OutboundWorkerObservation | undefined { + if (callback == null) return undefined; + let documentLoader: DocumentLoader | undefined; + let contextLoader: DocumentLoader | undefined; + return new OutboundWorkerObservation( + () => { + const loaderOptions = this.#getLoaderOptions(message.baseUrl); + return new ContextImpl({ + [outboundProvenance]: provenance, + federation: this, + url: new URL("/", message.baseUrl), + data: contextData, + documentLoader: (...args) => + (documentLoader ??= this.documentLoaderFactory(loaderOptions))( + ...args, + ), + contextLoader: (...args) => + (contextLoader ??= this.contextLoaderFactory(loaderOptions))( + ...args, + ), + }); + }, + callback, + message, + provenance, + ); + } + /** * Opens a CONSUMER span for a queue worker and scopes the extracted trace * context around its body. The body—typically built by @@ -1317,12 +1436,14 @@ export class FederationImpl spanAttributes: Attributes, extractedContext: ReturnType, instrumentedBody: (span: Span) => () => Promise, + worker?: OutboundWorkerObservation, ): Promise { await this._getTracer().startActiveSpan( spanName, { kind: SpanKind.CONSUMER, attributes: spanAttributes }, extractedContext, async (span: Span) => { + worker?.setSpan(span, trace.getSpanContext(extractedContext)); const spanCtx = span.spanContext(); return await withContext( { traceId: spanCtx.traceId, spanId: spanCtx.spanId }, @@ -1384,7 +1505,13 @@ export class FederationImpl async #listenFanoutMessage( data: TContextData, message: FanoutMessage, + provenance: OutboundProvenance, + span: Span, + parentSpanContext: SpanContext | undefined, + observers: OutboundObservers, + worker?: OutboundWorkerObservation, ): Promise { + worker?.enableActions(); const logger = getLogger(["fedify", "federation", "fanout"]); logger.debug( "Fanning out activity {activityId} to {inboxes} inbox(es)...", @@ -1393,47 +1520,75 @@ export class FederationImpl inboxes: globalThis.Object.keys(message.inboxes).length, }, ); - const keys: SenderKeyPair[] = await Promise.all( - message.keys.map(async ({ keyId, privateKey }) => ({ - keyId: new URL(keyId), - privateKey: await importJwk(privateKey, "private"), - })), - ); - const activity = await Activity.fromJsonLd(message.activity, { - contextLoader: this.contextLoaderFactory({ - allowPrivateAddress: this.allowPrivateAddress, - userAgent: this.userAgent, - }), - documentLoader: this.documentLoaderFactory({ - allowPrivateAddress: this.allowPrivateAddress, - userAgent: this.userAgent, - }), - tracerProvider: this.tracerProvider, - }); - const context = this.#createContext( - new URL(message.baseUrl), - data, - { + const importKeys = () => + Promise.all( + message.keys.map(async ({ keyId, privateKey }) => ({ + keyId: new URL(keyId), + privateKey: await importJwk(privateKey, "private"), + })), + ); + const keys: SenderKeyPair[] = worker == null + ? await importKeys() + : await worker.prepare("key-import", importKeys); + const parseActivity = () => + Activity.fromJsonLd(message.activity, { + contextLoader: this.contextLoaderFactory({ + allowPrivateAddress: this.allowPrivateAddress, + userAgent: this.userAgent, + }), documentLoader: this.documentLoaderFactory({ allowPrivateAddress: this.allowPrivateAddress, userAgent: this.userAgent, }), - }, + tracerProvider: this.tracerProvider, + }); + const activity = worker == null + ? await parseActivity() + : await worker.prepare("activity-parsing", parseActivity); + const createContext = () => + this.#createContext( + new URL(message.baseUrl), + data, + { + [outboundProvenance]: provenance, + documentLoader: this.documentLoaderFactory({ + allowPrivateAddress: this.allowPrivateAddress, + userAgent: this.userAgent, + }), + }, + ); + const context = worker == null + ? createContext() + : await worker.prepare("context-preparation", createContext); + worker?.useContext(context); + const observation = await OutboundBroadcastObservation.resumeFanout( + context, + observers, + message, + span, + parentSpanContext, ); await this.sendActivity(keys, message.inboxes, activity, { collectionSync: message.collectionSync, orderingKey: message.orderingKey, normalizeExistingProofs: message.normalizeExistingProofs, activityJsonLd: message.activity, + observation, + worker, context, }); } async #listenOutboxMessage( - _: TContextData, + contextData: TContextData, message: OutboxMessage, span: Span, + provenance: OutboundProvenance, + observers: OutboundObservers, + worker?: OutboundWorkerObservation, ): Promise { + const { onAttempt, onHttpExchange } = observers; + worker?.enableActions(); const logger = getLogger(["fedify", "federation", "outbox"]); const logData = { keyIds: message.keys.map((pair) => pair.keyId), @@ -1446,10 +1601,13 @@ export class FederationImpl const keys: SenderKeyPair[] = []; let rsaKeyPair: SenderKeyPair | null = null; for (const { keyId, privateKey } of message.keys) { - const pair: SenderKeyPair = { + const importPair = async (): Promise => ({ keyId: new URL(keyId), privateKey: await importJwk(privateKey, "private"), - }; + }); + const pair = worker == null + ? await importPair() + : await worker.prepare("key-import", importPair); if ( rsaKeyPair == null && pair.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" @@ -1459,6 +1617,33 @@ export class FederationImpl keys.push(pair); } const loaderOptions = this.#getLoaderOptions(message.baseUrl); + let observationContext: ContextImpl | undefined; + let observationDocumentLoader: DocumentLoader | undefined; + let observationContextLoader: DocumentLoader | undefined; + const observation = OutboundAttemptObserver.resumeOutbox( + () => + worker?.getContext() ?? (observationContext ??= new ContextImpl({ + [outboundProvenance]: provenance, + federation: this, + url: new URL("/", message.baseUrl), + data: contextData, + // Observing settlement must not create loaders or perform lookups. + documentLoader: (...args) => + (observationDocumentLoader ??= this.documentLoaderFactory( + loaderOptions, + ))(...args), + contextLoader: (...args) => + (observationContextLoader ??= this.contextLoaderFactory( + loaderOptions, + ))(...args), + })), + onAttempt, + message, + provenance, + onHttpExchange, + observers, + observers.responseBodyCapture, + ); let parsedActorIds: URL[] | undefined; const getActorIds = () => { parsedActorIds ??= (message.actorIds ?? []).flatMap((id) => { @@ -1493,6 +1678,7 @@ export class FederationImpl const enqueueHeldOutboxMessage = async ( delay: Temporal.Duration, heldSince: Temporal.Instant, + preparationFailures?: readonly OutboundWorkerPreparationDiagnostic[], ) => { const { outboxQueue } = this; if (outboxQueue == null) return; @@ -1501,10 +1687,19 @@ export class FederationImpl circuitHeld: true, circuitHeldSince: heldSince.toString(), } satisfies OutboxMessage; - await outboxQueue.enqueue(heldMessage, { + const enqueueOptions = { delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, - }); + }; + await (worker == null + ? outboxQueue.enqueue(heldMessage, enqueueOptions) + : worker.enqueue( + "defer", + heldMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(heldMessage, enqueueOptions), + preparationFailures, + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -1532,8 +1727,9 @@ export class FederationImpl if (this.outboxPermanentFailureHandler != null) { const ctx = this.#createContext( new URL(message.baseUrl), - _, + contextData, { + [outboundProvenance]: provenance, documentLoader: this.documentLoaderFactory(loaderOptions), }, ); @@ -1553,6 +1749,7 @@ export class FederationImpl actorIds: getActorIds(), }); } catch (handlerError) { + worker?.dropFailure([inbox.href], handlerError); logger.error( "An unexpected error occurred in " + "outboxPermanentFailureHandler:\n{error}", @@ -1569,13 +1766,16 @@ export class FederationImpl const { gatewayInboxes } = message; if (gatewayInboxes != null && gatewayInboxes.length > 1) { await this.#listenGatewayOutboxMessage( - _, + contextData, { ...message, gatewayInboxes, }, span, { + observation, + worker, + provenance, keys, logData, getActorIds, @@ -1586,7 +1786,10 @@ export class FederationImpl return; } try { - const inbox = new URL(message.inbox); + const parseInbox = () => new URL(message.inbox); + const inbox = worker == null + ? parseInbox() + : await worker.prepare("validation", parseInbox); const circuit = this.outboxQueue == null ? undefined : this.circuitBreaker; @@ -1610,14 +1813,18 @@ export class FederationImpl return; } if (decision.type === "drop") { - const activity = await parseActivity(); - await dropHeldOutboxMessage( - circuit, - remoteHost, - inbox, - decision.heldSince, - activity, - ); + worker?.abandon("held-ttl-expired", [inbox.href]); + const drop = async () => { + const activity = await parseActivity(); + await dropHeldOutboxMessage( + circuit, + remoteHost, + inbox, + decision.heldSince, + activity, + ); + }; + await (worker == null ? drop() : worker.drop([inbox.href], drop)); return; } if (decision.stateChange != null) { @@ -1629,6 +1836,8 @@ export class FederationImpl } } await sendActivity({ + attemptStarted: worker?.attemptStarted, + observation, allowPrivateAddress: this.allowPrivateAddress, keys, activity: message.activity, @@ -1636,7 +1845,10 @@ export class FederationImpl activityType: message.activityType, inbox, sharedInbox: message.sharedInbox, - headers: new Headers(message.headers), + headers: worker == null + ? new Headers(message.headers) + : await worker.prepare("request-preparation", () => + new Headers(message.headers)), specDeterminer: new KvSpecDeterminer( this.kv, this.kvPrefixes.httpMessageSignaturesSpec, @@ -1661,6 +1873,10 @@ export class FederationImpl } } } catch (error) { + const preparationFailures = worker?.recordPreparationFailure( + error, + message.inbox, + ) ?? []; span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); const remoteHost = (() => { if (error instanceof SendActivityError) { @@ -1668,7 +1884,7 @@ export class FederationImpl } try { return getRemoteHost(new URL(message.inbox)); - } catch (_) { + } catch { logger.warn( "Invalid inbox URL in queued outbox message: {inbox}", logData, @@ -1791,13 +2007,22 @@ export class FederationImpl } if (circuitDrop != null) { - await dropHeldOutboxMessage( - circuitDrop.circuit, - circuitDrop.remoteHost, - circuitDrop.inbox, - circuitDrop.heldSince, - activity, + worker?.abandon( + "held-ttl-expired", + [circuitDrop.inbox.href], + preparationFailures, ); + const drop = () => + dropHeldOutboxMessage( + circuitDrop.circuit, + circuitDrop.remoteHost, + circuitDrop.inbox, + circuitDrop.heldSince, + activity, + ); + await (worker == null + ? drop() + : worker.drop([circuitDrop.inbox.href], drop)); return; } @@ -1805,6 +2030,11 @@ export class FederationImpl if ( isPermanentFailure ) { + worker?.abandon( + "permanent-failure", + [message.inbox], + preparationFailures, + ); this.metrics.recordPermanentFailure( error.inbox, error.statusCode, @@ -1818,29 +2048,34 @@ export class FederationImpl }, ); if (this.outboxPermanentFailureHandler != null) { - const ctx = this.#createContext( - new URL(message.baseUrl), - _, - { - documentLoader: this.documentLoaderFactory(loaderOptions), - }, - ); - try { - await this.outboxPermanentFailureHandler(ctx, { - reason: "http", - inbox: new URL(message.inbox), - activity, - error, - statusCode: error.statusCode, - actorIds: getActorIds(), - }); - } catch (handlerError) { - logger.error( - "An unexpected error occurred in " + - "outboxPermanentFailureHandler:\n{error}", - { ...logData, error: handlerError }, + const drop = async () => { + const ctx = this.#createContext( + new URL(message.baseUrl), + contextData, + { + [outboundProvenance]: provenance, + documentLoader: this.documentLoaderFactory(loaderOptions), + }, ); - } + try { + await this.outboxPermanentFailureHandler!(ctx, { + reason: "http", + inbox: new URL(message.inbox), + activity, + error, + statusCode: error.statusCode, + actorIds: getActorIds(), + }); + } catch (handlerError) { + worker?.dropFailure([message.inbox], handlerError); + logger.error( + "An unexpected error occurred in " + + "outboxPermanentFailureHandler:\n{error}", + { ...logData, error: handlerError }, + ); + } + }; + await (worker == null ? drop() : worker.drop([message.inbox], drop)); } recordOutboxActivity( this.meterProvider, @@ -1871,6 +2106,7 @@ export class FederationImpl await enqueueHeldOutboxMessage( holdDelay, circuitHold.heldSince, + preparationFailures, ); return; } @@ -1898,13 +2134,19 @@ export class FederationImpl } satisfies OutboxMessage; const { outboxQueue } = this; if (outboxQueue != null) { - await outboxQueue.enqueue( - retryMessage, - { - delay: clampNegativeDelay(delay), - orderingKey: message.orderingKey, - }, - ); + const enqueueOptions = { + delay: clampNegativeDelay(delay), + orderingKey: message.orderingKey, + }; + await (worker == null + ? outboxQueue.enqueue(retryMessage, enqueueOptions) + : worker.enqueue( + "retry", + retryMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(retryMessage, enqueueOptions), + preparationFailures, + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -1918,8 +2160,19 @@ export class FederationImpl "retried", retryMessage.activityType, ); + } else { + worker?.abandon( + "queue-unavailable", + [message.inbox], + preparationFailures, + ); } } else { + worker?.abandon( + "retry-policy-declined", + [message.inbox], + preparationFailures, + ); logger.error( "Failed to send activity {activityId} to {inbox} after {attempt} " + "attempts; giving up:\n{error}", @@ -1957,6 +2210,9 @@ export class FederationImpl message: OutboxMessage & { readonly gatewayInboxes: readonly string[] }, span: Span, helpers: { + readonly observation: SendActivityParameters["observation"]; + readonly worker: OutboundWorkerObservation | undefined; + readonly provenance: OutboundProvenance; readonly keys: readonly SenderKeyPair[]; readonly logData: Record; readonly getActorIds: () => URL[]; @@ -1972,8 +2228,16 @@ export class FederationImpl ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); const circuitLogger = getLogger(["fedify", "federation", "circuit"]); - const { keys, logData, getActorIds, parseActivity, dropHeldOutboxMessage } = - helpers; + const { + observation, + worker, + provenance, + keys, + logData, + getActorIds, + parseActivity, + dropHeldOutboxMessage, + } = helpers; const circuit = this.outboxQueue == null ? undefined : this.circuitBreaker; const specDeterminer = new KvSpecDeterminer( this.kv, @@ -1994,6 +2258,11 @@ export class FederationImpl readonly retryAfter?: Temporal.Duration; } const survivors: Survivor[] = []; + const preparationFailuresByRoute = new Map< + string, + readonly OutboundWorkerPreparationDiagnostic[] + >(); + const permanentRoutes = new Set(); let lastError: unknown; let lastPermanent: { error: SendActivityError; inbox: URL } | undefined; let activity: Activity | undefined; @@ -2004,13 +2273,23 @@ export class FederationImpl inbox: URL, heldSince: Temporal.Instant, ) => { - await dropHeldOutboxMessage( - circuit, - remoteHost, - inbox, - heldSince, - await getActivity(), + const routes = message.gatewayInboxes.filter((route) => + !permanentRoutes.has(route) + ); + worker?.abandon( + "held-ttl-expired", + routes, + routes.flatMap((route) => preparationFailuresByRoute.get(route) ?? []), ); + const execute = async () => + dropHeldOutboxMessage( + circuit, + remoteHost, + inbox, + heldSince, + await getActivity(), + ); + await (worker == null ? execute() : worker.drop(routes, execute)); }; // Checked up front, as gateways deferred by Retry-After are skipped // without asking the circuit breaker, which would otherwise tell: @@ -2022,12 +2301,18 @@ export class FederationImpl Temporal.Now.instant(), ) <= 0 ) { - const inbox = new URL(message.gatewayInboxes[0]); + const parseInbox = () => new URL(message.gatewayInboxes[0]); + const inbox = worker == null + ? parseInbox() + : await worker.prepare("validation", parseInbox); await drop(circuit, getRemoteHost(inbox), inbox, heldSince); return; } for (const inboxHref of message.gatewayInboxes) { - const inbox = new URL(inboxHref); + const parseInbox = () => new URL(inboxHref); + const inbox = worker == null + ? parseInbox() + : await worker.prepare("validation", parseInbox); const remoteHost = getRemoteHost(inbox); const now = Temporal.Now.instant(); const notBefore = parseInstant(message.gatewayNotBefore?.[inboxHref]); @@ -2074,6 +2359,8 @@ export class FederationImpl } try { await sendActivity({ + attemptStarted: worker?.attemptStarted, + observation, allowPrivateAddress: this.allowPrivateAddress, keys, activity: message.activity, @@ -2081,12 +2368,22 @@ export class FederationImpl activityType: message.activityType, inbox, sharedInbox: message.sharedInbox, - headers: new Headers(message.headers), + headers: worker == null + ? new Headers(message.headers) + : await worker.prepare("request-preparation", () => + new Headers(message.headers)), specDeterminer, meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, }); } catch (error) { + const preparationFailures = worker?.recordPreparationFailure( + error, + inboxHref, + ) ?? []; + if (preparationFailures.length > 0) { + preparationFailuresByRoute.set(inboxHref, preparationFailures); + } lastError = error; const isPermanentFailure = error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode); @@ -2154,6 +2451,12 @@ export class FederationImpl ); } if (isPermanentFailure) { + permanentRoutes.add(inboxHref); + worker?.abandon( + "permanent-failure", + [inboxHref], + preparationFailures, + ); logger.warn( "Permanent delivery failure for activity {activityId} to " + "{inbox} ({status}); not trying the gateway again.", @@ -2227,31 +2530,38 @@ export class FederationImpl { ...logData, portableInbox: message.portableInbox }, ); if (this.outboxPermanentFailureHandler != null) { - const ctx = this.#createContext( - new URL(message.baseUrl), - contextData, - { - documentLoader: this.documentLoaderFactory( - this.#getLoaderOptions(message.baseUrl), - ), - }, - ); - try { - await this.outboxPermanentFailureHandler(ctx, { - reason: "http", - inbox, - activity: await getActivity(), - error, - statusCode: error.statusCode, - actorIds: getActorIds(), - }); - } catch (handlerError) { - logger.error( - "An unexpected error occurred in " + - "outboxPermanentFailureHandler:\n{error}", - { ...logData, error: handlerError }, + const drop = async () => { + const ctx = this.#createContext( + new URL(message.baseUrl), + contextData, + { + [outboundProvenance]: provenance, + documentLoader: this.documentLoaderFactory( + this.#getLoaderOptions(message.baseUrl), + ), + }, ); - } + try { + await this.outboxPermanentFailureHandler!(ctx, { + reason: "http", + inbox, + activity: await getActivity(), + error, + statusCode: error.statusCode, + actorIds: getActorIds(), + }); + } catch (handlerError) { + worker?.dropFailure(message.gatewayInboxes, handlerError); + logger.error( + "An unexpected error occurred in " + + "outboxPermanentFailureHandler:\n{error}", + { ...logData, error: handlerError }, + ); + } + }; + await (worker == null + ? drop() + : worker.drop(message.gatewayInboxes, drop)); } recordOutboxActivity( this.meterProvider, @@ -2312,10 +2622,21 @@ export class FederationImpl "until one of its gateways can be tried again.", { ...logData, portableInbox: message.portableInbox }, ); - await outboxQueue.enqueue(heldMessage, { + const enqueueOptions = { delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, - }); + }; + await (worker == null + ? outboxQueue.enqueue(heldMessage, enqueueOptions) + : worker.enqueue( + "defer", + heldMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(heldMessage, enqueueOptions), + survivors.flatMap((route) => + preparationFailuresByRoute.get(route.inbox) ?? [] + ), + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -2342,6 +2663,13 @@ export class FederationImpl throw lastError; } if (policyDelay == null || outboxQueue == null) { + worker?.abandon( + outboxQueue == null ? "queue-unavailable" : "retry-policy-declined", + survivors.map((route) => route.inbox), + survivors.flatMap((route) => + preparationFailuresByRoute.get(route.inbox) ?? [] + ), + ); logger.error( "Failed to send activity {activityId} to every gateway of " + "{portableInbox} after {attempt} attempts; giving up:\n{error}", @@ -2368,10 +2696,21 @@ export class FederationImpl "{portableInbox} (attempt #{attempt}); retry...:\n{error}", { ...logData, portableInbox: message.portableInbox, error: lastError }, ); - await outboxQueue.enqueue(retryMessage, { + const enqueueOptions = { delay: clampNegativeDelay(delay), orderingKey: message.orderingKey, - }); + }; + await (worker == null + ? outboxQueue.enqueue(retryMessage, enqueueOptions) + : worker.enqueue( + "retry", + retryMessage, + enqueueOptions.delay, + () => outboxQueue.enqueue(retryMessage, enqueueOptions), + survivors.flatMap((route) => + preparationFailuresByRoute.get(route.inbox) ?? [] + ), + )); this.metrics.recordQueueTaskEnqueued( { role: "outbox", @@ -2391,11 +2730,14 @@ export class FederationImpl ctxData: TContextData, message: InboxMessage, span: Span, + provenance: OutboundProvenance, onActivityType?: (activityType: string) => void, ): Promise { const logger = getLogger(["fedify", "federation", "inbox"]); const baseUrl = new URL(message.baseUrl); - let context = this.#createContext(baseUrl, ctxData); + let context = this.#createContext(baseUrl, ctxData, { + [outboundProvenance]: provenance, + }); // A portable actor does not necessarily have key pairs for authorized // fetch, so fall back to the default document loader without them: if ( @@ -2403,6 +2745,7 @@ export class FederationImpl this.actorCallbacks?.keyPairsDispatcher != null ) { context = this.#createContext(baseUrl, ctxData, { + [outboundProvenance]: provenance, documentLoader: await context.getDocumentLoader({ identifier: message.identifier, }), @@ -2413,6 +2756,7 @@ export class FederationImpl const identity = await this.sharedInboxKeyDispatcher(context); if (identity != null) { context = this.#createContext(baseUrl, ctxData, { + [outboundProvenance]: provenance, documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity), @@ -3093,13 +3437,17 @@ export class FederationImpl #createContext( baseUrl: URL, contextData: TContextData, - opts?: { documentLoader?: DocumentLoader }, + opts?: { + documentLoader?: DocumentLoader; + [outboundProvenance]?: OutboundProvenance; + }, ): ContextImpl; #createContext( request: Request, contextData: TContextData, opts?: { + [outboundProvenance]?: OutboundProvenance; documentLoader?: DocumentLoader; invokedFromActorDispatcher?: { identifier: string }; invokedFromObjectDispatcher?: { @@ -3114,6 +3462,7 @@ export class FederationImpl urlOrRequest: Request | URL, contextData: TContextData, opts: { + [outboundProvenance]?: OutboundProvenance; documentLoader?: DocumentLoader; invokedFromActorDispatcher?: { identifier: string }; invokedFromObjectDispatcher?: { @@ -3134,6 +3483,7 @@ export class FederationImpl } const loaderOptions = this.#getLoaderOptions(url.origin); const ctxOptions: ContextOptions = { + [outboundProvenance]: opts[outboundProvenance], url, federation: this, data: contextData, @@ -3179,68 +3529,105 @@ export class FederationImpl options: SendActivityInternalOptions, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); - const { immediate, collectionSync, orderingKey, context: ctx } = options; - if (activity.id == null) { - throw new TypeError("The activity to send must have an id."); - } - if (activity.actorId == null) { - throw new TypeError( - "The activity to send must have at least one actor property.", - ); - } else if (keys.length < 1) { - throw new TypeError("The keys must not be empty."); - } - const contextLoader = this.contextLoaderFactory( - this.#getLoaderOptions(ctx.origin), - ); - const activityId = activity.id.href; - let rsaKey: { keyId: URL; privateKey: CryptoKey } | null = null; + const { + immediate, + collectionSync, + orderingKey, + context: ctx, + observation, + worker, + provenance = getOutboundProvenance(ctx), + } = options; + observation?.setPhase("validation"); + const validateActivity = () => { + if (activity.id == null) { + throw new TypeError("The activity to send must have an id."); + } + if (activity.actorId == null) { + throw new TypeError( + "The activity to send must have at least one actor property.", + ); + } else if (keys.length < 1) { + throw new TypeError("The keys must not be empty."); + } + return activity.id.href; + }; + const activityId = worker == null + ? validateActivity() + : await worker.prepare("validation", validateActivity); + observation?.setPhase("serialization"); + const createContextLoader = () => + this.contextLoaderFactory(this.#getLoaderOptions(ctx.origin)); + const contextLoader = worker == null + ? createContextLoader() + : await worker.prepare("serialization", createContextLoader); // A Linked Data Signature attests authorship, which neither a gateway key // nor any key other than the DID's can do for a portable actor's // activity; its Object Integrity Proof does that instead: - const ldSignable = !hasPortableActor(activity); - for (const { keyId, privateKey } of keys) { - validateCryptoKey(privateKey, "private"); - if ( - ldSignable && rsaKey == null && - privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && - !isPortableKeyId(keyId) - ) { - rsaKey = { keyId, privateKey }; + const resolveRsaKey = () => { + const ldSignable = !hasPortableActor(activity); + let rsaKey: { keyId: URL; privateKey: CryptoKey } | null = null; + for (const { keyId, privateKey } of keys) { + validateCryptoKey(privateKey, "private"); + if ( + ldSignable && rsaKey == null && + privateKey.algorithm.name === "RSASSA-PKCS1-v1_5" && + !isPortableKeyId(keyId) + ) { + rsaKey = { keyId, privateKey }; + } } - } + return { rsaKey, ldSignable }; + }; + observation?.setPhase("key-resolution"); + const { rsaKey, ldSignable } = worker == null + ? resolveRsaKey() + : await worker.prepare("key-resolution", resolveRsaKey); // If Object Integrity Proofs were already created before fanout (e.g., in // sendActivityInternal()), skip signing to avoid duplicates. - const signed = await signOutgoingActivity( - activity, - keys.map(({ keyId, privateKey }) => ({ - verificationMethod: keyId, - privateKey, - })), - { - contextLoader, - tracerProvider: this.tracerProvider, - appendToExistingProofs: false, - }, - ); + observation?.setPhase("signing"); + const signActivity = () => + signOutgoingActivity( + activity, + keys.map(({ keyId, privateKey }) => ({ + verificationMethod: keyId, + privateKey, + })), + { + contextLoader, + tracerProvider: this.tracerProvider, + appendToExistingProofs: false, + }, + ); + const signed = worker == null + ? await signActivity() + : await worker.prepare("signing", signActivity); activity = signed.activity; const { hasProof, proofCreated } = signed; - let jsonLd = !proofCreated && options.activityJsonLd != null - ? options.activityJsonLd - : await activity.toJsonLd({ - format: "compact", - contextLoader, - }); - // Existing proofs are preserved by default because they may have been - // created over the compact JSON-LD bytes exactly as supplied. Fedify can - // safely normalize unsigned activities, proofs it just created, or - // locally pre-signed activities when callers opt in. - if (proofCreated || !hasProof || options.normalizeExistingProofs) { - jsonLd = await normalizeOutgoingActivityJsonLd(jsonLd, contextLoader, { - preserveNestedSecuredDocuments: true, - }); - } - assertSupportedCompoundProofShape(jsonLd, activityId); + observation?.setPhase("serialization"); + const serializeActivity = async () => { + let result = !proofCreated && options.activityJsonLd != null + ? options.activityJsonLd + : await activity.toJsonLd({ format: "compact", contextLoader }); + // Existing proofs are preserved by default because they may have been + // created over the compact JSON-LD bytes exactly as supplied. Fedify can + // safely normalize unsigned activities, proofs it just created, or + // locally pre-signed activities when callers opt in. + if (proofCreated || !hasProof || options.normalizeExistingProofs) { + result = await normalizeOutgoingActivityJsonLd(result, contextLoader, { + preserveNestedSecuredDocuments: true, + }); + } + return result; + }; + let jsonLd = worker == null + ? await serializeActivity() + : await worker.prepare("serialization", serializeActivity); + observation?.setPhase("validation"); + const validateProofShape = () => + assertSupportedCompoundProofShape(jsonLd, activityId); + if (worker == null) validateProofShape(); + else await worker.prepare("validation", validateProofShape); if (rsaKey == null && !ldSignable) { logger.debug( "The activity {activityId} is performed by a portable actor, so it " + @@ -3262,20 +3649,27 @@ export class FederationImpl }, ); } else { - try { - jsonLd = await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, { - contextLoader, - tracerProvider: this.tracerProvider, - }); - } catch (error) { - if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error; - logger.warn( - "Failed to create a Linked Data signature for the activity " + - "{activityId}. The activity will be sent without a Linked " + - "Data signature.", - { activityId, error }, - ); - } + const signLinkedData = async () => { + try { + observation?.setPhase("signing"); + return await signJsonLd(jsonLd, rsaKey.privateKey, rsaKey.keyId, { + contextLoader, + tracerProvider: this.tracerProvider, + }); + } catch (error) { + if (!isLinkedDataSignatureJsonLdProcessingError(error)) throw error; + logger.warn( + "Failed to create a Linked Data signature for the activity " + + "{activityId}. The activity will be sent without a Linked " + + "Data signature.", + { activityId, error }, + ); + return jsonLd; + } + }; + jsonLd = worker == null + ? await signLinkedData() + : await worker.prepare("signing", signLinkedData); } if (!hasProof) { logger.warn( @@ -3291,6 +3685,7 @@ export class FederationImpl }, ); } + observation?.setPhase("request-preparation"); if (immediate || this.outboxQueue == null) { if (immediate) { logger.debug( @@ -3306,32 +3701,71 @@ export class FederationImpl } const promises: Promise[] = []; for (const inbox in inboxes) { - promises.push( - sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, { - allowPrivateAddress: this.allowPrivateAddress, - keys, - activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, - sharedInbox: inboxes[inbox].sharedInbox, - headers: collectionSync == null ? undefined : new Headers({ - "Collection-Synchronization": - await buildCollectionSynchronizationHeader( - collectionSync, - inboxes[inbox].actorIds, - ), - }), - specDeterminer: new KvSpecDeterminer( - this.kv, - this.kvPrefixes.httpMessageSignaturesSpec, - this.firstKnock, - { specTtl: this.httpMessageSignaturesSpecTtl }, - ), - meterProvider: this.meterProvider, - tracerProvider: this.tracerProvider, - }), - ); + const delivery = observation?.delivery(inbox); + let sending: Promise; + try { + const createHeaders = collectionSync == null + ? undefined + : async () => + new Headers({ + "Collection-Synchronization": + await buildCollectionSynchronizationHeader( + collectionSync, + inboxes[inbox].actorIds, + ), + }); + const sendActivityId = activity.id?.href; + const activityType = getTypeId(activity).href; + observation?.setActivity({ + ...(sendActivityId == null ? {} : { id: sendActivityId }), + type: activityType, + }); + sending = sendActivityToInbox( + inbox, + inboxes[inbox].gatewayInboxes, + { + attemptStarted: worker == null + ? undefined + : (target) => + worker.directAttemptStarted( + target, + observation?.continuation(inbox)?.outbound, + ), + observation: delivery, + allowPrivateAddress: this.allowPrivateAddress, + keys, + activity: jsonLd, + activityId: sendActivityId, + activityType, + sharedInbox: inboxes[inbox].sharedInbox, + headers: createHeaders == null + ? undefined + : await (worker == null + ? createHeaders() + : worker.prepare("request-preparation", createHeaders)), + specDeterminer: new KvSpecDeterminer( + this.kv, + this.kvPrefixes.httpMessageSignaturesSpec, + this.firstKnock, + { specTtl: this.httpMessageSignaturesSpecTtl }, + ), + meterProvider: this.meterProvider, + tracerProvider: this.tracerProvider, + }, + ); + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } + if (observation != null) { + // The preparation callback may await while an earlier send rejects. + // Keep its original rejection for Promise.all without an unhandled one. + sending.catch(() => {}); + } + if (delivery != null) await delivery.handoff({ type: "direct" }); + promises.push(sending); } + if (observation != null) await observation.prepared("delivery"); await Promise.all(promises); return; } @@ -3339,46 +3773,77 @@ export class FederationImpl "Enqueuing activity {activityId} to send later.", { activityId: activity.id!.href, activity: jsonLd }, ); + observation?.setPhase("key-resolution"); const keyJwkPairs: SenderKeyJwkPair[] = []; for (const { keyId, privateKey } of keys) { - const privateKeyJwk = await exportJwk(privateKey); + const exportPrivateKey = () => exportJwk(privateKey); + const privateKeyJwk = worker == null + ? await exportPrivateKey() + : await worker.prepare("key-export", exportPrivateKey); keyJwkPairs.push({ keyId: keyId.href, privateKey: privateKeyJwk }); } if (!this.manuallyStartQueue) this._startQueueInternal(ctx.data); + observation?.setPhase("request-preparation"); const carrier: Record = {}; - propagation.inject(context.active(), carrier); + const injectTraceContext = () => + propagation.inject(context.active(), carrier); + if (worker == null) injectTraceContext(); + else await worker.prepare("request-preparation", injectTraceContext); const messages: { message: OutboxMessage; orderingKey?: string }[] = []; for (const inbox in inboxes) { - const { portableInbox, gatewayInboxes } = inboxes[inbox]; - const messageOrderingKey = orderingKey == null - ? undefined - : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; - const message: OutboxMessage = { - type: "outbox", - id: crypto.randomUUID(), - baseUrl: ctx.origin, - keys: keyJwkPairs, - activity: jsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, - inbox, - sharedInbox: inboxes[inbox].sharedInbox, - actorIds: [...inboxes[inbox].actorIds], - started: new Date().toISOString(), - attempt: 0, - headers: collectionSync == null ? {} : { - "Collection-Synchronization": - await buildCollectionSynchronizationHeader( - collectionSync, - inboxes[inbox].actorIds, - ), - }, - orderingKey: messageOrderingKey, - ...getPortableMessageFields(portableInbox, gatewayInboxes), - traceContext: carrier, - }; - messages.push({ message, orderingKey: messageOrderingKey }); + try { + const prepareMessage = async () => { + const { portableInbox, gatewayInboxes } = inboxes[inbox]; + const messageOrderingKey = orderingKey == null + ? undefined + : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`; + const messageActivityId = activity.id?.href; + const messageActivityType = getTypeId(activity).href; + observation?.setActivity({ + ...(messageActivityId == null ? {} : { id: messageActivityId }), + type: messageActivityType, + }); + const message: OutboxMessage = { + type: "outbox", + ...serializeProvenance(provenance), + id: crypto.randomUUID(), + baseUrl: ctx.origin, + keys: keyJwkPairs, + activity: jsonLd, + activityId: messageActivityId, + activityType: messageActivityType, + inbox, + ...observation?.continuation(inbox), + sharedInbox: inboxes[inbox].sharedInbox, + actorIds: [...inboxes[inbox].actorIds], + started: new Date().toISOString(), + attempt: 0, + headers: collectionSync == null ? {} : { + "Collection-Synchronization": + await buildCollectionSynchronizationHeader( + collectionSync, + inboxes[inbox].actorIds, + ), + }, + orderingKey: messageOrderingKey, + ...getPortableMessageFields(portableInbox, gatewayInboxes), + traceContext: carrier, + }; + return { message, orderingKey: messageOrderingKey }; + }; + messages.push( + await (worker == null + ? prepareMessage() + : worker.prepare("request-preparation", prepareMessage)), + ); + } catch (error) { + if (observation != null) { + await observation.delivery(inbox)?.failed(error); + } + throw error; + } } + if (observation != null) await observation.prepared("delivery"); const { outboxQueue } = this; // enqueueMany does not support per-message orderingKey, so fall back to // individual enqueues whenever orderingKey is specified or the backend @@ -3386,7 +3851,21 @@ export class FederationImpl if (outboxQueue.enqueueMany == null || orderingKey != null) { const promises: PromiseSettledResult[] = await Promise.allSettled( messages.map(async (m) => { - await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }); + const enqueue = () => + worker == null + ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) + : worker.enqueueChildren( + [m.message], + () => + outboxQueue.enqueue(m.message, { + orderingKey: m.orderingKey, + }), + ); + await (observation == null ? enqueue() : observation.enqueue( + [{ inbox: m.message.inbox, queueMessageId: m.message.id }], + "outbox", + enqueue, + )); recordOutboxEnqueue(this.meterProvider, outboxQueue, m.message); }), ); @@ -3408,7 +3887,21 @@ export class FederationImpl } } else { try { - await outboxQueue.enqueueMany(messages.map((m) => m.message)); + const enqueue = () => + worker == null + ? outboxQueue.enqueueMany!(messages.map((m) => m.message)) + : worker.enqueueChildren( + messages.map((m) => m.message), + () => outboxQueue.enqueueMany!(messages.map((m) => m.message)), + ); + await (observation == null ? enqueue() : observation.enqueue( + messages.map(({ message }) => ({ + inbox: message.inbox, + queueMessageId: message.id, + })), + "outbox", + enqueue, + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to send later: {error}", @@ -3427,9 +3920,17 @@ export class FederationImpl options: FederationFetchOptions, ): Promise { const requestId = getRequestId(request); - return withContext({ requestId }, async () => { + return withContext({ + requestId, + // Each HTTP ingress owns its current identity; async ancestry is not + // evidence that this request came from another inbox request. + inboxRequestId: undefined, + dispatchId: undefined, + sourceInboxRequestId: undefined, + sourceRequestId: undefined, + }, async () => { const tracer = this._getTracer(); - const metricState: HttpMetricState = {}; + const metricState: HttpMetricState = { requestId }; const metricStart = performance.now(); return await tracer.startActiveSpan( request.method, @@ -3493,7 +3994,14 @@ export class FederationImpl logger.error( "An error occurred while serving request " + "{method} {url}: {error}", - { method: request.method, url: request.url, error }, + { + method: request.method, + url: request.url, + error, + ...(metricState.inboxRequest == null ? {} : { + inboxRequestId: metricState.inboxRequest.inboxRequestId, + }), + }, ); throw error; } @@ -3525,6 +4033,9 @@ export class FederationImpl const url = new URL(request.url); const logTpl = "{method} {path}: {status}"; const values = { + ...(metricState.inboxRequest == null ? {} : { + inboxRequestId: metricState.inboxRequest.inboxRequestId, + }), method: request.method, path: `${url.pathname}${url.search}`, url: request.url, @@ -3611,9 +4122,24 @@ export class FederationImpl metricState.endpoint = getEndpointCategory(route.name); span.updateName(`${request.method} ${route.template}`); const routeName = route.name.replace(/:.*$/, ""); + const source = request.method === "POST" && + (routeName === "inbox" || routeName === "sharedInbox") + ? this.#createInboxRequestSource(request, metricState) + : undefined; + const provenance = globalThis.Object.freeze( + source == null + ? { originatingRequestId: metricState.requestId } + : { inboxRequest: source }, + ); let context: RequestContextImpl; try { - context = this.#createContext(request, contextData); + context = withContext( + source == null ? {} : { inboxRequestId: source.inboxRequestId }, + () => + this.#createContext(request, contextData, { + [outboundProvenance]: provenance, + }), + ); } catch (error) { if ( request.method !== "POST" || @@ -3731,36 +4257,42 @@ export class FederationImpl const observation = new InboxObservation({ kind: routeName === "inbox" ? "personal" : "shared", recipient, - }); + }, source); this.#setInboxCompletion(metricState, observation, () => context); - return await observation.run( - async () => { - if (routeName === "inbox") { - context = this.#createContext(request, contextData, { - documentLoader: await context.getDocumentLoader({ - identifier: recipient!, - }), - }); - } else if (this.sharedInboxKeyDispatcher != null) { - const identity = await this.sharedInboxKeyDispatcher(context); - if (identity != null) { - context = this.#createContext(request, contextData, { - documentLoader: - "identifier" in identity || "username" in identity - ? await context.getDocumentLoader(identity) - : context.getDocumentLoader(identity), + return await withContext( + { inboxRequestId: source!.inboxRequestId }, + () => + observation.run( + async () => { + if (routeName === "inbox") { + context = this.#createContext(request, contextData, { + [outboundProvenance]: getOutboundProvenance(context), + documentLoader: await context.getDocumentLoader({ + identifier: recipient!, + }), + }); + } else if (this.sharedInboxKeyDispatcher != null) { + const identity = await this.sharedInboxKeyDispatcher(context); + if (identity != null) { + context = this.#createContext(request, contextData, { + [outboundProvenance]: getOutboundProvenance(context), + documentLoader: + "identifier" in identity || "username" in identity + ? await context.getDocumentLoader(identity) + : context.getDocumentLoader(identity), + }); + } + } + return await this.#handleInbox(request, { + recipient, + context, + contextData, + onNotFound, + observation, }); - } - } - return await this.#handleInbox(request, { - recipient, - context, - contextData, - onNotFound, - observation, - }); - }, - span, + }, + span, + ), ); } switch (routeName) { @@ -3770,6 +4302,7 @@ export class FederationImpl ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier; context = this.#createContext(request, contextData, { + [outboundProvenance]: getOutboundProvenance(context), invokedFromActorDispatcher: { identifier }, }); return await handleActor(request, { @@ -3786,6 +4319,7 @@ export class FederationImpl const callbacks = this.objectCallbacks[typeId]; const cls = this.objectTypeIds[typeId]; context = this.#createContext(request, contextData, { + [outboundProvenance]: getOutboundProvenance(context), invokedFromObjectDispatcher: { cls, values: route.values }, }); return await handleObject(request, { @@ -3854,6 +4388,7 @@ export class FederationImpl const identity = await this.sharedInboxKeyDispatcher(context); if (identity != null) { context = this.#createContext(request, contextData, { + [outboundProvenance]: getOutboundProvenance(context), documentLoader: "identifier" in identity || "username" in identity ? await context.getDocumentLoader(identity) : context.getDocumentLoader(identity), @@ -4052,11 +4587,15 @@ export class FederationImpl } return response; } + const provenance = globalThis.Object.freeze({ + originatingRequestId: metricState.requestId, + }); if (collectionRoute != null) { return await this.#fetchPortableCollection(request, portable, route, { onNotFound, onUnauthorized, contextData, + provenance, }); } if (isActor) { @@ -4064,6 +4603,7 @@ export class FederationImpl ? route.name.substring(ACTOR_ALIAS_PREFIX.length) : route.values.identifier; const context = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, invokedFromActorDispatcher: { identifier }, portableRequest: portable.portableRequest, }); @@ -4088,6 +4628,7 @@ export class FederationImpl const callbacks = this.objectCallbacks[typeId]; const cls = this.objectTypeIds[typeId]; const context = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, invokedFromObjectDispatcher: { cls, values: route.values }, portableRequest: portable.portableRequest, }); @@ -4111,6 +4652,7 @@ export class FederationImpl onNotFound, onUnauthorized, contextData, + provenance, }: & Required< Pick< @@ -4118,7 +4660,7 @@ export class FederationImpl "onNotFound" | "onUnauthorized" > > - & { contextData: TContextData }, + & { contextData: TContextData; provenance: OutboundProvenance }, ): Promise { const logger = getLogger(["fedify", "federation", "collection"]); const url = new URL(request.url); @@ -4127,8 +4669,11 @@ export class FederationImpl // that its ID comes from the application's data rather than from // the authority of the request, which the portable ID helpers would // otherwise default to: - const ownerContext = this.#createContext(request, contextData); + const ownerContext = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, + }); const context = this.#createContext(request, contextData, { + [outboundProvenance]: provenance, portableRequest: portable.portableRequest, }); const reject = async (reason: string, identifier?: string) => { @@ -4276,6 +4821,19 @@ export class FederationImpl return await onNotFound(request); } + #createInboxRequestSource( + request: Request, + metricState: HttpMetricState, + ): OutboundInboxRequest { + const source = createInboxRequestSource( + request, + metricState.requestId ?? getRequestId(request), + this.origin?.webOrigin, + ); + metricState.inboxRequest = source; + return source; + } + #setInboxCompletion( metricState: HttpMetricState, observation: InboxObservation, @@ -4327,6 +4885,9 @@ export class FederationImpl ): Promise { const unavailableLoader: DocumentLoader = () => Promise.reject(error); const context = new RequestContextImpl({ + [outboundProvenance]: globalThis.Object.freeze({ + inboxRequest: metricState.inboxRequest, + }), url: new URL(request.url), request, federation: this, @@ -4334,7 +4895,7 @@ export class FederationImpl documentLoader: unavailableLoader, contextLoader: unavailableLoader, }); - const observation = new InboxObservation(inbox); + const observation = new InboxObservation(inbox, metricState.inboxRequest); this.#setInboxCompletion(metricState, observation, () => context); return await observation.run( () => Promise.reject(error), @@ -4431,9 +4992,18 @@ export class FederationImpl metricState.endpoint = "inbox"; span.updateName(`${request.method} ${metricState.routeTemplate}`); const identifier = route.values.identifier; + const source = this.#createInboxRequestSource(request, metricState); let context: RequestContextImpl; try { - context = this.#createContext(request, contextData); + context = withContext( + { inboxRequestId: source.inboxRequestId }, + () => + this.#createContext(request, contextData, { + [outboundProvenance]: globalThis.Object.freeze({ + inboxRequest: source, + }), + }), + ); } catch (error) { return await this.#reportInboxPreparationError( request, @@ -4447,104 +5017,128 @@ export class FederationImpl const observation = new InboxObservation({ kind: "portable", recipient: identifier, - }); + }, source); this.#setInboxCompletion(metricState, observation, () => context); - return await observation.run( - async () => { - // The actor is looked up only by its identifier, as for ordinary inbox - // deliveries, so that the identifier alone determines the recipient, - // also for inbox listeners and queued deliveries: - const actor = this.actorCallbacks?.dispatcher == null - ? null - : await this.actorCallbacks.dispatcher(context, identifier); - const localOrigin = context.canonicalOrigin; - const resolution = resolvePortableInboxRecipient( - actor == null || actor instanceof Tombstone ? null : actor, - { - authority: portable.portableRequest.authority, - canonicalInboxId: portable.canonicalId, - localOrigin, - }, - ); - if (resolution.status === "rejected") { - logger.debug( - "Not accepting a delivery to the portable inbox {inbox} on behalf " + - "of the actor {identifier}: {reason}.", - { - inbox: portable.canonicalId, - identifier, - reason: resolution.reason, - }, - ); - return await onNotFound(request); - } - const { recipient } = resolution; - // A portable actor does not necessarily have key pairs for authorized - // fetch, so fall back to the default document loader without them: - if (this.actorCallbacks?.keyPairsDispatcher != null) { - context = this.#createContext(request, contextData, { - documentLoader: await context.getDocumentLoader({ identifier }), - }); - } - const excludedOrigins = [localOrigin, new URL(request.url).origin]; - return await this.#handleInbox(request, { - recipient: identifier, - observation, - context, - contextData, - onNotFound, - portableInbox: { - recipient, - forward: async (activity, activityId, activityType, signatures) => { - if (this.portableInboxForwarding.maxTargets < 1) return; - if (this.kv.cas == null) { - if (!this.#portableInboxForwardingWarned) { - this.#portableInboxForwardingWarned = true; - logger.warn( - "Activities delivered to FEP-ef61 portable inboxes are not " + - "forwarded to the other gateways, as the key–value store " + - "does not support compare-and-swap (KvStore.cas()), which " + - "is needed to forward each activity at most once.", - ); - } - return; - } - await forwardPortableInboxActivity({ + return await withContext( + { inboxRequestId: source.inboxRequestId }, + () => + observation.run( + async () => { + // The actor is looked up only by its identifier, as for ordinary inbox + // deliveries, so that the identifier alone determines the recipient, + // also for inbox listeners and queued deliveries: + const actor = this.actorCallbacks?.dispatcher == null + ? null + : await this.actorCallbacks.dispatcher(context, identifier); + const localOrigin = context.canonicalOrigin; + const resolution = resolvePortableInboxRecipient( + actor == null || actor instanceof Tombstone ? null : actor, + { + authority: portable.portableRequest.authority, + canonicalInboxId: portable.canonicalId, + localOrigin, + }, + ); + if (resolution.status === "rejected") { + logger.debug( + "Not accepting a delivery to the portable inbox {inbox} on behalf " + + "of the actor {identifier}: {reason}.", + { + inbox: portable.canonicalId, + identifier, + reason: resolution.reason, + }, + ); + return await onNotFound(request); + } + const { recipient } = resolution; + // A portable actor does not necessarily have key pairs for authorized + // fetch, so fall back to the default document loader without them: + if (this.actorCallbacks?.keyPairsDispatcher != null) { + context = this.#createContext(request, contextData, { + [outboundProvenance]: getOutboundProvenance(context), + documentLoader: await context.getDocumentLoader({ identifier }), + }); + } + const excludedOrigins = [localOrigin, new URL(request.url).origin]; + return await this.#handleInbox(request, { + recipient: identifier, + observation, + context, + contextData, + onNotFound, + portableInbox: { recipient, - activity, - activityId, - activityType, - excludedOrigins, - baseUrl: context.origin, - kv: this.kv, - kvPrefix: this.kvPrefixes.portableInboxForwarding, - outboxQueue: this.outboxQueue, - startQueue: this.manuallyStartQueue - ? undefined - : () => this._startQueueInternal(contextData), - allowPrivateAddress: this.allowPrivateAddress, - getKeys: () => - this.#getPortableGatewayKeyPairs( + forward: (activity, activityId, activityType, signatures) => + observeOutboundBroadcast( context, - identifier, - recipient, + this.outboundObservers, + async (observation) => { + observation?.setActivity({ + ...(activityId == null ? {} : { id: activityId.href }), + type: activityType, + }); + if (this.portableInboxForwarding.maxTargets < 1) { + if (observation != null) { + await observation.skipped("forwarding-disabled"); + } + return; + } + if (this.kv.cas == null) { + if (!this.#portableInboxForwardingWarned) { + this.#portableInboxForwardingWarned = true; + logger.warn( + "Activities delivered to FEP-ef61 portable inboxes are not " + + "forwarded to the other gateways, as the key–value store " + + "does not support compare-and-swap (KvStore.cas()), which " + + "is needed to forward each activity at most once.", + ); + } + if (observation != null) { + await observation.skipped("forwarding-unavailable"); + } + return; + } + await forwardPortableInboxActivity({ + provenance: getOutboundProvenance(context), + recipient, + activity, + activityId, + activityType, + excludedOrigins, + baseUrl: context.origin, + kv: this.kv, + kvPrefix: this.kvPrefixes.portableInboxForwarding, + outboxQueue: this.outboxQueue, + startQueue: this.manuallyStartQueue + ? undefined + : () => this._startQueueInternal(contextData), + allowPrivateAddress: this.allowPrivateAddress, + getKeys: () => + this.#getPortableGatewayKeyPairs( + context, + identifier, + recipient, + ), + specDeterminer: new KvSpecDeterminer( + this.kv, + this.kvPrefixes.httpMessageSignaturesSpec, + this.firstKnock, + { specTtl: this.httpMessageSignaturesSpecTtl }, + ), + signatures, + options: this.portableInboxForwarding, + meterProvider: this.meterProvider, + tracerProvider: this.tracerProvider, + }, observation); + }, + "portable-replication", ), - specDeterminer: new KvSpecDeterminer( - this.kv, - this.kvPrefixes.httpMessageSignaturesSpec, - this.firstKnock, - { specTtl: this.httpMessageSignaturesSpecTtl }, - ), - signatures, - options: this.portableInboxForwarding, - meterProvider: this.meterProvider, - tracerProvider: this.tracerProvider, - }); - }, + }, + }); }, - }); - }, - span, + span, + ), ); } @@ -4653,7 +5247,11 @@ export class FederationImpl }, ); } - const context = this.#createContext(request, contextData); + const context = this.#createContext(request, contextData, { + [outboundProvenance]: globalThis.Object.freeze({ + originatingRequestId: metricState.requestId, + }), + }); const response = await this.hashlinkMediaDispatcher!(context, media.media); // The onNotFound response is returned as is, since integrations may // recognize it by its identity: @@ -4769,6 +5367,8 @@ type FedifyEndpoint = | "error"; interface HttpMetricState { + requestId?: string; + inboxRequest?: OutboundInboxRequest; inboxCompletion?: { observation: InboxObservation; finish( @@ -4847,6 +5447,7 @@ function getCollectionMetricRoute(routeName: string): } interface ContextOptions { + [outboundProvenance]?: OutboundProvenance; url: URL; federation: FederationImpl; data: TContextData; @@ -4909,9 +5510,13 @@ export class ContextImpl implements Context { documentLoader, contextLoader, invokedFromActorKeyPairsDispatcher, + [outboundProvenance]: provenance = emptyProvenance, }: ContextOptions, ) { this.url = url; + // Keep internal symbol metadata enumerable for context spreads, without + // exposing a computed class field in generated declarations. + globalThis.Object.assign(this, { [outboundProvenance]: provenance }); this.federation = federation; this.data = data; this.documentLoader = documentLoader; @@ -4941,6 +5546,7 @@ export class ContextImpl implements Context { clone(data: TContextData): Context { return new ContextImpl({ + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -4959,6 +5565,7 @@ export class ContextImpl implements Context { metadata?: InboxContextMetadata, ): InboxContextImpl { return new InboxContextImpl(recipient, activity, activityId, activityType, { + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data: this.data, @@ -4981,6 +5588,7 @@ export class ContextImpl implements Context { activityId, activityType, { + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data: this.data, @@ -5646,45 +6254,71 @@ export class ContextImpl implements Context { activity: Activity, options: SendActivityOptionsForCollection = {}, ): Promise { - const tracer = this.tracerProvider.getTracer( - metadata.name, - metadata.version, - ); - return tracer.startActiveSpan( - this.federation.outboxQueue == null || options.immediate - ? "activitypub.outbox" - : "activitypub.fanout", - { - kind: this.federation.outboxQueue == null || options.immediate - ? SpanKind.CLIENT - : SpanKind.PRODUCER, - attributes: { - "activitypub.activity.type": getTypeId(activity).href, - "activitypub.activity.to": activity.toIds.map((to) => to.href), - "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), - "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href), - "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href), - }, - }, - async (span) => { - try { - if (activity.id != null) { - span.setAttribute("activitypub.activity.id", activity.id.href); - } - await this.sendActivityInternal( - sender, - recipients, - activity, - options, - span, - ); - } catch (e) { - span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); - throw e; - } finally { - span.end(); - } + return observeOutboundBroadcast( + this, + this.federation.outboundObservers, + (observation, provenance) => { + const tracer = this.tracerProvider.getTracer( + metadata.name, + metadata.version, + ); + const parentSpanContext = trace.getSpanContext(context.active()); + const activityType = getTypeId(activity).href; + observation?.setActivity({ type: activityType }); + return tracer.startActiveSpan( + this.federation.outboxQueue == null || options.immediate + ? "activitypub.outbox" + : "activitypub.fanout", + { + kind: this.federation.outboxQueue == null || options.immediate + ? SpanKind.CLIENT + : SpanKind.PRODUCER, + attributes: { + "activitypub.activity.type": activityType, + "activitypub.activity.to": activity.toIds.map((to) => to.href), + "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), + "activitypub.activity.bto": activity.btoIds.map((bto) => + bto.href + ), + "activitypub.activity.bcc": activity.bccIds.map((bcc) => + bcc.href + ), + }, + }, + async (span) => { + observation?.setSpan(span, parentSpanContext); + try { + const activityId = activity.id?.href; + if (activityId != null) { + span.setAttribute("activitypub.activity.id", activityId); + } + observation?.setActivity({ + type: activityType, + ...(activityId == null ? {} : { id: activityId }), + }); + await this.sendActivityInternal( + sender, + recipients, + activity, + options, + span, + observation, + provenance, + ); + } catch (e) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: String(e), + }); + throw e; + } finally { + span.end(); + } + }, + ); }, + "send", + options.dispatchId, ); } @@ -5698,8 +6332,11 @@ export class ContextImpl implements Context { activity: Activity, options: SendActivityOptionsForCollection, span: Span, + observation?: OutboundBroadcastObservation, + provenance: OutboundProvenance = getOutboundProvenance(this), ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); + observation?.setPhase("sender-resolution"); let keys: SenderKeyPair[]; let identifier: string | null = null; let actorKeyPairs: ActorKeyPair[] | null = null; @@ -5726,6 +6363,7 @@ export class ContextImpl implements Context { } } span.setAttribute("fedify.actor.identifier", identifier); + observation?.setPhase("key-resolution"); if (this.federation.actorCallbacks?.keyPairsDispatcher == null) { throw new Error("No actor key pairs dispatcher registered."); } @@ -5740,6 +6378,7 @@ export class ContextImpl implements Context { privateKey: kp.privateKey, })); } else if (Array.isArray(sender)) { + observation?.setPhase("key-resolution"); if (sender.length < 1) { throw new Error("The sender's key pairs are empty."); } @@ -5750,9 +6389,11 @@ export class ContextImpl implements Context { if (keys.length < 1) { throw new TypeError("The sender's keys must not be empty."); } + observation?.setPhase("key-resolution"); for (const { privateKey } of keys) { validateCryptoKey(privateKey, "private"); } + observation?.setPhase("recipient-resolution"); let expandedRecipients: Recipient[]; let collectionSync: string | undefined; if (Array.isArray(recipients)) { @@ -5784,16 +6425,43 @@ export class ContextImpl implements Context { } const opts: SendActivityInternalOptions = { context: this, + observation, + provenance, orderingKey: options.orderingKey, collectionSync, immediate: options.immediate, normalizeExistingProofs: options.normalizeExistingProofs, }; span.setAttribute("activitypub.inboxes", expandedRecipients.length); + observation?.setPhase("transformation"); + if (this.federation.activityTransformers.length > 0) { + // A transformer can replace the Activity; do not retain its old ID/type. + observation?.setActivity({}); + } for (const activityTransformer of this.federation.activityTransformers) { activity = activityTransformer(activity, this); } - span?.setAttribute("activitypub.activity.id", activity?.id?.href ?? ""); + let transformedActivityId: string | undefined; + try { + transformedActivityId = activity?.id?.href; + } finally { + if (observation != null) { + let activityType: string | undefined; + try { + activityType = getTypeId(activity)?.href; + } catch { + // Observation must not replace an error from reading the Activity. + } + observation.setActivity({ + ...(transformedActivityId == null + ? {} + : { id: transformedActivityId }), + ...(activityType == null ? {} : { type: activityType }), + }); + } + } + span?.setAttribute("activitypub.activity.id", transformedActivityId ?? ""); + observation?.setPhase("validation"); if (activity.actorId == null) { logger.error( "Activity {activityId} to send does not have an actor.", @@ -5820,6 +6488,7 @@ export class ContextImpl implements Context { // does not accept. This mirrors the guard `FederationImpl.sendActivity()` // applies before signing. Actor key pairs keep appending outside that // profile, as they always have. + observation?.setPhase("signing"); const { activity: signedActivity, proofCreated } = await signOutgoingActivity( activity, @@ -5841,16 +6510,19 @@ export class ContextImpl implements Context { }, ); activity = signedActivity; + observation?.setPhase("destination-planning"); const inboxes = extractInboxes({ recipients: expandedRecipients, preferSharedInbox: options.preferSharedInbox, excludeBaseUris: options.excludeBaseUris, }); + if (observation != null) await observation.planDestinations(inboxes); if (globalThis.Object.keys(inboxes).length < 1) { logger.debug("No inboxes found for activity {activityId}.", { activityId: activity.id?.href, activity, }); + if (observation != null) await observation.skipped(); return false; } logger.debug("Sending activity {activityId} to inboxes:\n{inboxes}", { @@ -5858,6 +6530,7 @@ export class ContextImpl implements Context { activityId: activity.id?.href, activity, }); + observation?.setPhase("serialization"); const activityJsonLd = await activity.toJsonLd({ format: "compact", contextLoader: this.contextLoader, @@ -5890,17 +6563,27 @@ export class ContextImpl implements Context { } // Reject before anything is enqueued, so the caller learns about it // instead of the fanout worker. + observation?.setPhase("validation"); assertSupportedCompoundProofShape(activityJsonLd, activity.id?.href); + observation?.setPhase("key-resolution"); const keyJwkPairs = await Promise.all( keys.map(async ({ keyId, privateKey }) => ({ keyId: keyId.href, privateKey: await exportJwk(privateKey), })), ); + observation?.setPhase("request-preparation"); const carrier: Record = {}; propagation.inject(context.active(), carrier); + const activityId = activity.id?.href; + const activityType = getTypeId(activity).href; + observation?.setActivity({ + ...(activityId == null ? {} : { id: activityId }), + type: activityType, + }); const message: FanoutMessage = { type: "fanout", + ...serializeProvenance(provenance), id: crypto.randomUUID(), baseUrl: this.origin, keys: keyJwkPairs, @@ -5909,13 +6592,14 @@ export class ContextImpl implements Context { [k, { actorIds, sharedInbox, portableInbox, gatewayInboxes }], ) => [k, { actorIds: [...actorIds], + ...observation?.continuation(k), sharedInbox, ...getPortableMessageFields(portableInbox, gatewayInboxes), }]), ), activity: activityJsonLd, - activityId: activity.id?.href, - activityType: getTypeId(activity).href, + activityId, + activityType, collectionSync: opts.collectionSync, orderingKey: options.orderingKey, normalizeExistingProofs: proofCreated || options.normalizeExistingProofs, @@ -5924,10 +6608,22 @@ export class ContextImpl implements Context { if (!this.federation.manuallyStartQueue) { this.federation._startQueueInternal(this.data); } - await this.federation.fanoutQueue.enqueue( - message, - { orderingKey: options.orderingKey }, - ); + if (observation != null) await observation.prepared("fanout"); + await (observation == null + ? this.federation.fanoutQueue.enqueue(message, { + orderingKey: options.orderingKey, + }) + : observation.enqueue( + globalThis.Object.keys(inboxes).map((inbox) => ({ + inbox, + queueMessageId: message.id, + })), + "fanout", + () => + this.federation.fanoutQueue!.enqueue(message, { + orderingKey: options.orderingKey, + }), + )); getFederationMetrics(this.federation.meterProvider).recordQueueTaskEnqueued( { role: "fanout", @@ -6364,6 +7060,7 @@ class RequestContextImpl extends ContextImpl override clone(data: TContextData): RequestContext { return new RequestContextImpl({ + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -6729,39 +7426,55 @@ function forwardActivity( recipients: Recipient | Recipient[] | "followers", options?: ForwardActivityOptions, ): Promise { - const tracer = ctx.tracerProvider.getTracer( - metadata.name, - metadata.version, - ); - return tracer.startActiveSpan( - ctx.federation.outboxQueue == null || options?.immediate - ? `activitypub.${loggerCategory}` - : "activitypub.fanout", - { - kind: ctx.federation.outboxQueue == null || options?.immediate - ? SpanKind.CLIENT - : SpanKind.PRODUCER, - attributes: { "activitypub.activity.type": ctx.activityType }, - }, - async (span) => { - try { - if (ctx.activityId != null) { - span.setAttribute("activitypub.activity.id", ctx.activityId); - } - return await forwardActivityInternal( - ctx, - loggerCategory, - forwarder, - recipients, - options, - ); - } catch (e) { - span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); - throw e; - } finally { - span.end(); - } + return observeOutboundBroadcast( + ctx, + ctx.federation.outboundObservers, + (observation, provenance) => { + observation?.setActivity({ + ...(ctx.activityId == null ? {} : { id: ctx.activityId }), + type: ctx.activityType, + }); + const tracer = ctx.tracerProvider.getTracer( + metadata.name, + metadata.version, + ); + const parentSpanContext = trace.getSpanContext(context.active()); + return tracer.startActiveSpan( + ctx.federation.outboxQueue == null || options?.immediate + ? `activitypub.${loggerCategory}` + : "activitypub.fanout", + { + kind: ctx.federation.outboxQueue == null || options?.immediate + ? SpanKind.CLIENT + : SpanKind.PRODUCER, + attributes: { "activitypub.activity.type": ctx.activityType }, + }, + async (span) => { + observation?.setSpan(span, parentSpanContext); + try { + if (ctx.activityId != null) { + span.setAttribute("activitypub.activity.id", ctx.activityId); + } + return await forwardActivityInternal( + ctx, + loggerCategory, + forwarder, + recipients, + options, + observation, + provenance, + ); + } catch (e) { + span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); + throw e; + } finally { + span.end(); + } + }, + ); }, + "forward", + options?.dispatchId, ); } @@ -6775,8 +7488,11 @@ async function forwardActivityInternal( | { username: string }, recipients: Recipient | Recipient[] | "followers", options?: ForwardActivityOptions, + observation?: OutboundBroadcastObservation, + provenance: OutboundProvenance = getOutboundProvenance(ctx), ): Promise { const logger = getLogger(["fedify", "federation", loggerCategory]); + observation?.setPhase("sender-resolution"); let keys: SenderKeyPair[]; let identifier: string | null = null; if ( @@ -6803,6 +7519,7 @@ async function forwardActivityInternal( identifier = mapped; } } + observation?.setPhase("key-resolution"); const actorKeyPairs = await ctx.getActorKeyPairs(identifier); if (actorKeyPairs.length < 1) { throw new Error( @@ -6814,6 +7531,7 @@ async function forwardActivityInternal( privateKey: kp.privateKey, })); } else if (Array.isArray(forwarder)) { + observation?.setPhase("key-resolution"); if (forwarder.length < 1) { throw new Error("The forwarder's key pairs are empty."); } @@ -6821,10 +7539,14 @@ async function forwardActivityInternal( } else { keys = [forwarder]; } + observation?.setPhase("validation"); if (!hasSignatureLike(ctx.activity)) { const hasProof = hasProofLike(ctx.activity); if (!hasProof) { - if (options?.skipIfUnsigned) return false; + if (options?.skipIfUnsigned) { + if (observation != null) await observation.skipped("unsigned"); + return false; + } logger.warn( "The activity {activityId} is not signed; even if it is " + "forwarded to other servers as is, it may not be accepted by " + @@ -6837,6 +7559,7 @@ async function forwardActivityInternal( ); } } + observation?.setPhase("recipient-resolution"); if (recipients === "followers") { if (identifier == null) { throw new Error( @@ -6850,17 +7573,20 @@ async function forwardActivityInternal( } recipients = followers; } + observation?.setPhase("destination-planning"); const inboxes = extractInboxes({ recipients: Array.isArray(recipients) ? recipients : [recipients], preferSharedInbox: options?.preferSharedInbox, excludeBaseUris: options?.excludeBaseUris, }); + if (observation != null) await observation.planDestinations(inboxes); if (globalThis.Object.keys(inboxes).length < 1) { logger.debug("No inboxes found for activity {activityId}.", { activityId: ctx.activityId, activityType: ctx.activityType, identifier: identifier ?? undefined, }); + if (observation != null) await observation.skipped(); return false; } logger.debug("Forwarding activity {activityId} to inboxes:\n{inboxes}", { @@ -6880,27 +7606,45 @@ async function forwardActivityInternal( "set.", ); } + observation?.setPhase("request-preparation"); const promises: Promise[] = []; for (const inbox in inboxes) { - promises.push( - sendActivityToInbox(inbox, inboxes[inbox].gatewayInboxes, { - allowPrivateAddress: ctx.federation.allowPrivateAddress, - keys, - activity: ctx.activity, - activityId: ctx.activityId, - activityType: ctx.activityType, - sharedInbox: inboxes[inbox].sharedInbox, - meterProvider: ctx.meterProvider, - tracerProvider: ctx.tracerProvider, - specDeterminer: new KvSpecDeterminer( - ctx.federation.kv, - ctx.federation.kvPrefixes.httpMessageSignaturesSpec, - ctx.federation.firstKnock, - { specTtl: ctx.federation.httpMessageSignaturesSpecTtl }, - ), - }), - ); + const delivery = observation?.delivery(inbox); + let sending: Promise; + try { + sending = sendActivityToInbox( + inbox, + inboxes[inbox].gatewayInboxes, + { + observation: delivery, + allowPrivateAddress: ctx.federation.allowPrivateAddress, + keys, + activity: ctx.activity, + activityId: ctx.activityId, + activityType: ctx.activityType, + sharedInbox: inboxes[inbox].sharedInbox, + meterProvider: ctx.meterProvider, + tracerProvider: ctx.tracerProvider, + specDeterminer: new KvSpecDeterminer( + ctx.federation.kv, + ctx.federation.kvPrefixes.httpMessageSignaturesSpec, + ctx.federation.firstKnock, + { specTtl: ctx.federation.httpMessageSignaturesSpecTtl }, + ), + }, + ); + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } + if (observation != null) { + // Preserve early rejections while the preparation observer is awaited. + sending.catch(() => {}); + } + if (delivery != null) await delivery.handoff({ type: "direct" }); + promises.push(sending); } + if (observation != null) await observation.prepared("delivery"); await Promise.all(promises); return true; } @@ -6908,51 +7652,69 @@ async function forwardActivityInternal( "Enqueuing activity {activityId} to forward later.", { activityId: ctx.activityId, activity: ctx.activity }, ); + observation?.setPhase("request-preparation"); if (!ctx.federation.manuallyStartQueue) { ctx.federation._startQueueInternal(ctx.data); } + observation?.setPhase("key-resolution"); const keyJwkPairs: SenderKeyJwkPair[] = []; for (const { keyId, privateKey } of keys) { const privateKeyJwk = await exportJwk(privateKey); keyJwkPairs.push({ keyId: keyId.href, privateKey: privateKeyJwk }); } + observation?.setPhase("request-preparation"); const carrier: Record = {}; propagation.inject(context.active(), carrier); const orderingKey = options?.orderingKey; const started = new Date().toISOString(); const messages: { message: OutboxMessage; orderingKey?: string }[] = []; for (const inbox in inboxes) { - const { portableInbox, gatewayInboxes } = inboxes[inbox]; - const message: OutboxMessage = { - type: "outbox", - id: crypto.randomUUID(), - baseUrl: ctx.origin, - keys: keyJwkPairs, - activity: ctx.activity, - activityId: ctx.activityId, - activityType: ctx.activityType, - inbox, - sharedInbox: inboxes[inbox].sharedInbox, - actorIds: [...inboxes[inbox].actorIds], - started, - attempt: 0, - headers: {}, - orderingKey: orderingKey == null - ? undefined - : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`, - ...getPortableMessageFields(portableInbox, gatewayInboxes), - traceContext: carrier, - }; - messages.push({ - message, - orderingKey: message.orderingKey, - }); + try { + const { portableInbox, gatewayInboxes } = inboxes[inbox]; + const message: OutboxMessage = { + type: "outbox", + ...serializeProvenance(provenance), + id: crypto.randomUUID(), + baseUrl: ctx.origin, + keys: keyJwkPairs, + activity: ctx.activity, + activityId: ctx.activityId, + activityType: ctx.activityType, + inbox, + ...observation?.continuation(inbox), + sharedInbox: inboxes[inbox].sharedInbox, + actorIds: [...inboxes[inbox].actorIds], + started, + attempt: 0, + headers: {}, + orderingKey: orderingKey == null + ? undefined + : `${orderingKey}\n${getOrderingDestination(inbox, portableInbox)}`, + ...getPortableMessageFields(portableInbox, gatewayInboxes), + traceContext: carrier, + }; + messages.push({ + message, + orderingKey: message.orderingKey, + }); + } catch (error) { + if (observation != null) await observation.delivery(inbox)?.failed(error); + throw error; + } } + if (observation != null) await observation.prepared("delivery"); const { outboxQueue } = ctx.federation; if (outboxQueue.enqueueMany == null || orderingKey != null) { const promises: PromiseSettledResult[] = await Promise.allSettled( messages.map(async (m) => { - await outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }); + await (observation == null + ? outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }) + : observation.enqueue( + [{ inbox: m.message.inbox, queueMessageId: m.message.id }], + "outbox", + () => + outboxQueue.enqueue(m.message, { orderingKey: m.orderingKey }), + )); recordOutboxEnqueue( ctx.federation.meterProvider, outboxQueue, @@ -6978,7 +7740,16 @@ async function forwardActivityInternal( } } else { try { - await outboxQueue.enqueueMany(messages.map((m) => m.message)); + await (observation == null + ? outboxQueue.enqueueMany(messages.map((m) => m.message)) + : observation.enqueue( + messages.map(({ message }) => ({ + inbox: message.inbox, + queueMessageId: message.id, + })), + "outbox", + () => outboxQueue.enqueueMany!(messages.map((m) => m.message)), + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to forward later:\n{error}", @@ -7036,6 +7807,7 @@ export class InboxContextImpl extends ContextImpl this.activityId, this.activityType, { + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -7116,46 +7888,72 @@ export class OutboxContextImpl extends ContextImpl activity: Activity, options: SendActivityOptionsForCollection = {}, ): Promise { - const tracer = this.tracerProvider.getTracer( - metadata.name, - metadata.version, - ); - return tracer.startActiveSpan( - this.federation.outboxQueue == null || options.immediate - ? "activitypub.outbox" - : "activitypub.fanout", - { - kind: this.federation.outboxQueue == null || options.immediate - ? SpanKind.CLIENT - : SpanKind.PRODUCER, - attributes: { - "activitypub.activity.type": getTypeId(activity).href, - "activitypub.activity.to": activity.toIds.map((to) => to.href), - "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), - "activitypub.activity.bto": activity.btoIds.map((bto) => bto.href), - "activitypub.activity.bcc": activity.bccIds.map((bcc) => bcc.href), - }, - }, - async (span) => { - try { - if (activity.id != null) { - span.setAttribute("activitypub.activity.id", activity.id.href); - } - const delivered = await this.sendActivityInternal( - sender, - recipients, - activity, - options, - span, - ); - if (delivered) this.#deliveryState.delivered = true; - } catch (e) { - span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); - throw e; - } finally { - span.end(); - } + return observeOutboundBroadcast( + this, + this.federation.outboundObservers, + (observation, provenance) => { + const tracer = this.tracerProvider.getTracer( + metadata.name, + metadata.version, + ); + const parentSpanContext = trace.getSpanContext(context.active()); + const activityType = getTypeId(activity).href; + observation?.setActivity({ type: activityType }); + return tracer.startActiveSpan( + this.federation.outboxQueue == null || options.immediate + ? "activitypub.outbox" + : "activitypub.fanout", + { + kind: this.federation.outboxQueue == null || options.immediate + ? SpanKind.CLIENT + : SpanKind.PRODUCER, + attributes: { + "activitypub.activity.type": activityType, + "activitypub.activity.to": activity.toIds.map((to) => to.href), + "activitypub.activity.cc": activity.ccIds.map((cc) => cc.href), + "activitypub.activity.bto": activity.btoIds.map((bto) => + bto.href + ), + "activitypub.activity.bcc": activity.bccIds.map((bcc) => + bcc.href + ), + }, + }, + async (span) => { + observation?.setSpan(span, parentSpanContext); + try { + const activityId = activity.id?.href; + if (activityId != null) { + span.setAttribute("activitypub.activity.id", activityId); + } + observation?.setActivity({ + type: activityType, + ...(activityId == null ? {} : { id: activityId }), + }); + const delivered = await this.sendActivityInternal( + sender, + recipients, + activity, + options, + span, + observation, + provenance, + ); + if (delivered) this.#deliveryState.delivered = true; + } catch (e) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: String(e), + }); + throw e; + } finally { + span.end(); + } + }, + ); }, + "send", + options.dispatchId, ); } @@ -7197,6 +7995,7 @@ export class OutboxContextImpl extends ContextImpl this.activityId, this.activityType, { + [outboundProvenance]: getOutboundProvenance(this), url: this.url, federation: this.federation, data, @@ -7211,6 +8010,9 @@ export class OutboxContextImpl extends ContextImpl } interface SendActivityInternalOptions { + readonly worker?: OutboundWorkerObservation; + readonly provenance?: OutboundProvenance; + readonly observation?: OutboundBroadcastObservation; readonly immediate?: boolean; readonly collectionSync?: string; readonly orderingKey?: string; diff --git a/packages/fedify/src/federation/mod.ts b/packages/fedify/src/federation/mod.ts index 79ecf68a3..dac2763b5 100644 --- a/packages/fedify/src/federation/mod.ts +++ b/packages/fedify/src/federation/mod.ts @@ -22,6 +22,7 @@ export { type IsActorAuthenticatedOptions, } from "./inbox-authentication.ts"; export * from "./inbox-report.ts"; +export * from "./outbound-report.ts"; export * from "./kv.ts"; export { createFederation, diff --git a/packages/fedify/src/federation/outbound-body.test.ts b/packages/fedify/src/federation/outbound-body.test.ts new file mode 100644 index 000000000..40d1afb4a --- /dev/null +++ b/packages/fedify/src/federation/outbound-body.test.ts @@ -0,0 +1,138 @@ +import { test } from "@fedify/fixture"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { ResponseBodyPump } from "./outbound-body.ts"; + +test("response body pump pulls once on demand and releases at EOF", async () => { + let pulls = 0; + let cancels = 0; + let controller!: ReadableStreamDefaultController; + let pulled!: () => void; + const pulling = new Promise((resolve) => { + pulled = resolve; + }); + const body = new ReadableStream({ + start(value) { + controller = value; + }, + pull() { + pulls++; + pulled(); + }, + cancel() { + cancels++; + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 1000); + try { + strictEqual(pulls, 0); + ok(body.locked); + const first = pump.read(); + const concurrent = pump.read(); + strictEqual(first, concurrent); + await pulling; + strictEqual(pulls, 1); + controller.enqueue(new Uint8Array([1, 2])); + const result = await first; + deepStrictEqual(result, { done: false, value: new Uint8Array([1, 2]) }); + strictEqual(await concurrent, result); + strictEqual(pulls, 1); + controller.close(); + deepStrictEqual(await pump.read(), { done: true, value: undefined }); + strictEqual(body.locked, false); + strictEqual(cancels, 0); + deepStrictEqual(await pump.read(), { done: true, value: undefined }); + } finally { + pump.close(); + } +}); + +test("response body pump deadline bounds stalled reads and cancellation cleanup", async (t) => { + for (const mode of ["no-pull", "stalled", "late-rejection"] as const) { + await t.step(mode, async () => { + let pulls = 0; + let cancels = 0; + let cancelled!: () => void; + const cancelling = new Promise((resolve) => { + cancelled = resolve; + }); + let rejectCancel!: (error: unknown) => void; + const cleanup = new Promise((_resolve, reject) => { + rejectCancel = reject; + }); + const body = new ReadableStream({ + pull() { + pulls++; + }, + cancel() { + cancels++; + cancelled(); + return cleanup; + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 25); + try { + if (mode === "no-pull") await cancelling; + else strictEqual(await pump.read(), undefined); + strictEqual(cancels, 1); + strictEqual(body.locked, false); + strictEqual(pulls, mode === "no-pull" ? 0 : 1); + strictEqual(await pump.read(), undefined); + pump.close(); + strictEqual(cancels, 1); + if (mode === "late-rejection") { + rejectCancel(new Error("late cleanup failure")); + // Flush rejection handlers; the late failure must remain observed. + await Promise.resolve(); + await Promise.resolve(); + } + } finally { + pump.close(); + } + }); + } +}); + +test("response body pump deadline also stops continuously ready empty chunks", async () => { + let pulls = 0; + let cancels = 0; + const body = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array()); + }, + cancel() { + cancels++; + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 25); + try { + while ( + await pump.read() != null + ) { /* Keep pulling a non-terminating source. */ } + ok(pulls > 0); + strictEqual(cancels, 1); + strictEqual(body.locked, false); + const stoppedAt = pulls; + strictEqual(await pump.read(), undefined); + strictEqual(pulls, stoppedAt); + } finally { + pump.close(); + } +}); + +test("response body pump preserves read failure and releases ownership", async () => { + const failure = new Error("broken body"); + const body = new ReadableStream({ + pull(controller) { + controller.error(failure); + }, + }, { highWaterMark: 0 }); + const pump = new ResponseBodyPump(body, 1000); + try { + await rejects(pump.read(), (error: unknown) => error === failure); + strictEqual(body.locked, false); + deepStrictEqual(await pump.read(), { done: true, value: undefined }); + } finally { + pump.close(); + } +}); diff --git a/packages/fedify/src/federation/outbound-body.ts b/packages/fedify/src/federation/outbound-body.ts new file mode 100644 index 000000000..803c12eb4 --- /dev/null +++ b/packages/fedify/src/federation/outbound-body.ts @@ -0,0 +1,93 @@ +/** One pull-driven response reader with an absolute lifetime and no extra queue. */ +export class ResponseBodyPump { + readonly #reader: ReadableStreamDefaultReader; + #expiresAt: number; + readonly #deadline: Promise; + #timer: ReturnType; + #expireDeadline!: () => void; + #pending: + | Promise | undefined> + | undefined; + #timedOut = false; + #closed = false; + #ended = false; + + constructor(body: ReadableStream, timeoutMs: number) { + this.#reader = body.getReader(); + this.#expiresAt = performance.now() + timeoutMs; + this.#deadline = new Promise((resolve) => { + this.#expireDeadline = () => resolve(undefined); + }); + this.#timer = setTimeout(() => this.#expire(), timeoutMs); + } + + /** Undefined means the absolute lifetime expired, independently of EOF. */ + read(): Promise | undefined> { + if (this.#closed) { + return Promise.resolve( + this.#timedOut ? undefined : { done: true, value: undefined }, + ); + } + if (performance.now() >= this.#expiresAt) { + this.#expire(); + return Promise.resolve(undefined); + } + // Concurrent consumers share this read instead of issuing another one. + this.#pending ??= Promise.race([this.#reader.read(), this.#deadline]).then( + (result) => { + this.#pending = undefined; + // A continuously ready stream can keep timers from running. Check the + // same monotonic deadline on every read, including empty chunks. + if (this.#timedOut || performance.now() >= this.#expiresAt) { + this.#expire(); + return undefined; + } + if (this.#closed) return { done: true as const, value: undefined }; + if (result?.done) { + this.#ended = true; + this.close(); + } + return result; + }, + (error) => { + this.#pending = undefined; + this.close(); + throw error; + }, + ); + return this.#pending; + } + + /** Shorten the remaining lifetime when ownership passes to diagnostic cleanup. */ + limitLifetime(timeoutMs: number): void { + if (this.#closed) return; + this.#expiresAt = Math.min( + this.#expiresAt, + performance.now() + Math.max(0, timeoutMs), + ); + clearTimeout(this.#timer); + if (performance.now() >= this.#expiresAt) this.#expire(); + else {this.#timer = setTimeout(() => + this.#expire(), this.#expiresAt - performance.now());} + } + + /** Stop pulling and release ownership without awaiting arbitrary cancellation. */ + close(): void { + if (this.#closed) return; + this.#closed = true; + clearTimeout(this.#timer); + try { + if (!this.#ended) this.#reader.cancel().catch(() => {}); + } finally { + // Pending read rejection and late cancellation rejection are both observed. + this.#reader.releaseLock(); + } + } + + #expire(): void { + if (this.#timedOut) return; + this.#timedOut = true; + this.#expireDeadline(); + this.close(); + } +} diff --git a/packages/fedify/src/federation/outbound-capture.test.ts b/packages/fedify/src/federation/outbound-capture.test.ts new file mode 100644 index 000000000..cdb7254cb --- /dev/null +++ b/packages/fedify/src/federation/outbound-capture.test.ts @@ -0,0 +1,402 @@ +import { test } from "@fedify/fixture"; +import { + deepStrictEqual, + ok, + rejects, + strictEqual, + throws, +} from "node:assert/strict"; +import { + captureResponseBody, + snapshotBodyCapture, + takeCapturedDiagnostic, +} from "./outbound-capture.ts"; +import { createFederationBuilder, FederationBuilderImpl } from "./builder.ts"; + +const limits = { maxBytes: 32, timeoutMs: 1000 }; +async function drain(body: ReadableStream): Promise { + const reader = body.getReader(); + const bytes: number[] = []; + try { + while (true) { + const value = await reader.read(); + if (value.done) return bytes; + bytes.push(...value.value); + } + } catch { + return bytes; + } finally { + reader.releaseLock(); + } +} + +test("response body capture validates explicit limits before replacing registration", () => { + const config = { maxBytes: 32, timeoutMs: 0.5, onBody() {} }; + const builder = createFederationBuilder(); + ok(builder instanceof FederationBuilderImpl); + builder.setOutboundObservers({ responseBodyCapture: config }); + const original = builder.outboundObservers; + deepStrictEqual(original.responseBodyCapture, { + ...config, + timeoutMs: 1, + responses: "all", + }); + ok(Object.isFrozen(original.responseBodyCapture)); + config.maxBytes = 1; + strictEqual(original.responseBodyCapture?.maxBytes, 32); + for ( + const [value, error] of [ + [null, TypeError], + [{ ...limits }, TypeError], + [{ onBody() {}, timeoutMs: 1 }, TypeError], + [{ onBody() {}, maxBytes: 1 }, TypeError], + [{ ...limits, onBody() {}, maxBytes: "1" }, TypeError], + [{ ...limits, onBody() {}, maxBytes: 0 }, RangeError], + [{ ...limits, onBody() {}, maxBytes: 1.5 }, RangeError], + [ + { ...limits, onBody() {}, maxBytes: Number.MAX_SAFE_INTEGER + 1 }, + RangeError, + ], + [{ ...limits, onBody() {}, timeoutMs: Infinity }, RangeError], + [{ ...limits, onBody() {}, timeoutMs: NaN }, RangeError], + [{ ...limits, onBody() {}, timeoutMs: 0 }, RangeError], + [{ ...limits, onBody() {}, timeoutMs: 2147483647.1 }, RangeError], + [{ ...limits, onBody() {}, responses: "failures" }, TypeError], + ] as const + ) { + throws( + () => + builder.setOutboundObservers({ responseBodyCapture: value as never }), + error, + ); + strictEqual(builder.outboundObservers, original); + } + strictEqual(snapshotBodyCapture(undefined), undefined); +}); + +test("response body capture distinguishes selection EOF and exact byte limits", async (t) => { + for ( + const mode of [ + "filtered", + "no-body", + "empty", + "short", + "exact", + "over", + ] as const + ) { + await t.step(mode, async () => { + const response = new Response( + mode === "no-body" + ? null + : mode === "empty" + ? "" + : mode === "over" + ? "abcd" + : "abc", + ); + let calls = 0; + let exposed: number[] = []; + let signal: AbortSignal | undefined; + const result = await captureResponseBody(response, { + maxBytes: mode === "exact" || mode === "over" ? 3 : 32, + timeoutMs: 1000, + ...(mode === "filtered" + ? { responses: "non-success-http-responses" as const } + : {}), + }, async (body, current) => { + calls++; + signal = current; + exposed = await drain(body); + }); + strictEqual( + result.type, + mode === "filtered" || mode === "no-body" + ? mode + : mode === "exact" || mode === "over" + ? "limit-reached" + : "complete", + ); + strictEqual( + result.bytesExposed, + mode === "filtered" || mode === "no-body" || mode === "empty" ? 0 : 3, + ); + strictEqual(calls, mode === "filtered" || mode === "no-body" ? 0 : 1); + if (calls) { + ok(signal?.aborted); + strictEqual(response.body?.locked, false); + } + if (result.bytesExposed) deepStrictEqual(exposed, [97, 98, 99]); + ok(Object.isFrozen(result)); + }); + } +}); + +test("response body capture revokes locked readers and bounds callback cleanup", async (t) => { + for ( + const mode of [ + "returned", + "cancelled", + "throws", + "hangs", + "pending-read", + "body-error", + ] as const + ) { + await t.step(mode, async () => { + const failure = new Error(mode); + let signal!: AbortSignal; + let retained!: ReadableStreamDefaultReader; + let pending: Promise | undefined; + let rejectLate!: (error: unknown) => void; + const late = new Promise((_resolve, reject) => { + rejectLate = reject; + }); + const body = new ReadableStream({ + pull(controller) { + if (mode === "body-error") controller.error(failure); + }, + cancel() { + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const result = await captureResponseBody(new Response(body), { + ...limits, + timeoutMs: 25, + }, async (stream, current) => { + signal = current; + retained = stream.getReader(); + if (mode === "throws") throw failure; + if (mode === "cancelled") await retained.cancel(failure); + if (mode === "hangs") await late; + if (mode === "pending-read") { + pending = retained.read().catch((error) => error); + } + if (mode === "body-error") await retained.read(); + }); + const expected = mode === "returned" || mode === "pending-read" + ? "callback-returned" + : mode === "cancelled" + ? "consumer-stopped" + : mode === "hangs" + ? "deadline" + : "failed"; + strictEqual(result.type, expected); + strictEqual(result.bytesExposed, 0); + ok(signal.aborted); + strictEqual(body.locked, false); + if (result.type === "failed") { + strictEqual(result.phase, mode === "throws" ? "observer" : "body"); + strictEqual(result.error, failure); + } + if (result.type === "consumer-stopped") { + strictEqual(result.reason, failure); + } + if (mode !== "cancelled") await rejects(retained.read()); + if (pending) ok(await pending instanceof Error); + retained.releaseLock(); + if (mode === "hangs") { + rejectLate(failure); + await Promise.resolve(); + await Promise.resolve(); + } + }); + } +}); + +test("response body capture shares one reader with bounded diagnostics", async () => { + const response = new Response("diagnostic failure", { status: 503 }); + ok(response.body); + const getReader = response.body.getReader.bind(response.body); + let acquisitions = 0; + Object.defineProperty(response.body, "getReader", { + value: () => { + acquisitions++; + return getReader(); + }, + }); + const summary = await captureResponseBody(response, { + ...limits, + maxBytes: 4, + }, async (body) => { + await drain(body); + }); + strictEqual(summary.type, "limit-reached"); + strictEqual(summary.bytesExposed, 4); + strictEqual(acquisitions, 1); + strictEqual(takeCapturedDiagnostic(response), "diagnostic failure"); + strictEqual(takeCapturedDiagnostic(response), undefined); +}); + +test("response body capture isolates unavailable readers and diagnostic deadlines", async (t) => { + await t.step("already locked source", async () => { + const response = new Response("locked"); + const reader = response.body!.getReader(); + try { + const result = await captureResponseBody(response, limits, () => { + throw new Error("must not invoke callback"); + }); + ok(result.type === "failed"); + strictEqual(result.phase, "body"); + strictEqual(result.bytesExposed, 0); + } finally { + await reader.cancel(); + reader.releaseLock(); + } + }); + await t.step("diagnostics retain their own deadline", async () => { + let cancels = 0; + const body = new ReadableStream({ + pull() {}, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const response = new Response(body, { status: 503 }); + const result = await captureResponseBody(response, { + ...limits, + timeoutMs: 10000, + }, () => {}); + deepStrictEqual(result, { type: "callback-returned", bytesExposed: 0 }); + strictEqual(takeCapturedDiagnostic(response), "… (timed out)"); + strictEqual(cancels, 1); + strictEqual(body.locked, false); + }); +}); + +test("response body capture bounds oversized and nonterminating sources", async (t) => { + for (const mode of ["oversized", "infinite", "empty-infinite"] as const) { + await t.step(mode, async () => { + let pulls = 0; + let cancels = 0; + const source = new Uint8Array( + mode === "oversized" ? 1024 * 1024 : mode === "infinite" ? 1 : 0, + ).fill(255); + const original = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(source); + }, + cancel() { + cancels++; + }, + }, { highWaterMark: 0 }); + let exposed = 0; + const result = await captureResponseBody(new Response(original), { + maxBytes: 17, + timeoutMs: 50, + }, async (body) => { + const reader = body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + exposed += chunk.value.length; + ok(chunk.value.buffer.byteLength <= 17); + strictEqual(chunk.value[0], 255); + chunk.value.fill(0); + } + } catch { + /* Cap/deadline revokes the reader. */ + } finally { + reader.releaseLock(); + } + }); + strictEqual( + result.type, + mode === "empty-infinite" ? "deadline" : "limit-reached", + ); + strictEqual(result.bytesExposed, mode === "empty-infinite" ? 0 : 17); + strictEqual(exposed, result.bytesExposed); + strictEqual(cancels, 1); + strictEqual(original.locked, false); + if (mode !== "empty-infinite") { + strictEqual(pulls, mode === "oversized" ? 1 : 17); + strictEqual(source[0], 255); + } + }); + } +}); + +test("response body capture applies backpressure and revokes a paused consumer", async () => { + let pulls = 0; + let cancels = 0; + let retained!: ReadableStreamDefaultReader; + let exposed = 0; + const source = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array([1])); + }, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const result = await captureResponseBody(new Response(source), { + maxBytes: 32, + timeoutMs: 50, + }, async (body, signal) => { + retained = body.getReader(); + const first = await retained.read(); + ok(!first.done); + exposed += first.value.length; + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else signal.addEventListener("abort", () => resolve(), { once: true }); + }); + strictEqual(pulls, 1); + await rejects(retained.read()); + }); + deepStrictEqual(result, { type: "deadline", bytesExposed: 1 }); + strictEqual(exposed, 1); + strictEqual(pulls, 1); + strictEqual(cancels, 1); + strictEqual(source.locked, false); + await rejects(retained.read()); + retained.releaseLock(); +}); + +test("response body capture rejects late source data and observes late pull failure", async () => { + let controller!: ReadableStreamDefaultController; + let rejectPull!: (error: unknown) => void; + const pulling = new Promise((_resolve, reject) => { + rejectPull = reject; + }); + let cancels = 0; + const source = new ReadableStream({ + start(value) { + controller = value; + }, + pull() { + return pulling; + }, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + let retained!: ReadableStreamDefaultReader; + let exposed = 0; + const result = await captureResponseBody(new Response(source), { + maxBytes: 32, + timeoutMs: 25, + }, async (body) => { + retained = body.getReader(); + try { + const chunk = await retained.read(); + if (!chunk.done) exposed += chunk.value.length; + } catch { /* Revoked at deadline. */ } + }); + deepStrictEqual(result, { type: "deadline", bytesExposed: 0 }); + strictEqual(exposed, 0); + strictEqual(cancels, 1); + throws(() => controller.enqueue(new Uint8Array([1])), TypeError); + rejectPull(new Error("late upstream failure")); + await Promise.resolve(); + await Promise.resolve(); + await rejects(retained.read()); + retained.releaseLock(); + strictEqual(source.locked, false); +}); diff --git a/packages/fedify/src/federation/outbound-capture.ts b/packages/fedify/src/federation/outbound-capture.ts new file mode 100644 index 000000000..523ce5538 --- /dev/null +++ b/packages/fedify/src/federation/outbound-capture.ts @@ -0,0 +1,261 @@ +import { ResponseBodyPump } from "./outbound-body.ts"; +import type { + OutboundBodyCaptureSummary, + OutboundResponseBodyCapture, +} from "./outbound-report.ts"; + +const diagnosticTimeoutMs = 1000; +const diagnosticMaxBytes = 1024; +const diagnostics = new WeakMap(); + +/** Reuse a selected capture's diagnostic preview instead of acquiring a second reader. */ +export function takeCapturedDiagnostic(response: Response): string | undefined { + const value = diagnostics.get(response); + diagnostics.delete(response); + return value; +} + +/** Validate and copy before replacing the current observer registration. */ +export function snapshotBodyCapture( + value: OutboundResponseBodyCapture | undefined, +): OutboundResponseBodyCapture | undefined { + if (value === undefined) return undefined; + if ( + value == null || typeof value !== "object" || + typeof value.onBody !== "function" + ) { + throw new TypeError("Response body capture requires an onBody callback."); + } + const { maxBytes, timeoutMs, responses = "all", onBody } = value; + if (typeof maxBytes !== "number" || typeof timeoutMs !== "number") { + throw new TypeError( + "Response body capture requires numeric maxBytes and timeoutMs.", + ); + } + if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) { + throw new RangeError( + "Response body capture maxBytes must be a positive safe integer.", + ); + } + if ( + !Number.isFinite(timeoutMs) || timeoutMs <= 0 || + Math.ceil(timeoutMs) > 2147483647 + ) { + throw new RangeError( + "Response body capture timeoutMs is outside the timer range.", + ); + } + if (responses !== "all" && responses !== "non-success-http-responses") { + throw new TypeError("Invalid response body capture selector."); + } + return Object.freeze({ + maxBytes, + timeoutMs: Math.ceil(timeoutMs), + responses, + onBody, + }); +} + +type WithoutBytes = T extends unknown ? Omit : never; +type CaptureResult = WithoutBytes; + +/** One source reader feeds both the revocable application view and bounded diagnostics. */ +export async function captureResponseBody( + response: Response, + options: { + maxBytes: number; + timeoutMs: number; + responses?: "all" | "non-success-http-responses"; + }, + onBody: ( + body: ReadableStream, + signal: AbortSignal, + ) => void | Promise, +): Promise { + if (options.responses === "non-success-http-responses" && response.ok) { + return Object.freeze({ type: "filtered", bytesExposed: 0 }); + } + if (response.body == null) { + return Object.freeze({ type: "no-body", bytesExposed: 0 }); + } + const start = performance.now(); + const expires = start + options.timeoutMs; + let pump: ResponseBodyPump; + try { + pump = new ResponseBodyPump( + response.body, + Math.max(options.timeoutMs, response.ok ? 0 : diagnosticTimeoutMs), + ); + } catch (error) { + return Object.freeze({ + type: "failed" as const, + phase: "body" as const, + error, + bytesExposed: 0, + }); + } + const signal = new AbortController(); + let controller!: ReadableStreamDefaultController; + let active = true; + let bytesExposed = 0; + let result: CaptureResult | undefined; + let eof = false; + let diagnosticBytes = 0; + let diagnosticTruncated = false; + let diagnosticFailed = false; + const preview: string[] = []; + const decoder = new TextDecoder(); + let pending: + | Promise | undefined> + | undefined; + const read = () => + pending ??= pump.read().then((value) => { + pending = undefined; + if (value?.done) eof = true; + if ( + !response.ok && value != null && !value.done && + performance.now() - start < diagnosticTimeoutMs + ) { + const remaining = diagnosticMaxBytes - diagnosticBytes; + const chunk = value.value; + if (chunk.length > remaining) diagnosticTruncated = true; + if (remaining > 0 && chunk.length > 0) { + const selected = chunk.subarray(0, remaining); + preview.push(decoder.decode(selected, { stream: true })); + diagnosticBytes += selected.length; + } + } + return value; + }, (error) => { + pending = undefined; + diagnosticFailed = true; + throw error; + }); + const revoke = (reason: unknown) => { + active = false; + if (!signal.signal.aborted) signal.abort(reason); + // Controller access works even while the application retains a locked reader. + try { + controller.error(reason); + } catch { /* The consumer may already have cancelled. */ } + }; + const body = new ReadableStream({ + start(value) { + controller = value; + }, + async pull() { + try { + while (active) { + if (performance.now() >= expires) { + result = { type: "deadline" }; + revoke( + new DOMException( + "Response body capture deadline exceeded.", + "TimeoutError", + ), + ); + return; + } + const value = await read(); + if (!active) return; + if (performance.now() >= expires || value == null) { + result = { type: "deadline" }; + revoke( + new DOMException( + "Response body capture deadline exceeded.", + "TimeoutError", + ), + ); + return; + } + if (value.done) { + result = { type: "complete" }; + active = false; + controller.close(); + return; + } + if (value.value.length === 0) continue; + const chunk = value.value.slice(0, options.maxBytes - bytesExposed); + bytesExposed += chunk.length; + controller.enqueue(chunk); + if (bytesExposed === options.maxBytes) { + result = { type: "limit-reached" }; + revoke( + new DOMException( + "Response body capture byte limit reached.", + "QuotaExceededError", + ), + ); + } + return; + } + } catch (error) { + if (active) { + result = { type: "failed", phase: "body", error }; + revoke(error); + } + } + }, + cancel(reason) { + result = { type: "consumer-stopped", reason }; + revoke(reason); + }, + }, { highWaterMark: 0 }); + let timer!: ReturnType; + const deadline = new Promise((resolve) => { + timer = setTimeout(() => { + result = { type: "deadline" }; + revoke( + new DOMException( + "Response body capture deadline exceeded.", + "TimeoutError", + ), + ); + resolve(); + }, Math.max(0, expires - performance.now())); + }); + const callback = Promise.resolve().then(() => onBody(body, signal.signal)) + .then( + () => {}, + (error) => { + if (result == null || result.type === "complete") { + result = { type: "failed", phase: "observer", error }; + } + }, + ); + try { + await Promise.race([callback, deadline]); + clearTimeout(timer); + if (performance.now() >= expires) result = { type: "deadline" }; + result ??= { type: "callback-returned" }; + revoke(new DOMException("Response body capture finished.", "AbortError")); + // Diagnostics may continue under their own budget, never exposing more app bytes. + if (!response.ok) { + pump.limitLifetime(diagnosticTimeoutMs - (performance.now() - start)); + try { + while ( + !eof && !diagnosticTruncated && + performance.now() - start < diagnosticTimeoutMs + ) { + if (await read() == null) break; + } + } catch { /* Diagnostic failure does not replace HTTP evidence. */ } + diagnostics.set( + response, + preview.join("") + + (diagnosticTruncated + ? "… (truncated)" + : !eof && !diagnosticFailed + ? "… (timed out)" + : ""), + ); + } + return Object.freeze({ + ...result, + bytesExposed, + }) as OutboundBodyCaptureSummary; + } finally { + clearTimeout(timer); + pump.close(); + } +} diff --git a/packages/fedify/src/federation/outbound-dispatch.test.ts b/packages/fedify/src/federation/outbound-dispatch.test.ts new file mode 100644 index 000000000..b1acc5f96 --- /dev/null +++ b/packages/fedify/src/federation/outbound-dispatch.test.ts @@ -0,0 +1,452 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Person } from "@fedify/vocab"; +import { + configure, + getLogger, + type LogRecord, + reset, + withContext, +} from "@logtape/logtape"; +import fetchMock from "fetch-mock"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; +import type { SendActivityOptions } from "./context.ts"; +import type { Federation } from "./federation.ts"; +import { MemoryKvStore } from "./kv.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; +import type { MessageQueue } from "./mq.ts"; +import type { + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; +import type { Message } from "./queue.ts"; +import type { SenderKeyPair } from "./send.ts"; + +const activity = new Create({ + id: new URL("https://example.com/dispatch"), + actor: new URL("https://example.com/person2"), +}); +const target = new Person({ + id: new URL("https://remote.example/bob"), + inbox: new URL("https://remote.example/inbox"), +}); +const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; +const loaders = { + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, +}; +const modes = [ + "send", + "outbox-send", + "inbox-forward", + "outbox-forward", +] as const; +type Mode = typeof modes[number]; +function entry( + federation: Federation, + mode: Mode, + identity: SenderKeyPair | { identifier: string } = sender, +) { + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const outbox = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const inbox = base.toInboxContext( + null, + {}, + activity.id!.href, + Create.typeId.href, + ); + if (mode === "send") { + return { + context: base, + invoke: (options: SendActivityOptions) => + base.sendActivity( + identity, + target, + activity, + Object.assign(options, { fanout: "force" as const }), + ), + }; + } + if (mode === "outbox-send") { + return { + context: outbox, + invoke: (options: SendActivityOptions) => + outbox.sendActivity( + identity, + target, + activity, + Object.assign(options, { fanout: "skip" as const }), + ), + }; + } + const context = mode === "inbox-forward" ? inbox : outbox; + return { + context, + invoke: (options: SendActivityOptions) => + context.forwardActivity( + identity, + target, + Object.assign(options, { skipIfUnsigned: false }), + ), + }; +} + +test("dispatch ID validation precedes send and forward side effects", async (t) => { + for ( + const { mode, observed } of [ + { mode: "send" as const, observed: false }, + ...modes.map((mode) => ({ mode, observed: true })), + ] + ) { + await t.step( + `${mode}: ${observed ? "observed" : "unobserved"}`, + async () => { + let sideEffects = 0; + const queue: MessageQueue = { + enqueue() { + sideEffects++; + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { outbox: queue, fanout: queue }, + }); + federation.setActorDispatcher("/users/{identifier}", () => { + sideEffects++; + return new Person({}); + }).setKeyPairsDispatcher(() => { + sideEffects++; + return []; + }); + const { context, invoke } = entry(federation, mode, { + identifier: "alice", + }); + const provider = context.tracerProvider; + Object.defineProperty(context, "tracerProvider", { + get() { + sideEffects++; + return provider; + }, + }); + const events: OutboundBroadcastEvent[] = []; + let deliveries = 0; + if (observed) { + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + onDelivery() { + deliveries++; + }, + }); + } + for (const value of [null, "x".repeat(1025), "😀".repeat(257)]) { + const start = events.length; + let original: unknown; + await rejects( + invoke({ dispatchId: value as string }), + (error: unknown) => { + original = error; + ok( + value === null + ? error instanceof TypeError + : error instanceof RangeError, + ); + ok((error as Error).message.includes("dispatchId")); + return true; + }, + ); + strictEqual(sideEffects, 0); + strictEqual(deliveries, 0); + if (observed) { + deepStrictEqual(events.slice(start).map((event) => event.stage), [ + "started", + "preparation-finished", + ]); + const finished = events.at(-1)!; + ok( + finished.stage === "preparation-finished" && + finished.result.type === "failed", + ); + strictEqual(finished.result.failure.phase, "validation"); + strictEqual(finished.result.failure.error, original); + strictEqual(finished.provenance.dispatchId, undefined); + } + } + }, + ); + } +}); + +test("dispatch IDs correlate send and forward entries without deduplication or context leakage", async (t) => { + for (const mode of modes) { + await t.step(mode, async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { fanout: queue, outbox: queue }, + }); + const { context, invoke } = entry(federation, mode); + const broadcasts: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const contexts: unknown[] = []; + let current: { dispatchId?: string } = {}; + federation.setOutboundObservers({ + onBroadcast(ctx, event) { + broadcasts.push(event); + contexts.push(ctx); + if (event.stage === "started") { + current.dispatchId = "observer-mutated"; + } + }, + onDelivery(ctx, event) { + deliveries.push(event); + contexts.push(ctx); + }, + }); + const dispatchId = ' app\\dispatch"\n한글 '; + for (const value of [dispatchId, dispatchId, undefined]) { + current = { dispatchId: value }; + await invoke(current); + } + strictEqual(messages.length, 3); + const starts = broadcasts.filter((event) => event.stage === "started"); + strictEqual(starts.length, 3); + strictEqual(new Set(starts.map((event) => event.broadcastId)).size, 3); + deepStrictEqual(starts.map((event) => event.activity), [{}, {}, {}]); + const finishes = broadcasts.filter((event) => + event.stage === "preparation-finished" + ); + strictEqual(finishes.length, 3); + for (const event of finishes) { + deepStrictEqual(event.activity, { + id: activity.id!.href, + type: Create.typeId.href, + }); + ok(Object.isFrozen(event.activity)); + } + for (const ctx of contexts) strictEqual(ctx, context); + for (let index = 0; index < 3; index++) { + const message = messages[index]; + ok(message.type === "outbox" || message.type === "fanout"); + const expected = index < 2 ? dispatchId : undefined; + strictEqual(message.provenance?.dispatchId, expected); + strictEqual(starts[index].provenance.dispatchId, expected); + deepStrictEqual( + JSON.parse(JSON.stringify(message.provenance ?? {})), + message.provenance ?? {}, + ); + const delivery = deliveries[index * 2]; + strictEqual(delivery.provenance.dispatchId, expected); + } + if (mode === "send") { + const fanout = messages[0]; + ok(fanout.type === "fanout"); + await federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(fanout)), + ); + await federation.processQueuedTask(undefined, structuredClone(fanout)); + for (const child of messages.slice(3)) { + ok(child.type === "outbox"); + strictEqual(child.provenance?.dispatchId, dispatchId); + deepStrictEqual( + child.outbound, + fanout.inboxes[target.inboxId!.href].outbound, + ); + } + strictEqual(messages.length, 5); + strictEqual(broadcasts.length, 6); + } + }); + } +}); + +test("dispatch ID byte limits and queue fallback preserve exact values", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const producer = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { outbox: queue }, + }); + const { invoke } = entry(producer, "outbox-send"); + const valid = ["", "x".repeat(1024), "😀".repeat(256)]; + for (const dispatchId of valid) await invoke({ dispatchId }); + strictEqual(messages.length, 3); + const records: LogRecord[] = []; + await reset(); + await configure({ + sinks: { buffer: (record) => records.push(record) }, + loggers: [{ + category: ["test", "dispatch"], + sinks: ["buffer"], + lowestLevel: "info", + }, { + category: ["fedify", "federation", "queue"], + sinks: ["buffer"], + lowestLevel: "warning", + }, { category: ["logtape", "meta"], sinks: [] }], + contextLocalStorage: new AsyncLocalStorage(), + }); + const consumer = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { outbox: queue }, + }); + const descendants: OutboundBroadcastEvent[] = []; + consumer.setOutboundObservers({ + onBroadcast(_ctx, event) { + descendants.push(event); + }, + }); + consumer.setOutboxPermanentFailureHandler(async (ctx) => { + getLogger(["test", "dispatch"]).info("original dispatch"); + // A new public invocation needs its own application-owned correlation. + await ctx.sendActivity(sender, [], activity); + }); + fetchMock.mockGlobal().post(target.inboxId!.href, 410); + try { + const queued = messages[0]; + ok(queued.type === "outbox"); + const malformed = [ + { dispatchId: null }, + { dispatchId: "x".repeat(1025) }, + { dispatchId: "😀".repeat(257) }, + { + inboxRequest: { + inboxRequestId: "", + requestId: "bad", + context: { + origin: "x", + host: "x", + hostname: "x", + canonicalOrigin: "x", + }, + }, + }, + ]; + for (const provenance of malformed) { + await withContext( + { requestId: "current-request", dispatchId: "ambient-dispatch" }, + () => + consumer.processQueuedTask(undefined, { + ...queued, + provenance: provenance as NonNullable, + }), + ); + } + ok(fetchMock.callHistory.calls().length >= malformed.length); + const warnings = records.filter((record) => + record.rawMessage === + "Ignoring invalid provenance in queue message {messageId}: {error}" + ); + strictEqual(warnings.length, malformed.length); + for (const warning of warnings) { + strictEqual(warning.properties.messageId, queued.id); + } + for (let index = 0; index < messages.length; index++) { + const message = messages[index]; + ok(message.type === "outbox"); + strictEqual(message.provenance?.dispatchId, valid[index]); + await withContext( + { requestId: "current-request", dispatchId: "ambient-dispatch" }, + () => + consumer.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ), + ); + } + const logs = records.filter((record) => + record.rawMessage === "original dispatch" + ); + deepStrictEqual( + logs.map((record) => record.properties.dispatchId), + [...malformed.map(() => undefined), ...valid], + ); + for (const record of logs) { + strictEqual(record.properties.requestId, "current-request"); + } + strictEqual(descendants.length, 2 * (malformed.length + valid.length)); + for (const event of descendants) { + strictEqual(event.provenance.dispatchId, undefined); + } + } finally { + fetchMock.hardReset(); + await reset(); + } +}); + +test("malformed queued provenance does not poison a retry", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }), + queue: { outbox: queue }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, target, activity, { fanout: "skip" }); + const original = messages[0]; + ok(original.type === "outbox"); + const malformed = { + ...original, + provenance: { inboxRequest: { inboxRequestId: "" } }, + } as Message; + fetchMock.mockGlobal().post(target.inboxId!.href, 503); + try { + await federation.processQueuedTask(undefined, malformed); + strictEqual(messages.length, 2); + const retry = messages[1]; + ok(retry.type === "outbox"); + deepStrictEqual(retry.provenance, {}); + strictEqual(retry.attempt, 1); + await federation.processQueuedTask(undefined, retry); + strictEqual(messages.length, 3); + const next = messages[2]; + ok(next.type === "outbox"); + deepStrictEqual(next.provenance, {}); + } finally { + fetchMock.hardReset(); + } +}); diff --git a/packages/fedify/src/federation/outbound-document.test.ts b/packages/fedify/src/federation/outbound-document.test.ts new file mode 100644 index 000000000..b5c1ee78e --- /dev/null +++ b/packages/fedify/src/federation/outbound-document.test.ts @@ -0,0 +1,140 @@ +import { test } from "@fedify/fixture"; +import { deepStrictEqual, ok, strictEqual, throws } from "node:assert/strict"; +import { projectSenderPayload } from "./outbound-document.ts"; +import { createFederationBuilder, FederationBuilderImpl } from "./builder.ts"; + +const as = "https://www.w3.org/ns/activitystreams"; +const secret = "https://private.example/recipient"; + +test("sender payload projects nested compact expanded and proof-bearing blind fields", () => { + const source = { + "@context": [as, "https://w3id.org/security/data-integrity/v2"], + type: "Create", + bto: [secret], + object: { + bcc: [secret], + children: [{ + "as:bto": secret, + [`${as}#bcc`]: secret, + content: "visible", + }], + proof: { + "@context": "https://w3id.org/identity/v1", + type: "DataIntegrityProof", + proofValue: "retained-proof", + "as:bcc": secret, + }, + }, + signature: { signatureValue: "retained-signature", [`${as}#bto`]: secret }, + }; + const original = structuredClone(source); + const result = projectSenderPayload(source); + ok(result.type === "available"); + strictEqual(result.representation, "privacy-projection"); + strictEqual(result.redacted, true); + strictEqual(JSON.stringify(result.value).includes(secret), false); + ok(JSON.stringify(result.value).includes("retained-proof")); + ok(JSON.stringify(result.value).includes("retained-signature")); + deepStrictEqual(source, original); + ok(Object.isFrozen(result)); + ok(Object.isFrozen(result.value)); + const value = result.value as Record; + ok(Object.isFrozen(value.object)); + source.object.children[0].content = "changed"; + ok(JSON.stringify(result.value).includes("visible")); + const expanded = projectSenderPayload([{ + "@type": [`${as}#Create`], + [`${as}#object`]: [{ + [`${as}#bcc`]: [{ "@id": secret }], + [`${as}#content`]: [{ "@value": "visible" }], + }], + }]); + ok(expanded.type === "available"); + strictEqual(expanded.redacted, true); + strictEqual(JSON.stringify(expanded.value).includes(secret), false); +}); + +test("sender payload fails closed without evaluating getters or JSON conversion", () => { + let getters = 0; + let conversions = 0; + const getter = { + "@context": as, + get object() { + getters++; + return "private"; + }, + }; + const conversion = { "@context": as }; + Object.defineProperty(conversion, "toJSON", { + value() { + conversions++; + return {}; + }, + }); + const context = [as]; + Object.defineProperty(context, "includes", { + get() { + getters++; + throw new Error("must not run"); + }, + enumerable: true, + }); + const cycle: Record = { "@context": as }; + cycle.object = cycle; + let deep: unknown = "leaf"; + for (let i = 0; i < 65; i++) deep = { object: deep }; + for ( + const source of [ + { "@context": [as, { private: `${as}#bcc` }], private: secret }, + { + "@context": as, + object: { + "@context": { as: "https://other.example/" }, + "as:bcc": secret, + }, + }, + { "@context": [as, "https://unknown.example/context"], private: secret }, + { "@context": [as, { sensitive: "as:bcc" }], sensitive: secret }, + { + "@context": [as, { toot: `${as}#`, Emoji: "toot:bcc" }], + Emoji: secret, + }, + { type: "Create", object: secret }, + getter, + conversion, + { "@context": context }, + cycle, + { "@context": as, object: deep }, + { "@context": as, object: NaN }, + { "@context": as, object: new Date() }, + { "@context": as, object: [undefined] }, + ] + ) { + deepStrictEqual(projectSenderPayload(source), { + stage: "sender-payload", + type: "unavailable", + reason: "privacy-filter-unavailable", + }); + } + strictEqual(getters, 0); + strictEqual(conversions, 0); +}); + +test("sender payload opt-in validates atomically and retains builder selection", () => { + const builder = createFederationBuilder(); + ok(builder instanceof FederationBuilderImpl); + const config = { senderPayload: true, onAttempt() {} }; + builder.setOutboundObservers(config); + const original = builder.outboundObservers; + config.senderPayload = false; + strictEqual(original.senderPayload, true); + for (const value of [null, 1, "true", {}]) { + throws( + () => builder.setOutboundObservers({ senderPayload: value as never }), + TypeError, + ); + strictEqual(builder.outboundObservers, original); + } + builder.setOutboundObservers({}); + strictEqual(builder.outboundObservers.senderPayload, false); +}); diff --git a/packages/fedify/src/federation/outbound-document.ts b/packages/fedify/src/federation/outbound-document.ts new file mode 100644 index 000000000..420bbd8fc --- /dev/null +++ b/packages/fedify/src/federation/outbound-document.ts @@ -0,0 +1,267 @@ +import { preloadedContexts } from "@fedify/vocab-runtime"; +import type { + OutboundJsonValue, + OutboundSenderPayload, +} from "./outbound-report.ts"; + +const asContext = "https://www.w3.org/ns/activitystreams"; +const asNamespace = `${asContext}#`; +const blind = new Set([ + "bto", + "bcc", + "as:bto", + "as:bcc", + `${asNamespace}bto`, + `${asNamespace}bcc`, +]); +const contextCandidates = [ + asContext, + "https://w3id.org/security/v1", + "https://w3id.org/identity/v1", + "https://gotosocial.org/ns", + "https://w3id.org/security/data-integrity/v1", + "https://w3id.org/security/data-integrity/v2", +]; + +// This audit is specific to blind addressing, unlike the attachment/Public +// normalization predicates. Unknown definitions never trigger context loading. +function safeContext(context: unknown): boolean { + if (Array.isArray(context)) return context.every(safeContext); + if (context == null || typeof context !== "object") return false; + for (const [name, definition] of Object.entries(context)) { + if (name === "@import") return false; + const mapping = typeof definition === "string" + ? definition + : definition != null && typeof definition === "object" + ? (definition as Record)["@id"] + : undefined; + if (name === "as" && mapping !== asNamespace) return false; + if ( + name !== "as" && typeof mapping === "string" && + mapping.startsWith(asNamespace) && + (mapping.endsWith("#") || mapping.endsWith("/") || + typeof definition === "object" && + (definition as Record)["@prefix"] === true) + ) return false; + if ( + typeof mapping === "string" && blind.has(mapping) && name !== "bto" && + name !== "bcc" + ) return false; + if ( + (name === "bto" || name === "bcc") && mapping !== `as:${name}` && + mapping !== `${asNamespace}${name}` + ) return false; + if ( + name === "@vocab" && typeof mapping === "string" && + mapping.startsWith(asNamespace) + ) return false; + if ( + definition != null && typeof definition === "object" && + "@context" in definition && + !safeContext((definition as Record)["@context"]) + ) return false; + } + return true; +} +const contexts = new Set(contextCandidates.filter((url) => { + const document = preloadedContexts[url]; + return document != null && typeof document === "object" && + safeContext((document as Record)["@context"]); +})); +// Exact generated extension definitions, not an interpreter for arbitrary inline +// contexts. In particular, callers cannot redefine prefixes or add blind aliases. +const generatedTerms: Record = { + toot: "http://joinmastodon.org/ns#", + misskey: "https://misskey-hub.net/ns#", + fedibird: "http://fedibird.com/ns#", + sensitive: "as:sensitive", + votersCount: { + "@id": "toot:votersCount", + "@type": "http://www.w3.org/2001/XMLSchema#nonNegativeInteger", + }, + Emoji: "toot:Emoji", + Hashtag: "as:Hashtag", + ChatMessage: "http://litepub.social/ns#ChatMessage", + quote: { "@id": "https://w3id.org/fep/044f#quote", "@type": "@id" }, + quoteUrl: "as:quoteUrl", + _misskey_quote: "misskey:_misskey_quote", + quoteUri: "fedibird:quoteUri", + QuoteAuthorization: "https://w3id.org/fep/044f#QuoteAuthorization", + QuoteRequest: "https://w3id.org/fep/044f#QuoteRequest", + quoteAuthorization: { + "@id": "https://w3id.org/fep/044f#quoteAuthorization", + "@type": "@id", + }, + emojiReactions: { "@id": "fedibird:emojiReactions", "@type": "@id" }, +}; +function sameDefinition(value: OutboundJsonValue, expected: unknown): boolean { + if (value === expected) return true; + if ( + value == null || expected == null || typeof value !== "object" || + typeof expected !== "object" || Array.isArray(value) + ) return false; + const actual = value as Record; + const reference = expected as Record; + return Object.keys(actual).length === Object.keys(reference).length && + Object.keys(actual).every((name) => + Object.hasOwn(reference, name) && + sameDefinition(actual[name], reference[name]) + ); +} +function generatedContext(value: OutboundJsonValue): boolean { + if (value == null || typeof value !== "object" || Array.isArray(value)) { + return false; + } + return Object.entries(value).every(([name, definition]) => + Object.hasOwn(generatedTerms, name) && + sameDefinition(definition, generatedTerms[name]) + ); +} +const keywords = new Set([ + "@id", + "@type", + "@value", + "@language", + "@direction", + "@index", + "@list", + "@set", + "@graph", + "@reverse", + "@included", + "@nest", +]); + +/** Validate the explicit opt-in before replacing observer registration. */ +export function snapshotSenderPayloadOption( + value: boolean | undefined, +): boolean { + if (value === undefined) return false; + if (typeof value !== "boolean") { + throw new TypeError("Outbound senderPayload must be a boolean."); + } + return value; +} + +/** Project existing plain JSON only; no getters, JSON-LD processing or serializers. */ +export function projectSenderPayload(input: unknown): OutboundSenderPayload { + try { + let compact = false; + let redacted = false; + const seen = new WeakSet(); + const copy = ( + value: unknown, + depth: number, + contextData = false, + ): OutboundJsonValue => { + if (depth >= 64) throw new TypeError("Unsafe document depth."); + if ( + value === null || typeof value === "string" || + typeof value === "boolean" + ) return value; + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "object" || value == null || seen.has(value)) { + throw new TypeError("Not independent JSON data."); + } + const array = Array.isArray(value); + const prototype = Object.getPrototypeOf(value); + if ( + array + ? prototype !== Array.prototype + : prototype !== Object.prototype && prototype !== null + ) throw new TypeError("Unsupported document object."); + seen.add(value); + try { + const descriptors = Object.getOwnPropertyDescriptors(value); + if (Object.hasOwn(descriptors, "toJSON")) { + throw new TypeError("Custom JSON conversion is not projected."); + } + const names = Reflect.ownKeys(descriptors).filter((name) => + descriptors[name as string].enumerable + ); + if ( + names.some((name) => + typeof name !== "string" || !("value" in descriptors[name]) + ) + ) throw new TypeError("Not plain JSON properties."); + let contextValue: OutboundJsonValue | undefined; + if ( + !contextData && !array && + descriptors["@context"]?.enumerable + ) { + const context = copy(descriptors["@context"].value, depth + 1, true); + contextValue = context; + const urls = typeof context === "string" ? [context] : context; + if ( + !Array.isArray(urls) || urls.length === 0 || + (depth === 0 ? !urls.includes(asContext) : !compact) || + !urls.every((url) => + typeof url === "string" + ? contexts.has(url) + : depth === 0 && generatedContext(url) + ) + ) { + throw new TypeError("Unsupported document context."); + } + compact = true; + } + if (array) { + const length = descriptors.length.value as number; + if ( + names.length !== length || + names.some((name, index) => name !== String(index)) + ) throw new TypeError("Not a JSON array."); + return Object.freeze( + names.map((name) => + copy(descriptors[name as string].value, depth + 1, contextData) + ), + ); + } + const result: Record = Object.create(null); + for (const name of names as string[]) { + if (contextData) { + result[name] = copy(descriptors[name].value, depth + 1, true); + continue; + } + if (name === "@context") { + if (contextValue === undefined || !compact) { + throw new TypeError("Scoped context not supported."); + } + result[name] = contextValue!; + continue; + } + if ( + !compact && !keywords.has(name) && + !/^[A-Za-z][A-Za-z0-9+.-]*:/.test(name) + ) throw new TypeError("Unresolved document property."); + const next = copy(descriptors[name].value, depth + 1); + if (blind.has(name)) { + redacted = true; + continue; + } + result[name] = next; + } + return Object.freeze(result); + } finally { + seen.delete(value); + } + }; + if (input == null || typeof input !== "object") { + throw new TypeError("Not an outgoing document."); + } + const value = copy(input, 0); + return Object.freeze({ + stage: "sender-payload", + type: "available", + representation: "privacy-projection", + redacted, + value, + }); + } catch { + return Object.freeze({ + stage: "sender-payload", + type: "unavailable", + reason: "privacy-filter-unavailable", + }); + } +} diff --git a/packages/fedify/src/federation/outbound-headers.ts b/packages/fedify/src/federation/outbound-headers.ts new file mode 100644 index 000000000..afbd6d172 --- /dev/null +++ b/packages/fedify/src/federation/outbound-headers.ts @@ -0,0 +1,86 @@ +import type { + OutboundHeaderPolicy, + OutboundHeaderSnapshot, +} from "./outbound-report.ts"; + +const protectedHeaders = new Set([ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "signature", + "signature-input", + "authentication-info", + "proxy-authentication-info", + "x-api-key", + "x-auth-token", +]); + +function snapshotNames(names: readonly string[]): readonly string[] { + if (!Array.isArray(names)) { + throw new TypeError("Outbound header names must be an array."); + } + const normalized = new Set(); + for (const name of names) { + if ( + typeof name !== "string" || + !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name) || name.includes("*") + ) { + throw new TypeError("Outbound header names must be literal HTTP tokens."); + } + normalized.add(name.toLowerCase()); + } + return Object.freeze([...normalized].sort()); +} + +/** Validate/copy policy before registration replaces the selected observers. */ +export function snapshotHeaderPolicy( + policy: OutboundHeaderPolicy | undefined, +): OutboundHeaderPolicy | undefined { + if (policy === undefined) return undefined; + if (policy == null || typeof policy !== "object") { + throw new TypeError("Outbound header policy must be an object."); + } + const { allow, redact, maxBytes } = policy; + if (typeof maxBytes !== "number") { + throw new TypeError("Outbound header maxBytes must be a number."); + } + if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) { + throw new RangeError( + "Outbound header maxBytes must be a positive safe integer.", + ); + } + return Object.freeze({ + allow: snapshotNames(allow), + ...(redact === undefined ? {} : { redact: snapshotNames(redact) }), + maxBytes, + }); +} + +/** Copy only selected/redacted entries; budgets never truncate a header value. */ +export function snapshotHeaders( + headers: Headers, + policy: OutboundHeaderPolicy, +): OutboundHeaderSnapshot { + const redact = new Set([...protectedHeaders, ...(policy.redact ?? [])]); + const entries: (readonly [string, string])[] = []; + const encoder = new TextEncoder(); + let bytes = 0; + let limitReached = false; + for (const name of policy.allow) { + if (!headers.has(name)) continue; + // Preserve the runtime's entry order without retaining unselected values. + for (const [key, rawValue] of headers) { + if (key !== name) continue; + const value = redact.has(name) ? "[redacted]" : rawValue; + const size = encoder.encode(`${name}: ${value}\r\n`).byteLength; + if (size > policy.maxBytes - bytes) { + limitReached = true; + continue; + } + entries.push(Object.freeze([name, value] as const)); + bytes += size; + } + } + return Object.freeze({ entries: Object.freeze(entries), limitReached }); +} diff --git a/packages/fedify/src/federation/outbound-observation.ts b/packages/fedify/src/federation/outbound-observation.ts new file mode 100644 index 000000000..7be7949d8 --- /dev/null +++ b/packages/fedify/src/federation/outbound-observation.ts @@ -0,0 +1,1298 @@ +import { projectSenderPayload } from "./outbound-document.ts"; +import { captureResponseBody } from "./outbound-capture.ts"; +import { snapshotHeaders } from "./outbound-headers.ts"; +import { + getOutboundProvenance, + invocationProvenance, + snapshotLocalContext, + sourceLogContext, +} from "./outbound-provenance.ts"; +import { + isSpanContextValid, + type Span, + type SpanContext, +} from "@opentelemetry/api"; +import { encodeHex } from "byte-encodings/hex"; +import { getLogger, withContext } from "@logtape/logtape"; +import type { Context } from "./context.ts"; +import type { + FanoutMessage, + OutboxMessage, + SerializedOutboundDelivery, +} from "./queue.ts"; +import type { + OutboundActivityReference, + OutboundAttemptReport, + OutboundBodyCaptureSummary, + OutboundBroadcastEvent, + OutboundDeliveryEvent, + OutboundHttpExchangeReport, + OutboundObservers, + OutboundPreparationResult, + OutboundProvenance, + OutboundResponseBodyEvent, + OutboundWorkerPreparationDiagnostic, + OutboundWorkerReport, +} from "./outbound-report.ts"; + +type WithoutSource = T extends unknown + ? Omit + : never; + +type PreparationFailure = Extract< + OutboundPreparationResult, + { type: "failed" } +>; + +interface PlannedDestination { + readonly actorIds: Iterable; + readonly sharedInbox: boolean; + readonly portableInbox?: string; + readonly mode?: "portable-replication"; +} + +function snapshotDeliveryIdentity( + identity: SerializedOutboundDelivery, +): SerializedOutboundDelivery { + if ( + identity == null || typeof identity.deliveryId !== "string" || + identity.deliveryId.length < 1 || + typeof identity.destinationInbox !== "string" || + identity.destinationInbox.length < 1 || + identity.broadcast == null || + !(identity.broadcast.type === "unknown" || + identity.broadcast.type === "known" && + typeof identity.broadcast.id === "string" && + identity.broadcast.id.length > 0) || + !(identity.portableInbox == null || + typeof identity.portableInbox === "string") + ) { + throw new TypeError( + "Invalid outbound delivery identity in queue metadata.", + ); + } + return Object.freeze({ + broadcast: Object.freeze( + identity.broadcast.type === "known" + ? { type: "known" as const, id: identity.broadcast.id } + : { type: "unknown" as const }, + ), + deliveryId: identity.deliveryId, + destinationInbox: identity.destinationInbox, + ...(identity.portableInbox == null + ? {} + : { portableInbox: identity.portableInbox }), + }); +} + +async function legacyFanoutIdentity( + taskId: string, + inbox: string, + portableInbox?: string, +): Promise { + // A legacy task establishes destination obligations, not its originating + // invocation. JSON tuple encoding keeps different task/destination pairs + // unambiguous; hashing keeps the derived identity opaque and bounded. + const bytes = new TextEncoder().encode(JSON.stringify([ + "fedify-legacy-fanout-delivery", + taskId, + inbox, + ])); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return { + broadcast: { type: "unknown" }, + deliveryId: `legacy-fanout:${encodeHex(new Uint8Array(digest))}`, + destinationInbox: inbox, + ...(portableInbox == null ? {} : { portableInbox }), + }; +} + +async function notifyObserver< + TContextData, + TEvent extends { readonly provenance: OutboundProvenance }, +>( + context: Context, + callback: + | ((context: Context, event: TEvent) => void | Promise) + | undefined, + event: TEvent, + unit: "broadcast" | "delivery" | "attempt" | "http-exchange" | "worker", +): Promise { + if (callback == null) return; + try { + await withContext( + sourceLogContext(event.provenance), + () => callback(context, event), + ); + } catch (error) { + logObserverFailure(unit, error); + } +} + +function logObserverFailure(unit: string, error: unknown): void { + try { + getLogger(["fedify", "federation", "outbox"]).error( + "An unexpected error occurred in outbound {unit} observer:\n{error}", + { unit, error }, + ); + } catch { + // A logging sink must not replace the execution's result either. + } +} + +function warnInvalidDeliveryIdentity(messageId: string, error: unknown): void { + try { + getLogger(["fedify", "federation", "outbox"]).warn( + "Ignoring invalid outbound delivery identity in queue message {messageId}: {error}", + { messageId, error }, + ); + } catch { + // Logging must not prevent a queued delivery either. + } +} + +function snapshotTrace( + span: Span, + parentSpanContext?: SpanContext, +): OutboundBroadcastEvent["trace"] { + const spanContext = span.spanContext(); + // A no-op tracer can return the parent's context instead of creating a span. + if ( + !isSpanContextValid(spanContext) || spanContext.isRemote || + spanContext.traceId === parentSpanContext?.traceId && + spanContext.spanId === parentSpanContext.spanId + ) return undefined; + return Object.freeze({ + traceId: spanContext.traceId, + spanId: spanContext.spanId, + }); +} + +type AttemptFailure = Extract< + OutboundAttemptReport["outcome"], + { type: "failed" } +>["failure"]; +type AttemptFacts = Pick< + OutboundAttemptReport, + | "attemptId" + | "inboxUrl" + | "httpExchangeCount" + | "senderPayload" + | "observedAt" + | "trace" + | "outcome" +>; + +type HeaderPolicies = Pick< + OutboundObservers, + "requestHeaders" | "responseHeaders" | "senderPayload" +>; + +type ExchangeFacts = Pick< + OutboundHttpExchangeReport, + | "attemptId" + | "inboxUrl" + | "exchangeIndex" + | "observedAt" + | "request" + | "outcome" + | "bodyCapture" +>; + +type BodyFacts = Pick< + OutboundResponseBodyEvent, + | "attemptId" + | "inboxUrl" + | "exchangeIndex" + | "observedAt" + | "request" + | "response" + | "body" + | "signal" +>; +type BodyCapture = { + maxBytes: number; + timeoutMs: number; + responses?: "all" | "non-success-http-responses"; + onBody: (event: BodyFacts) => void | Promise; +}; + +type AttemptResponse = OutboundAttemptReport["outcome"]["response"]; +type FinalResponse = Extract; +type ObservedResponse = Pick; + +/** State owned by one concrete send, never persisted in queue metadata. */ +export class OutboundAttemptObservation { + readonly #attemptId = crypto.randomUUID(); + readonly #failures: AttemptFailure[] = []; + #response: AttemptResponse = Object.freeze({ type: "not-observed" }); + #finalResponse: FinalResponse | undefined; + readonly #responses = new WeakMap(); + #trace: OutboundAttemptReport["trace"]; + #finished = false; + #exchangeCount = 0; + #senderPayload: OutboundAttemptReport["senderPayload"]; + + constructor( + readonly inboxUrl: string, + readonly report: (facts: AttemptFacts) => Promise, + readonly reportExchange: + | ((facts: ExchangeFacts) => Promise) + | undefined = undefined, + readonly headerPolicies: HeaderPolicies = {}, + readonly bodyCapture: BodyCapture | undefined = undefined, + readonly senderPayload = false, + ) { + this.#senderPayload = Object.freeze({ + stage: "sender-payload", + type: "unavailable", + reason: senderPayload ? "not-materialized" : "disabled", + }); + } + + snapshotSenderPayload(input: unknown): void { + if (this.senderPayload) this.#senderPayload = projectSenderPayload(input); + } + + setSpan(span: Span, parentSpanContext?: SpanContext): void { + this.#trace = snapshotTrace(span, parentSpanContext); + } + + async fetch( + request: Request, + submit: () => Promise, + ): Promise { + const exchangeIndex = this.#exchangeCount++; + const snapshot = Object.freeze({ + url: request.url, + method: request.method, + ...(this.headerPolicies.requestHeaders == null ? {} : { + headers: snapshotHeaders( + request.headers, + this.headerPolicies.requestHeaders, + ), + }), + }); + let response: Response; + try { + response = await submit(); + } catch (error) { + await this.#exchange(exchangeIndex, snapshot, { type: "failed", error }); + throw error; + } + const metadata = Object.freeze({ + status: response.status, + statusText: response.statusText, + ...(response.url === "" ? {} : { url: response.url }), + }); + const observed = Object.freeze({ exchangeIndex, metadata }); + this.#responses.set(response, observed); + this.#response = Object.freeze({ + type: "undetermined", + lastObserved: observed, + }); + const responseSnapshot = Object.freeze({ + ...metadata, + ...(this.headerPolicies.responseHeaders == null ? {} : { + headers: snapshotHeaders( + response.headers, + this.headerPolicies.responseHeaders, + ), + }), + }); + const bodyCapture = this.bodyCapture == null + ? undefined + : await captureResponseBody( + response, + this.bodyCapture, + (body, signal) => + this.bodyCapture!.onBody({ + attemptId: this.#attemptId, + inboxUrl: this.inboxUrl, + exchangeIndex, + observedAt: Temporal.Now.instant().epochMilliseconds, + request: snapshot, + response: responseSnapshot, + body, + signal, + }), + ); + if (bodyCapture?.type === "failed" && bodyCapture.phase === "observer") { + logObserverFailure("response-body", bodyCapture.error); + } + await this.#exchange(exchangeIndex, snapshot, { + type: "response", + response: responseSnapshot, + }, bodyCapture); + return response; + } + + #exchange( + exchangeIndex: number, + request: OutboundHttpExchangeReport["request"], + outcome: OutboundHttpExchangeReport["outcome"], + bodyCapture?: OutboundBodyCaptureSummary, + ): Promise { + return this.reportExchange?.({ + attemptId: this.#attemptId, + inboxUrl: this.inboxUrl, + exchangeIndex, + observedAt: Temporal.Now.instant().epochMilliseconds, + request, + outcome: Object.freeze(outcome), + ...(bodyCapture == null ? {} : { bodyCapture }), + }) ?? Promise.resolve(); + } + + response(response: Response, disposition: "final" | "intermediate"): void { + const observed = this.#responses.get(response); + if (observed == null) return; + if (disposition === "final") { + this.#finalResponse = Object.freeze({ type: "final", ...observed }); + this.#response = this.#finalResponse; + } else { + this.#response = Object.freeze({ + type: "intermediate-only", + lastObserved: observed, + }); + } + } + + failure(failure: AttemptFailure): void { + this.#failures.push(Object.freeze(failure)); + } + + unknown(error: unknown): void { + if ( + !this.#failures.some((failure) => + failure.error === error || failure.phase !== "cleanup" + ) + ) { + this.failure({ kind: "unknown", phase: "unknown", error }); + } + } + + succeeded(): Promise { + if (this.#finished) return Promise.resolve(); + if (this.#finalResponse == null) { + throw new Error( + "A successful outbound attempt must have an observed final response.", + ); + } + return this.#finish({ type: "succeeded", response: this.#finalResponse }); + } + + failed(error: unknown): Promise { + let failure: AttemptFailure; + let diagnostics: AttemptFailure[]; + if ( + this.#response.type === "final" && + (this.#response.metadata.status < 200 || + this.#response.metadata.status >= 300) + ) { + failure = this.#failures.find((failure) => failure.kind === "http") ?? + Object.freeze({ kind: "http", phase: "response" }); + diagnostics = this.#failures.filter((failure) => failure.kind !== "http"); + } else { + failure = this.#failures.find((failure) => failure.phase !== "cleanup") ?? + this.#failures[0] ?? + Object.freeze({ kind: "unknown", phase: "unknown", error }); + diagnostics = this.#failures.filter((item) => item !== failure); + } + return this.#finish({ + type: "failed", + failure, + error, + response: this.#response, + ...(diagnostics.length === 0 + ? {} + : { diagnostics: Object.freeze(diagnostics) }), + }); + } + + #finish(outcome: OutboundAttemptReport["outcome"]): Promise { + if (this.#finished) return Promise.resolve(); + this.#finished = true; + return this.report({ + attemptId: this.#attemptId, + inboxUrl: this.inboxUrl, + httpExchangeCount: this.#exchangeCount, + senderPayload: this.#senderPayload, + observedAt: Temporal.Now.instant().epochMilliseconds, + ...(this.#trace == null ? {} : { trace: this.#trace }), + outcome: Object.freeze(outcome), + }); + } +} + +type AttemptIdentity = Pick< + OutboundAttemptReport, + "broadcast" | "deliveryId" | "destinationInbox" | "portableInbox" +>; + +function snapshotOutboxIdentity(message: OutboxMessage): AttemptIdentity { + if (message.outbound != null) { + try { + return snapshotDeliveryIdentity(message.outbound); + } catch (error) { + warnInvalidDeliveryIdentity(message.id, error); + } + } + return Object.freeze({ + broadcast: Object.freeze({ type: "unknown" as const }), + deliveryId: message.id, + ...(message.portableInbox == null + ? {} + : { portableInbox: message.portableInbox }), + }); +} + +type WorkerPreparationFailure = Extract< + OutboundWorkerReport["result"], + { type: "preparation-failed" } +>["failure"]; + +type WorkerAction = NonNullable[number]; +type WorkerAbandonment = Extract; +type WorkerPreparationDiagnostic = + readonly OutboundWorkerPreparationDiagnostic[]; + +/** Per-execution state, independent of concrete attempt/exchange callbacks. */ +export class OutboundWorkerObservation { + #attemptsStarted = 0; + #actions: WorkerAction[] | undefined; + #trace: OutboundWorkerReport["trace"]; + #preparationFailure: WorkerPreparationFailure | undefined; + #preparationFailures: OutboundWorkerPreparationDiagnostic[] = []; + readonly #queue: OutboundWorkerReport["queue"]; + readonly #delivery: AttemptIdentity | undefined; + readonly #role: OutboundWorkerReport["role"]; + #currentContext: Context | undefined; + + constructor( + readonly context: () => Context, + readonly callback: OutboundObservers["onWorker"], + message: FanoutMessage | OutboxMessage, + readonly provenance: OutboundProvenance, + ) { + this.#role = message.type; + this.#queue = Object.freeze({ + queueMessageId: message.id, + ...(message.type === "outbox" ? { queueAttempt: message.attempt } : {}), + }); + if (message.type === "outbox") { + this.#delivery = snapshotOutboxIdentity(message); + } + } + + readonly attemptStarted: () => void = () => { + this.#attemptsStarted++; + }; + + enableActions(): void { + this.#actions ??= []; + } + + directAttemptStarted( + inboxUrl: string, + identity?: SerializedOutboundDelivery, + ): void { + this.attemptStarted(); + this.#actions?.push(Object.freeze({ + type: "handoff", + action: "direct", + inboxUrls: Object.freeze([inboxUrl]), + ...(identity == null + ? {} + : { delivery: snapshotDeliveryIdentity(identity) }), + })); + } + + async enqueueChildren( + messages: readonly OutboxMessage[], + enqueue: () => Promise, + ): Promise { + if (this.#actions == null) return await enqueue(); + const facts = messages.map((message) => ({ + action: "enqueue" as const, + queue: Object.freeze({ + queueMessageId: message.id, + queueAttempt: message.attempt, + }), + inboxUrls: Object.freeze( + message.gatewayInboxes != null && message.gatewayInboxes.length > 1 + ? [...message.gatewayInboxes] + : [message.inbox], + ), + ...(message.outbound == null + ? {} + : { delivery: snapshotDeliveryIdentity(message.outbound) }), + })); + try { + await enqueue(); + } catch (error) { + for (const fact of facts) { + this.#actions.push(Object.freeze({ + type: "action-failed", + ...fact, + acceptance: "unknown" as const, + error, + })); + } + throw error; + } + for (const fact of facts) { + this.#actions.push(Object.freeze({ type: "scheduled", ...fact })); + } + } + + async enqueue( + action: "retry" | "defer", + message: OutboxMessage, + delay: Temporal.Duration, + enqueue: () => Promise, + preparationFailures?: WorkerPreparationDiagnostic, + ): Promise { + if (this.#actions == null) return await enqueue(); + const facts = { + action, + queue: Object.freeze({ + queueMessageId: message.id, + queueAttempt: message.attempt, + }), + inboxUrls: Object.freeze( + message.gatewayInboxes != null && message.gatewayInboxes.length > 1 + ? [...message.gatewayInboxes] + : [message.inbox], + ), + delay: delay.toString(), + ...(preparationFailures == null || preparationFailures.length === 0 + ? {} + : { + preparationFailures: Object.freeze( + [...preparationFailures], + ), + }), + ...(message.gatewayNotBefore == null ? {} : { + notBefore: Object.freeze({ ...message.gatewayNotBefore }), + }), + }; + try { + await enqueue(); + } catch (error) { + this.#actions.push( + Object.freeze({ + type: "action-failed", + ...facts, + acceptance: "unknown" as const, + error, + }), + ); + throw error; + } + this.#actions.push(Object.freeze({ type: "scheduled", ...facts })); + } + + abandon( + reason: WorkerAbandonment["reason"], + inboxUrls: readonly string[], + preparationFailures?: WorkerPreparationDiagnostic, + ): void { + this.#actions?.push( + Object.freeze({ + type: "abandoned", + reason, + inboxUrls: Object.freeze([...inboxUrls]), + ...(preparationFailures == null || preparationFailures.length === 0 + ? {} + : { + preparationFailures: Object.freeze( + [...preparationFailures], + ), + }), + }), + ); + } + + dropFailure(inboxUrls: readonly string[], error: unknown): void { + this.#actions?.push( + Object.freeze({ + type: "action-failed", + action: "drop", + inboxUrls: Object.freeze([...inboxUrls]), + error, + }), + ); + } + + async drop( + inboxUrls: readonly string[], + operation: () => Promise, + ): Promise { + try { + await operation(); + } catch (error) { + this.dropFailure(inboxUrls, error); + throw error; + } + } + + getContext(): Context { + return this.#currentContext ??= this.context(); + } + + useContext(context: Context): void { + this.#currentContext = context; + } + + setSpan(span: Span, parentSpanContext?: SpanContext): void { + this.#trace = snapshotTrace(span, parentSpanContext); + } + + async prepare( + phase: WorkerPreparationFailure["phase"], + operation: () => TResult | Promise, + ): Promise { + try { + const result = await operation(); + this.#preparationFailure = undefined; + return result; + } catch (error) { + this.#preparationFailure = Object.freeze({ kind: "local", phase, error }); + throw error; + } + } + + recordPreparationFailure( + error: unknown, + inboxUrl: string, + ): WorkerPreparationDiagnostic { + const failure = this.#preparationFailure; + if (failure == null || !Object.is(failure.error, error)) return []; + const diagnostic = Object.freeze({ inboxUrl, failure }); + this.#preparationFailures.push(diagnostic); + return [diagnostic]; + } + + async run(operation: () => Promise): Promise { + try { + await operation(); + } catch (error) { + await this.#notify( + this.#preparationFailure == null || + !Object.is(this.#preparationFailure.error, error) + ? { type: "failed", error } + : { + type: "preparation-failed", + failure: this.#preparationFailure, + error, + }, + ); + throw error; + } + await this.#notify({ type: "completed" }); + } + + async #notify(result: OutboundWorkerReport["result"]): Promise { + const observedAt = Temporal.Now.instant().epochMilliseconds; + try { + const context = this.getContext(); + await notifyObserver( + context, + this.callback, + Object.freeze({ + role: this.#role, + queue: this.#queue, + ...(this.#delivery == null ? {} : { delivery: this.#delivery }), + attemptsStarted: this.#attemptsStarted, + actions: Object.freeze([...(this.#actions ?? [])]), + ...(this.#preparationFailures.length === 0 ? {} : { + preparationFailures: Object.freeze([...this.#preparationFailures]), + }), + observedAt, + ...(this.#trace == null ? {} : { trace: this.#trace }), + provenance: this.provenance, + local: snapshotLocalContext(context), + result: Object.freeze(result), + }), + "worker", + ); + } catch (error) { + logObserverFailure("worker", error); + } + } +} + +/** Selected callback and restored grouping, shared by concrete gateway attempts. */ +export class OutboundAttemptObserver { + readonly #queue: OutboundAttemptReport["queue"]; + + constructor( + readonly context: () => Context, + readonly callback: OutboundObservers["onAttempt"], + readonly identity: AttemptIdentity, + readonly provenance: OutboundProvenance, + queue?: OutboundAttemptReport["queue"], + readonly onHttpExchange: OutboundObservers["onHttpExchange"] = + undefined, + readonly headerPolicies: HeaderPolicies = {}, + readonly bodyCapture: OutboundObservers< + TContextData + >["responseBodyCapture"] = undefined, + ) { + this.#queue = queue == null ? undefined : Object.freeze({ ...queue }); + } + + static resumeOutbox( + context: () => Context, + callback: OutboundObservers["onAttempt"], + message: OutboxMessage, + provenance: OutboundProvenance, + onHttpExchange?: OutboundObservers["onHttpExchange"], + headerPolicies: HeaderPolicies = {}, + bodyCapture?: OutboundObservers["responseBodyCapture"], + ): OutboundAttemptObserver | undefined { + if (callback == null && onHttpExchange == null && bodyCapture == null) { + return undefined; + } + const identity = snapshotOutboxIdentity(message); + return new OutboundAttemptObserver( + context, + callback, + identity, + provenance, + { + queueMessageId: message.id, + queueAttempt: message.attempt, + }, + onHttpExchange, + headerPolicies, + bodyCapture, + ); + } + + startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { + if ( + this.callback == null && this.onHttpExchange == null && + this.bodyCapture == null + ) return undefined; + const context = this.context(); + return new OutboundAttemptObservation( + inboxUrl, + (facts) => + notifyObserver( + context, + this.callback, + Object.freeze({ + ...this.identity, + ...facts, + ...(this.#queue == null ? {} : { queue: this.#queue }), + provenance: this.provenance, + local: snapshotLocalContext(context), + }), + "attempt", + ), + (facts) => + notifyObserver( + context, + this.onHttpExchange, + Object.freeze({ + ...this.identity, + ...facts, + ...(this.#queue == null ? {} : { queue: this.#queue }), + provenance: this.provenance, + local: snapshotLocalContext(context), + }), + "http-exchange", + ), + this.onHttpExchange == null && this.bodyCapture == null + ? {} + : this.headerPolicies, + this.bodyCapture == null ? undefined : { + ...this.bodyCapture, + onBody: (event) => + withContext( + sourceLogContext(this.provenance), + () => + this.bodyCapture!.onBody( + context, + Object.freeze({ + ...this.identity, + ...event, + ...(this.#queue == null ? {} : { queue: this.#queue }), + provenance: this.provenance, + local: snapshotLocalContext(context), + }), + ), + ), + }, + this.callback != null && this.headerPolicies.senderPayload === true, + ); + } +} + +/** One producer's destination, without callbacks or live inputs in queues. */ +class OutboundDeliveryObservation { + #finished = false; + readonly #reference: SerializedOutboundDelivery; + readonly #plan; + readonly #attempts: OutboundAttemptObserver; + + constructor( + readonly context: Context, + readonly observer: OutboundObservers["onDelivery"], + identity: SerializedOutboundDelivery, + destination: PlannedDestination, + readonly provenance: OutboundProvenance, + readonly trace: OutboundBroadcastEvent["trace"], + onAttempt: OutboundObservers["onAttempt"], + onHttpExchange: OutboundObservers["onHttpExchange"], + queue?: OutboundAttemptReport["queue"], + headerPolicies: HeaderPolicies = {}, + bodyCapture?: OutboundObservers["responseBodyCapture"], + ) { + this.#reference = snapshotDeliveryIdentity(identity); + this.#attempts = new OutboundAttemptObserver( + () => this.context, + onAttempt, + this.#reference, + this.provenance, + queue, + onHttpExchange, + headerPolicies, + bodyCapture, + ); + this.#plan = Object.freeze({ + actorIds: Object.freeze(Array.from(destination.actorIds)), + sharedInbox: destination.sharedInbox, + mode: destination.mode ?? + (destination.portableInbox == null ? "ordinary" : "portable-fallback"), + }); + } + + startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined { + return this.#attempts.startAttempt(inboxUrl); + } + + get identity(): SerializedOutboundDelivery { + return this.#reference; + } + + planned(): Promise { + return notifyObserver( + this.context, + this.observer, + Object.freeze({ + ...(this.trace == null ? {} : { trace: this.trace }), + observedAt: Temporal.Now.instant().epochMilliseconds, + provenance: this.provenance, + local: snapshotLocalContext(this.context), + ...this.#reference, + ...this.#plan, + stage: "planned" as const, + }), + "delivery", + ); + } + + handoff( + result: Extract["result"], + ): Promise { + if (this.#finished) return Promise.resolve(); + this.#finished = true; + return notifyObserver( + this.context, + this.observer, + Object.freeze({ + ...(this.trace == null ? {} : { trace: this.trace }), + observedAt: Temporal.Now.instant().epochMilliseconds, + provenance: this.provenance, + local: snapshotLocalContext(this.context), + ...this.#reference, + stage: "handoff" as const, + result: Object.freeze(result), + }), + "delivery", + ); + } + + failed(error: unknown): Promise { + if (this.#finished) return Promise.resolve(); + this.#finished = true; + return notifyObserver( + this.context, + this.observer, + Object.freeze({ + ...(this.trace == null ? {} : { trace: this.trace }), + observedAt: Temporal.Now.instant().epochMilliseconds, + provenance: this.provenance, + local: snapshotLocalContext(this.context), + ...this.#reference, + stage: "preparation-failed" as const, + failure: Object.freeze({ + kind: "local" as const, + phase: "request-preparation" as const, + error, + }), + }), + "delivery", + ); + } +} + +/** State owned by one outgoing invocation, never serialized into queues. */ +export class OutboundBroadcastObservation { + readonly #broadcast: SerializedOutboundDelivery["broadcast"]; + #finished = false; + #phase: PreparationFailure["failure"]["phase"] = "unknown"; + #planning: PreparationFailure["planning"] = "not-started"; + #destinationCount = 0; + #trace: OutboundBroadcastEvent["trace"]; + #activity: OutboundActivityReference = Object.freeze({}); + + readonly #onBroadcast: OutboundObservers["onBroadcast"]; + readonly #onDelivery: OutboundObservers["onDelivery"]; + readonly #onAttempt: OutboundObservers["onAttempt"]; + readonly #onHttpExchange: OutboundObservers["onHttpExchange"]; + readonly #headerPolicies: HeaderPolicies; + readonly #bodyCapture: OutboundObservers["responseBodyCapture"]; + readonly #deliveries = new Map< + string, + OutboundDeliveryObservation + >(); + + constructor( + readonly context: Context, + observers: OutboundObservers, + readonly operation: OutboundBroadcastEvent["operation"], + readonly provenance: OutboundProvenance = getOutboundProvenance(context), + broadcast: SerializedOutboundDelivery["broadcast"] = { + type: "known", + id: crypto.randomUUID(), + }, + readonly queue: OutboundAttemptReport["queue"] = undefined, + ) { + this.#broadcast = broadcast; + this.#onBroadcast = observers.onBroadcast; + this.#onDelivery = observers.onDelivery; + this.#onAttempt = observers.onAttempt; + this.#onHttpExchange = observers.onHttpExchange; + this.#bodyCapture = observers.responseBodyCapture; + this.#headerPolicies = { + requestHeaders: observers.requestHeaders, + responseHeaders: observers.responseHeaders, + senderPayload: observers.senderPayload, + }; + } + + /** Continue an existing fanout; do not emit a new broadcast invocation. */ + static async resumeFanout( + context: Context, + observers: OutboundObservers, + message: FanoutMessage, + span?: Span, + parentSpanContext?: SpanContext, + ): Promise | undefined> { + if ( + observers.onDelivery == null && observers.onAttempt == null && + observers.onHttpExchange == null && + observers.responseBodyCapture == null && observers.onWorker == null && + !Object.values(message.inboxes).some((entry) => entry.outbound != null) + ) { + return undefined; + } + const observation = new OutboundBroadcastObservation( + context, + { + onDelivery: observers.onDelivery, + onAttempt: observers.onAttempt, + onHttpExchange: observers.onHttpExchange, + requestHeaders: observers.requestHeaders, + responseHeaders: observers.responseHeaders, + responseBodyCapture: observers.responseBodyCapture, + senderPayload: observers.senderPayload, + }, + "send", + getOutboundProvenance(context), + { type: "unknown" }, + { queueMessageId: message.id }, + ); + if (span != null) observation.setSpan(span, parentSpanContext); + observation.#finished = true; + observation.planned(Object.keys(message.inboxes).length); + for (const [inbox, destination] of Object.entries(message.inboxes)) { + let identity: SerializedOutboundDelivery | undefined; + if (destination.outbound != null) { + try { + const snapshot = snapshotDeliveryIdentity(destination.outbound); + if (snapshot.destinationInbox !== inbox) { + throw new TypeError( + "Outbound delivery identity does not match its fanout destination.", + ); + } + identity = snapshot; + } catch (error) { + warnInvalidDeliveryIdentity(message.id, error); + } + } + identity ??= await legacyFanoutIdentity( + message.id, + inbox, + destination.portableInbox, + ); + observation.#deliveries.set( + inbox, + observation.#createDelivery(inbox, destination, identity), + ); + } + return observation; + } + + #createDelivery( + inbox: string, + destination: PlannedDestination, + identity?: SerializedOutboundDelivery, + ): OutboundDeliveryObservation { + return new OutboundDeliveryObservation( + this.context, + this.#onDelivery, + identity ?? { + broadcast: this.#broadcast, + deliveryId: crypto.randomUUID(), + destinationInbox: inbox, + ...(destination.portableInbox == null + ? {} + : { portableInbox: destination.portableInbox }), + }, + destination, + this.provenance, + this.#trace, + this.#onAttempt, + this.#onHttpExchange, + this.queue, + this.#headerPolicies, + this.#bodyCapture, + ); + } + + continuation( + inbox: string, + ): { readonly outbound: SerializedOutboundDelivery } | undefined { + const delivery = this.delivery(inbox); + return delivery == null ? undefined : { outbound: delivery.identity }; + } + + async planDestinations( + inboxes: Readonly>, + ): Promise { + this.planned(Object.keys(inboxes).length); + for (const [inbox, destination] of Object.entries(inboxes)) { + if (!this.#deliveries.has(inbox)) { + this.#deliveries.set(inbox, this.#createDelivery(inbox, destination)); + } + } + for (const delivery of this.#deliveries.values()) await delivery.planned(); + } + + delivery( + inbox: string, + ): OutboundDeliveryObservation | undefined { + return this.#deliveries.get(inbox); + } + + async enqueue( + entries: readonly { + readonly inbox: string; + readonly queueMessageId: string; + }[], + queue: "fanout" | "outbox", + enqueue: () => Promise, + ): Promise { + let result: TResult; + try { + result = await enqueue(); + } catch (error) { + await Promise.all( + entries.map(({ inbox }) => + this.delivery(inbox)?.handoff({ + type: "enqueue-failed", + acceptance: "unknown", + error, + }) + ), + ); + throw error; + } + // Acceptance is established before unrelated metrics or logging can fail. + await Promise.all( + entries.map(({ inbox, queueMessageId }) => + this.delivery(inbox)?.handoff({ + type: "enqueued", + queue, + queueMessageId, + }) + ), + ); + return result; + } + + /** Attach only the span supplied by this execution, never an ambient fallback. */ + setSpan(span: Span, parentSpanContext?: SpanContext): void { + this.#trace = snapshotTrace(span, parentSpanContext); + } + + setPhase(phase: PreparationFailure["failure"]["phase"]): void { + this.#phase = phase; + if (phase === "destination-planning") this.#planning = "incomplete"; + } + + /** Replace the Activity facts captured by the producer path. */ + setActivity(reference: OutboundActivityReference): void { + this.#activity = Object.freeze({ ...reference }); + } + + planned(destinationCount: number): void { + this.#planning = "complete"; + this.#destinationCount = destinationCount; + } + + async planDestination( + inbox: string, + destination: PlannedDestination, + ): Promise { + this.#destinationCount++; + const delivery = this.#createDelivery(inbox, destination); + this.#deliveries.set(inbox, delivery); + await delivery.planned(); + } + + prepared(preparedFor: "fanout" | "delivery"): Promise { + return this.#finish(Object.freeze({ + type: "prepared", + preparedFor, + destinationCount: this.#destinationCount, + })); + } + + skipped( + reason: Extract["reason"] = + "no-destinations", + ): Promise { + return this.#finish(Object.freeze({ + type: "skipped", + reason, + })); + } + + async run( + send: ( + observation: OutboundBroadcastObservation | undefined, + provenance: OutboundProvenance, + ) => Promise, + ): Promise { + if (this.#broadcast.type === "known") { + await this.#notify(Object.freeze({ + broadcastId: this.#broadcast.id, + operation: this.operation, + stage: "started", + activity: this.#activity, + })); + } + try { + return await send(this, this.provenance); + } catch (error) { + await this.failed(error); + throw error; + } + } + + failed(error: unknown): Promise { + const failure: PreparationFailure["failure"] = { + kind: this.#phase === "unknown" ? "unknown" : "local", + phase: this.#phase, + error, + }; + return this.#finish(Object.freeze({ + type: "failed", + failure: Object.freeze(failure), + planning: this.#planning, + plannedDestinationCount: this.#destinationCount, + })); + } + + async #finish(result: OutboundPreparationResult): Promise { + if (this.#finished) return; + this.#finished = true; + if (this.#broadcast.type === "known") { + await this.#notify(Object.freeze({ + broadcastId: this.#broadcast.id, + operation: this.operation, + stage: "preparation-finished", + result, + activity: this.#activity, + })); + } + } + + #notify( + event: WithoutSource, + ): Promise { + return notifyObserver( + this.context, + this.#onBroadcast, + Object.freeze({ + ...event, + ...(this.#trace == null ? {} : { trace: this.#trace }), + observedAt: Temporal.Now.instant().epochMilliseconds, + provenance: this.provenance, + local: snapshotLocalContext(this.context), + }), + "broadcast", + ); + } +} + +/** Observe preparation without changing the unobserved send's execution path. */ +export function observeOutboundBroadcast( + context: Context, + observers: OutboundObservers, + send: ( + observation: OutboundBroadcastObservation | undefined, + provenance: OutboundProvenance, + ) => Promise, + operation: OutboundBroadcastEvent["operation"] = "send", + dispatchId?: string, +): Promise { + let provenance: OutboundProvenance; + let invalid: { error: unknown } | undefined; + try { + provenance = invocationProvenance(context, dispatchId); + } catch (error) { + provenance = invocationProvenance(context, undefined); + invalid = { error }; + } + const execute = async ( + observation?: OutboundBroadcastObservation, + ) => { + if (invalid != null) { + observation?.setPhase("validation"); + throw invalid.error; + } + return await send(observation, provenance); + }; + return withContext(sourceLogContext(provenance), () => { + if ( + observers.onBroadcast == null && observers.onDelivery == null && + observers.onAttempt == null && observers.onHttpExchange == null && + observers.responseBodyCapture == null + ) { + return execute(); + } + return new OutboundBroadcastObservation( + context, + observers, + operation, + provenance, + ).run(execute); + }); +} diff --git a/packages/fedify/src/federation/outbound-provenance.test.ts b/packages/fedify/src/federation/outbound-provenance.test.ts new file mode 100644 index 000000000..1d1faa211 --- /dev/null +++ b/packages/fedify/src/federation/outbound-provenance.test.ts @@ -0,0 +1,765 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Note, Person } from "@fedify/vocab"; +import { + computeDigestMultibase, + exportDidKey, + parseIri, +} from "@fedify/vocab-runtime"; +import { + configure, + getLogger, + type LogRecord, + reset, + withContext, +} from "@logtape/logtape"; +import fetchMock from "fetch-mock"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { + deepStrictEqual, + notStrictEqual, + ok, + rejects, + strictEqual, +} from "node:assert/strict"; +import { signObject } from "../sig/proof.ts"; +import { + ed25519Multikey, + ed25519PrivateKey, + ed25519PublicKey, + rsaPrivateKey2, + rsaPublicKey2, +} from "../testing/keys.ts"; +import type { Context, RequestContext } from "./context.ts"; +import type { InboxRequestReport } from "./inbox-report.ts"; +import { MemoryKvStore } from "./kv.ts"; +import { ContextImpl, createFederation } from "./middleware.ts"; +import type { + OutboundBroadcastEvent, + OutboundDeliveryEvent, + OutboundProvenance, +} from "./outbound-report.ts"; +import type { Message } from "./queue.ts"; +import type { MessageQueue } from "./mq.ts"; + +const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; +const outgoing = new Create({ + id: new URL("https://example.com/outgoing"), + actor: new URL("https://example.com/person2"), +}); +const target = new Person({ + id: new URL("https://remote.example/bob"), + inbox: new URL("https://remote.example/inbox"), +}); +const loaders = { + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, + authenticatedDocumentLoaderFactory: () => mockDocumentLoader, +}; +function post(path: string, body: unknown) { + return new Request(`https://inbox.example:8443${path}`, { + method: "POST", + headers: { + "Content-Type": "application/activity+json", + "X-Request-Id": "shared-external-label", + }, + body: JSON.stringify(body), + }); +} +async function payload() { + return await new Create({ + id: new URL("https://example.com/incoming"), + actor: new URL("https://example.com/person2"), + }).toJsonLd({ contextLoader: mockDocumentLoader }); +} + +test("outbound inbox provenance follows clones, conversions, loaders, sends, and forwarding", async (t) => { + for (const path of ["/users/alice/inbox", "/inbox"]) { + await t.step(path, async () => { + const reports: InboxRequestReport[] = []; + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + const messages: Message[] = []; + let retained: Context | undefined; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + origin: "https://canonical.example", + skipSignatureVerification: true, + manuallyStartQueue: true, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + onDelivery(_ctx, event) { + events.push(event); + }, + }); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .setKeyPairsDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return []; + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .setSharedKeyDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return sender; + }) + .on(Create, async (ctx) => { + ok(ctx instanceof ContextImpl); + const clone = ctx.clone(undefined); + ok(clone instanceof ContextImpl); + const outbox = clone.toOutboxContext( + "alice", + {}, + outgoing.id!.href, + Create.typeId.href, + ); + const inbox = outbox.toInboxContext( + null, + {}, + outgoing.id!.href, + Create.typeId.href, + ).clone(undefined); + // All derived contexts share a URL object; historical source facts must not. + ctx.url.host = "changed.example:9443"; + retained = inbox; + await clone.sendActivity(sender, target, outgoing); + await outbox.clone(undefined).sendActivity(sender, target, outgoing); + await inbox.forwardActivity(sender, target, { + skipIfUnsigned: false, + }); + }) + .onRequestFinished(async (ctx, report) => { + reports.push(report); + await ctx.clone(undefined).sendActivity(sender, [], outgoing); + }); + const body = await payload(); + for (let index = 0; index < 2; index++) { + const start = events.length; + strictEqual( + (await federation.fetch(post(path, body), { contextData: undefined })) + .status, + 202, + ); + const report = reports[index]; + ok(report.inboxRequestId); + strictEqual(report.requestId, "shared-external-label"); + const observed = events.slice(start); + ok(observed.length > 0); + for (const event of observed) { + const source = event.provenance.inboxRequest; + ok(source); + strictEqual(source.inboxRequestId, report.inboxRequestId); + strictEqual(source.requestId, report.requestId); + deepStrictEqual(source.context, { + origin: "https://inbox.example:8443", + host: "inbox.example:8443", + hostname: "inbox.example", + canonicalOrigin: "https://canonical.example", + }); + ok(Object.isFrozen(event.provenance)); + ok(Object.isFrozen(source)); + ok(Object.isFrozen(source.context)); + ok(Object.isFrozen(event.local)); + } + } + notStrictEqual(reports[0].inboxRequestId, reports[1].inboxRequestId); + strictEqual(messages.length, 3); // The repeated Activity is an inbox duplicate. + ok(events.some((event) => event.local.host === "changed.example:9443")); + ok(events.some((event) => event.local.host === "inbox.example:8443")); + ok( + events.some((event) => + "operation" in event && event.operation === "forward" + ), + ); + ok(retained); + const count = events.length; + await retained.sendActivity(sender, [], outgoing); + strictEqual( + events[count].provenance.inboxRequest?.inboxRequestId, + reports[0].inboxRequestId, + ); + // Same-looking Request URLs and programmatic inbox conversion establish no HTTP cause. + for ( + const input of [new URL("https://inbox.example:8443"), post(path, body)] + ) { + const independent = input instanceof URL + ? federation.createContext(input, undefined) + : federation.createContext(input, undefined); + ok(independent instanceof ContextImpl); + await independent.toInboxContext( + null, + body, + "https://example.com/manual", + Create.typeId.href, + ).sendActivity(sender, [], outgoing); + deepStrictEqual(events.at(-1)!.provenance, {}); + } + }); + } +}); + +test("outbound inbox provenance exists before loader preparation fails", async () => { + const failure = new Error("loader construction failed"); + const reports: InboxRequestReport[] = []; + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + contextLoaderFactory() { + throw failure; + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .onRequestFinished(async (ctx, report) => { + reports.push(report); + await ctx.sendActivity(sender, [], outgoing); + }); + for (const path of ["/users/alice/inbox", "/inbox"]) { + await rejects( + federation.fetch(post(path, {}), { contextData: undefined }), + (error) => error === failure, + ); + const report = reports.at(-1)!; + ok(report.inboxRequestId); + strictEqual(report.requestId, "shared-external-label"); + strictEqual(report.outcome.type, "exception"); + strictEqual( + events.at(-1)?.provenance.inboxRequest?.inboxRequestId, + report.inboxRequestId, + ); + } + notStrictEqual(reports[0].inboxRequestId, reports[1].inboxRequestId); +}); + +test("outbound provenance keeps source log labels separate from current request labels", async () => { + const records: LogRecord[] = []; + await reset(); + await configure({ + sinks: { buffer: (record) => records.push(record) }, + loggers: [{ category: [], sinks: ["buffer"], lowestLevel: "info" }, { + category: ["logtape", "meta"], + sinks: [], + }], + contextLocalStorage: new AsyncLocalStorage(), + }); + try { + const events: OutboundBroadcastEvent[] = []; + let retained: RequestContext | undefined; + let report: InboxRequestReport | undefined; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + skipSignatureVerification: true, + }); + federation.setActorDispatcher("/users/{identifier}", async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return new Person({}); + }).setKeyPairsDispatcher(() => []); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + getLogger(["test", "provenance"]).info("outbound callback"); + }, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox").on( + Create, + async () => { + await withContext({ + sourceInboxRequestId: "ambient-inbox", + sourceRequestId: "ambient-source", + }, () => + federation.fetch( + new Request("https://inbox.example/users/alice?nested=1", { + headers: { + Accept: "application/activity+json", + "X-Request-Id": "nested-request", + }, + }), + { contextData: undefined }, + )); + }, + ) + .onRequestFinished((ctx, value) => { + retained = ctx; + report = value; + getLogger(["test", "provenance"]).info("inbox callback"); + }); + strictEqual( + (await federation.fetch(post("/users/alice/inbox", await payload()), { + contextData: undefined, + })).status, + 202, + ); + ok(retained && report?.inboxRequestId); + const ingress = records.find((record) => + record.rawMessage === "inbox callback" + ); + ok(ingress); + strictEqual(ingress.properties.inboxRequestId, report.inboxRequestId); + strictEqual(ingress.properties.requestId, report.requestId); + const nested = records.find((record) => + record.properties.url === "https://inbox.example/users/alice?nested=1" && + record.properties.requestId === "nested-request" + ); + ok(nested); + strictEqual(nested.properties.inboxRequestId, undefined); + strictEqual(nested.properties.sourceInboxRequestId, undefined); + strictEqual(nested.properties.sourceRequestId, undefined); + const original = records.find((record) => + record.properties.url === + "https://inbox.example:8443/users/alice/inbox" && + record.properties.status === 202 + ); + ok(original); + strictEqual(original.properties.inboxRequestId, report.inboxRequestId); + await withContext( + { requestId: "unrelated-current" }, + () => retained!.sendActivity(sender, [], outgoing), + ); + const descendant = records.filter((record) => + record.rawMessage === "outbound callback" + ).at(-1); + ok(descendant); + strictEqual(descendant.properties.requestId, "unrelated-current"); + strictEqual(descendant.properties.sourceRequestId, report.requestId); + strictEqual( + descendant.properties.sourceInboxRequestId, + report.inboxRequestId, + ); + const independent = federation.createContext( + new URL("https://inbox.example"), + undefined, + ); + await withContext({ + requestId: "independent-current", + sourceRequestId: "ambient-source", + sourceInboxRequestId: "ambient-inbox", + }, () => independent.sendActivity(sender, [], outgoing)); + deepStrictEqual(events.at(-1)!.provenance, {}); + const independentLog = records.filter((record) => + record.rawMessage === "outbound callback" + ).at(-1); + ok(independentLog); + strictEqual(independentLog.properties.requestId, "independent-current"); + strictEqual(independentLog.properties.sourceRequestId, undefined); + strictEqual(independentLog.properties.sourceInboxRequestId, undefined); + // Ordinary HTTP-derived work retains its logging label without an inbox cause. + const start = events.length; + await federation.fetch( + new Request("https://inbox.example/users/alice", { + headers: { + Accept: "application/activity+json", + "X-Request-Id": "actor-request", + }, + }), + { contextData: undefined }, + ); + ok(events.length > start); + for (const event of events.slice(start)) { + deepStrictEqual(event.provenance, { + originatingRequestId: "actor-request", + }); + } + } finally { + await reset(); + } +}); + +test("outbound provenance does not invent an inbox cause for programmatic routing", async () => { + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox").on( + Create, + async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + }, + ); + const ctx = federation.createContext( + new URL("https://inbox.example"), + undefined, + ); + const signed = await signObject( + new Create({ + id: new URL("https://example.com/programmatic"), + actor: new URL("https://example.com/person2"), + }), + ed25519PrivateKey, + ed25519Multikey.id!, + { contextLoader: mockDocumentLoader }, + ); + strictEqual(await ctx.routeActivity(null, signed), true); + strictEqual(events.length, 2); + for (const event of events) deepStrictEqual(event.provenance, {}); +}); + +test("outbound provenance retains request labels on gateway dispatchers", async (t) => { + const did = await exportDidKey(ed25519PublicKey.publicKey); + const digest = await computeDigestMultibase( + new TextEncoder().encode("gateway media"), + ); + for (const route of ["actor", "object", "collection", "media"] as const) { + await t.step(route, async () => { + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + }); + const send = (ctx: Context) => + ctx.clone(undefined).sendActivity(sender, [], outgoing); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + federation.setActorDispatcher("/users/{identifier}", async (ctx) => { + await send(ctx); + return route === "collection" + ? new Person({ + id: parseIri(`ap+ef61://${did}/users/alice`), + followers: parseIri(`ap+ef61://${did}/users/alice/followers`), + }) + : null; + }); + federation.setObjectDispatcher(Note, "/notes/{id}", async (ctx) => { + await send(ctx); + return null; + }); + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + async (ctx) => { + await send(ctx); + return { items: [] }; + }, + ); + federation.setHashlinkMediaDispatcher(async (ctx) => { + await send(ctx); + return null; + }); + const suffix = route === "actor" + ? `${did}/users/alice` + : route === "object" + ? `${did}/notes/1` + : route === "collection" + ? `${did}/users/alice/followers` + : `hl:${digest}`; + const response = await federation.fetch( + new Request(`https://example.com/.well-known/apgateway/${suffix}`, { + headers: { + Accept: "application/activity+json", + "X-Request-Id": `${route}-request`, + }, + }), + { contextData: undefined }, + ); + strictEqual(response.status, route === "collection" ? 200 : 404); + strictEqual(events.length, route === "collection" ? 4 : 2); + for (const event of events) { + deepStrictEqual(event.provenance, { + originatingRequestId: `${route}-request`, + }); + strictEqual(event.local.host, "example.com"); + } + }); + } +}); + +test("queued provenance survives inbox retries, fanout, and outbox worker reconstruction", async (t) => { + const records: LogRecord[] = []; + await reset(); + await configure({ + sinks: { buffer: (record) => records.push(record) }, + loggers: [{ category: [], sinks: ["buffer"], lowestLevel: "info" }, { + category: ["logtape", "meta"], + sinks: [], + }], + contextLocalStorage: new AsyncLocalStorage(), + }); + try { + // Source propagation covers both retry owners. Legacy absence is + // independent of retry ownership, so one legacy execution is sufficient. + for ( + const { legacy, native } of [ + { legacy: false, native: false }, + { legacy: false, native: true }, + { legacy: true, native: false }, + ] + ) { + await t.step( + `${legacy ? "legacy" : "source"}: ${ + native ? "native redelivery" : "scheduled retry" + }`, + async () => { + const messages: Message[] = []; + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + const reports: InboxRequestReport[] = []; + const queue: MessageQueue = { + nativeRetrial: native, + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + origin: "https://canonical.example", + skipSignatureVerification: true, + manuallyStartQueue: true, + queue: { inbox: queue, fanout: queue, outbox: queue }, + }); + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ).setKeyPairsDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return []; + }); + let calls = 0; + const failure = new Error("retry listener"); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .setSharedKeyDispatcher(async (ctx) => { + await ctx.sendActivity(sender, [], outgoing); + return sender; + }) + .on(Create, async (ctx) => { + getLogger(["test", "queued-source"]).info("queued listener"); + await ctx.sendActivity(sender, [], outgoing); + if (++calls === 1) throw failure; + ok(ctx instanceof ContextImpl); + const clone = ctx.clone(undefined); + await clone.sendActivity(sender, target, outgoing, { + fanout: "force", + }); + const outbox = ctx.toOutboxContext( + "alice", + {}, + outgoing.id!.href, + Create.typeId.href, + ).clone(undefined); + await outbox.sendActivity(sender, target, outgoing, { + fanout: "skip", + }); + await clone.forwardActivity(sender, target, { + skipIfUnsigned: false, + }); + }).onRequestFinished((_ctx, report) => { + reports.push(report); + }); + // Producer metadata must not depend on observer registration. + strictEqual( + (await federation.fetch( + post(native ? "/inbox" : "/users/alice/inbox", await payload()), + { contextData: undefined }, + )).status, + 202, + ); + const produced = messages[0]; + ok(produced.type === "inbox"); + ok(produced.provenance?.inboxRequest); + strictEqual( + produced.provenance.inboxRequest.inboxRequestId, + reports[0].inboxRequestId, + ); + deepStrictEqual( + JSON.parse(JSON.stringify(produced.provenance)), + produced.provenance, + ); + const task = JSON.parse( + JSON.stringify(produced), + ) as typeof produced; + if (legacy) delete (task as { provenance?: unknown }).provenance; + const expected: OutboundProvenance = task.provenance ?? {}; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + onDelivery(_ctx, event) { + events.push(event); + }, + }); + const execute = (message: Message) => + withContext({ + requestId: "worker-current", + inboxRequestId: "unrelated-inbox", + sourceInboxRequestId: "ambient-source", + sourceRequestId: "ambient-label", + }, () => federation.processQueuedTask(undefined, message)); + messages.length = 0; + const logStart = records.length; + if (native) { + await rejects(execute(task), (error) => error === failure); + } else await execute(task); + const retry = native + ? task + : messages.find((message) => message.type === "inbox"); + ok(retry?.type === "inbox"); + deepStrictEqual(retry.provenance ?? {}, expected); + strictEqual(retry.attempt, native ? 0 : 1); + await execute({ + ...structuredClone(retry), + id: crypto.randomUUID(), + }); + strictEqual(calls, 2); + const fanout = messages.find((message) => message.type === "fanout"); + ok(fanout?.type === "fanout"); + deepStrictEqual(fanout.provenance ?? {}, expected); + await execute(JSON.parse(JSON.stringify(fanout))); + const outgoingTasks = messages.filter((message) => + message.type === "outbox" + ); + strictEqual(outgoingTasks.length, 3); + for (const message of outgoingTasks) { + deepStrictEqual(message.provenance ?? {}, expected); + strictEqual("provenance" in message, !legacy); + } + const consumer = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + origin: "https://consumer.example", + manuallyStartQueue: true, + queue: { outbox: queue }, + }); + consumer.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + let permanent = 0; + consumer.setOutboxPermanentFailureHandler(async (ctx) => { + permanent++; + getLogger(["test", "queued-source"]).info("permanent callback"); + await ctx.clone(undefined).sendActivity(sender, [], outgoing); + }); + fetchMock.mockGlobal().post(target.inboxId!.href, 410); + try { + await withContext({ + requestId: "worker-current", + sourceInboxRequestId: "ambient-source", + sourceRequestId: "ambient-label", + }, () => + consumer.processQueuedTask(undefined, { + ...structuredClone(outgoingTasks[0]), + baseUrl: "https://worker.example", + })); + } finally { + fetchMock.hardReset(); + } + strictEqual(permanent, 1); + ok(events.length > 0); + for (const event of events) { + deepStrictEqual(event.provenance, expected); + ok(Object.isFrozen(event.provenance)); + if (!legacy) { + ok(Object.isFrozen(event.provenance.inboxRequest?.context)); + } + } + strictEqual(events.at(-1)!.local.host, "worker.example"); + strictEqual( + events.at(-1)!.local.canonicalOrigin, + "https://consumer.example", + ); + const workerLogs = records.slice(logStart).filter((record) => + record.rawMessage === "queued listener" || + record.rawMessage === "permanent callback" + ); + strictEqual(workerLogs.length, 3); + for (const record of workerLogs) { + strictEqual(record.properties.requestId, "worker-current"); + strictEqual(record.properties.inboxRequestId, undefined); + strictEqual( + record.properties.sourceInboxRequestId, + expected.inboxRequest?.inboxRequestId, + ); + strictEqual( + record.properties.sourceRequestId, + expected.inboxRequest?.requestId, + ); + } + }, + ); + } + } finally { + await reset(); + } +}); + +test("queued provenance retains a non-inbox HTTP request label without inventing an inbox cause", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...loaders, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { fanout: queue, outbox: queue }, + }); + federation.setActorDispatcher("/users/{identifier}", async (ctx) => { + await ctx.sendActivity(sender, target, outgoing, { fanout: "force" }); + return null; + }); + strictEqual( + (await federation.fetch( + new Request("https://example.com/users/alice", { + headers: { + Accept: "application/activity+json", + "X-Request-Id": "original-http-label", + }, + }), + { contextData: undefined }, + )).status, + 404, + ); + const message = messages[0]; + ok(message?.type === "fanout"); + deepStrictEqual(message.provenance, { + originatingRequestId: "original-http-label", + }); + const events: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); + await withContext( + { requestId: "worker-current", sourceInboxRequestId: "unrelated-inbox" }, + () => + federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ), + ); + strictEqual(events.length, 1); + deepStrictEqual(events[0].provenance, message.provenance); + const child = messages[1]; + ok(child?.type === "outbox"); + deepStrictEqual(child.provenance, message.provenance); +}); diff --git a/packages/fedify/src/federation/outbound-provenance.ts b/packages/fedify/src/federation/outbound-provenance.ts new file mode 100644 index 000000000..d035739f8 --- /dev/null +++ b/packages/fedify/src/federation/outbound-provenance.ts @@ -0,0 +1,147 @@ +import type { Context } from "./context.ts"; +import type { + OutboundInboxRequest, + OutboundLocalContext, + OutboundProvenance, +} from "./outbound-report.ts"; + +// An own symbol property survives context spreads without adding public API. +export const outboundProvenance: unique symbol = Symbol("outboundProvenance"); +export const emptyProvenance: OutboundProvenance = Object.freeze({}); + +export function getOutboundProvenance(context: object): OutboundProvenance { + return (context as { [outboundProvenance]?: OutboundProvenance })[ + outboundProvenance + ] ?? emptyProvenance; +} + +export function snapshotLocalContext( + context: Pick< + Context, + "origin" | "host" | "hostname" | "canonicalOrigin" + >, +): OutboundLocalContext { + return Object.freeze({ + origin: context.origin, + host: context.host, + hostname: context.hostname, + canonicalOrigin: context.canonicalOrigin, + }); +} + +export function createInboxRequestSource( + request: Request, + requestId: string, + canonicalOrigin?: string, +): OutboundInboxRequest { + const url = new URL(request.url); + return Object.freeze({ + inboxRequestId: crypto.randomUUID(), + requestId, + context: snapshotLocalContext({ + origin: url.origin, + host: url.host, + hostname: url.hostname, + canonicalOrigin: canonicalOrigin ?? url.origin, + }), + }); +} + +/** The same exact-string budget applies to public input and queued metadata. */ +export function validateDispatchId( + value: unknown, +): asserts value is string | undefined { + if (value === undefined) return; + if (typeof value !== "string") { + throw new TypeError("dispatchId must be a string."); + } + if ( + value.length > 1024 || new TextEncoder().encode(value).byteLength > 1024 + ) { + throw new RangeError("dispatchId must not exceed 1,024 UTF-8 bytes."); + } +} + +/** A new public invocation keeps its HTTP source, with its own dispatch ID. */ +export function invocationProvenance( + context: object, + dispatchId: unknown, +): OutboundProvenance { + validateDispatchId(dispatchId); + const { dispatchId: _previous, ...source } = getOutboundProvenance(context); + return Object.freeze({ + ...source, + ...(dispatchId === undefined ? {} : { dispatchId }), + }); +} + +/** Restore only explicit source facts; never infer them from queue/trace IDs. */ +export function restoreProvenance(value: unknown): OutboundProvenance { + if (value === undefined) return emptyProvenance; + const record = (v: unknown): v is Record => + v != null && typeof v === "object" && !Array.isArray(v); + const invalid = () => new TypeError("Invalid provenance in queue metadata."); + if (!record(value)) throw invalid(); + validateDispatchId(value.dispatchId); + if ( + value.originatingRequestId !== undefined && + typeof value.originatingRequestId !== "string" + ) throw invalid(); + let inboxRequest: OutboundInboxRequest | undefined; + if (value.inboxRequest !== undefined) { + const source = value.inboxRequest; + if ( + !record(source) || typeof source.inboxRequestId !== "string" || + source.inboxRequestId.length < 1 || + typeof source.requestId !== "string" || !record(source.context) + ) throw invalid(); + const facts = source.context; + if ( + typeof facts.origin !== "string" || typeof facts.host !== "string" || + typeof facts.hostname !== "string" || + typeof facts.canonicalOrigin !== "string" + ) throw invalid(); + inboxRequest = Object.freeze({ + inboxRequestId: source.inboxRequestId, + requestId: source.requestId, + context: Object.freeze({ + origin: facts.origin, + host: facts.host, + hostname: facts.hostname, + canonicalOrigin: facts.canonicalOrigin, + }), + }); + } + return Object.freeze({ + ...(value.dispatchId === undefined ? {} : { dispatchId: value.dispatchId }), + ...(inboxRequest == null ? {} : { inboxRequest }), + ...(value.originatingRequestId === undefined + ? {} + : { originatingRequestId: value.originatingRequestId }), + }); +} + +/** Omit absent provenance on legacy and independent producers. */ +export function serializeProvenance( + value: OutboundProvenance | undefined, +): { readonly provenance?: OutboundProvenance } { + const provenance = restoreProvenance(value); + return provenance.dispatchId === undefined && + provenance.inboxRequest == null && + provenance.originatingRequestId == null + ? {} + : { provenance }; +} + +export function sourceLogContext(source: OutboundProvenance): { + dispatchId: string | undefined; + sourceInboxRequestId: string | undefined; + sourceRequestId: string | undefined; +} { + return { + dispatchId: source.dispatchId, + sourceInboxRequestId: source.inboxRequest?.inboxRequestId, + sourceRequestId: source.inboxRequest?.requestId ?? + source.originatingRequestId, + }; +} diff --git a/packages/fedify/src/federation/outbound-report.test.ts b/packages/fedify/src/federation/outbound-report.test.ts new file mode 100644 index 000000000..3028d1aab --- /dev/null +++ b/packages/fedify/src/federation/outbound-report.test.ts @@ -0,0 +1,7077 @@ +import { + createTestMeterProvider, + createTestTracerProvider, + mockDocumentLoader, + test, +} from "@fedify/fixture"; +import { Create, Follow, Person } from "@fedify/vocab"; +import { FetchError, UrlError } from "@fedify/vocab-runtime"; +import { configure, reset } from "@logtape/logtape"; +import { + isSpanContextValid, + propagation, + SpanKind, + SpanStatusCode, + trace, + TraceFlags, +} from "@opentelemetry/api"; +import fetchMock from "fetch-mock"; +import { + deepStrictEqual, + notStrictEqual, + ok, + rejects, + strictEqual, + throws, +} from "node:assert/strict"; +import { createFederationBuilder, FederationBuilderImpl } from "./builder.ts"; +import { MemoryKvStore } from "./kv.ts"; +import { + ContextImpl, + createFederation, + FederationImpl, + InboxContextImpl, + OutboxContextImpl, +} from "./middleware.ts"; +import type { + Federation, + Message, + MessageQueue, + OutboundAttemptReport, + OutboundBroadcastEvent, + OutboundDeliveryEvent, + OutboundHttpExchangeReport, + OutboundObservers, + OutboundResponseBodyEvent, + OutboundWorkerReport, +} from "./mod.ts"; +import type { FanoutMessage, OutboxMessage } from "./queue.ts"; +import { + sendActivity as sendConcreteActivity, + SendActivityError, +} from "./send.ts"; +import { + OutboundAttemptObservation, + OutboundBroadcastObservation, +} from "./outbound-observation.ts"; +import { snapshotHeaderPolicy, snapshotHeaders } from "./outbound-headers.ts"; +import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; +import { exportJwk } from "../sig/key.ts"; + +const activity = new Create({ + id: new URL("https://example.com/create"), + actor: new URL("https://example.com/alice"), +}); +const sender = { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }; +const options = { + kv: new MemoryKvStore(), + documentLoaderFactory: () => mockDocumentLoader, + contextLoaderFactory: () => mockDocumentLoader, +}; + +test("successful outbound attempts require an observed final response", async () => { + const reports: unknown[] = []; + const observation = new OutboundAttemptObservation( + "https://receiver.example/inbox", + async (facts) => { + await Promise.resolve(); + reports.push(facts); + }, + ); + throws(() => observation.succeeded(), { + message: + "A successful outbound attempt must have an observed final response.", + }); + observation.response(new Response(null, { status: 202 }), "final"); + throws(() => observation.succeeded(), { + message: + "A successful outbound attempt must have an observed final response.", + }); + strictEqual(reports.length, 0); + + const response = new Response(null, { status: 202 }); + await observation.fetch( + new Request("https://receiver.example/inbox", { method: "POST" }), + async () => { + await Promise.resolve(); + return response; + }, + ); + observation.response(response, "final"); + await observation.succeeded(); + await observation.succeeded(); + strictEqual(reports.length, 1); +}); + +test("outbound observation timestamps snapshot each boundary and refresh on fanout replay", async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + const timestamps: number[] = []; + let now = 1_800_000_000_000; + const originalNow = Temporal.Now.instant; + const observe = async ( + _context: unknown, + event: OutboundBroadcastEvent | OutboundDeliveryEvent, + ) => { + const observedAt = now; + strictEqual(event.observedAt, observedAt); + ok(Object.isFrozen(event)); + ok(Object.isFrozen(event.provenance)); + ok(Object.isFrozen(event.local)); + events.push(event); + timestamps.push(observedAt); + // Time spent awaiting a recorder must not change this event's snapshot. + now += 1_000; + await Promise.resolve(); + strictEqual(event.observedAt, observedAt); + }; + const observers = { onBroadcast: observe, onDelivery: observe }; + const inbox = "https://receiver.example/inbox"; + const otherInbox = "https://receiver.example/other"; + try { + Temporal.Now.instant = () => Temporal.Instant.fromEpochMilliseconds(now); + const observation = new OutboundBroadcastObservation( + ctx, + observers, + "send", + ); + await observation.run(async () => { + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + [otherInbox]: { actorIds: [], sharedInbox: false }, + }); + await observation.delivery(inbox)!.handoff({ type: "direct" }); + await observation.delivery(otherInbox)!.failed(new Error("local setup")); + await observation.prepared("delivery"); + }); + deepStrictEqual(events.map((event) => event.stage), [ + "started", + "planned", + "planned", + "handoff", + "preparation-failed", + "preparation-finished", + ]); + deepStrictEqual(timestamps, [ + 1_800_000_000_000, + 1_800_000_001_000, + 1_800_000_002_000, + 1_800_000_003_000, + 1_800_000_004_000, + 1_800_000_005_000, + ]); + const identity = observation.continuation(inbox)!.outbound; + strictEqual("observedAt" in identity, false); + const message: FanoutMessage = JSON.parse(JSON.stringify({ + type: "fanout", + id: crypto.randomUUID(), + baseUrl: "https://example.com/", + keys: [], + inboxes: { + [inbox]: { actorIds: [], sharedInbox: false, outbound: identity }, + }, + activity: {}, + activityType: Create.typeId.href, + traceContext: {}, + })); + // Replaying the same task preserves grouping, but observes a new instant. + for (let replay = 0; replay < 2; replay++) { + now += 60_000; + const resumed = await OutboundBroadcastObservation.resumeFanout( + ctx, + observers, + message, + ); + ok(resumed); + await resumed.delivery(inbox)!.handoff({ + type: "enqueued", + queue: "outbox", + queueMessageId: crypto.randomUUID(), + }); + const event = events.at(-1)!; + ok("deliveryId" in event); + strictEqual(event.deliveryId, identity.deliveryId); + strictEqual(event.provenance, events[0].provenance); + deepStrictEqual(event.local, events[0].local); + } + deepStrictEqual(timestamps.slice(6), [ + 1_800_000_066_000, + 1_800_000_127_000, + ]); + deepStrictEqual(events.map((event) => event.observedAt), timestamps); + strictEqual("observedAt" in message.inboxes[inbox].outbound!, false); + } finally { + Temporal.Now.instant = originalNow; + } +}); + +test("outbound exchange-only observers await snapshots across unsigned worker redirects", async () => { + const builder = createFederationBuilder<{ label: string }>(); + const exchanges: OutboundHttpExchangeReport[] = []; + const contexts: unknown[] = []; + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let replacements = 0; + builder.setOutboundObservers({ + async onHttpExchange(ctx, report) { + exchanges.push(report); + contexts.push(ctx.data); + entered(); + await gate; + throw new Error("exchange callback rejection must be isolated"); + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + builder.setOutboundObservers({ + onHttpExchange() { + replacements++; + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + const data = { label: "worker" }; + const next = new URL("/next", message.inbox).href; + const bodies: unknown[] = []; + fetchMock.mockGlobal() + .post(message.inbox, (call) => { + bodies.push(JSON.parse(String(call.options.body))); + return { status: 303, headers: { Location: next } }; + }) + .get(next, 202); + let settled = false; + const processing = federation.processQueuedTask(data, message) + .finally(() => { + settled = true; + }); + try { + await observing; + strictEqual(settled, false); + strictEqual(fetchMock.callHistory.calls().length, 1); + strictEqual(exchanges.length, 1); + strictEqual( + Reflect.set(exchanges[0].request, "url", "https://changed.example"), + false, + ); + federation.setOutboundObservers({ + onHttpExchange() { + replacements++; + }, + }); + release(); + await processing; + strictEqual(replacements, 0); + strictEqual(exchanges.length, 2); + deepStrictEqual(contexts, [data, data]); + deepStrictEqual(bodies, [message.activity]); + deepStrictEqual(exchanges.map((report) => report.exchangeIndex), [0, 1]); + deepStrictEqual(exchanges.map((report) => report.request), [ + { url: message.inbox, method: "POST" }, + { url: next, method: "GET" }, + ]); + strictEqual(exchanges[0].attemptId, exchanges[1].attemptId); + for (const [index, report] of exchanges.entries()) { + strictEqual(report.deliveryId, message.outbound!.deliveryId); + deepStrictEqual(report.broadcast, message.outbound!.broadcast); + deepStrictEqual(report.provenance, message.provenance); + deepStrictEqual(report.queue, { + queueMessageId: message.id, + queueAttempt: message.attempt, + }); + strictEqual(report.outcome.type, "response"); + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, index === 0 ? 303 : 202); + ok(Object.isFrozen(report)); + ok(Object.isFrozen(report.request)); + ok(Object.isFrozen(report.outcome)); + ok(Object.isFrozen(report.outcome.response)); + strictEqual("trace" in report, false); + } + strictEqual(fetchMock.callHistory.calls().length, 2); + } finally { + release(); + await processing; + fetchMock.hardReset(); + } +}); + +test("outbound unsigned exchanges report only actual submissions and retain received responses", async (t) => { + for ( + const phase of [ + "http", + "request", + "policy", + "serialization", + "redirect-policy", + "redirect-cleanup", + "final-cleanup", + ] as const + ) { + await t.step(phase, async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + throw new Error("recorder must not change the exchange result"); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = phase.includes("policy") + ? new UrlError("destination denied") + : undefined; + let submissions = 0; + let closed = 0; + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: phase === "serialization" + ? { + toJSON() { + throw failure; + }, + } + : {}, + keys: [], + inbox: new URL(inbox), + }, { + openHop(url) { + if ( + phase === "policy" || phase === "redirect-policy" && url !== inbox + ) { + return Promise.reject(failure); + } + return Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + if (phase === "request") return Promise.reject(failure); + const redirect = phase.startsWith("redirect-"); + return Promise.resolve( + new Response(null, { + status: redirect ? 307 : phase === "http" ? 503 : 202, + statusText: "Observed response", + headers: redirect ? { Location: "/next" } : {}, + }), + ); + }, + close() { + closed++; + return phase.endsWith("cleanup") + ? Promise.reject(failure) + : Promise.resolve(); + }, + }); + }, + }), + (error: unknown) => { + outward = error; + return true; + }, + ); + strictEqual(attempts.length, 1); + ok(attempts[0].outcome.type === "failed"); + strictEqual(attempts[0].outcome.error, outward); + const count = phase === "policy" || phase === "serialization" ? 0 : 1; + strictEqual(submissions, count); + strictEqual(exchanges.length, count); + strictEqual(attempts[0].httpExchangeCount, count); + const evidence = attempts[0].outcome.response; + if (count === 0 || phase === "request") { + deepStrictEqual(evidence, { type: "not-observed" }); + } else if (phase.startsWith("redirect-")) { + deepStrictEqual(evidence, { + type: "intermediate-only", + lastObserved: { + exchangeIndex: 0, + metadata: { status: 307, statusText: "Observed response" }, + }, + }); + } else { + ok(evidence.type === "final"); + strictEqual(evidence.exchangeIndex, 0); + strictEqual(evidence.metadata.status, phase === "http" ? 503 : 202); + } + if (count === 0) return; + const report = exchanges[0]; + strictEqual(report.attemptId, attempts[0].attemptId); + strictEqual(report.exchangeIndex, 0); + deepStrictEqual(report.request, { url: inbox, method: "POST" }); + if (phase === "request") { + strictEqual(report.outcome.type, "failed"); + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, failure); + } else { + ok(report.outcome.type === "response"); + deepStrictEqual(report.outcome.response, { + status: phase.startsWith("redirect-") + ? 307 + : phase === "http" + ? 503 + : 202, + statusText: "Observed response", + }); + } + ok(closed > 0); + }); + } +}); + +test("outbound signed exchanges follow actual challenge, fallback, and redirect submissions", async (t) => { + const challenge = + 'sig1=("@method" "@target-uri");created;nonce="observer-nonce"'; + for ( + const mode of [ + "success", + "fallback", + "challenge", + "challenge-fallback", + "challenge-redirect", + "unfulfillable", + ] as const + ) { + await t.step(mode, async () => { + const inbox = recipient.inboxId!.href; + const next = new URL("/next", inbox).href; + const responses: ResponseInit[] = mode === "success" + ? [{ status: 202 }] + : mode === "fallback" + ? [{ status: 401 }, { status: 202 }] + : mode === "unfulfillable" + ? [ + { + status: 401, + headers: { + "Accept-Signature": 'sig1=("@method" "x-absent");created', + }, + }, + { status: 202 }, + ] + : [ + { status: 401, headers: { "Accept-Signature": challenge } }, + ...(mode === "challenge" + ? [{ status: 202 }] + : mode === "challenge-fallback" + ? [{ status: 401 }, { status: 202 }] + : [{ status: 307, headers: { Location: next } }, { status: 202 }]), + ]; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + throw new Error("observer must not alter signature negotiation"); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const requests: Request[] = []; + const origins: string[] = []; + const payload = { id: "https://example.com/activity", extra: "original" }; + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: payload, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { + determineSpec(origin) { + origins.push(origin); + return "rfc9421"; + }, + rememberSpec() {}, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch(input) { + ok(input instanceof Request); + const response = responses[requests.length]; + requests.push(input); + ok( + response, + "observation must not create extra HTTP submissions", + ); + return Promise.resolve(new Response(null, response)); + }, + close: () => Promise.resolve(), + }), + }); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(exchanges.length, responses.length); + strictEqual(requests.length, responses.length); + strictEqual(attempts[0].httpExchangeCount, requests.length); + const final = attempts[0].outcome.response; + ok(final.type === "final"); + strictEqual(final.exchangeIndex, requests.length - 1); + ok(Object.isFrozen(final)); + ok(Object.isFrozen(final.metadata)); + strictEqual(final.metadata.status, 202); + strictEqual(origins.length, mode === "challenge-redirect" ? 2 : 1); + for (const [index, report] of exchanges.entries()) { + strictEqual(report.exchangeIndex, index); + strictEqual(report.attemptId, attempts[0].attemptId); + deepStrictEqual(report.request, { + url: requests[index].url, + method: requests[index].method, + }); + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, responses[index].status); + deepStrictEqual( + JSON.parse(await requests[index].clone().text()), + payload, + ); + ok(requests[index].headers.has("Signature")); + } + ok(requests[0].headers.has("Signature-Input")); + if (mode.startsWith("challenge")) { + ok( + requests[1].headers.get("Signature-Input")?.includes( + 'nonce="observer-nonce"', + ), + ); + } + if ( + mode === "fallback" || mode === "challenge-fallback" || + mode === "unfulfillable" + ) { + strictEqual(requests.at(-1)!.headers.has("Signature-Input"), false); + } + if (mode === "challenge-redirect") strictEqual(requests[2].url, next); + }); + } +}); + +test("outbound signed exchanges preserve earlier responses when subsequent operations fail", async (t) => { + for ( + const phase of [ + "spec-lookup", + "signing", + "later-spec-lookup", + "later-signing", + "post-response", + "request", + "handled-signing", + ] as const + ) { + await t.step(phase, async () => { + const inbox = recipient.inboxId!.href; + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = Symbol(phase); + const keyId = new URL(sender.keyId); + const href = keyId.href; + let submissions = 0; + let lookups = 0; + Object.defineProperty(keyId, "href", { + get() { + if ( + phase === "signing" || phase === "later-signing" && submissions > 0 + ) throw failure; + return href; + }, + }); + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [{ ...sender, keyId }], + inbox: new URL(inbox), + specDeterminer: { + determineSpec() { + lookups++; + if ( + phase === "spec-lookup" || + phase === "later-spec-lookup" && lookups > 1 + ) throw failure; + return "rfc9421"; + }, + rememberSpec() { + if (phase === "post-response") return Promise.reject(failure); + }, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + if ( + phase === "request" || + phase === "handled-signing" && submissions > 1 + ) return Promise.reject(failure); + return Promise.resolve( + new Response(null, { + status: phase === "later-spec-lookup" + ? 307 + : phase === "post-response" + ? 202 + : 401, + headers: phase === "later-spec-lookup" + ? { Location: "/next" } + : phase === "handled-signing" + ? { + "Accept-Signature": + 'sig1=("@method" "x-absent");created', + } + : {}, + }), + ); + }, + close: () => Promise.resolve(), + }), + }), + (error: unknown) => { + outward = error; + return error instanceof FetchError; + }, + ); + strictEqual(exchanges.length, submissions); + strictEqual(attempts.length, 1); + const outcome = attempts[0].outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + strictEqual(outcome.failure.error, failure); + const transport = phase === "request" || phase === "handled-signing"; + strictEqual(outcome.failure.kind, transport ? "transport" : "local"); + strictEqual( + outcome.failure.phase, + transport ? "request" : phase.replace("later-", ""), + ); + const count = phase === "spec-lookup" || phase === "signing" + ? 0 + : phase === "handled-signing" + ? 2 + : 1; + strictEqual(submissions, count); + strictEqual(attempts[0].httpExchangeCount, count); + if (phase === "post-response") { + ok(outcome.response.type === "final"); + strictEqual(outcome.response.exchangeIndex, 0); + strictEqual(outcome.response.metadata.status, 202); + } else if (phase.startsWith("later-") || phase === "handled-signing") { + ok(outcome.response.type === "intermediate-only"); + strictEqual(outcome.response.lastObserved.exchangeIndex, 0); + strictEqual( + outcome.response.lastObserved.metadata.status, + phase === "later-spec-lookup" ? 307 : 401, + ); + } else deepStrictEqual(outcome.response, { type: "not-observed" }); + for (const [index, report] of exchanges.entries()) { + strictEqual(report.attemptId, attempts[0].attemptId); + strictEqual(report.exchangeIndex, index); + if (transport && index === count - 1) { + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, failure); + } else { + ok(report.outcome.type === "response"); + strictEqual( + report.outcome.response.status, + phase === "later-spec-lookup" + ? 307 + : phase === "post-response" + ? 202 + : 401, + ); + } + } + }); + } +}); + +test("outbound attempt finality preserves HTTP, cache, and cleanup failures together", async (t) => { + for (const status of [202, 503]) { + for (const phase of ["cache", "cleanup", "both"] as const) { + await t.step(`${status}: ${phase}`, async () => { + const inbox = recipient.inboxId!.href; + const reports: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + reports.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const cacheError = Symbol("cache failure"); + const cleanupError = Symbol("cleanup failure"); + let submissions = 0; + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() { + if (phase !== "cleanup") throw cacheError; + }, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + return Promise.resolve( + new Response(null, { + status: ++submissions === 1 ? 401 : status, + statusText: "Observed response", + }), + ); + }, + close() { + return phase === "cache" + ? Promise.resolve() + : Promise.reject(cleanupError); + }, + }), + }), + (error: unknown) => { + outward = error; + return error instanceof FetchError; + }, + ); + strictEqual(reports.length, 1); + strictEqual(submissions, 2); + const report = reports[0]; + strictEqual(report.httpExchangeCount, 2); + const outcome = report.outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + strictEqual( + (outward as FetchError).cause, + phase === "cleanup" ? cleanupError : cacheError, + ); + deepStrictEqual(outcome.response, { + type: "final", + exchangeIndex: 1, + metadata: { status, statusText: "Observed response" }, + }); + const diagnostics = []; + if (phase !== "cleanup") { + diagnostics.push({ + kind: "local", + phase: "post-response", + error: cacheError, + }); + } + if (phase !== "cache") { + diagnostics.push({ + kind: "local", + phase: "cleanup", + error: cleanupError, + }); + } + if (status === 503) { + deepStrictEqual(outcome.failure, { kind: "http", phase: "response" }); + deepStrictEqual(outcome.diagnostics, diagnostics); + } else { + deepStrictEqual(outcome.failure, diagnostics[0]); + deepStrictEqual( + outcome.diagnostics, + diagnostics.length > 1 ? diagnostics.slice(1) : undefined, + ); + } + ok(Object.isFrozen(outcome.response)); + ok(Object.isFrozen(outcome.response.metadata)); + if (outcome.diagnostics != null) { + ok(Object.isFrozen(outcome.diagnostics)); + } + strictEqual(exchanges.length, 2); + ok(exchanges[1].outcome.type === "response"); + deepStrictEqual( + exchanges[1].outcome.response, + outcome.response.metadata, + ); + }); + } + } +}); + +test("outbound attempt finality follows branch decisions rather than status or last index", async (t) => { + for ( + const mode of [ + "challenge-cache", + "final-401", + "unsigned-300", + "signed-300", + "undetermined", + ] as const + ) { + await t.step(mode, async () => { + const inbox = recipient.inboxId!.href; + const reports: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + reports.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = Symbol(mode); + const denied = new UrlError("redirect denied"); + const keyId = new URL(sender.keyId); + const href = keyId.href; + let submissions = 0; + let cacheWrites = 0; + Object.defineProperty(keyId, "href", { + get() { + if (mode === "undetermined" && submissions > 0) throw failure; + return href; + }, + }); + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: mode === "unsigned-300" ? [] : [{ ...sender, keyId }], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() { + cacheWrites++; + if (mode === "challenge-cache") throw failure; + }, + }, + }, { + openHop(url) { + if (url !== inbox) return Promise.reject(denied); + return Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return Promise.resolve( + new Response(null, { + status: mode.endsWith("300") + ? 300 + : mode === "challenge-cache" && submissions > 1 + ? 202 + : 401, + statusText: "Observed response", + headers: mode.endsWith("300") + ? { Location: "/next" } + : mode === "challenge-cache" || mode === "undetermined" + ? { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created', + } + : {}, + }), + ); + }, + close: () => Promise.resolve(), + }); + }, + }), + (error: unknown) => { + outward = error; + return true; + }, + ); + strictEqual(reports.length, 1); + const report = reports[0]; + strictEqual(report.httpExchangeCount, submissions); + strictEqual( + submissions, + mode === "challenge-cache" || mode === "final-401" ? 2 : 1, + ); + strictEqual(exchanges.length, submissions); + const outcome = report.outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + const evidence = outcome.response; + if (mode === "signed-300" || mode === "undetermined") { + ok( + evidence.type === + (mode === "signed-300" ? "intermediate-only" : "undetermined"), + ); + strictEqual(evidence.lastObserved.exchangeIndex, 0); + strictEqual( + evidence.lastObserved.metadata.status, + mode === "signed-300" ? 300 : 401, + ); + strictEqual( + outcome.failure.kind, + mode === "signed-300" ? "policy" : "unknown", + ); + } else { + ok(evidence.type === "final"); + strictEqual(evidence.exchangeIndex, submissions - 1); + strictEqual( + evidence.metadata.status, + mode === "challenge-cache" ? 202 : mode === "final-401" ? 401 : 300, + ); + strictEqual( + outcome.failure.kind, + mode === "challenge-cache" ? "local" : "http", + ); + } + if (mode === "final-401") strictEqual(cacheWrites, 0); + if (mode === "challenge-cache" || mode === "undetermined") { + strictEqual(outcome.failure.error, failure); + } + }); + } +}); + +test("outbound attempts report ordinary and outbox sends with attempt-only builder registration", async (t) => { + for (const kind of ["ordinary", "outbox"] as const) { + for (const status of [202, 503, "policy"] as const) { + await t.step(`${kind}: ${status}`, async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + try { + const builder = createFederationBuilder(); + const reports: OutboundAttemptReport[] = []; + const contexts: unknown[] = []; + let recorded = false; + let replacementCalls = 0; + builder.setOutboundObservers({ + async onAttempt(context, report) { + contexts.push(context); + reports.push(report); + await Promise.resolve(); + recorded = true; + throw new Error("observer rejection must be isolated"); + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + tracerProvider, + allowPrivateAddress: status !== "policy", + }); + builder.setOutboundObservers({ + onAttempt() { + replacementCalls++; + }, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = kind === "ordinary" + ? base + : base.toOutboxContext("alice", {}, undefined, Create.typeId.href); + const target = status === "policy" + ? recipient.clone({ + inbox: new URL("http://127.0.0.1/inbox"), + }) + : recipient; + if (status !== "policy") { + fetchMock.mockGlobal().post(target.inboxId!.href, status); + } + const sending = ctx.sendActivity(sender, target, activity, { + immediate: true, + dispatchId: "application-dispatch", + }); + let outward: unknown; + if (status === 202) await sending; + else { + await rejects(sending, (error: unknown) => { + outward = error; + return status === "policy" + ? error instanceof UrlError + : error instanceof SendActivityError && + error.statusCode === status; + }); + } + strictEqual(recorded, true); + strictEqual(replacementCalls, 0); + strictEqual(reports.length, 1); + strictEqual(contexts[0], ctx); + const report = reports[0]; + ok(report.attemptId.length > 0); + ok(report.deliveryId.length > 0); + ok(report.broadcast.type === "known"); + strictEqual(report.destinationInbox, target.inboxId!.href); + strictEqual(report.inboxUrl, target.inboxId!.href); + strictEqual(report.provenance.dispatchId, "application-dispatch"); + ok(Object.isFrozen(report)); + ok(Object.isFrozen(report.outcome)); + ok(Object.isFrozen(report.local)); + ok(Object.isFrozen(report.provenance)); + const span = exporter.getSpan("activitypub.send_activity"); + ok(span); + deepStrictEqual(report.trace, { + traceId: span.spanContext().traceId, + spanId: span.spanContext().spanId, + }); + strictEqual( + report.httpExchangeCount, + status === "policy" ? 0 : status === 202 ? 1 : 2, + ); + if (status === 202) { + strictEqual(report.outcome.type, "succeeded"); + ok(report.outcome.response.type === "final"); + strictEqual(report.outcome.response.metadata.status, status); + strictEqual(report.outcome.response.exchangeIndex, 0); + } else { + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, outward); + strictEqual(report.outcome.failure.error, outward); + strictEqual( + report.outcome.failure.kind, + status === "policy" ? "policy" : "http", + ); + if (status === "policy") { + deepStrictEqual(report.outcome.response, { + type: "not-observed", + }); + } else { + ok(report.outcome.response.type === "final"); + strictEqual(report.outcome.response.metadata.status, status); + strictEqual(report.outcome.response.exchangeIndex, 1); + } + } + } finally { + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } + } +}); + +test("outbound attempts await the selected observer without inventing pre-entry attempts", async () => { + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + const reports: OutboundAttemptReport[] = []; + let replacements = 0; + let settled = false; + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + if (event.stage === "started") { + federation.setOutboundObservers({ + onAttempt() { + replacements++; + }, + }); + } + }, + async onAttempt(_ctx, report) { + reports.push(report); + entered(); + await gate; + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sending = ctx.sendActivity(sender, recipient, activity, { + immediate: true, + }) + .finally(() => { + settled = true; + }); + await observing; + strictEqual(settled, false); + strictEqual(replacements, 0); + release(); + await sending; + strictEqual(reports.length, 1); + await ctx.sendActivity(sender, [], activity); + await rejects( + ctx.sendActivity(sender, recipient, new Create({})), + TypeError, + ); + strictEqual(replacements, 0); + strictEqual(reports.length, 1); + } finally { + release(); + fetchMock.hardReset(); + } +}); + +test("outbound attempts preserve boundary failures and allocate fresh IDs per concrete entry", async (t) => { + for ( + const phase of [ + "success", + "serialization", + "dns", + "request", + "cleanup", + "http-cleanup", + "spec-lookup", + ] as const + ) { + await t.step(phase, async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const reports: OutboundAttemptReport[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onAttempt(_ctx, report) { + reports.push(report); + }, + onDelivery(_ctx, event) { + deliveries.push(event); + }, + }, "send"); + const inbox = "https://receiver.example/inbox"; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const failure = phase === "dns" + ? new UrlError(inbox, { reason: "dns" }) + : Symbol(phase); + const delivery = observation.delivery(inbox)!; + let submissions = 0; + const send = () => + sendConcreteActivity({ + observation: delivery, + activity: phase === "serialization" + ? { + toJSON() { + throw failure; + }, + } + : {}, + keys: phase === "spec-lookup" ? [sender] : [], + inbox: new URL(inbox), + specDeterminer: { + determineSpec() { + throw failure; + }, + rememberSpec() { + return Promise.resolve(); + }, + }, + }, { + openHop(url) { + if (phase === "dns") return Promise.reject(failure); + return Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return phase === "request" + ? Promise.reject(failure) + : Promise.resolve( + new Response(null, { + status: phase === "http-cleanup" ? 503 : 202, + }), + ); + }, + close() { + return phase === "cleanup" || phase === "http-cleanup" + ? Promise.reject(failure) + : Promise.resolve(); + }, + }); + }, + }); + let outward: unknown; + if (phase === "success") { + await send(); + await send(); + strictEqual(reports.length, 2); + notStrictEqual(reports[0].attemptId, reports[1].attemptId); + strictEqual(reports[0].deliveryId, reports[1].deliveryId); + strictEqual(submissions, 2); + return; + } + await rejects(send(), (error: unknown) => { + outward = error; + return phase === "serialization" + ? error === failure + : error instanceof FetchError; + }); + strictEqual(reports.length, 1); + const report = reports[0]; + strictEqual(report.deliveryId, deliveries[0].deliveryId); + strictEqual(report.outcome.type, "failed"); + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.error, outward); + if (phase === "http-cleanup") { + deepStrictEqual(report.outcome.failure, { + kind: "http", + phase: "response", + }); + deepStrictEqual(report.outcome.diagnostics, [{ + kind: "local", + phase: "cleanup", + error: failure, + }]); + deepStrictEqual(report.outcome.response, { + type: "final", + exchangeIndex: 0, + metadata: { status: 503, statusText: "" }, + }); + } else { + deepStrictEqual(report.outcome.failure, { + kind: phase === "dns" || phase === "request" ? "transport" : "local", + phase: phase === "dns" ? "validation" : phase, + error: failure, + }); + } + strictEqual( + submissions, + ["request", "cleanup", "http-cleanup"].includes(phase) ? 1 : 0, + ); + strictEqual("trace" in report, false); + }); + } +}); + +test("outbound observers report fresh immutable starts without tracing or extra loading", async (t) => { + const { AlwaysOffSampler, BasicTracerProvider } = await import( + "@opentelemetry/sdk-trace-base" + ); + const provider = new BasicTracerProvider({ sampler: new AlwaysOffSampler() }); + try { + for (const tracerProvider of [undefined, provider]) { + await t.step( + tracerProvider == null ? "no provider" : "sampling disabled", + async () => { + let loads = 0; + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + tracerProvider, + documentLoaderFactory: () => () => { + loads++; + throw new Error("unexpected observation lookup"); + }, + contextLoaderFactory: () => () => { + loads++; + throw new Error("unexpected observation lookup"); + }, + }); + const ctx = federation.createContext(new URL("https://example.com"), { + label: "app", + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(context, event) { + strictEqual(context, ctx); + strictEqual(context.data.label, "app"); + strictEqual(event.operation, "send"); + ok(Object.isFrozen(event)); + events.push(event); + }, + }); + await ctx.sendActivity(sender, [], activity); + await ctx.sendActivity(sender, [], activity); + deepStrictEqual(preparation(events.slice(0, 2)), { + type: "skipped", + reason: "no-destinations", + }); + deepStrictEqual(preparation(events.slice(2)), { + type: "skipped", + reason: "no-destinations", + }); + for (const event of events) { + if (event.stage === "started" || tracerProvider == null) { + strictEqual("trace" in event, false); + } else { + ok(event.trace); + ok( + isSpanContextValid({ + ...event.trace, + traceFlags: TraceFlags.NONE, + }), + ); + } + } + ok(events[0].broadcastId.length > 0); + notStrictEqual(events[0].broadcastId, events[2].broadcastId); + strictEqual(loads, 0); + }, + ); + } + } finally { + await provider.shutdown(); + } +}); + +test("outbound trace snapshots identify producer and fanout spans without changing callbacks", async (t) => { + for ( + const mode of [ + "send", + "outbox-send", + "inbox-forward", + "outbox-forward", + ] as const + ) { + await t.step(mode, async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + try { + const queued: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + tracerProvider, + manuallyStartQueue: true, + queue: { fanout: queue, outbox: queue }, + }); + const events: (OutboundBroadcastEvent | OutboundDeliveryEvent)[] = []; + let replacements = 0; + const replacement = () => { + replacements++; + }; + const record = async ( + _ctx: unknown, + event: OutboundBroadcastEvent | OutboundDeliveryEvent, + ) => { + events.push(event); + if (event.stage === "started") { + federation.setOutboundObservers({ + onBroadcast: replacement, + onDelivery: replacement, + }); + } + await Promise.resolve(); + }; + const observers = { onBroadcast: record, onDelivery: record }; + federation.setOutboundObservers(observers); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(ctx instanceof ContextImpl); + if (mode.endsWith("forward")) { + await forwardingContext( + federation, + mode === "inbox-forward" ? "inbox" : "outbox", + forwardingPayload, + undefined, + ).forwardActivity(sender, recipient, { skipIfUnsigned: true }); + } else { + const producer = mode === "outbox-send" + ? ctx.toOutboxContext("alice", {}, undefined, Create.typeId.href) + : ctx; + await producer.sendActivity(sender, recipient, activity, { + fanout: mode === "send" ? "force" : "skip", + }); + } + strictEqual(replacements, 0); + strictEqual(events[0].stage, "started"); + strictEqual("trace" in events[0], false); + ok(events.length > 2); + const producerEvents = events.length; + if (mode === "send") { + federation.setOutboundObservers(observers); + const task = queued.find((message) => message.type === "fanout"); + ok(task); + await federation.processQueuedTask(undefined, structuredClone(task)); + ok(events.length > producerEvents); + } + for (const [index, event] of events.entries()) { + if (index === 0) continue; + ok(event.trace); + ok(Object.isFrozen(event.trace)); + const span = exporter.spans.find((span) => + span.spanContext().spanId === event.trace!.spanId + ); + ok(span); + deepStrictEqual(event.trace, { + traceId: span.spanContext().traceId, + spanId: span.spanContext().spanId, + }); + strictEqual( + span.kind, + index < producerEvents ? SpanKind.PRODUCER : SpanKind.CONSUMER, + ); + } + } finally { + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound trace snapshots omit invalid and inherited span identities", async () => { + const federation = createFederation(options); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const parent = { + traceId: "11111111111111111111111111111111", + spanId: "2222222222222222", + traceFlags: TraceFlags.NONE, + }; + for ( + const spanContext of [ + { ...parent, traceId: "00000000000000000000000000000000" }, + parent, + { ...parent, isRemote: true }, + ] + ) { + const events: OutboundBroadcastEvent[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + onBroadcast(_ctx, event) { + events.push(event); + }, + }, "send"); + observation.setSpan(trace.wrapSpanContext(spanContext), parent); + await observation.skipped(); + strictEqual(events.length, 1); + strictEqual("trace" in events[0], false); + } +}); + +test("outbound observers await entry before preparation and retain the selected callback", async () => { + const federation = createFederation(options); + let preparations = 0; + const failure = Symbol("original preparation failure"); + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .mapHandle(() => { + preparations++; + throw failure; + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + let entered!: () => void; + const entry = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const calls: string[] = []; + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + calls.push(`original:${event.stage}`); + entered(); + await gate; + throw new Error("asynchronous observer failure"); + }, + }); + const pending = rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + await entry; + strictEqual(preparations, 0); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + calls.push(`replacement:${event.stage}`); + }, + }); + release(); + await pending; + await rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + deepStrictEqual(calls, [ + "original:started", + "original:preparation-finished", + "replacement:started", + "replacement:preparation-finished", + ]); + strictEqual(preparations, 2); +}); + +test("outbound observers isolate callback and logging errors from success and original failures", async () => { + let logCalls = 0; + await configure({ + reset: true, + sinks: { + broken() { + logCalls++; + throw new Error("sink failed"); + }, + }, + loggers: [ + { + category: ["fedify", "federation", "outbox"], + sinks: ["broken"], + lowestLevel: "error", + }, + { category: ["logtape", "meta"], sinks: [] }, + ], + }); + try { + const federation = createFederation(options); + const failure = { original: true }; + let calls = 0; + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .mapHandle(() => { + throw failure; + }); + federation.setOutboundObservers({ + onBroadcast() { + calls++; + throw "observer failed"; + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await ctx.sendActivity(sender, [], activity); + await rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + strictEqual(calls, 4); + strictEqual(logCalls, 4); + } finally { + await reset(); + } +}); + +test("outbound observers copy registration and isolate builder instances", async () => { + const builder = createFederationBuilder(); + const calls: string[] = []; + const registration = { + onBroadcast() { + calls.push("original"); + }, + } satisfies OutboundObservers; + builder.setOutboundObservers(registration); + registration.onBroadcast = () => { + calls.push("mutated"); + }; + const first = await builder.build(options); + const second = await builder.build(options); + builder.setOutboundObservers({ + onBroadcast() { + calls.push("new builder"); + }, + }); + const third = await builder.build(options); + first.setOutboundObservers({ + onBroadcast() { + calls.push("first only"); + }, + }); + for (const federation of [first, second, third]) { + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, [], activity); + } + deepStrictEqual(calls, [ + "first only", + "first only", + "original", + "original", + "new builder", + "new builder", + ]); + second.setOutboundObservers({}); + await second.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, [], activity); + strictEqual(calls.length, 6); +}); + +test("outbound observers do not veto an actual immediate delivery", async () => { + const order: string[] = []; + fetchMock.mockGlobal().post("https://receiver.example/inbox", (call) => { + order.push("fetch"); + strictEqual(JSON.parse(String(call.options.body)).id, activity.id!.href); + return 202; + }); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + order.push(event.stage); + throw new Error("recording failed"); + }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + sender, + new Person({ + id: new URL("https://receiver.example/actor"), + inbox: new URL("https://receiver.example/inbox"), + }), + activity, + { immediate: true }, + ); + strictEqual(order[0], "started"); + strictEqual(order.filter((entry) => entry === "fetch").length, 1); + strictEqual( + order.filter((entry) => entry === "preparation-finished").length, + 1, + ); + } finally { + fetchMock.hardReset(); + } +}); + +function preparation(events: OutboundBroadcastEvent[]) { + deepStrictEqual(events.map((event) => event.stage), [ + "started", + "preparation-finished", + ]); + strictEqual(events[0].broadcastId, events[1].broadcastId); + ok(events.every((event) => Object.isFrozen(event))); + const event = events[1]; + ok(event.stage === "preparation-finished"); + ok(Object.isFrozen(event.result)); + return event.result; +} + +const recipient = new Person({ + id: new URL("https://receiver.example/actor"), + inbox: new URL("https://receiver.example/inbox"), +}); + +test("outbound preparation preserves failure provenance and plan completeness", async (t) => { + for ( + const phase of [ + "sender-resolution", + "key-resolution", + "recipient-resolution", + "transformation", + "signing", + "destination-planning", + "serialization", + ] as const + ) { + await t.step(phase, async () => { + const failure = Symbol(phase); + const events: OutboundBroadcastEvent[] = []; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + ...(phase === "transformation" + ? { + activityTransformers: [() => { + throw failure; + }], + } + : { activityTransformers: [] }), + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + let sentActivity = activity; + let target = recipient; + if (phase === "serialization") { + sentActivity = activity.clone({}); + Object.defineProperty(sentActivity, "toJsonLd", { + value: () => Promise.reject(failure), + }); + } + if (phase === "signing") { + sentActivity = activity.clone({}); + Object.defineProperty(sentActivity, "getProofs", { + value() { + throw failure; + }, + }); + } + if (phase === "destination-planning") { + target = recipient.clone({}); + Object.defineProperty(target, "inboxId", { + get() { + throw failure; + }, + }); + } + if (phase === "sender-resolution") { + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .mapHandle(() => { + throw failure; + }); + } + let keyDispatches = 0; + if (phase === "key-resolution" || phase === "recipient-resolution") { + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .setKeyPairsDispatcher(() => { + keyDispatches++; + if (phase === "key-resolution") throw failure; + return [{ + privateKey: rsaPrivateKey2, + publicKey: rsaPublicKey2.publicKey!, + }]; + }); + } + if (phase === "recipient-resolution") { + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + () => { + throw failure; + }, + ); + } + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sending = phase === "recipient-resolution" + ? ctx.sendActivity({ identifier: "alice" }, "followers", sentActivity) + : ctx.sendActivity( + phase === "sender-resolution" + ? { username: "alice" } + : phase === "key-resolution" + ? { identifier: "alice" } + : sender, + target, + sentActivity, + ); + let originalError: unknown; + await rejects(sending, (error: unknown) => { + originalError = error; + if (phase === "key-resolution") { + // Key-dispatcher errors are already handled by getActorKeyPairs(). + // Preserve the public send error from the resulting empty key set. + return error instanceof Error && + error.message === 'No key pair found for actor "alice".'; + } + return error === failure; + }); + if (phase === "key-resolution" || phase === "recipient-resolution") { + strictEqual(keyDispatches, 1); + } + const result = preparation(events); + ok(result.type === "failed"); + strictEqual(result.failure.error, originalError); + deepStrictEqual(result, { + type: "failed", + failure: { kind: "local", phase, error: originalError }, + planning: phase === "serialization" + ? "complete" + : phase === "destination-planning" + ? "incomplete" + : "not-started", + plannedDestinationCount: phase === "serialization" ? 1 : 0, + }); + }); + } +}); + +test("broadcast activity references are immutable and follow transformed activities", async () => { + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const transformedId = "https://example.com/transformed"; + const federation = createFederation({ + ...options, + queue: { fanout: queue }, + manuallyStartQueue: true, + activityTransformers: [() => + new Follow({ + id: new URL(transformedId), + actor: new URL("https://example.com/alice"), + object: new URL("https://example.com/bob"), + })], + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await ctx.sendActivity(sender, recipient, activity, { fanout: "force" }); + await ctx.sendActivity(sender, recipient, activity, { fanout: "force" }); + await ctx.sendActivity( + sender, + new Person({ + id: new URL("https://receiver.example/no-inbox"), + }), + activity, + ); + + strictEqual(messages.length, 2); + const starts = events.filter((event) => event.stage === "started"); + const finishes = events.filter((event) => + event.stage === "preparation-finished" + ); + strictEqual(starts.length, 3); + strictEqual(finishes.length, 3); + deepStrictEqual(starts.map((event) => event.activity), [{}, {}, {}]); + strictEqual(new Set(starts.map((event) => event.broadcastId)).size, 3); + for (const event of finishes.slice(0, 2)) { + deepStrictEqual(event.activity, { + id: transformedId, + type: Follow.typeId.href, + }); + ok(Object.isFrozen(event.activity)); + } + deepStrictEqual(finishes[2].activity, { + id: transformedId, + type: Follow.typeId.href, + }); + ok(Object.isFrozen(starts[0].activity)); +}); + +test("transformed activity type capture does not mask validation failures", async () => { + const validationMessage = + "The activity to send must have at least one actor property."; + class UnreadableTypeFollow extends Follow { + static override get typeId(): URL { + throw new Error("type must not be read"); + } + } + for (const unreadableType of [false, true]) { + const transformedId = unreadableType + ? "https://example.com/unreadable" + : "https://example.com/missing-actor"; + const federation = createFederation({ + ...options, + activityTransformers: [() => { + const ActivityClass = unreadableType ? UnreadableTypeFollow : Follow; + return new ActivityClass({ id: new URL(transformedId) } as never); + }], + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + + let rejected: unknown; + await rejects( + ctx.sendActivity(sender, recipient, activity), + (error: unknown) => { + rejected = error; + return error instanceof TypeError && + error.message === validationMessage; + }, + ); + const finished = events.find((event) => + event.stage === "preparation-finished" + ); + ok(finished); + strictEqual(finished.activity.id, transformedId); + if (unreadableType) { + ok(!("type" in finished.activity)); + } else { + strictEqual(finished.activity.type, Follow.typeId.href); + } + strictEqual(finished.result.type, "failed"); + if (finished.result.type === "failed") { + strictEqual(finished.result.failure.phase, "validation"); + strictEqual(finished.result.failure.error, rejected); + } + ok(Object.isFrozen(finished.activity)); + } +}); + +test("transformed activity ID getter failures retain the readable type", async () => { + const failure = new Error("transformed ID is unreadable"); + for (const unreadableType of [false, true]) { + let idReads = 0; + const transformed = new Follow({ + id: new URL("https://example.com/transformed"), + actor: new URL("https://example.com/alice"), + object: new URL("https://example.com/bob"), + }); + Object.defineProperty(transformed, "id", { + get() { + idReads++; + throw failure; + }, + }); + if (unreadableType) { + Object.defineProperty(transformed, "constructor", { + get() { + throw new Error("transformed type is unreadable"); + }, + }); + } + const federation = createFederation({ + ...options, + activityTransformers: [() => transformed], + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + + await rejects(ctx.sendActivity(sender, recipient, activity), (error) => { + strictEqual(error, failure); + return true; + }); + + strictEqual(idReads, 1); + const started = events.find((event) => event.stage === "started"); + ok(started); + deepStrictEqual(started.activity, {}); + ok(Object.isFrozen(started.activity)); + const finished = events.find((event) => + event.stage === "preparation-finished" + ); + ok(finished); + deepStrictEqual( + finished.activity, + unreadableType ? {} : { type: Follow.typeId.href }, + ); + ok(Object.isFrozen(finished.activity)); + strictEqual(finished.result.type, "failed"); + if (finished.result.type === "failed") { + strictEqual(finished.result.failure.phase, "transformation"); + strictEqual(finished.result.failure.error, failure); + } + } +}); + +test("outbound preparation precedes queue acceptance and is not rewritten by enqueue failure", async (t) => { + for (const role of ["outbox", "fanout"] as const) { + for (const rejected of [false, true]) { + await t.step( + `${role}: ${rejected ? "enqueue rejects" : "enqueue accepts"}`, + async () => { + const failure = Symbol("enqueue failure"); + const events: OutboundBroadcastEvent[] = []; + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + if (rejected) return Promise.reject(failure); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { [role]: queue }, + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await gate; + } + }, + }); + const sending = federation.createContext( + new URL("https://example.com"), + undefined, + ) + .sendActivity(sender, [recipient, recipient], activity, { + fanout: role === "fanout" ? "force" : "skip", + }); + const settled = rejected + ? rejects(sending, (error: unknown) => error === failure) + : sending; + await prepared; + strictEqual(messages.length, 0); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: role === "fanout" ? "fanout" : "delivery", + destinationCount: 1, + }); + release(); + await settled; + strictEqual(messages.length, 1); + strictEqual(messages[0].type, role); + const message = messages[0]; + const identity = message.type === "fanout" + ? message.inboxes[recipient.inboxId!.href].outbound + : message.type === "outbox" + ? message.outbound + : undefined; + ok(identity); + deepStrictEqual(identity.broadcast, { + type: "known", + id: events[0].broadcastId, + }); + strictEqual(identity.destinationInbox, recipient.inboxId!.href); + deepStrictEqual(structuredClone(identity), identity); + strictEqual(events.length, 2); + ok(!("observation" in messages[0])); + }, + ); + } + } +}); + +test("outbound preparation finishes before immediate sends settle without reporting later failures twice", async (t) => { + for (const status of [202, 503]) { + await t.step(`HTTP ${status}`, async () => { + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let releaseResponse!: () => void; + const responseGate = new Promise((resolve) => { + releaseResponse = resolve; + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let releaseObserver!: () => void; + const observerGate = new Promise((resolve) => { + releaseObserver = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const [tracerProvider, spanExporter] = createTestTracerProvider(); + let sendEnded!: () => void; + const concreteSendEnded = new Promise((resolve) => { + sendEnded = resolve; + }); + const exportSpans = spanExporter.export.bind(spanExporter); + spanExporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (spans.some((span) => span.name === "activitypub.send_activity")) { + sendEnded(); + } + }; + fetchMock.mockGlobal().post( + "https://receiver.example/inbox", + async () => { + fetched(); + await responseGate; + return { status, body: "receiver response" }; + }, + ); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + }); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await observerGate; + } + }, + }); + let settled = false; + const sending = federation.createContext( + new URL("https://example.com"), + undefined, + ) + .sendActivity(sender, recipient, activity, { immediate: true }) + .finally(() => { + settled = true; + }); + const result = status === 202 + ? sending + : rejects(sending, (error: unknown) => { + return error instanceof Error && "statusCode" in error && + error.statusCode === status; + }); + await Promise.all([prepared, fetching]); + strictEqual(settled, false); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + releaseResponse(); + // Wait for the actual concrete send to finish while the preparation + // observer remains blocked, rather than assuming a timer is long enough. + await concreteSendEnded; + const span = spanExporter.getSpan("activitypub.send_activity"); + ok(span); + if (status === 503) strictEqual(span.status.code, SpanStatusCode.ERROR); + // Only now yield a turn so an unhandled rejection would be detected + // before releasing the observer and joining the send promises. + await new Promise((resolve) => setTimeout(resolve, 0)); + strictEqual(settled, false); + releaseObserver(); + await result; + deepStrictEqual(deliveries.map((event) => event.stage), [ + "planned", + "handoff", + ]); + const handoff = deliveries[1]; + ok(handoff.stage === "handoff"); + deepStrictEqual(handoff.result, { type: "direct" }); + strictEqual(events.length, 2); + } finally { + releaseResponse(); + releaseObserver(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound preparation preserves earlier handoffs when later destination setup fails", async (t) => { + for (const immediate of [true, false]) { + await t.step( + immediate + ? "immediate handoff survives" + : "queued messages are not handed off yet", + async () => { + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const messages: Message[] = []; + const firstInbox = "https://receiver.example/first-inbox"; + const broken = recipient.clone({ + inbox: new URL("https://receiver.example/second-inbox"), + }); + // Fault injection: planning can retain this value, but collection-sync + // header construction cannot interpret it as a URL. + Object.defineProperty(broken, "id", { + value: { href: "invalid actor URL" }, + }); + let firstFetched!: () => void; + const fetching = new Promise((resolve) => { + firstFetched = resolve; + }); + fetchMock.mockGlobal().post(firstInbox, () => { + firstFetched(); + return new Response(null, { status: 202 }); + }); + try { + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + ...(immediate ? {} : { queue: { outbox: queue } }), + }); + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .setKeyPairsDispatcher( + () => [{ + privateKey: rsaPrivateKey2, + publicKey: rsaPublicKey2.publicKey!, + }], + ); + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + () => ({ + items: [recipient.clone({ inbox: new URL(firstInbox) }), broken], + }), + ); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + let originalError: unknown; + await rejects( + federation.createContext(new URL("https://example.com"), undefined) + .sendActivity({ identifier: "alice" }, "followers", activity, { + immediate, + syncCollection: true, + }), + (error: unknown) => { + originalError = error; + return error instanceof TypeError; + }, + ); + deepStrictEqual(preparation(events), { + type: "failed", + failure: { + kind: "local", + phase: "request-preparation", + error: originalError, + }, + planning: "complete", + plannedDestinationCount: 2, + }); + strictEqual(messages.length, 0); + if (immediate) { + // Prove the earlier send still reaches fetch after producer failure. + await fetching; + } + const planned = deliveries.filter((event) => + event.stage === "planned" + ); + strictEqual(planned.length, 2); + const first = deliveries.filter((event) => + event.destinationInbox === firstInbox + ); + deepStrictEqual( + first.map((event) => event.stage), + immediate ? ["planned", "handoff"] : ["planned"], + ); + if (immediate) { + const handoff = first[1]; + ok(handoff.stage === "handoff"); + deepStrictEqual(handoff.result, { type: "direct" }); + } + const failed = deliveries.find((event) => + event.stage === "preparation-failed" + ); + ok(failed?.stage === "preparation-failed"); + strictEqual( + failed.destinationInbox, + "https://receiver.example/second-inbox", + ); + strictEqual(failed.failure.error, originalError); + strictEqual( + deliveries.filter((event) => event.stage === "handoff").length, + immediate ? 1 : 0, + ); + strictEqual(events.length, 2); + } finally { + fetchMock.hardReset(); + } + }, + ); + } +}); + +test("outbox-context observers await starts and preserve context, registration, and failures", async () => { + const federation = createFederation<{ label: string }>(options); + const failure = Symbol("sender resolution failed"); + let resolutions = 0; + federation.setActorDispatcher("/users/{identifier}", () => new Person({})) + .mapHandle(() => { + resolutions++; + throw failure; + }); + const base = federation.createContext(new URL("https://example.com"), { + label: "app", + }); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + let entered!: () => void; + const entry = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + let replacementCalls = 0; + federation.setOutboundObservers({ + async onBroadcast(context, event) { + contexts.push(context); + events.push(event); + if (event.stage === "started") { + entered(); + await gate; + } + throw new Error("recorder failed"); + }, + }); + const sending = rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + await entry; + strictEqual(resolutions, 0); + strictEqual(ctx.hasDeliveredActivity(), false); + federation.setOutboundObservers({ + onBroadcast() { + replacementCalls++; + }, + }); + release(); + await sending; + strictEqual(resolutions, 1); + strictEqual(replacementCalls, 0); + strictEqual(contexts.length, 2); + for (const context of contexts) strictEqual(context, ctx); + strictEqual(ctx.data.label, "app"); + deepStrictEqual(preparation(events), { + type: "failed", + failure: { kind: "local", phase: "sender-resolution", error: failure }, + planning: "not-started", + plannedDestinationCount: 0, + }); + strictEqual(ctx.hasDeliveredActivity(), false); + federation.setOutboundObservers({}); + await rejects( + ctx.sendActivity({ username: "alice" }, [], activity), + (error: unknown) => error === failure, + ); + strictEqual(resolutions, 2); + strictEqual(events.length, 2); + strictEqual(ctx.hasDeliveredActivity(), false); +}); + +test("outbox-context observers preserve empty sends without extra loading", async () => { + let loads = 0; + const federation = createFederation({ + kv: new MemoryKvStore(), + documentLoaderFactory: () => () => { + loads++; + throw new Error("unexpected lookup"); + }, + contextLoaderFactory: () => () => { + loads++; + throw new Error("unexpected lookup"); + }, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + await ctx.sendActivity(sender, [], activity); + deepStrictEqual(preparation(events), { + type: "skipped", + reason: "no-destinations", + }); + strictEqual(ctx.hasDeliveredActivity(), false); + federation.setOutboundObservers({}); + await ctx.sendActivity(sender, [], activity); + strictEqual(ctx.hasDeliveredActivity(), false); + strictEqual(events.length, 2); + strictEqual(loads, 0); +}); + +test("outbox-context observers report queue readiness before acceptance and preserve delivery state", async (t) => { + for (const role of ["outbox", "fanout"] as const) { + for (const rejected of [false, true]) { + await t.step( + `${role}: ${rejected ? "rejected" : "accepted"}`, + async () => { + const failure = Symbol("enqueue failed"); + const events: OutboundBroadcastEvent[] = []; + const messages: Message[] = []; + const checkpoints: { + context: unknown; + delivered: boolean; + enqueued: number; + }[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { [role]: queue }, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const clone = ctx.clone(undefined); + federation.setOutboundObservers({ + onBroadcast(context, event) { + events.push(event); + checkpoints.push({ + context, + delivered: ctx.hasDeliveredActivity(), + enqueued: messages.length, + }); + // Callback failure must not change acceptance or delivery state. + throw new Error("recorder failed"); + }, + }); + const sending = ctx.sendActivity( + sender, + [recipient, recipient], + activity, + { + fanout: role === "fanout" ? "force" : "skip", + }, + ); + if (rejected) { + await rejects(sending, (error: unknown) => error === failure); + } else await sending; + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: role === "fanout" ? "fanout" : "delivery", + destinationCount: 1, + }); + strictEqual(checkpoints.length, 2); + for (const checkpoint of checkpoints) { + strictEqual(checkpoint.context, ctx); + strictEqual(checkpoint.delivered, false); + strictEqual(checkpoint.enqueued, 0); + } + strictEqual(messages.length, 1); + strictEqual(messages[0].type, role); + strictEqual(ctx.hasDeliveredActivity(), !rejected); + strictEqual(clone.hasDeliveredActivity(), !rejected); + // Clearing observers keeps successful sends and the shared state working. + if (!rejected) { + federation.setOutboundObservers({}); + await ctx.sendActivity(sender, recipient, activity, { + fanout: role === "fanout" ? "force" : "skip", + }); + await ctx.sendActivity(sender, [], activity); + strictEqual(messages.length, 2); + strictEqual(events.length, 2); + strictEqual(ctx.hasDeliveredActivity(), true); + strictEqual(clone.hasDeliveredActivity(), true); + } + }, + ); + } + } +}); + +test("outbox-context observers report immediate readiness before transport settles", async (t) => { + for (const status of [202, 503]) { + await t.step(`HTTP ${status}`, async () => { + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + const [tracerProvider, spanExporter] = createTestTracerProvider(); + fetchMock.mockGlobal().post( + "https://receiver.example/inbox", + async () => { + fetched(); + await gate; + return new Response(null, { status }); + }, + ); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + }); + const base = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(base instanceof ContextImpl); + const ctx = base.toOutboxContext( + "alice", + {}, + activity.id!.href, + Create.typeId.href, + ); + const clone = ctx.clone(undefined); + federation.setOutboundObservers({ + onBroadcast(context, event) { + contexts.push(context); + events.push(event); + if (event.stage === "preparation-finished") ready(); + }, + }); + let settled = false; + const sending = ctx.sendActivity(sender, recipient, activity, { + immediate: true, + }) + .finally(() => { + settled = true; + }); + const result = status === 202 + ? sending + : rejects(sending, (error: unknown) => { + return error instanceof Error && "statusCode" in error && + error.statusCode === status; + }); + await Promise.all([fetching, prepared]); + strictEqual(settled, false); + strictEqual(ctx.hasDeliveredActivity(), false); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + release(); + await result; + strictEqual(events.length, 2); + for (const context of contexts) strictEqual(context, ctx); + strictEqual(ctx.hasDeliveredActivity(), status === 202); + strictEqual(clone.hasDeliveredActivity(), status === 202); + const spans = spanExporter.getSpans("activitypub.outbox"); + strictEqual(spans.length, 1); + strictEqual( + spans[0].attributes["activitypub.activity.id"], + activity.id!.href, + ); + if (status === 503) { + strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + } + } finally { + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +// Keep extension fields and signature-shaped data opaque; forwarding observes +// their existing shape checks, not authentication or a vocabulary round trip. +const forwardingPayload = { + "@context": ["https://www.w3.org/ns/activitystreams", { + extension: "https://example.com/extension", + }], + type: "Create", + id: "https://sender.example/create", + actor: "https://sender.example/actor", + extension: { values: ["original", "payload"] }, + signature: { + type: "RsaSignature2017", + creator: "https://sender.example/key", + signatureValue: "opaque signature", + }, +}; + +function forwardingContext( + federation: Federation, + kind: "inbox" | "outbox", + payload: unknown, + data: TContextData, +) { + const base = federation.createContext(new URL("https://example.com"), data); + ok(base instanceof ContextImpl); + return kind === "inbox" + ? base.toInboxContext( + null, + payload, + forwardingPayload.id, + Create.typeId.href, + ) + : base.toOutboxContext( + "alice", + payload, + forwardingPayload.id, + Create.typeId.href, + ); +} + +test("forwarding observers distinguish skips from preparation failures without extra loading", async (t) => { + for (const kind of ["inbox", "outbox"] as const) { + for (const scenario of ["unsigned", "empty", "failed"] as const) { + await t.step(`${kind}: ${scenario}`, async () => { + let loads = 0; + const failure = Symbol("resolution failed"); + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + documentLoaderFactory: () => () => { + loads++; + throw new Error("extra lookup"); + }, + contextLoaderFactory: () => () => { + loads++; + throw new Error("extra lookup"); + }, + }); + federation.setActorDispatcher( + "/users/{identifier}", + () => new Person({}), + ) + .mapHandle(() => { + throw failure; + }); + const { signature: _signature, ...unsigned } = forwardingPayload; + const ctx = forwardingContext( + federation, + kind, + scenario === "unsigned" ? unsigned : forwardingPayload, + { label: "app" }, + ); + const events: OutboundBroadcastEvent[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onBroadcast(context, event) { + contexts.push(context); + events.push(event); + throw new Error("recorder failed"); + }, + }); + const forward = () => + ctx.forwardActivity( + scenario === "failed" ? { username: "alice" } : sender, + scenario === "empty" ? [] : recipient, + { skipIfUnsigned: true }, + ); + if (scenario === "failed") { + await rejects(forward(), (error: unknown) => error === failure); + } else await forward(); + deepStrictEqual( + preparation(events), + scenario === "failed" + ? { + type: "failed", + failure: { + kind: "local", + phase: "sender-resolution", + error: failure, + }, + planning: "not-started", + plannedDestinationCount: 0, + } + : { + type: "skipped", + reason: scenario === "unsigned" ? "unsigned" : "no-destinations", + }, + ); + for (const event of events) strictEqual(event.operation, "forward"); + for (const context of contexts) strictEqual(context, ctx); + strictEqual(ctx.data.label, "app"); + strictEqual(loads, 0); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), false); + } + federation.setOutboundObservers({}); + if (scenario === "failed") { + await rejects(forward(), (error: unknown) => error === failure); + } else await forward(); + strictEqual(events.length, 2); + strictEqual(loads, 0); + }); + } + } +}); + +test("forwarding observers await readiness before enqueue while preserving raw payloads and errors", async (t) => { + for (const batch of [false, true]) { + for (const rejected of [false, true]) { + await t.step( + `${batch ? "batch" : "single"}: ${rejected ? "rejected" : "accepted"}`, + async () => { + const failure = Symbol("enqueue failure"); + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + ...(batch + ? { + enqueueMany(batch: readonly Message[]) { + messages.push(...batch); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + } + : {}), + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + queue: { outbox: queue }, + manuallyStartQueue: true, + activityTransformers: [() => { + throw new Error("forwarding must not transform"); + }], + }); + const ctx = forwardingContext( + federation, + batch ? "outbox" : "inbox", + forwardingPayload, + undefined, + ); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const events: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await gate; + } + throw new Error("recorder failed"); + }, + }); + const sending = ctx.forwardActivity(sender, [recipient, recipient], { + skipIfUnsigned: true, + }); + const result = rejected + ? rejects(sending, (error: unknown) => error === failure) + : sending; + await prepared; + strictEqual(messages.length, 0); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), false); + } + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + release(); + await result; + strictEqual(events.length, 2); + strictEqual(messages.length, 1); + ok(messages[0].type === "outbox"); + strictEqual(messages[0].activity, forwardingPayload); + ok(!("broadcastId" in messages[0])); + deepStrictEqual(events[0].activity, {}); + deepStrictEqual(events[1].activity, { + id: forwardingPayload.id, + type: Create.typeId.href, + }); + ok(Object.isFrozen(events[0].activity)); + ok(Object.isFrozen(events[1].activity)); + for (const event of events) strictEqual(event.operation, "forward"); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), !rejected); + } + federation.setOutboundObservers({}); + const unobserved = ctx.forwardActivity(sender, recipient, { + skipIfUnsigned: true, + }); + if (rejected) { + await rejects(unobserved, (error: unknown) => error === failure); + } else await unobserved; + strictEqual(messages.length, 2); + ok(messages[1].type === "outbox"); + strictEqual(messages[1].activity, forwardingPayload); + strictEqual(events.length, 2); + }, + ); + } + } +}); + +test("forwarding observers retain readiness and registration through late immediate results", async (t) => { + for (const status of [202, 503]) { + await t.step(`HTTP ${status}`, async () => { + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let respond!: () => void; + const responseGate = new Promise((resolve) => { + respond = resolve; + }); + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const observerGate = new Promise((resolve) => { + release = resolve; + }); + const [tracerProvider, exporter] = createTestTracerProvider(); + let ended!: () => void; + const concreteSendEnded = new Promise((resolve) => { + ended = resolve; + }); + const exportSpans = exporter.export.bind(exporter); + exporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (spans.some((span) => span.name === "activitypub.send_activity")) { + ended(); + } + }; + let submitted: unknown; + fetchMock.mockGlobal().post( + "https://receiver.example/inbox", + async (call) => { + submitted = JSON.parse(String(call.options.body)); + fetched(); + await responseGate; + return new Response(null, { status }); + }, + ); + try { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + }); + const ctx = forwardingContext( + federation, + status === 202 ? "inbox" : "outbox", + forwardingPayload, + undefined, + ); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const attempts: OutboundAttemptReport[] = []; + let replacements = 0; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + onDelivery(_ctx, event) { + deliveries.push(event); + }, + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await observerGate; + } + }, + }); + let settled = false; + const sending = ctx.forwardActivity(sender, recipient, { + immediate: true, + skipIfUnsigned: true, + }) + .finally(() => { + settled = true; + }); + const result = status === 202 + ? sending + : rejects(sending, (error: unknown) => { + return error instanceof Error && "statusCode" in error && + error.statusCode === status; + }); + await Promise.all([prepared, fetching]); + deepStrictEqual(submitted, forwardingPayload); + strictEqual(settled, false); + deepStrictEqual(preparation(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + federation.setOutboundObservers({ + onAttempt() { + replacements++; + }, + onBroadcast() { + replacements++; + }, + }); + respond(); + await concreteSendEnded; + await new Promise((resolve) => setTimeout(resolve, 0)); + strictEqual(settled, false); + release(); + await result; + deepStrictEqual(deliveries.map((event) => event.stage), [ + "planned", + "handoff", + ]); + const handoff = deliveries[1]; + ok(handoff.stage === "handoff"); + deepStrictEqual(handoff.result, { type: "direct" }); + strictEqual(attempts.length, 1); + const attempt = attempts[0]; + strictEqual(attempt.deliveryId, handoff.deliveryId); + deepStrictEqual(attempt.broadcast, handoff.broadcast); + strictEqual(attempt.inboxUrl, recipient.inboxId!.href); + strictEqual( + attempt.outcome.type, + status === 202 ? "succeeded" : "failed", + ); + ok(attempt.outcome.response.type === "final"); + strictEqual(attempt.outcome.response.metadata.status, status); + strictEqual(attempt.httpExchangeCount, status === 202 ? 1 : 2); + if (attempt.outcome.type === "failed") { + strictEqual(attempt.outcome.failure.kind, "http"); + strictEqual(attempt.outcome.failure.error, attempt.outcome.error); + } + strictEqual(events.length, 2); + strictEqual(replacements, 0); + for (const event of events) strictEqual(event.operation, "forward"); + if (ctx instanceof OutboxContextImpl) { + strictEqual(ctx.hasDeliveredActivity(), false); + } + const spans = exporter.getSpans( + status === 202 ? "activitypub.inbox" : "activitypub.outbox", + ); + strictEqual(spans.length, 1); + if (status === 503) { + strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + } + } finally { + respond(); + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound delivery observers snapshot registration and deduplicated destination facts", async () => { + const events: OutboundDeliveryEvent[] = []; + const contexts: unknown[] = []; + const messages: Message[] = []; + let replacements = 0; + let mutationApplied: boolean | undefined; + const registration: { + onDelivery: NonNullable["onDelivery"]>; + } = { + onDelivery(context, event) { + contexts.push(context); + events.push(event); + if (event.stage === "planned") { + mutationApplied = Reflect.set(event.actorIds, 0, "changed"); + federation.setOutboundObservers({ + onDelivery() { + replacements++; + }, + }); + } + }, + }; + const builder = createFederationBuilder(); + builder.setOutboundObservers(registration); + // The caller's object and later builder changes cannot replace the snapshot. + registration.onDelivery = () => { + replacements++; + }; + const federation = await builder.build({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + builder.setOutboundObservers({}); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const shared = new URL("https://receiver.example/shared"); + const recipients = ["alice", "bob"].map((name) => ({ + id: new URL(`https://receiver.example/${name}`), + inboxId: new URL(`https://receiver.example/${name}/inbox`), + endpoints: { sharedInbox: shared }, + })); + await ctx.sendActivity(sender, [...recipients, recipient], activity, { + preferSharedInbox: true, + }); + strictEqual(mutationApplied, false); + strictEqual(replacements, 0); + strictEqual(events.length, 4); + for (const context of contexts) strictEqual(context, ctx); + const planned = events.filter((event) => event.stage === "planned"); + strictEqual(planned.length, 2); + const sharedPlan = planned.find((event) => + event.destinationInbox === shared.href + ); + ok(sharedPlan?.stage === "planned"); + deepStrictEqual( + sharedPlan.actorIds, + recipients.map((recipient) => recipient.id.href), + ); + strictEqual(sharedPlan.sharedInbox, true); + strictEqual(sharedPlan.mode, "ordinary"); + notStrictEqual(planned[0].deliveryId, planned[1].deliveryId); + ok(planned[0].broadcast.type === "known"); + ok(planned[1].broadcast.type === "known"); + strictEqual(planned[0].broadcast.id, planned[1].broadcast.id); + for (const event of events) { + ok(Object.isFrozen(event)); + ok(Object.isFrozen(event.broadcast)); + const plan = planned.find((plan) => plan.deliveryId === event.deliveryId); + ok(plan); + strictEqual(event.destinationInbox, plan.destinationInbox); + if (event.stage === "handoff") { + ok(Object.isFrozen(event.result)); + const result = event.result; + ok(result.type === "enqueued"); + strictEqual(result.queue, "outbox"); + const message = messages.find((message) => + message.id === result.queueMessageId + ); + ok(message?.type === "outbox"); + strictEqual(message.inbox, event.destinationInbox); + ok(!message.actorIds?.includes("changed")); + } + } +}); + +test("outbound delivery observers report queue acceptance separately from producer readiness", async (t) => { + for (const mode of ["single", "batch", "fanout", "forward"] as const) { + for (const rejected of [false, true]) { + await t.step( + `${mode}: ${rejected ? "rejected" : "accepted"}`, + async () => { + const failure = Symbol("enqueue failed"); + const messages: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + messages.push(message); + const fails = rejected && + (mode !== "single" || messages.length === 1); + return fails ? Promise.reject(failure) : Promise.resolve(); + }, + ...(mode === "batch" || mode === "forward" + ? { + enqueueMany(batch: readonly Message[]) { + messages.push(...batch); + return rejected ? Promise.reject(failure) : Promise.resolve(); + }, + } + : {}), + listen: () => Promise.resolve(), + }; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: mode === "fanout" ? { fanout: queue } : { outbox: queue }, + }); + const deliveries: OutboundDeliveryEvent[] = []; + const broadcasts: OutboundBroadcastEvent[] = []; + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onDelivery(_ctx, event) { + deliveries.push(event); + throw new Error("recorder failed"); + }, + }); + const targets = [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ]; + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + ok(ctx instanceof ContextImpl); + const sending = mode === "forward" + ? ctx.toInboxContext( + null, + forwardingPayload, + forwardingPayload.id, + Create.typeId.href, + ) + .forwardActivity(sender, targets, { skipIfUnsigned: true }) + : ctx.sendActivity(sender, targets, activity, { + fanout: mode === "fanout" ? "force" : "skip", + }); + if (rejected) { + await rejects(sending, (error: unknown) => error === failure); + } else await sending; + deepStrictEqual(preparation(broadcasts), { + type: "prepared", + preparedFor: mode === "fanout" ? "fanout" : "delivery", + destinationCount: 2, + }); + const planned = deliveries.filter((event) => + event.stage === "planned" + ); + const handoffs = deliveries.filter((event) => + event.stage === "handoff" + ); + strictEqual(planned.length, 2); + strictEqual(handoffs.length, 2); + for (const event of handoffs) { + ok(event.stage === "handoff"); + const fails = rejected && + (mode !== "single" || + event.destinationInbox === recipient.inboxId!.href); + if (fails) { + ok(event.result.type === "enqueue-failed"); + strictEqual(event.result.acceptance, "unknown"); + strictEqual(event.result.error, failure); + } else { + const result = event.result; + ok(result.type === "enqueued"); + strictEqual( + result.queue, + mode === "fanout" ? "fanout" : "outbox", + ); + ok( + messages.some((message) => + message.id === result.queueMessageId + ), + ); + } + ok(event.broadcast.type === "known"); + strictEqual(event.broadcast.id, broadcasts[0].broadcastId); + const message = messages.find((message) => + message.type === "fanout" + ? event.destinationInbox in message.inboxes + : message.type === "outbox" && + message.inbox === event.destinationInbox + ); + ok(message); + const identity = message.type === "fanout" + ? message.inboxes[event.destinationInbox].outbound + : message.type === "outbox" + ? message.outbound + : undefined; + deepStrictEqual(identity, { + broadcast: event.broadcast, + deliveryId: event.deliveryId, + destinationInbox: event.destinationInbox, + }); + deepStrictEqual(JSON.parse(JSON.stringify(identity)), identity); + deepStrictEqual(structuredClone(identity), identity); + strictEqual( + event.deliveryId, + planned.find((plan) => + plan.destinationInbox === event.destinationInbox + )!.deliveryId, + ); + } + }, + ); + } + } +}); + +test("outbound delivery observers retain accepted handoffs after bookkeeping failure", async () => { + const failure = Symbol("metrics failed after acceptance"); + const events: OutboundDeliveryEvent[] = []; + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { + outbox: { + enqueue() { + Object.defineProperty(federation, "meterProvider", { + get() { + throw failure; + }, + }); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); + await rejects( + federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, recipient, activity), + (error: unknown) => error === failure, + ); + deepStrictEqual(events.map((event) => event.stage), ["planned", "handoff"]); + const event = events[1]; + ok(event.stage === "handoff"); + ok(event.result.type === "enqueued"); +}); + +test("outbound queue identities survive fanout replay and legacy messages", async (t) => { + for (const legacy of [false, true]) { + await t.step( + legacy ? "legacy fanout" : "serialized producer identities", + async () => { + const fanouts: Message[] = []; + const outboxes: Message[] = []; + const recordingQueue = (messages: Message[]): MessageQueue => ({ + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }); + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + queue: { + fanout: recordingQueue(fanouts), + outbox: recordingQueue(outboxes), + }, + }); + const broadcasts: OutboundBroadcastEvent[] = []; + // Identity propagation also works when only the broadcast callback exists. + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const targets = [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ]; + await ctx.sendActivity(sender, targets, activity, { fanout: "force" }); + const message = fanouts[0]; + ok(message?.type === "fanout"); + const original = structuredClone(message); + if (legacy) { + for ( + const entry of Object.values(message.inboxes) + ) delete (entry as { outbound?: unknown }).outbound; + } + const events: OutboundDeliveryEvent[] = []; + let newBroadcasts = 0; + federation.setOutboundObservers({ + onBroadcast() { + newBroadcasts++; + }, + onDelivery(_ctx, event) { + events.push(event); + }, + }); + await federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ); + // Preserve metadata even when the consumer's callbacks are disabled. + if (!legacy) federation.setOutboundObservers({}); + await federation.processQueuedTask(undefined, structuredClone(message)); + strictEqual(newBroadcasts, 0); + strictEqual(outboxes.length, 4); + strictEqual(new Set(outboxes.map((message) => message.id)).size, 4); + const identities = new Set(); + for (const inbox of Object.keys(message.inboxes)) { + const children = outboxes.filter((message) => + message.type === "outbox" && message.inbox === inbox + ); + strictEqual(children.length, 2); + ok(children[0].type === "outbox" && children[1].type === "outbox"); + const identity = children[0].outbound; + ok(identity); + identities.add(identity.deliveryId); + deepStrictEqual(children[1].outbound, identity); + strictEqual(identity.destinationInbox, inbox); + deepStrictEqual( + identity.broadcast, + legacy + ? { type: "unknown" } + : { type: "known", id: broadcasts[0].broadcastId }, + ); + if (!legacy) { + deepStrictEqual(identity, original.inboxes[inbox].outbound); + } + for ( + const event of events.filter((event) => + event.destinationInbox === inbox + ) + ) { + strictEqual(event.stage, "handoff"); + strictEqual(event.deliveryId, identity.deliveryId); + deepStrictEqual(event.broadcast, identity.broadcast); + ok(event.stage === "handoff" && event.result.type === "enqueued"); + const queueMessageId = event.result.queueMessageId; + ok(children.some((child) => child.id === queueMessageId)); + } + } + strictEqual(identities.size, 2); + strictEqual(events.length, legacy ? 4 : 2); + if (!legacy) { + for (const mismatch of [false, true]) { + const invalid = structuredClone(original); + const first = Object.values(invalid.inboxes)[0]; + ok(first.outbound); + Object.assign( + first.outbound, + mismatch + ? { destinationInbox: "https://other.example/inbox" } + : { deliveryId: "" }, + ); + await federation.processQueuedTask(undefined, invalid); + const fallback = outboxes.at(-2); + ok(fallback?.type === "outbox" && fallback.outbound); + deepStrictEqual(fallback.outbound.broadcast, { type: "unknown" }); + strictEqual(fallback.outbound.destinationInbox, fallback.inbox); + strictEqual( + fallback.outbound.deliveryId.startsWith("legacy-fanout:"), + true, + ); + const retained = outboxes.at(-1); + ok(retained?.type === "outbox"); + deepStrictEqual( + retained.outbound, + Object.values(original.inboxes)[1].outbound, + ); + } + strictEqual(outboxes.length, 8); + } + }, + ); + } +}); + +test("outbound observers preserve processing of legacy outbox messages", async () => { + const messages: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const ctx = federation.createContext( + new URL("https://example.com"), + undefined, + ); + await ctx.sendActivity(sender, recipient, activity, { fanout: "skip" }); + const message = messages[0]; + ok(message?.type === "outbox"); + strictEqual(message.outbound, undefined); + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + onDelivery() { + throw new Error("unused worker observer"); + }, + }); + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202); + try { + await federation.processQueuedTask( + undefined, + JSON.parse(JSON.stringify(message)), + ); + strictEqual(fetchMock.callHistory.calls().length, 1); + strictEqual(messages.length, 1); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].deliveryId, message.id); + deepStrictEqual(attempts[0].broadcast, { type: "unknown" }); + strictEqual("destinationInbox" in attempts[0], false); + deepStrictEqual(attempts[0].provenance, {}); + } finally { + fetchMock.hardReset(); + } +}); + +test("invalid outbox grouping does not block delivery or observers", async () => { + const task = continuationMessage(recipient.inboxId!.href, true); + task.outbound && Object.assign(task.outbound, { deliveryId: "" }); + const attempts: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const workers: OutboundWorkerReport[] = []; + const bodies: OutboundResponseBodyEvent[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onWorker(_ctx, report) { + workers.push(report); + }, + responseBodyCapture: { + maxBytes: 128, + timeoutMs: 1000, + onBody(_ctx, event) { + bodies.push(event); + }, + }, + }); + fetchMock.mockGlobal().post(task.inbox, { status: 202, body: "accepted" }); + try { + await federation.processQueuedTask(undefined, task); + strictEqual(fetchMock.callHistory.calls().length, 1); + strictEqual(attempts.length, 1); + strictEqual(exchanges.length, 1); + strictEqual(workers.length, 1); + strictEqual(bodies.length, 1); + for (const report of [...attempts, ...exchanges, ...bodies]) { + strictEqual(report.deliveryId, task.id); + deepStrictEqual(report.broadcast, { type: "unknown" }); + strictEqual("destinationInbox" in report, false); + } + strictEqual(workers[0].delivery?.deliveryId, task.id); + deepStrictEqual(workers[0].delivery?.broadcast, { type: "unknown" }); + } finally { + fetchMock.hardReset(); + } +}); + +function continuationMessage( + inbox: string, + observed: boolean, + portableInbox?: string, +): OutboxMessage { + return { + type: "outbox", + id: crypto.randomUUID(), + baseUrl: "https://example.com", + keys: [], + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + id: activity.id!.href, + type: "Create", + actor: activity.actorId!.href, + }, + activityId: activity.id!.href, + activityType: Create.typeId.href, + inbox, + sharedInbox: false, + started: new Date().toISOString(), + attempt: 0, + headers: {}, + orderingKey: "original-order", + traceContext: {}, + ...(portableInbox == null ? {} : { portableInbox }), + ...(observed + ? { + provenance: { + dispatchId: "application-dispatch", + inboxRequest: { + inboxRequestId: crypto.randomUUID(), + requestId: "original-request-label", + context: { + origin: "https://source.example:8443", + host: "source.example:8443", + hostname: "source.example", + canonicalOrigin: "https://canonical.example", + }, + }, + }, + outbound: { + broadcast: { type: "known" as const, id: crypto.randomUUID() }, + deliveryId: crypto.randomUUID(), + destinationInbox: inbox, + ...(portableInbox == null ? {} : { portableInbox }), + }, + } + : {}), + }; +} + +test("gateway retry preparation diagnostics include only the failed route", async () => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + ]; + const queued: Message[] = []; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + gatewayNotBefore: { [gateways[0]]: "2999-01-01T00:00:00Z" }, + headers: { "bad\nname": "value" }, + }; + try { + await federation.processQueuedTask(undefined, message); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 0); + strictEqual(queued.length, 1); + const action = reports[0].actions?.[0]; + ok(action?.type === "scheduled"); + strictEqual(action.action, "retry"); + deepStrictEqual(action.inboxUrls, gateways); + deepStrictEqual(action.preparationFailures?.map((item) => item.inboxUrl), [ + gateways[1], + ]); + strictEqual( + action.preparationFailures?.[0].failure.phase, + "request-preparation", + ); + } finally { + fetchMock.hardReset(); + } +}); + +test("gateway worker actions retain route abandonment without claiming an unqueued deferral", async () => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + "https://gw3.example/inbox", + ]; + const queued: Message[] = []; + const workers: OutboundWorkerReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: false, + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + workers.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + }; + fetchMock.mockGlobal().post(gateways[0], 404) + .post(gateways[1], { status: 429, headers: { "Retry-After": "3600" } }) + .post(gateways[2], 202); + try { + await federation.processQueuedTask(undefined, message); + strictEqual(workers.length, 1); + strictEqual(workers[0].attemptsStarted, 3); + deepStrictEqual(workers[0].result, { type: "completed" }); + deepStrictEqual(workers[0].actions, [{ + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [gateways[0]], + }]); + strictEqual(queued.length, 0); + deepStrictEqual(attempts.map((report) => report.outcome.type), [ + "failed", + "failed", + "succeeded", + ]); + for (const report of attempts) { + strictEqual(report.deliveryId, message.outbound!.deliveryId); + } + } finally { + fetchMock.hardReset(); + } +}); + +test("gateway worker actions preserve abandonment and available drop errors", async (t) => { + const gateways = ["https://gw1.example/inbox", "https://gw2.example/inbox"]; + for ( + const mode of [ + "permanent", + "permanent-handler", + "declined", + "queue-unavailable", + "ttl", + "ttl-parse", + ] as const + ) { + await t.step(mode, async () => { + const ttl = mode.startsWith("ttl"); + const failure = Symbol(mode); + const reports: OutboundWorkerReport[] = []; + const queued: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: ttl ? { heldActivityTtl: { seconds: 1 } } : false, + outboxRetryPolicy: () => + mode === "declined" ? null : Temporal.Duration.from({ seconds: 1 }), + ...(mode === "queue-unavailable" ? {} : { + queue: { + outbox: { + enqueue(message: Message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }), + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + if (mode === "permanent-handler") { + federation.setOutboxPermanentFailureHandler(() => { + throw failure; + }); + } + if (mode === "ttl-parse") { + Object.defineProperty(federation, "documentLoaderFactory", { + value: () => { + throw failure; + }, + }); + } + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + ...(ttl + ? { + circuitHeld: true as const, + circuitHeldSince: "2000-01-01T00:00:00Z", + } + : {}), + }; + fetchMock.mockGlobal(); + for (const inbox of gateways) { + fetchMock.post(inbox, mode.startsWith("permanent") ? 404 : 503); + } + try { + const processing = federation.processQueuedTask(undefined, message); + if (mode === "ttl-parse") { + await rejects(processing, (error: unknown) => error === failure); + } else await processing; + strictEqual(reports.length, 1); + strictEqual(queued.length, 0); + const report = reports[0]; + strictEqual(report.attemptsStarted, ttl ? 0 : 2); + deepStrictEqual( + report.result, + mode === "ttl-parse" + ? { type: "failed", error: failure } + : { type: "completed" }, + ); + const actions = report.actions; + ok(actions); + const abandonments = actions.filter((action) => + action.type === "abandoned" + ); + if (mode.startsWith("permanent")) { + deepStrictEqual( + abandonments, + gateways.map((inbox) => ({ + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [inbox], + })), + ); + } else {deepStrictEqual(abandonments, [{ + type: "abandoned", + reason: ttl + ? "held-ttl-expired" + : mode === "declined" + ? "retry-policy-declined" + : "queue-unavailable", + inboxUrls: gateways, + }]);} + const failed = actions.filter((action) => + action.type === "action-failed" + ); + if (mode === "permanent-handler" || mode === "ttl-parse") { + strictEqual(failed.length, 1); + strictEqual(failed[0].action, "drop"); + strictEqual(failed[0].error, failure); + deepStrictEqual(failed[0].inboxUrls, gateways); + strictEqual("acceptance" in failed[0], false); + } else strictEqual(failed.length, 0); + if (ttl) strictEqual(fetchMock.callHistory.calls().length, 0); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("gateway continuation failures preserve partial actions and settled attempts", async (t) => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + "https://gw3.example/inbox", + ]; + for ( + const mode of ["retry-reject", "defer-reject", "retry-bookkeeping"] as const + ) { + await t.step(mode, async () => { + const [meterProvider] = createTestMeterProvider(); + const failure = Symbol(mode); + const queued: Message[] = []; + const workers: OutboundWorkerReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + meterProvider, + circuitBreaker: mode === "defer-reject" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return mode === "retry-bookkeeping" + ? Promise.resolve() + : Promise.reject(failure); + }, + listen: () => Promise.resolve(), + }, + }, + }); + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskEnqueued; + if (mode === "retry-bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + } + federation.setOutboundObservers({ + onWorker(_ctx, report) { + workers.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const message = { + ...continuationMessage(gateways[0], true), + gatewayInboxes: gateways, + }; + fetchMock.mockGlobal().post(gateways[0], 404).post(gateways[1], 503).post( + gateways[2], + 503, + ); + try { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + strictEqual(queued.length, 1); + strictEqual(workers.length, 1); + strictEqual(workers[0].attemptsStarted, 3); + deepStrictEqual(workers[0].result, { type: "failed", error: failure }); + const actions = workers[0].actions; + ok(actions); + strictEqual(actions.length, 2); + deepStrictEqual(actions[0], { + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [gateways[0]], + }); + const action = actions[1]; + ok( + action.type === "scheduled" || + action.type === "action-failed" && action.action !== "drop", + ); + strictEqual(action.action, mode === "defer-reject" ? "defer" : "retry"); + deepStrictEqual(action.inboxUrls, gateways.slice(1)); + const task = queued[0]; + ok(task.type === "outbox"); + deepStrictEqual(action.queue, { + queueMessageId: task.id, + queueAttempt: task.attempt, + }); + if (mode === "retry-bookkeeping") strictEqual(action.type, "scheduled"); + else { + ok(action.type === "action-failed"); + strictEqual(action.acceptance, "unknown"); + strictEqual(action.error, failure); + } + strictEqual(attempts.length, 3); + for (const report of attempts) { + ok(report.outcome.type === "failed"); + strictEqual(report.outcome.failure.kind, "http"); + } + } finally { + if (mode === "retry-bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + value: original, + }); + } + fetchMock.hardReset(); + } + }); + } +}); + +test("ordinary outbox worker actions record actual retry and hold enqueues", async (t) => { + for (const mode of ["retry", "retry-after", "hold"] as const) { + await t.step(mode, async () => { + const queued: { + message: Message; + options: Parameters[1]; + }[] = []; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => Temporal.Duration.from({ milliseconds: 250 }), + circuitBreaker: mode === "hold" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + queue: { + outbox: { + enqueue(message, options) { + queued.push({ message, options }); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + Object.assign(message, { + outbound: { + ...message.outbound!, + destinationInbox: "https://original.example/inbox", + }, + }); + fetchMock.mockGlobal().post( + message.inbox, + mode === "retry-after" + ? { + status: 429, + headers: { "Retry-After": "1" }, + } + : 503, + ); + try { + await federation.processQueuedTask(undefined, message); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 1); + deepStrictEqual(reports[0].result, { type: "completed" }); + strictEqual(queued.length, 1); + const task = queued[0].message; + ok(task.type === "outbox"); + const action = reports[0].actions?.[0]; + ok(action?.type === "scheduled"); + strictEqual(action.action, mode === "hold" ? "defer" : "retry"); + deepStrictEqual(action.queue, { + queueMessageId: task.id, + queueAttempt: mode === "hold" ? 0 : 1, + }); + strictEqual(reports[0].queue.queueAttempt, 0); + ok(action.action === "retry" || action.action === "defer"); + strictEqual(action.delay, queued[0].options?.delay?.toString()); + deepStrictEqual(action.inboxUrls, [message.inbox]); + strictEqual( + reports[0].delivery?.destinationInbox, + "https://original.example/inbox", + ); + strictEqual(reports[0].actions?.length, 1); + ok(Object.isFrozen(reports[0].actions)); + ok(Object.isFrozen(action)); + ok(Object.isFrozen(action.queue)); + ok(Object.isFrozen(action.inboxUrls)); + if (mode === "retry-after") { + strictEqual( + Temporal.Duration.from(action.delay).total("milliseconds"), + 1_000, + ); + } + if (mode === "hold") { + const requests = fetchMock.callHistory.calls().length; + await federation.processQueuedTask(undefined, structuredClone(task)); + strictEqual(reports.length, 2); + strictEqual(reports[1].attemptsStarted, 0); + strictEqual(fetchMock.callHistory.calls().length, requests); + const held = reports[1].actions?.[0]; + ok(held?.type === "scheduled"); + strictEqual(held.action, "defer"); + strictEqual(held.queue.queueAttempt, 0); + } + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("handled header preparation failures stay attached to actual retry actions", async (t) => { + for ( + const mode of [ + "accepted", + "enqueue-failed", + "declined", + "queue-unavailable", + ] as const + ) { + await t.step(mode, async () => { + const reports: OutboundWorkerReport[] = []; + const enqueueError = new Error("queue unavailable"); + const preparationErrors: unknown[] = []; + let enqueueCalls = 0; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => + mode === "declined" + ? null + : Temporal.Duration.from({ milliseconds: 250 }), + onOutboxError(error) { + preparationErrors.push(error); + }, + ...(mode === "queue-unavailable" ? {} : { + queue: { + outbox: { + enqueue() { + enqueueCalls++; + return mode === "enqueue-failed" + ? Promise.reject(enqueueError) + : Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }), + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = { + ...continuationMessage(recipient.inboxId!.href, true), + headers: { "bad\nname": "value" }, + }; + try { + if (mode === "enqueue-failed") { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === enqueueError, + ); + } else await federation.processQueuedTask(undefined, message); + const report = reports[0]; + if (mode === "enqueue-failed") { + ok(report.result.type === "failed"); + strictEqual(report.result.error, enqueueError); + } else deepStrictEqual(report.result, { type: "completed" }); + strictEqual(report.attemptsStarted, 0); + strictEqual( + enqueueCalls, + mode === "declined" || mode === "queue-unavailable" ? 0 : 1, + ); + strictEqual(report.actions?.length, 1); + const action = report.actions?.[0]; + if (mode === "declined" || mode === "queue-unavailable") { + ok(action?.type === "abandoned"); + strictEqual( + action.reason, + mode === "declined" ? "retry-policy-declined" : "queue-unavailable", + ); + } else { + ok( + action?.type === + (mode === "accepted" ? "scheduled" : "action-failed"), + ); + strictEqual(action.action, "retry"); + } + deepStrictEqual(action.inboxUrls, [message.inbox]); + const [diagnostic] = action.preparationFailures!; + const [workerDiagnostic] = report.preparationFailures!; + strictEqual(workerDiagnostic, diagnostic); + strictEqual(diagnostic.inboxUrl, message.inbox); + strictEqual(diagnostic.failure.kind, "local"); + strictEqual(diagnostic.failure.phase, "request-preparation"); + strictEqual(diagnostic.failure.error, preparationErrors[0]); + if (mode === "enqueue-failed") { + ok(action.type === "action-failed"); + strictEqual(action.error, enqueueError); + strictEqual(diagnostic.failure.error === enqueueError, false); + } else { + strictEqual(Object.isFrozen(action.preparationFailures), true); + } + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("ordinary outbox worker actions preserve enqueue uncertainty and accepted bookkeeping failures", async (t) => { + for (const mode of ["rejected", "accepted"] as const) { + await t.step(mode, async () => { + const [meterProvider] = createTestMeterProvider(); + const failure = Symbol(mode); + const queued: Message[] = []; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + meterProvider, + circuitBreaker: false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return mode === "rejected" + ? Promise.reject(failure) + : Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskEnqueued; + if (mode === "accepted") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + } + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + fetchMock.mockGlobal().post(message.inbox, 503); + try { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + strictEqual(queued.length, 1); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 1); + deepStrictEqual(reports[0].result, { type: "failed", error: failure }); + const action = reports[0].actions?.[0]; + ok(action); + strictEqual(reports[0].actions?.length, 1); + strictEqual( + action.type, + mode === "rejected" ? "action-failed" : "scheduled", + ); + ok( + action.type === "scheduled" || + action.type === "action-failed" && action.action === "retry", + ); + strictEqual(action.action, "retry"); + deepStrictEqual(action.queue, { + queueMessageId: queued[0].id, + queueAttempt: 1, + }); + if (action.type === "action-failed") { + ok("acceptance" in action); + strictEqual(action.acceptance, "unknown"); + strictEqual(action.error, failure); + } else strictEqual("acceptance" in action, false); + } finally { + if (mode === "accepted") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + value: original, + }); + } + fetchMock.hardReset(); + } + }); + } +}); + +test("ordinary outbox worker actions retain local abandonment and failed drop steps", async (t) => { + for ( + const mode of [ + "permanent", + "declined", + "queue-unavailable", + "ttl", + "ttl-parse", + "ttl-handler", + "permanent-handler", + ] as const + ) { + await t.step(mode, async () => { + const reports: OutboundWorkerReport[] = []; + const queued: Message[] = []; + const ttl = mode.startsWith("ttl"); + const failure = Symbol(mode); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + outboxRetryPolicy: () => + mode === "queue-unavailable" + ? Temporal.Duration.from({ seconds: 1 }) + : null, + circuitBreaker: ttl ? { heldActivityTtl: { seconds: 1 } } : false, + ...(mode === "queue-unavailable" ? {} : { + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }), + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + if (mode.endsWith("handler")) { + federation.setOutboxPermanentFailureHandler(() => { + throw failure; + }); + } + if (mode === "ttl-parse") { + Object.defineProperty(federation, "documentLoaderFactory", { + value: () => { + throw failure; + }, + }); + } + const original = continuationMessage(recipient.inboxId!.href, true); + const message = ttl + ? { + ...original, + circuitHeld: true as const, + circuitHeldSince: "2000-01-01T00:00:00Z", + } + : original; + fetchMock.mockGlobal().post( + message.inbox, + mode.startsWith("permanent") ? 404 : 503, + ); + try { + const processing = federation.processQueuedTask(undefined, message); + if (mode === "ttl-parse") { + await rejects(processing, (error: unknown) => error === failure); + } else await processing; + strictEqual(reports.length, 1); + strictEqual(queued.length, 0); + const report = reports[0]; + strictEqual(report.attemptsStarted, ttl ? 0 : 1); + deepStrictEqual( + report.result, + mode === "ttl-parse" + ? { type: "failed", error: failure } + : { type: "completed" }, + ); + const actions = report.actions; + ok(actions); + const abandoned = actions[0]; + ok(abandoned.type === "abandoned"); + strictEqual( + abandoned.reason, + ttl + ? "held-ttl-expired" + : mode === "declined" + ? "retry-policy-declined" + : mode === "queue-unavailable" + ? "queue-unavailable" + : "permanent-failure", + ); + deepStrictEqual(abandoned.inboxUrls, [message.inbox]); + if (mode.endsWith("handler") || mode === "ttl-parse") { + strictEqual(actions.length, 2); + const failed = actions[1]; + ok(failed.type === "action-failed" && failed.action === "drop"); + strictEqual(failed.error, failure); + strictEqual("acceptance" in failed, false); + } else strictEqual(actions.length, 1); + if (ttl) strictEqual(fetchMock.callHistory.calls().length, 0); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("outbound worker-only observers count real sends and report local settlement", async (t) => { + for ( + const mode of [ + "success", + "handled", + "native", + "gateway", + "recovered-preparation", + "recovered-preparation-success", + ] as const + ) { + await t.step(mode, async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + const builder = createFederationBuilder<{ label: string }>(); + const reports: OutboundWorkerReport[] = []; + const contexts: unknown[] = []; + const queued: Message[] = []; + let replacements = 0; + builder.setOutboundObservers({ + async onWorker(ctx, report) { + reports.push(report); + contexts.push(ctx.data); + await Promise.resolve(); + throw new Error("worker observer error must not replace settlement"); + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + tracerProvider, + queue: { + outbox: { + nativeRetrial: mode === "native" || + mode === "recovered-preparation", + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + builder.setOutboundObservers({ + onWorker() { + replacements++; + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + if ( + mode === "gateway" || mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { + Object.assign(message, { + gatewayInboxes: [message.inbox, "https://gateway.example/inbox"], + }); + } + let preparationError: Error | undefined; + if ( + mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { + let reads = 0; + preparationError = new Error("first gateway header preparation"); + Object.assign(message, { + headers: new Proxy({}, { + get(target, property, receiver) { + if (property === Symbol.iterator && ++reads === 1) { + throw preparationError; + } + return Reflect.get(target, property, receiver); + }, + }), + }); + } + const data = { label: "worker data" }; + fetchMock.mockGlobal().post( + message.inbox, + mode === "success" ? 202 : 503, + ); + if ( + mode === "gateway" || mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { + fetchMock.post( + "https://gateway.example/inbox", + mode === "gateway" || mode === "recovered-preparation-success" + ? 202 + : 503, + ); + } + let outward: unknown; + try { + const processing = federation.processQueuedTask(data, message); + if (mode === "native" || mode === "recovered-preparation") { + await rejects(processing, (error: unknown) => { + outward = error; + return error instanceof SendActivityError; + }); + } else await processing; + strictEqual(replacements, 0); + strictEqual(reports.length, 1); + deepStrictEqual(contexts, [data]); + const report = reports[0]; + strictEqual(report.role, "outbox"); + if (mode === "gateway" || mode === "recovered-preparation") { + deepStrictEqual(report.actions, []); + } else if (mode === "handled") { + ok(report.actions?.[0].type === "scheduled"); + } else deepStrictEqual(report.actions, []); + strictEqual( + report.attemptsStarted, + mode === "gateway" ? 2 : 1, + ); + if ( + mode === "recovered-preparation" || + mode === "recovered-preparation-success" + ) { + deepStrictEqual( + report.preparationFailures?.map((item) => item.inboxUrl), + [ + message.inbox, + ], + ); + strictEqual(report.preparationFailures?.length, 1); + strictEqual(Object.isFrozen(report.preparationFailures), true); + const [diagnostic] = report.preparationFailures!; + strictEqual(Object.isFrozen(diagnostic), true); + strictEqual(diagnostic.failure.error, preparationError); + strictEqual( + report.result.type, + mode === "recovered-preparation-success" ? "completed" : "failed", + ); + } else strictEqual("preparationFailures" in report, false); + deepStrictEqual(report.queue, { + queueMessageId: message.id, + queueAttempt: 0, + }); + deepStrictEqual(report.delivery, message.outbound); + deepStrictEqual(report.provenance, message.provenance); + strictEqual(report.local.origin, new URL(message.baseUrl).origin); + if (mode === "native" || mode === "recovered-preparation") { + deepStrictEqual(report.result, { type: "failed", error: outward }); + } else deepStrictEqual(report.result, { type: "completed" }); + strictEqual(queued.length, mode === "handled" ? 1 : 0); + const workerSpan = exporter.getSpan("activitypub.outbox"); + ok(workerSpan); + deepStrictEqual(report.trace, { + traceId: workerSpan.spanContext().traceId, + spanId: workerSpan.spanContext().spanId, + }); + ok(Object.isFrozen(report)); + ok(Object.isFrozen(report.queue)); + ok(Object.isFrozen(report.result)); + } finally { + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound worker observers report explicit key and fanout parse failures with zero sends", async (t) => { + for (const mode of ["key-import", "activity-parsing", "non-error"] as const) { + const phase = mode === "non-error" ? "activity-parsing" : mode; + await t.step(mode, async () => { + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + const reports: OutboundWorkerReport[] = []; + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const original = continuationMessage(recipient.inboxId!.href, true); + let loaderCalls = 0; + Object.defineProperties(federation, { + documentLoaderFactory: { + value: () => { + loaderCalls++; + return mockDocumentLoader; + }, + }, + contextLoaderFactory: { + value: () => { + loaderCalls++; + if (mode === "non-error") throw NaN; + return mockDocumentLoader; + }, + }, + }); + const message: Message = phase === "key-import" + ? { + ...original, + keys: [{ keyId: sender.keyId.href, privateKey: { kty: "invalid" } }], + } + : { + type: "fanout", + id: original.id, + baseUrl: original.baseUrl, + keys: [], + inboxes: {}, + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + }, + activityType: Create.typeId.href, + traceContext: {}, + provenance: original.provenance, + }; + let outward: unknown; + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => { + outward = error; + return true; + }, + ); + strictEqual(reports.length, 1); + const report = reports[0]; + strictEqual(report.attemptsStarted, 0); + deepStrictEqual(report.result, { + type: "preparation-failed", + failure: { kind: "local", phase, error: outward }, + error: outward, + }); + deepStrictEqual(report.provenance, original.provenance); + strictEqual( + loaderCalls, + mode === "non-error" ? 1 : phase === "key-import" ? 0 : 2, + ); + if (phase === "activity-parsing") { + strictEqual(report.role, "fanout"); + strictEqual(report.delivery, undefined); + } + }); + } +}); + +test("outbound worker classifies inbox URL parsing failures as validation", async (t) => { + for (const mode of ["ordinary", "gateway", "held-gateway"] as const) { + await t.step(mode, async () => { + const reports: OutboundWorkerReport[] = []; + const queued: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: mode === "held-gateway" + ? { heldActivityTtl: { seconds: 1 } } + : false, + outboxRetryPolicy: () => Temporal.Duration.from({ seconds: 1 }), + queue: { + outbox: { + nativeRetrial: true, + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const original = continuationMessage(recipient.inboxId!.href, true); + const malformed = "not a URL"; + const message: OutboxMessage = mode === "ordinary" + ? { ...original, inbox: malformed } + : { + ...original, + gatewayInboxes: [malformed, "https://gateway.example/inbox"], + ...(mode === "held-gateway" + ? { circuitHeld: true, circuitHeldSince: "2000-01-01T00:00:00Z" } + : {}), + }; + let outward: unknown; + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => { + outward = error; + return error instanceof TypeError; + }, + ); + strictEqual(reports.length, 1); + strictEqual(reports[0].attemptsStarted, 0); + deepStrictEqual(reports[0].result, { + type: "preparation-failed", + failure: { kind: "local", phase: "validation", error: outward }, + error: outward, + }); + strictEqual(queued.length, 0); + }); + } +}); + +test("fanout worker classifies sendActivity validation, signing, serialization, and normalization failures as preparation", async (t) => { + for ( + const phase of [ + "validation", + "signing", + "serialization", + "normalization", + "key-export", + ] as const + ) { + await t.step(phase, async () => { + const failure = Symbol(phase); + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + ...options, + ...(phase === "key-export" + ? { + queue: { + outbox: { + enqueue: () => Promise.resolve(), + listen: () => Promise.resolve(), + }, + }, + } + : {}), + }) as FederationImpl; + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + const originalSendActivity = federation.sendActivity; + Object.defineProperty(federation, "sendActivity", { + value: async (...args: Parameters) => { + const [keys, inboxes, outgoing, sendOptions] = args; + if (phase === "validation") { + Object.defineProperty(outgoing, "actorId", { + get() { + throw failure; + }, + }); + } else if (phase === "signing") { + Object.defineProperty(outgoing, "getProofs", { + value() { + throw failure; + }, + }); + } else if (phase === "serialization") { + Object.defineProperty(outgoing, "toJsonLd", { + value() { + throw failure; + }, + }); + } else if (phase === "normalization") { + Object.defineProperty(outgoing, "toJsonLd", { + value: () => + new Proxy({}, { + ownKeys() { + throw failure; + }, + }), + }); + } + const options = phase === "serialization" || phase === "normalization" + ? { ...sendOptions, activityJsonLd: undefined } + : sendOptions; + return await originalSendActivity.call( + federation, + keys, + inboxes, + outgoing, + options, + ); + }, + }); + const activityJsonLd = { + "@context": "https://www.w3.org/ns/activitystreams", + id: activity.id!.href, + type: "Create", + actor: activity.actorId!.href, + }; + const message: FanoutMessage = { + type: "fanout", + id: crypto.randomUUID(), + baseUrl: "https://example.com", + keys: [{ + keyId: sender.keyId.href, + privateKey: await exportJwk(sender.privateKey), + }], + inboxes: { + [recipient.inboxId!.href]: { + actorIds: [], + sharedInbox: false, + }, + }, + activity: activityJsonLd, + activityId: activity.id!.href, + activityType: Create.typeId.href, + traceContext: {}, + }; + let outward: unknown; + const exportKeyDescriptor = Object.getOwnPropertyDescriptor( + crypto.subtle, + "exportKey", + ); + try { + if (phase === "key-export") { + Object.defineProperty(crypto.subtle, "exportKey", { + configurable: true, + value: () => Promise.reject(failure), + }); + } + await rejects( + federation.processQueuedTask(undefined, message), + (error) => { + outward = error; + return error === failure; + }, + ); + } finally { + if (phase === "key-export") { + if (exportKeyDescriptor == null) { + Reflect.deleteProperty(crypto.subtle, "exportKey"); + } else { + Object.defineProperty( + crypto.subtle, + "exportKey", + exportKeyDescriptor, + ); + } + } + } + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].result, { + type: "preparation-failed", + failure: { + kind: "local", + phase: phase === "normalization" ? "serialization" : phase, + error: outward, + }, + error: outward, + }); + strictEqual(reports[0].attemptsStarted, 0); + }); + } +}); + +test("fanout worker preserves a trace propagator failure as request preparation", async () => { + const failure = Symbol("trace injection"); + const reports: OutboundWorkerReport[] = []; + const fanouts: Message[] = []; + const outboxMessages: Message[] = []; + const federation = createFederation({ + ...options, + manuallyStartQueue: true, + queue: { + fanout: { + enqueue(message) { + fanouts.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + outbox: { + enqueue(message) { + outboxMessages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + onWorker(_ctx, report) { + reports.push(report); + }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity(sender, [recipient], activity, { fanout: "force" }); + const message = fanouts[0]; + ok(message?.type === "fanout"); + try { + strictEqual( + propagation.setGlobalPropagator({ + inject() { + throw failure; + }, + extract(ctx) { + return ctx; + }, + fields() { + return []; + }, + }), + true, + ); + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + } finally { + propagation.disable(); + } + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].result, { + type: "preparation-failed", + failure: { kind: "local", phase: "request-preparation", error: failure }, + error: failure, + }); + strictEqual(reports[0].attemptsStarted, 0); + strictEqual(outboxMessages.length, 0); +}); + +test("outbound worker reports distinguish fanout child tasks from concrete sends", async (t) => { + for (const queued of [false, true]) { + await t.step(queued ? "child outbox tasks" : "direct sends", async () => { + const fanouts: Message[] = []; + const children: Message[] = []; + const queue = (messages: Message[]): MessageQueue => ({ + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }); + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { + fanout: queue(fanouts), + ...(queued ? { outbox: queue(children) } : {}), + }, + }); + const reports: OutboundWorkerReport[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onWorker(ctx, report) { + reports.push(report); + contexts.push(ctx.data); + }, + }); + const ctx = federation.createContext(new URL("https://example.com"), { + label: "producer", + }); + const targets = [ + recipient, + recipient.clone({ inbox: new URL("https://receiver.example/second") }), + ]; + await ctx.sendActivity(sender, targets, activity, { + fanout: "force", + dispatchId: "original invocation", + }); + strictEqual(reports.length, 0); + const task = fanouts[0]; + ok(task?.type === "fanout"); + const data = { label: "fanout worker" }; + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202).post( + "https://receiver.example/second", + 202, + ); + try { + await federation.processQueuedTask(data, task); + strictEqual(reports.length, 1); + strictEqual(reports[0].role, "fanout"); + strictEqual(reports[0].delivery, undefined); + strictEqual(reports[0].attemptsStarted, queued ? 0 : 2); + strictEqual(children.length, queued ? 2 : 0); + strictEqual(reports[0].actions.length, 2); + for (const action of reports[0].actions) { + strictEqual(action.type, queued ? "scheduled" : "handoff"); + ok(action.type === "scheduled" || action.type === "handoff"); + strictEqual(action.action, queued ? "enqueue" : "direct"); + const inbox = action.inboxUrls[0]; + strictEqual(action.delivery?.destinationInbox, inbox); + deepStrictEqual(action.delivery?.broadcast, { type: "unknown" }); + ok(action.delivery?.deliveryId); + if (action.type === "scheduled") { + const child = children.find((child) => + child.id === action.queue.queueMessageId + ); + ok(child?.type === "outbox"); + deepStrictEqual(action.delivery, child.outbound); + } + ok(Object.isFrozen(action)); + ok(Object.isFrozen(action.inboxUrls)); + ok(Object.isFrozen(action.delivery)); + } + deepStrictEqual(reports[0].queue, { queueMessageId: task.id }); + deepStrictEqual(reports[0].result, { type: "completed" }); + strictEqual(reports[0].provenance.dispatchId, "original invocation"); + deepStrictEqual(contexts, [data]); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("outbound worker callbacks retain selection, await settlement, and isolate logging failures", async () => { + let logs = 0; + await configure({ + reset: true, + sinks: { + broken() { + logs++; + throw new Error("logging failure"); + }, + }, + loggers: [{ + category: ["fedify", "federation", "outbox"], + sinks: ["broken"], + lowestLevel: "error", + }, { category: ["logtape", "meta"], sinks: [] }], + }); + let respond!: () => void; + const responseGate = new Promise((resolve) => { + respond = resolve; + }); + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let replacements = 0; + let settled = false; + const reports: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + async onWorker(_ctx, report) { + reports.push(report); + entered(); + await gate; + throw new Error("observer failure"); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, false); + fetchMock.mockGlobal().post(message.inbox, async () => { + fetched(); + await responseGate; + return 202; + }); + const processing = federation.processQueuedTask(undefined, message).finally( + () => { + settled = true; + }, + ); + try { + await fetching; + federation.setOutboundObservers({ + onWorker() { + replacements++; + }, + }); + respond(); + await observing; + strictEqual(settled, false); + release(); + await processing; + strictEqual(reports.length, 1); + strictEqual(replacements, 0); + strictEqual(logs, 1); + deepStrictEqual(reports[0].delivery, { + broadcast: { type: "unknown" }, + deliveryId: message.id, + }); + } finally { + respond(); + release(); + await processing; + fetchMock.hardReset(); + await reset(); + } +}); + +test("outbound worker failure after a completed sender does not rewrite its attempt", async () => { + const [meterProvider] = createTestMeterProvider(); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + meterProvider, + }); + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskOutcome; + const failure = Symbol("worker bookkeeping after sender settlement"); + Object.defineProperty(metrics, "recordQueueTaskOutcome", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + const attempts: OutboundAttemptReport[] = []; + const workers: OutboundWorkerReport[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onAttempt(ctx, report) { + attempts.push(report); + contexts.push(ctx); + }, + onWorker(ctx, report) { + workers.push(report); + contexts.push(ctx); + }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + fetchMock.mockGlobal().post(message.inbox, 202); + try { + await rejects( + federation.processQueuedTask(undefined, message), + (error: unknown) => error === failure, + ); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(workers.length, 1); + strictEqual(workers[0].attemptsStarted, 1); + deepStrictEqual(workers[0].result, { type: "failed", error: failure }); + strictEqual(contexts[0], contexts[1]); + } finally { + Object.defineProperty(metrics, "recordQueueTaskOutcome", { + value: original, + }); + fetchMock.hardReset(); + } +}); + +test("outbound worker attempts restore current context without creating loaders", async () => { + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + let loads = 0; + const unexpectedLoader = () => { + loads++; + throw new Error("observation must not create loaders"); + }; + Object.defineProperties(federation, { + documentLoaderFactory: { value: unexpectedLoader }, + contextLoaderFactory: { value: unexpectedLoader }, + }); + const message = continuationMessage(recipient.inboxId!.href, true); + const data = { label: "current worker" }; + const reports: OutboundAttemptReport[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onAttempt(ctx, report) { + reports.push(report); + contexts.push(ctx.data, ctx.clone(data).data); + throw new Error("worker observer failure must be isolated"); + }, + }); + fetchMock.mockGlobal().post(message.inbox, 202); + try { + await federation.processQueuedTask(data, structuredClone(message)); + strictEqual(loads, 0); + deepStrictEqual(contexts, [data, data]); + strictEqual(reports.length, 1); + deepStrictEqual(reports[0].provenance, message.provenance); + strictEqual(reports[0].local.origin, new URL(message.baseUrl).origin); + deepStrictEqual(reports[0].queue, { + queueMessageId: message.id, + queueAttempt: 0, + }); + strictEqual(reports[0].outcome.type, "succeeded"); + } finally { + fetchMock.hardReset(); + } +}); + +test("outbound fanout workers report fresh attempts with restored grouping and current data", async (t) => { + for (const legacy of [false, true]) { + await t.step(legacy ? "legacy fanout" : "populated identity", async () => { + const messages: Message[] = []; + const [tracerProvider, exporter] = createTestTracerProvider(); + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + tracerProvider, + manuallyStartQueue: true, + queue: { + fanout: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + if (!legacy) federation.setOutboundObservers({ onBroadcast() {} }); + const ctx = federation.createContext(new URL("https://example.com"), { + label: "producer", + }); + await ctx.sendActivity(sender, recipient, activity, { + fanout: "force", + dispatchId: "original-dispatch", + }); + const message = messages[0]; + ok(message?.type === "fanout"); + const identity = message.inboxes[recipient.inboxId!.href].outbound; + strictEqual(identity == null, legacy); + const reports: OutboundAttemptReport[] = []; + const contexts: unknown[] = []; + const data = { label: "worker" }; + federation.setOutboundObservers({ + onAttempt(context, report) { + reports.push(report); + contexts.push(context.data); + }, + }); + fetchMock.mockGlobal().post(recipient.inboxId!.href, 202); + try { + await federation.processQueuedTask(data, structuredClone(message)); + await federation.processQueuedTask(data, structuredClone(message)); + strictEqual(reports.length, 2); + deepStrictEqual(contexts, [data, data]); + notStrictEqual(reports[0].attemptId, reports[1].attemptId); + strictEqual(reports[0].deliveryId, reports[1].deliveryId); + if (identity != null) { + strictEqual(reports[0].deliveryId, identity.deliveryId); + } + for (const report of reports) { + strictEqual(report.destinationInbox, recipient.inboxId!.href); + deepStrictEqual( + report.broadcast, + identity?.broadcast ?? { type: "unknown" }, + ); + strictEqual(report.provenance.dispatchId, "original-dispatch"); + deepStrictEqual(report.queue, { queueMessageId: message.id }); + const span = exporter.spans.find((span) => + span.spanContext().spanId === report.trace?.spanId + ); + ok(span); + strictEqual(span.name, "activitypub.send_activity"); + } + } finally { + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }); + } +}); + +test("outbound workers omit attempts for key import, fanout parsing, and held-task drops", async (t) => { + for ( + const phase of [ + "key-import", + "fanout-parsing", + "drop", + "gateway-drop", + ] as const + ) { + await t.step(phase, async () => { + const messages: Message[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: { heldActivityTtl: { seconds: 1 } }, + queue: { + outbox: { + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const original = continuationMessage(recipient.inboxId!.href, true); + const message: Message = phase === "fanout-parsing" + ? { + type: "fanout", + id: original.id, + baseUrl: original.baseUrl, + keys: [], + inboxes: { [original.inbox]: { actorIds: [], sharedInbox: false } }, + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Person", + }, + activityType: Create.typeId.href, + traceContext: {}, + } + : phase === "key-import" + ? { + ...original, + keys: [{ keyId: sender.keyId.href, privateKey: { kty: "invalid" } }], + } + : { + ...original, + circuitHeld: true, + circuitHeldSince: "2000-01-01T00:00:00Z", + ...(phase === "gateway-drop" + ? { + gatewayInboxes: [original.inbox, "https://gateway.example/inbox"], + } + : {}), + }; + fetchMock.mockGlobal(); + try { + const processing = federation.processQueuedTask(undefined, message); + if (phase === "key-import" || phase === "fanout-parsing") { + await rejects(processing); + } else await processing; + strictEqual(attempts.length, 0); + strictEqual(fetchMock.callHistory.calls().length, 0); + strictEqual(messages.length, 0); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("outbound queue identities survive ordinary retries, holds, and native redelivery", async (t) => { + for (const observed of [false, true]) { + for (const mode of ["retry", "hold", "retry-after", "native"] as const) { + await t.step(`${mode}: ${observed ? "identity" : "legacy"}`, async () => { + const queued: { + message: Message; + options: Parameters[1]; + }[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + manuallyStartQueue: true, + allowPrivateAddress: true, + circuitBreaker: mode === "hold" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + queue: { + outbox: { + nativeRetrial: mode === "native", + enqueue(message, options) { + queued.push({ message, options }); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const message = continuationMessage(recipient.inboxId!.href, observed); + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const original = structuredClone(message); + const task: OutboxMessage = JSON.parse(JSON.stringify(message)); + fetchMock.mockGlobal().post( + message.inbox, + mode === "retry-after" + ? { status: 429, headers: { "Retry-After": "3600" } } + : 500, + ); + try { + if (mode === "native") { + await rejects( + federation.processQueuedTask( + undefined, + task, + ), + ); + strictEqual(queued.length, 0); + fetchMock.hardReset(); + fetchMock.mockGlobal().post(message.inbox, 202); + deepStrictEqual(task, original); + const redelivery = { + ...structuredClone(task), + id: crypto.randomUUID(), + }; + await federation.processQueuedTask(undefined, redelivery); + deepStrictEqual(redelivery.outbound, original.outbound); + strictEqual("outbound" in redelivery, observed); + ok(fetchMock.callHistory.calls().length > 0); + strictEqual(queued.length, 0); + } else { + await federation.processQueuedTask( + undefined, + task, + ); + strictEqual(queued.length, 1); + const next = queued[0].message; + ok(next.type === "outbox"); + deepStrictEqual(next.outbound, original.outbound); + deepStrictEqual(next.provenance, original.provenance); + strictEqual("provenance" in next, observed); + strictEqual("outbound" in next, observed); + strictEqual(next.attempt, mode === "hold" ? 0 : 1); + strictEqual(next.circuitHeld, mode === "hold" ? true : undefined); + strictEqual(queued[0].options?.orderingKey, message.orderingKey); + ok(queued[0].options?.delay); + const requests = fetchMock.callHistory.calls().length; + await federation.processQueuedTask(undefined, { + ...structuredClone(next), + id: crypto.randomUUID(), + }); + strictEqual(queued.length, 2); + const replay = queued[1].message; + ok(replay.type === "outbox"); + deepStrictEqual(replay.outbound, original.outbound); + deepStrictEqual(replay.provenance, original.provenance); + strictEqual("provenance" in replay, observed); + strictEqual("outbound" in replay, observed); + strictEqual(replay.attempt, mode === "hold" ? 0 : 2); + if (mode === "hold") { + strictEqual(fetchMock.callHistory.calls().length, requests); + } + } + strictEqual(attempts.length, mode === "hold" ? 1 : 2); + strictEqual( + new Set(attempts.map((report) => report.attemptId)).size, + attempts.length, + ); + for (const report of attempts) { + strictEqual( + report.destinationInbox, + original.outbound?.destinationInbox, + ); + deepStrictEqual( + report.broadcast, + original.outbound?.broadcast ?? { type: "unknown" }, + ); + strictEqual( + report.provenance.dispatchId, + original.provenance?.dispatchId, + ); + deepStrictEqual( + report.provenance.inboxRequest, + original.provenance?.inboxRequest, + ); + strictEqual(report.local.origin, new URL(message.baseUrl).origin); + ok(report.queue); + strictEqual( + report.deliveryId, + observed + ? original.outbound!.deliveryId + : report.queue.queueMessageId, + ); + ok(Object.isFrozen(report.queue)); + } + strictEqual(attempts[0].queue?.queueAttempt, 0); + if (mode !== "hold") { + strictEqual( + attempts[1].queue?.queueAttempt, + mode === "native" ? 0 : 1, + ); + } + deepStrictEqual(task, original); + deepStrictEqual(message, original); + } finally { + fetchMock.hardReset(); + } + }); + } + } +}); + +test("outbound queue identities retain planned destinations through gateway continuations", async (t) => { + const gateways = [ + "https://gw1.example/inbox", + "https://gw2.example/inbox", + "https://gw3.example/inbox", + ]; + const portableInbox = "ap+ef61://z6Mktest/inbox"; + for (const observed of [false, true]) { + for (const mode of ["retry", "hold", "defer", "native"] as const) { + await t.step(`${mode}: ${observed ? "identity" : "legacy"}`, async () => { + const queued: Message[] = []; + const enqueueOptions: Parameters[1][] = []; + const workers: OutboundWorkerReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + manuallyStartQueue: true, + allowPrivateAddress: true, + circuitBreaker: mode === "hold" + ? { failureThreshold: 1, recoveryDelay: { hours: 1 } } + : false, + queue: { + outbox: { + nativeRetrial: mode === "native", + enqueue(message, options) { + queued.push(message); + enqueueOptions.push(options); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + const message = { + ...continuationMessage(gateways[0], observed, portableInbox), + gatewayInboxes: gateways, + }; + const attempts: OutboundAttemptReport[] = []; + federation.setOutboundObservers({ + onWorker(_ctx, report) { + workers.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }); + const original = structuredClone(message); + const task: OutboxMessage = JSON.parse(JSON.stringify(message)); + fetchMock.mockGlobal(); + for (const [index, inbox] of gateways.entries()) { + fetchMock.post( + inbox, + mode !== "native" && index === 0 + ? 404 + : mode === "defer" + ? { status: 429, headers: { "Retry-After": "3600" } } + : 500, + ); + } + try { + if (mode === "native") { + await rejects( + federation.processQueuedTask( + undefined, + task, + ), + ); + strictEqual(queued.length, 0); + fetchMock.hardReset(); + fetchMock.mockGlobal().post(gateways[0], 202); + deepStrictEqual(task, original); + const redelivery = { + ...structuredClone(task), + id: crypto.randomUUID(), + }; + await federation.processQueuedTask(undefined, redelivery); + deepStrictEqual(redelivery.outbound, original.outbound); + strictEqual("outbound" in redelivery, observed); + ok(fetchMock.callHistory.calls().length > 0); + strictEqual(queued.length, 0); + } else { + await federation.processQueuedTask( + undefined, + task, + ); + strictEqual(queued.length, 1); + const next = queued[0]; + ok(next.type === "outbox"); + strictEqual(next.inbox, gateways[1]); + deepStrictEqual(next.gatewayInboxes, gateways.slice(1)); + deepStrictEqual(next.outbound, original.outbound); + deepStrictEqual(next.provenance, original.provenance); + strictEqual("provenance" in next, observed); + strictEqual("outbound" in next, observed); + strictEqual(next.portableInbox, portableInbox); + if (observed) { + strictEqual(next.outbound?.destinationInbox, gateways[0]); + } + strictEqual(next.attempt, mode === "hold" ? 0 : 1); + strictEqual(next.circuitHeld, mode === "hold" ? true : undefined); + if (mode === "defer") { + deepStrictEqual( + Object.keys(next.gatewayNotBefore ?? {}), + gateways.slice(1), + ); + } + const requests = fetchMock.callHistory.calls().length; + await federation.processQueuedTask(undefined, { + ...structuredClone(next), + id: crypto.randomUUID(), + }); + strictEqual(queued.length, 2); + const replay = queued[1]; + ok(replay.type === "outbox"); + deepStrictEqual(replay.outbound, original.outbound); + deepStrictEqual(replay.provenance, original.provenance); + strictEqual("provenance" in replay, observed); + strictEqual("outbound" in replay, observed); + strictEqual(replay.inbox, gateways[1]); + strictEqual(replay.attempt, mode === "retry" ? 2 : next.attempt); + if (mode !== "retry") { + strictEqual(fetchMock.callHistory.calls().length, requests); + } + } + strictEqual( + attempts.length, + mode === "native" ? 4 : mode === "retry" ? 5 : 3, + ); + strictEqual( + new Set(attempts.map((report) => report.attemptId)).size, + attempts.length, + ); + for (const report of attempts) { + strictEqual( + report.destinationInbox, + original.outbound?.destinationInbox, + ); + deepStrictEqual( + report.broadcast, + original.outbound?.broadcast ?? { type: "unknown" }, + ); + strictEqual( + report.provenance.dispatchId, + original.provenance?.dispatchId, + ); + deepStrictEqual( + report.provenance.inboxRequest, + original.provenance?.inboxRequest, + ); + strictEqual(report.local.origin, new URL(message.baseUrl).origin); + ok(report.queue); + strictEqual( + report.deliveryId, + observed + ? original.outbound!.deliveryId + : report.queue.queueMessageId, + ); + ok(Object.isFrozen(report.queue)); + } + strictEqual(attempts[0].queue?.queueAttempt, 0); + deepStrictEqual( + attempts.slice(0, 3).map((report) => report.inboxUrl), + gateways, + ); + for (const report of attempts) { + strictEqual(report.portableInbox, portableInbox); + } + if (mode === "retry") { + deepStrictEqual( + attempts.slice(3).map((report) => report.inboxUrl), + gateways.slice(1), + ); + strictEqual(attempts[3].queue?.queueAttempt, 1); + } + strictEqual(workers.length, 2); + for (const [index, report] of workers.entries()) { + if (mode === "native") { + deepStrictEqual(report.actions, []); + strictEqual( + report.result.type, + index === 0 ? "failed" : "completed", + ); + strictEqual(report.attemptsStarted, index === 0 ? 3 : 1); + continue; + } + const actions = report.actions; + ok(actions); + if (index === 0) { + deepStrictEqual(actions[0], { + type: "abandoned", + reason: "permanent-failure", + inboxUrls: [gateways[0]], + }); + } + strictEqual(actions.length, index === 0 ? 2 : 1); + const action = actions.at(-1)!; + ok(action.type === "scheduled"); + const task = queued[index]; + ok(task.type === "outbox"); + strictEqual( + action.action, + mode === "hold" || index === 1 && mode === "defer" + ? "defer" + : "retry", + ); + deepStrictEqual(action.inboxUrls, gateways.slice(1)); + deepStrictEqual(action.queue, { + queueMessageId: task.id, + queueAttempt: task.attempt, + }); + ok(action.action === "retry" || action.action === "defer"); + strictEqual(action.delay, enqueueOptions[index]?.delay?.toString()); + deepStrictEqual(action.notBefore, task.gatewayNotBefore); + if (action.notBefore != null) ok(Object.isFrozen(action.notBefore)); + strictEqual( + report.attemptsStarted, + index === 0 ? 3 : mode === "retry" ? 2 : 0, + ); + deepStrictEqual(report.result, { type: "completed" }); + } + deepStrictEqual(task, original); + deepStrictEqual(message, original); + } finally { + fetchMock.hardReset(); + } + }); + } + } +}); + +test("fanout worker actions retain child enqueue settlement and replay identities", async (t) => { + for ( + const mode of [ + "individual", + "partial", + "batch", + "batch-rejected", + "ordered", + "bookkeeping", + ] as const + ) { + await t.step(mode, async () => { + const fanouts: Message[] = []; + const submitted: OutboxMessage[] = []; + const accepted: OutboxMessage[] = []; + const reports: OutboundWorkerReport[] = []; + const broadcasts: OutboundBroadcastEvent[] = []; + const failure = Symbol(mode); + const [meterProvider] = createTestMeterProvider(); + let batches = 0; + const outbox: MessageQueue = { + enqueue(message) { + ok(message.type === "outbox"); + submitted.push(message); + if ( + mode === "partial" && reports.length === 0 && + message.inbox.endsWith("/second") + ) { + return Promise.reject(failure); + } + accepted.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + if (["batch", "batch-rejected", "ordered"].includes(mode)) { + outbox.enqueueMany = (messages) => { + batches++; + for (const message of messages) { + ok(message.type === "outbox"); + submitted.push(message); + } + // A rejected batch may already have accepted part of its input. + accepted.push( + ...submitted.slice(0, mode === "batch-rejected" ? 1 : 2), + ); + return mode === "batch-rejected" + ? Promise.reject(failure) + : Promise.resolve(); + }; + } + const federation = createFederation({ + ...options, + kv: new MemoryKvStore(), + manuallyStartQueue: true, + meterProvider, + queue: { + fanout: { + enqueue(message) { + fanouts.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + outbox, + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onWorker(_ctx, report) { + reports.push(report); + }, + }); + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + sender, + [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ], + activity, + { + fanout: "force", + ...(mode === "ordered" ? { orderingKey: "conversation" } : {}), + }, + ); + const task = fanouts[0]; + ok(task?.type === "fanout"); + const producerEvents = [...broadcasts]; + ok(federation instanceof FederationImpl); + const metrics = federation.metrics; + const original = metrics.recordQueueTaskEnqueued; + if (mode === "bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: (...args: Parameters) => { + original.call(metrics, ...args); + throw failure; + }, + }); + } + try { + for ( + let replay = 0; + replay < (mode === "individual" || mode === "partial" ? 2 : 1); + replay++ + ) { + submitted.length = 0; + accepted.length = 0; + const processing = federation.processQueuedTask( + undefined, + structuredClone(task), + ); + const rejected = mode === "partial" && replay === 0 || + mode === "batch-rejected" || mode === "bookkeeping"; + if (rejected) { + await rejects( + processing, + (error: unknown) => + mode === "bookkeeping" + ? error instanceof AggregateError && + error.errors.every((item) => item === failure) + : error === failure, + ); + } else await processing; + const report = reports[replay]; + strictEqual(report.result.type, rejected ? "failed" : "completed"); + strictEqual(report.attemptsStarted, 0); + strictEqual(report.delivery, undefined); + strictEqual(report.actions.length, 2); + strictEqual(submitted.length, 2); + strictEqual( + accepted.length, + mode === "partial" && replay === 0 || mode === "batch-rejected" + ? 1 + : 2, + ); + for (const child of submitted) { + const action = report.actions.find((action) => + "queue" in action && action.queue.queueMessageId === child.id + ); + ok( + action?.type === "scheduled" || action?.type === "action-failed", + ); + ok(action.action === "enqueue"); + deepStrictEqual(action.queue, { + queueMessageId: child.id, + queueAttempt: 0, + }); + deepStrictEqual(action.inboxUrls, [child.inbox]); + deepStrictEqual( + action.delivery, + task.inboxes[child.inbox].outbound, + ); + deepStrictEqual(action.delivery, child.outbound); + strictEqual("delay" in action, false); + strictEqual( + action.type, + mode === "batch-rejected" || + mode === "partial" && replay === 0 && + child.inbox.endsWith("/second") + ? "action-failed" + : "scheduled", + ); + if (action.type === "action-failed") { + strictEqual(action.acceptance, "unknown"); + strictEqual(action.error, failure); + } + if (replay > 0) { + const earlier = reports[0].actions.find((earlier) => + "delivery" in earlier && + earlier.delivery?.destinationInbox === child.inbox + ); + ok( + (earlier?.type === "scheduled" || + earlier?.type === "action-failed") && + earlier.action === "enqueue", + ); + notStrictEqual( + action.queue.queueMessageId, + earlier.queue.queueMessageId, + ); + deepStrictEqual(action.delivery, earlier.delivery); + } + } + deepStrictEqual(broadcasts, producerEvents); + } + strictEqual( + batches, + mode === "batch" || mode === "batch-rejected" ? 1 : 0, + ); + } finally { + if (mode === "bookkeeping") { + Object.defineProperty(metrics, "recordQueueTaskEnqueued", { + configurable: true, + value: original, + }); + } + } + }); + } +}); + +test("fanout worker actions preserve earlier direct entries across later setup failure", async (t) => { + for (const queued of [false, true]) { + await t.step( + queued + ? "prepare all children before enqueue" + : "earlier send still settles", + async () => { + const fanouts: Message[] = []; + const children: Message[] = []; + const reports: OutboundWorkerReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const broadcasts: OutboundBroadcastEvent[] = []; + const queue = (messages: Message[]): MessageQueue => ({ + enqueue(message) { + messages.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + queue: { + fanout: queue(fanouts), + ...(queued ? { outbox: queue(children) } : {}), + }, + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onWorker(_ctx, report) { + reports.push(report); + }, + }); + await federation.createContext( + new URL("https://example.com"), + undefined, + ) + .sendActivity( + sender, + [ + recipient, + recipient.clone({ + inbox: new URL("https://receiver.example/second"), + }), + ], + activity, + { fanout: "force" }, + ); + const originalTask = fanouts[0]; + ok(originalTask?.type === "fanout"); + const secondInbox = "https://receiver.example/second"; + const task: FanoutMessage = { + ...originalTask, + collectionSync: "https://example.com/followers", + inboxes: { + ...originalTask.inboxes, + [secondInbox]: { + ...originalTask.inboxes[secondInbox], + actorIds: ["invalid actor URL"], + }, + }, + }; + const producerEvents = [...broadcasts]; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let observed!: () => void; + const settled = new Promise((resolve) => { + observed = resolve; + }); + federation.setOutboundObservers({ + onBroadcast(_ctx, event) { + broadcasts.push(event); + }, + onWorker(_ctx, report) { + reports.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + observed(); + }, + }); + fetchMock.mockGlobal().post(recipient.inboxId!.href, async () => { + await gate; + return 202; + }); + try { + let failure: unknown; + await rejects( + federation.processQueuedTask(undefined, task), + (error: unknown) => { + failure = error; + return error instanceof TypeError; + }, + ); + strictEqual(reports.length, 1); + const report = reports[0]; + deepStrictEqual(report.result, { + type: "preparation-failed", + error: failure, + failure: { + kind: "local", + phase: "request-preparation", + error: failure, + }, + }); + strictEqual(report.attemptsStarted, queued ? 0 : 1); + strictEqual(children.length, 0); + strictEqual(report.actions.length, queued ? 0 : 1); + if (!queued) { + const action = report.actions[0]; + ok(action.type === "handoff"); + strictEqual(action.action, "direct"); + deepStrictEqual(action.inboxUrls, [recipient.inboxId!.href]); + deepStrictEqual( + action.delivery, + task.inboxes[recipient.inboxId!.href].outbound, + ); + strictEqual(attempts.length, 0); + release(); + await settled; + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(reports.length, 1); + } + deepStrictEqual(broadcasts, producerEvents); + } finally { + release(); + fetchMock.hardReset(); + } + }, + ); + } +}); + +test("outbound header policies validate atomically and copy registration", async () => { + const builder = createFederationBuilder(); + ok(builder instanceof FederationBuilderImpl); + const allow = ["X-Safe", "x-safe", "Content-Type"]; + const redact = ["X-Custom-Secret"]; + const policy = { allow, redact, maxBytes: 128 }; + const callback = () => {}; + builder.setOutboundObservers({ + requestHeaders: policy, + onHttpExchange: callback, + }); + const original = builder.outboundObservers; + deepStrictEqual(original.requestHeaders, { + allow: ["content-type", "x-safe"], + redact: ["x-custom-secret"], + maxBytes: 128, + }); + ok(Object.isFrozen(original.requestHeaders)); + ok(Object.isFrozen(original.requestHeaders?.allow)); + ok(Object.isFrozen(original.requestHeaders?.redact)); + allow.push("authorization"); + redact.length = 0; + policy.maxBytes = 1; + const invalid = [ + [{ allow: ["safe"], maxBytes: 0 }, RangeError], + [{ allow: ["safe"], maxBytes: -1 }, RangeError], + [{ allow: ["safe"], maxBytes: 1.5 }, RangeError], + [{ allow: ["safe"], maxBytes: Infinity }, RangeError], + [{ allow: ["safe"], maxBytes: NaN }, RangeError], + [{ allow: ["safe"], maxBytes: Number.MAX_SAFE_INTEGER + 1 }, RangeError], + [{ allow: ["safe"] }, TypeError], + [{ allow: ["safe"], maxBytes: "128" }, TypeError], + [{ allow: ["*"], maxBytes: 128 }, TypeError], + [{ allow: ["x-foo\nsecret"], maxBytes: 128 }, TypeError], + [{ allow: ["ümlaut"], maxBytes: 128 }, TypeError], + [{ allow: "x-safe", maxBytes: 128 }, TypeError], + [{ allow: [12], maxBytes: 128 }, TypeError], + [{ allow: [], redact: null, maxBytes: 128 }, TypeError], + [null, TypeError], + ] as const; + for (const [value, error] of invalid) { + throws(() => + builder.setOutboundObservers({ + requestHeaders: { allow: [], maxBytes: 1 }, + responseHeaders: value as never, + }), error); + strictEqual(builder.outboundObservers, original); + } + const federation = await builder.build(options); + ok(federation instanceof FederationImpl); + deepStrictEqual( + federation.outboundObservers.requestHeaders, + original.requestHeaders, + ); + builder.setOutboundObservers({}); + strictEqual(federation.outboundObservers.onHttpExchange, callback); + const registration = federation.outboundObservers; + throws( + () => + federation.setOutboundObservers({ + requestHeaders: { allow: [], maxBytes: 0 }, + }), + RangeError, + ); + strictEqual(federation.outboundObservers, registration); + federation.setOutboundObservers({}); + strictEqual(federation.outboundObservers.requestHeaders, undefined); +}); + +test("outbound header snapshots redact secrets and budget complete UTF-8 entries", () => { + const secrets = [ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "signature", + "signature-input", + "authentication-info", + "proxy-authentication-info", + "x-api-key", + "x-auth-token", + ]; + const headers = new Headers( + secrets.map((name): [string, string] => [name, "do-not-export"]), + ); + headers.append("X-Visible", "first"); + headers.append("x-visible", "second"); + headers.set("X-Custom", "also-secret"); + const policy = snapshotHeaderPolicy({ + allow: [...secrets, "X-Visible", "x-custom", "x-missing"], + redact: ["X-Custom"], + maxBytes: 4096, + }); + ok(policy); + const snapshot = snapshotHeaders(headers, policy); + for (const [name, value] of snapshot.entries) { + strictEqual(value, name === "x-visible" ? "first, second" : "[redacted]"); + } + deepStrictEqual( + snapshot.entries.map(([name]) => name), + [...secrets, "x-visible", "x-custom"].sort(), + ); + strictEqual(JSON.stringify(snapshot).includes("do-not-export"), false); + strictEqual(JSON.stringify(snapshot).includes("also-secret"), false); + ok(Object.isFrozen(snapshot)); + ok(Object.isFrozen(snapshot.entries)); + for (const entry of snapshot.entries) ok(Object.isFrozen(entry)); + headers.set("x-visible", "changed"); + strictEqual( + snapshot.entries.find(([name]) => name === "x-visible")?.[1], + "first, second", + ); + const values = new Headers({ "a": "oversized value", "b": "é", "c": "z" }); + const capture = (maxBytes: number) => + snapshotHeaders( + values, + snapshotHeaderPolicy({ allow: ["c", "a", "b"], maxBytes })!, + ); + // `b: é\r\n` is seven UTF-8 bytes; skip a large entry and keep considering later ones. + deepStrictEqual(capture(7), { entries: [["b", "é"]], limitReached: true }); + deepStrictEqual(capture(6), { entries: [["c", "z"]], limitReached: true }); + deepStrictEqual( + snapshotHeaders( + new Headers({ b: "é" }), + snapshotHeaderPolicy({ allow: ["b"], maxBytes: 7 })!, + ), + { entries: [["b", "é"]], limitReached: false }, + ); + deepStrictEqual( + snapshotHeaders(values, snapshotHeaderPolicy({ allow: [], maxBytes: 1 })!), + { entries: [], limitReached: false }, + ); + deepStrictEqual( + snapshotHeaders( + new Headers({ authorization: "very long secret" }), + snapshotHeaderPolicy({ allow: ["authorization"], maxBytes: 27 })!, + ), + { entries: [["authorization", "[redacted]"]], limitReached: false }, + ); +}); + +test("outbound header policies retain selected snapshots across unsigned redirects", async () => { + const exchanges: OutboundHttpExchangeReport[] = []; + const allow = ["X-Trace", "Content-Type", "Authorization"]; + const builder = createFederationBuilder(); + let entered!: () => void; + const observing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + builder.setOutboundObservers({ + requestHeaders: { allow, maxBytes: 512 }, + responseHeaders: { + allow: ["Location", "X-Reply", "Set-Cookie"], + maxBytes: 512, + }, + async onHttpExchange(_ctx, report) { + exchanges.push(report); + entered(); + await gate; + }, + }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + builder.setOutboundObservers({}); + const message = { + ...continuationMessage(recipient.inboxId!.href, true), + headers: { + "X-Trace": "worker", + "Authorization": "secret", + "Content-Type": "application/activity+json", + }, + }; + const next = new URL("/next", message.inbox).href; + fetchMock.mockGlobal().post(message.inbox, { + status: 303, + headers: { + Location: next, + "X-Reply": "first", + "Set-Cookie": "secret-cookie", + }, + }).get(next, { status: 202, headers: { "X-Reply": "second" } }); + const processing = federation.processQueuedTask(undefined, message); + try { + await observing; + allow.length = 0; + federation.setOutboundObservers({ + onHttpExchange() { + throw new Error("replacement must not run"); + }, + }); + release(); + await processing; + strictEqual(exchanges.length, 2); + for (const report of exchanges) { + strictEqual( + report.request.headers?.entries.find(([name]) => + name === "authorization" + )?.[1], + "[redacted]", + ); + strictEqual( + report.request.headers?.entries.find(([name]) => name === "x-trace") + ?.[1], + "worker", + ); + ok(report.outcome.type === "response"); + ok(Object.isFrozen(report.outcome.response.headers)); + strictEqual(JSON.stringify(report).includes("secret-cookie"), false); + } + strictEqual(exchanges[1].request.method, "GET"); + strictEqual( + exchanges[1].request.headers?.entries.some(([name]) => + name === "content-type" + ), + false, + ); + ok( + exchanges[0].outcome.type === "response" && + exchanges[1].outcome.type === "response", + ); + deepStrictEqual(exchanges[0].outcome.response.headers?.entries, [ + ["location", next], + ["set-cookie", "[redacted]"], + ["x-reply", "first"], + ]); + deepStrictEqual(exchanges[1].outcome.response.headers?.entries, [[ + "x-reply", + "second", + ]]); + } finally { + release(); + await processing; + fetchMock.hardReset(); + } +}); + +test("outbound header snapshots follow signed challenge and redirect requests", async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + requestHeaders: snapshotHeaderPolicy({ + allow: ["content-digest", "signature", "signature-input"], + maxBytes: 1024, + }), + responseHeaders: snapshotHeaderPolicy({ + allow: ["accept-signature", "location"], + maxBytes: 1024, + }), + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + const next = new URL("/next", inbox).href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const responses: ResponseInit[] = [ + { + status: 401, + headers: { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created;nonce="observer-nonce"', + }, + }, + { status: 307, headers: { Location: next } }, + { status: 202 }, + ]; + const requests: Request[] = []; + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: { id: "https://example.com/activity" }, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { determineSpec: () => "rfc9421", rememberSpec() {} }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch(input) { + ok(input instanceof Request); + requests.push(input); + return Promise.resolve( + new Response(null, responses[requests.length - 1]), + ); + }, + close: () => Promise.resolve(), + }), + }); + strictEqual(exchanges.length, 3); + for (const [index, report] of exchanges.entries()) { + const expected = Array.from(requests[index].headers).filter(([name]) => + ["content-digest", "signature", "signature-input"].includes(name) + ).map(( + [name, value], + ) => [name, name.startsWith("signature") ? "[redacted]" : value]); + deepStrictEqual(report.request.headers?.entries, expected); + strictEqual(report.request.headers?.limitReached, false); + ok(requests[index].headers.has("signature")); + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, responses[index].status); + } + strictEqual(exchanges[2].request.url, next); + ok(exchanges[0].outcome.type === "response"); + strictEqual( + exchanges[0].outcome.response.headers?.entries[0][0], + "accept-signature", + ); +}); + +test("outbound header capture keeps each side independent and disabled by default", async (t) => { + for (const mode of ["disabled", "request", "response", "empty"] as const) { + await t.step(mode, async () => { + const reports: OutboundHttpExchangeReport[] = []; + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + ...(mode === "request" || mode === "empty" + ? { + requestHeaders: { + allow: mode === "empty" ? [] : ["x-request"], + maxBytes: 128, + }, + } + : {}), + ...(mode === "response" + ? { responseHeaders: { allow: ["x-response"], maxBytes: 128 } } + : {}), + onHttpExchange(_ctx, report) { + reports.push(report); + }, + }); + const message = { + ...continuationMessage(recipient.inboxId!.href, false), + headers: { "X-Request": "request" }, + }; + fetchMock.mockGlobal().post(message.inbox, { + status: 202, + headers: { "X-Response": "response" }, + }); + try { + await federation.processQueuedTask(undefined, message); + strictEqual(reports.length, 1); + const report = reports[0]; + ok(report.outcome.type === "response"); + deepStrictEqual( + report.request.headers, + mode === "request" + ? { entries: [["x-request", "request"]], limitReached: false } + : mode === "empty" + ? { entries: [], limitReached: false } + : undefined, + ); + deepStrictEqual( + report.outcome.response.headers, + mode === "response" + ? { entries: [["x-response", "response"]], limitReached: false } + : undefined, + ); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +async function readCapturedText( + body: ReadableStream, +): Promise { + const reader = body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const result = await reader.read(); + if (result.done) return text; + text += decoder.decode(result.value, { stream: true }); + } + } catch { + return text; + } finally { + reader.releaseLock(); + } +} + +test("outbound body-only observers retain worker context and selection across redirects", async () => { + const events: OutboundResponseBodyEvent[] = []; + const texts: string[] = []; + const builder = createFederationBuilder<{ label: string }>(); + const config = { + maxBytes: 128, + timeoutMs: 1000, + async onBody( + ctx: import("./context.ts").Context<{ label: string }>, + event: OutboundResponseBodyEvent, + ) { + strictEqual(ctx.data.label, "worker"); + events.push(event); + texts.push(await readCapturedText(event.body)); + federation.setOutboundObservers({}); + }, + }; + builder.setOutboundObservers({ responseBodyCapture: config }); + const federation = await builder.build({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + config.maxBytes = 1; + config.onBody = () => { + throw new Error("mutated callback must not be selected"); + }; + builder.setOutboundObservers({}); + const task = continuationMessage(recipient.inboxId!.href, false); + const next = new URL("/captured-next", task.inbox).href; + fetchMock.mockGlobal().post(task.inbox, { + status: 303, + headers: { Location: next }, + body: "first", + }).get(next, { status: 202, body: "second" }); + try { + await federation.processQueuedTask({ label: "worker" }, task); + deepStrictEqual(texts, ["first", "second"]); + strictEqual(events.length, 2); + for (const [index, event] of events.entries()) { + ok(Object.isFrozen(event)); + ok(event.signal.aborted); + strictEqual(event.exchangeIndex, index); + strictEqual(event.queue?.queueMessageId, task.id); + strictEqual(event.deliveryId, task.id); + strictEqual(event.response.status, index === 0 ? 303 : 202); + strictEqual(event.request.url, index === 0 ? task.inbox : next); + } + strictEqual(events[0].attemptId, events[1].attemptId); + } finally { + fetchMock.hardReset(); + } +}); + +test("outbound capture settles before exchanges without changing signed negotiation", async (t) => { + for (const fails of [false, true]) { + await t.step( + fails ? "intermediate observer failure" : "complete bodies", + async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const order: string[] = []; + const failure = new Error("capture observer failed"); + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 128, + timeoutMs: 1000, + async onBody(_ctx, event) { + order.push(`body ${event.exchangeIndex}`); + if (fails && event.exchangeIndex === 0) throw failure; + await readCapturedText(event.body); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + order.push(`exchange ${report.exchangeIndex}`); + }, + onAttempt(_ctx, report) { + attempts.push(report); + order.push("attempt"); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + const next = new URL("/capture-signed", inbox).href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const responses: ResponseInit[] = [ + { + status: 401, + headers: { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created;nonce="capture-nonce"', + }, + }, + { status: 307, headers: { Location: next } }, + { status: 202 }, + ]; + let submissions = 0; + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: { id: "https://example.com/capture" }, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { determineSpec: () => "rfc9421", rememberSpec() {} }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + const index = submissions++; + return Promise.resolve( + new Response(`body ${index}`, responses[index]), + ); + }, + close: () => Promise.resolve(), + }), + }); + strictEqual(submissions, 3); + deepStrictEqual(order, [ + "body 0", + "exchange 0", + "body 1", + "exchange 1", + "body 2", + "exchange 2", + "attempt", + ]); + for (const [index, report] of exchanges.entries()) { + if (fails && index === 0) { + deepStrictEqual(report.bodyCapture, { + type: "failed", + phase: "observer", + error: failure, + bytesExposed: 0, + }); + } else {deepStrictEqual(report.bodyCapture, { + type: "complete", + bytesExposed: 6, + });} + ok(report.outcome.type === "response"); + strictEqual(report.outcome.response.status, responses[index].status); + strictEqual("body" in report, false); + } + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(attempts[0].httpExchangeCount, 3); + const final = attempts[0].outcome.response; + ok(final.type === "final"); + strictEqual(final.metadata.status, 202); + }, + ); + } +}); + +test("outbound capture limits retain the original final HTTP diagnostic", async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 4, + timeoutMs: 1000, + async onBody(_ctx, event) { + await readCapturedText(event.body); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + let submissions = 0; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [], + inbox: new URL(inbox), + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return Promise.resolve( + new Response("full diagnostic error", { + status: 503, + headers: { "Retry-After": "1" }, + }), + ); + }, + close: () => Promise.resolve(), + }), + }), + (error: unknown) => { + ok(error instanceof SendActivityError); + strictEqual(error.statusCode, 503); + strictEqual(error.responseBody, "full diagnostic error"); + return true; + }, + ); + strictEqual(submissions, 1); + deepStrictEqual(exchanges[0].bodyCapture, { + type: "limit-reached", + bytesExposed: 4, + }); + ok(attempts[0].outcome.type === "failed"); + strictEqual(attempts[0].outcome.failure.kind, "http"); + ok(attempts[0].outcome.response.type === "final"); + strictEqual(attempts[0].outcome.response.metadata.status, 503); +}); + +test("outbound capture deadlines select each real exchange without changing HTTP flow", async (t) => { + const measurements: { + signed: boolean; + selector: string; + selected: number; + timeoutMs: number; + elapsedMs: number; + }[] = []; + for (const signed of [false, true]) { + for (const responses of ["all", "non-success-http-responses"] as const) { + await t.step( + `${signed ? "signed" : "unsigned"}: ${responses}`, + async () => { + const inbox = recipient.inboxId!.href; + const next = new URL("/capture-deadline", inbox).href; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const signals: AbortSignal[] = []; + const timeoutMs = 25; + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 32, + timeoutMs, + responses, + onBody(_ctx, event) { + signals.push(event.signal); + // The archive callback never pulls or settles; each exchange still proceeds. + return new Promise(() => {}); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const steps: ResponseInit[] = signed + ? [ + { + status: 401, + headers: { + "Accept-Signature": + 'sig1=("@method" "@target-uri");created;nonce="deadline-nonce"', + }, + }, + { status: 307, headers: { Location: next } }, + { status: 202 }, + ] + : [{ status: 303, headers: { Location: next } }, { status: 202 }]; + let submissions = 0; + const start = performance.now(); + await sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: signed ? [sender] : [], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() {}, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + return Promise.resolve( + new Response("response", steps[submissions++]), + ); + }, + close: () => Promise.resolve(), + }), + }); + const elapsedMs = performance.now() - start; + const selected = steps.length - (responses === "all" ? 0 : 1); + strictEqual(submissions, steps.length); + strictEqual(signals.length, selected); + for (const signal of signals) ok(signal.aborted); + deepStrictEqual( + exchanges.map((event) => event.bodyCapture?.type), + steps.map((step) => + step.status === 202 && responses !== "all" + ? "filtered" + : "deadline" + ), + ); + for (const [index, exchange] of exchanges.entries()) { + strictEqual(exchange.bodyCapture?.bytesExposed, 0); + ok(exchange.outcome.type === "response"); + strictEqual(exchange.outcome.response.status, steps[index].status); + } + strictEqual(attempts[0].outcome.type, "succeeded"); + strictEqual(attempts[0].httpExchangeCount, steps.length); + const final = attempts[0].outcome.response; + ok(final.type === "final"); + strictEqual(final.metadata.status, 202); + measurements.push({ + signed, + selector: responses, + selected, + timeoutMs, + elapsedMs: Math.round(elapsedMs * 100) / 100, + }); + }, + ); + } + } + console.info( + "Selected-exchange capture latency (local test, not a benchmark):", + JSON.stringify(measurements), + ); +}); + +test("outbound capture evidence survives later final response bookkeeping failures", async (t) => { + for (const status of [202, 503]) { + await t.step(`${status}: cache and cleanup`, async () => { + const inbox = recipient.inboxId!.href; + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const exchanges: OutboundHttpExchangeReport[] = []; + const attempts: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + responseBodyCapture: { + maxBytes: 4, + timeoutMs: 1000, + async onBody(_ctx, event) { + await readCapturedText(event.body); + }, + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + }, "send"); + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const cache = Symbol("cache after capture"); + const cleanup = Symbol("cleanup after capture"); + let submissions = 0; + let outward: unknown; + await rejects( + sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: {}, + keys: [sender], + inbox: new URL(inbox), + specDeterminer: { + determineSpec: () => "rfc9421", + rememberSpec() { + throw cache; + }, + }, + }, { + openHop: (url) => + Promise.resolve({ + url: new URL(url), + fetch() { + return Promise.resolve( + new Response("response body", { + status: ++submissions === 1 ? 401 : status, + }), + ); + }, + close: () => Promise.reject(cleanup), + }), + }), + (error: unknown) => { + outward = error; + return error instanceof FetchError && error.cause === cache; + }, + ); + strictEqual(submissions, 2); + deepStrictEqual(exchanges.map((report) => report.bodyCapture), [{ + type: "limit-reached", + bytesExposed: 4, + }, { type: "limit-reached", bytesExposed: 4 }]); + const outcome = attempts[0].outcome; + ok(outcome.type === "failed"); + strictEqual(outcome.error, outward); + ok(outcome.response.type === "final"); + strictEqual(outcome.response.metadata.status, status); + strictEqual(outcome.failure.kind, status === 503 ? "http" : "local"); + const failures = [outcome.failure, ...(outcome.diagnostics ?? [])]; + ok( + failures.some((failure) => + failure.phase === "post-response" && failure.error === cache + ), + ); + ok( + failures.some((failure) => + failure.phase === "cleanup" && failure.error === cleanup + ), + ); + strictEqual(attempts[0].httpExchangeCount, 2); + }); + } +}); + +test("sender payload projections preserve actual send forward and queued inputs", async (t) => { + for ( + const mode of [ + "ordinary", + "outbox", + "queued", + "fanout", + "forward", + "forward-queued", + ] as const + ) { + await t.step(mode, async () => { + const tasks: Message[] = []; + const queue: MessageQueue = { + enqueue(message) { + tasks.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }; + const federation = createFederation<{ label: string }>({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + ...(mode === "queued" || mode === "forward-queued" || mode === "fanout" + ? { + queue: { + outbox: queue, + ...(mode === "fanout" ? { fanout: queue } : {}), + }, + } + : {}), + }); + const reports: OutboundAttemptReport[] = []; + const contexts: string[] = []; + federation.setOutboundObservers({ + senderPayload: true, + onAttempt(ctx, report) { + reports.push(report); + contexts.push(ctx.data.label); + }, + }); + const secret = "https://private.example/recipient"; + const payload = { + ...forwardingPayload, + "@context": "https://www.w3.org/ns/activitystreams", + bto: [secret], + object: { type: "Note", bcc: [secret], content: "visible" }, + }; + const original = structuredClone(payload); + const outgoing = activity.clone({ + btos: [new URL(secret)], + bccs: [new URL(secret)], + }); + const ctx = mode.startsWith("forward") + ? forwardingContext(federation, "inbox", payload, { label: "producer" }) + : mode === "outbox" + ? forwardingContext(federation, "outbox", payload, { + label: "producer", + }) + : federation.createContext(new URL("https://example.com"), { + label: "producer", + }); + const bodies: unknown[] = []; + fetchMock.mockGlobal().post(recipient.inboxId!.href, (call) => { + bodies.push(JSON.parse(String(call.options.body))); + return 202; + }); + try { + if (mode.startsWith("forward")) { + ok(ctx instanceof InboxContextImpl); + await ctx.forwardActivity(sender, recipient, { + skipIfUnsigned: false, + immediate: mode === "forward", + }); + } else { + ok(ctx instanceof ContextImpl); + const senderContext: ContextImpl<{ label: string }> = ctx; + await senderContext.sendActivity(sender, recipient, outgoing, { + immediate: mode === "ordinary" || mode === "outbox", + fanout: mode === "fanout" ? "force" : "skip", + }); + } + while (tasks.length) { + await federation.processQueuedTask( + { label: "worker" }, + tasks.shift()!, + ); + } + strictEqual(reports.length, 1); + strictEqual(bodies.length, 1); + const document = reports[0].senderPayload; + ok( + document.type === "available", + JSON.stringify({ + document, + context: (bodies[0] as Record)["@context"], + }), + ); + strictEqual(document.stage, "sender-payload"); + strictEqual(document.redacted, true); + strictEqual(JSON.stringify(document.value).includes(secret), false); + ok(JSON.stringify(bodies[0]).includes(secret)); + deepStrictEqual(payload, original); + deepStrictEqual(contexts, [ + mode === "queued" || mode === "forward-queued" || mode === "fanout" + ? "worker" + : "producer", + ]); + } finally { + fetchMock.hardReset(); + } + }); + } +}); + +test("sender payload availability never changes serialization or redirected bodies", async (t) => { + for ( + const mode of [ + "disabled", + "preparation", + "conversion", + "getter", + "redirect", + ] as const + ) { + await t.step(mode, async () => { + const ctx = createFederation(options).createContext( + new URL("https://example.com"), + undefined, + ); + const reports: OutboundAttemptReport[] = []; + const observation = new OutboundBroadcastObservation(ctx, { + senderPayload: mode !== "disabled", + onAttempt(_ctx, report) { + reports.push(report); + }, + }, "send"); + const inbox = recipient.inboxId!.href; + await observation.planDestinations({ + [inbox]: { actorIds: [], sharedInbox: false }, + }); + const payload = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + bcc: ["https://private.example/recipient"], + object: { content: "visible" }, + }; + let conversions = 0; + let getters = 0; + const input = mode === "conversion" + ? Object.defineProperty({}, "toJSON", { + // Bun ignores a non-enumerable converter; use a shared runtime fixture. + enumerable: true, + value() { + conversions++; + return payload; + }, + }) + : mode === "getter" + ? Object.defineProperty({ ...payload }, "object", { + enumerable: true, + get() { + getters++; + return payload.object; + }, + }) + : payload; + const requests: { method: string; body: string }[] = []; + const failure = new Error("session unavailable before serialization"); + const sending = sendConcreteActivity({ + observation: observation.delivery(inbox), + activity: input, + keys: [], + inbox: new URL(inbox), + }, { + openHop: (url) => + mode === "preparation" ? Promise.reject(failure) : Promise.resolve({ + url: new URL(url), + async fetch(request) { + ok(request instanceof Request); + requests.push({ + method: request.method, + body: await request.text(), + }); + return new Response( + null, + mode === "redirect" && requests.length === 1 + ? { status: 303, headers: { Location: "/payload-next" } } + : { status: 202 }, + ); + }, + close: () => Promise.resolve(), + }), + }); + if (mode === "preparation") { + await rejects(sending, (error: unknown) => error === failure); + } else await sending; + strictEqual(reports.length, 1); + const document = reports[0].senderPayload; + if (mode === "redirect") { + ok(document.type === "available"); + strictEqual(document.redacted, true); + strictEqual( + JSON.stringify(document.value).includes("private.example"), + false, + ); + strictEqual(requests.length, 2); + strictEqual(requests[1].method, "GET"); + strictEqual(requests[1].body, ""); + ok(requests[0].body.includes("private.example")); + } else {deepStrictEqual(document, { + stage: "sender-payload", + type: "unavailable", + reason: mode === "disabled" + ? "disabled" + : mode === "preparation" + ? "not-materialized" + : "privacy-filter-unavailable", + });} + strictEqual(conversions, mode === "conversion" ? 1 : 0); + strictEqual(getters, mode === "getter" ? 1 : 0); + if (mode !== "preparation") { + strictEqual(requests[0].body, JSON.stringify(payload)); + } + }); + } +}); diff --git a/packages/fedify/src/federation/outbound-report.ts b/packages/fedify/src/federation/outbound-report.ts new file mode 100644 index 000000000..25155b8b9 --- /dev/null +++ b/packages/fedify/src/federation/outbound-report.ts @@ -0,0 +1,771 @@ +import type { Context } from "./context.ts"; + +/** + * The current producer's preparation result, not delivery completion. + * @since 2.5.0 + */ +export type OutboundPreparationResult = + | { + /** The producer finished preparing its continuation. */ + readonly type: "prepared"; + /** Fanout-ready work still needs worker-side delivery preparation. */ + readonly preparedFor: "fanout" | "delivery"; + /** Number of deduplicated destinations, not actors or successful sends. */ + readonly destinationCount: number; + } + | { + /** A deliberate no-send decision. */ + readonly type: "skipped"; + /** The existing no-send decision made by this producer. */ + readonly reason: + | "no-destinations" + | "unsigned" + | "forwarding-disabled" + | "forwarding-unavailable" + | "no-forwarding-targets"; + } + | { + /** Preparation failed before the producer reached readiness. */ + readonly type: "failed"; + /** The operation that failed and its original thrown value. */ + readonly failure: { + /** Local processing failure, or unavailable operation provenance. */ + readonly kind: "local" | "unknown"; + /** The observed operation, not a diagnosis from the error message. */ + readonly phase: + | "sender-resolution" + | "key-resolution" + | "recipient-resolution" + | "transformation" + | "validation" + | "signing" + | "destination-planning" + | "serialization" + | "request-preparation" + | "unknown"; + /** Original thrown value, including non-Error values. */ + readonly error: unknown; + }; + /** Whether a complete destination plan was established before failure. */ + readonly planning: "not-started" | "incomplete" | "complete"; + /** Confirmed planned obligations so far, not an audience estimate. */ + readonly plannedDestinationCount: number; + }; + +/** + * Known Activity ID and type at a broadcast observation boundary. + * + * Fields are captured without serializing the Activity or loading contexts. + * Successful transformation refreshes both the ID and the Activity class type. + * A `started` event may have an empty reference, and later events may reflect + * Activity transformations that happened after that event. + * @since 2.5.0 + */ +export type OutboundActivityReference = { + /** Activity identifier, when available at the observation boundary. */ + readonly id?: string; + /** Activity class type IRI, when available at the observation boundary. */ + readonly type?: string; +}; + +/** + * A send, original-payload forwarding, or portable replication producer lifecycle. + * + * Preparation results do not report queue acceptance or delivery completion. + * @since 2.5.0 + */ +export type OutboundBroadcastEvent = + & OutboundSourceMetadata + & { + /** Fresh opaque identity shared by producer events, independent of tracing. */ + readonly broadcastId: string; + /** The outgoing operation being observed. */ + readonly operation: "send" | "forward" | "portable-replication"; + /** Immutable Activity identity facts captured by the send path. */ + readonly activity: OutboundActivityReference; + } + & ( + | { + /** Entry before sender/recipient resolution or signing. */ + readonly stage: "started"; + } + | { + /** The current producer's preparation settled. */ + readonly stage: "preparation-finished"; + /** Readiness, a deliberate skip, or the original preparation failure. */ + readonly result: OutboundPreparationResult; + } + ); + +/** + * A producer-side destination obligation, not delivery completion. + * + * IDs group producer handoffs and their fanout expansion across queue tasks. + * Concrete sends use separate {@link OutboundAttemptReport} values; these + * events do not establish delivery completion or worker scheduling outcomes. + * An unvisited destination stays planned when another destination's preparation + * interrupts the producer. + * @since 2.5.0 + */ +export type OutboundDeliveryEvent = + & OutboundSourceMetadata + & { + /** The originating invocation, or unknown for legacy fanout work. */ + readonly broadcast: + | { readonly type: "known"; readonly id: string } + | { readonly type: "unknown" }; + /** Opaque identity allocated after deduplication or a confirmed claim. */ + readonly deliveryId: string; + /** Planned destination, unchanged by concrete gateway selection. */ + readonly destinationInbox: string; + /** Canonical portable inbox when the selected route represents one. */ + readonly portableInbox?: string; + } + & ( + | { + /** This destination was established by the existing plan. */ + readonly stage: "planned"; + /** Known actors sharing the destination, not an exhaustive audience. */ + readonly actorIds: readonly string[]; + /** Whether the selected destination uses a shared inbox. */ + readonly sharedInbox: boolean; + /** Fallback routes share one obligation; replication targets do not. */ + readonly mode: "ordinary" | "portable-fallback" | "portable-replication"; + } + | { + /** Local execution entry or enqueue settlement, not remote success. */ + readonly stage: "handoff"; + /** Evidence from the actual producer handoff. */ + readonly result: + | { readonly type: "direct" } + | { + readonly type: "enqueued"; + readonly queue: "fanout" | "outbox"; + readonly queueMessageId: string; + } + | { + readonly type: "enqueue-failed"; + readonly acceptance: "unknown"; + readonly error: unknown; + }; + } + | { + /** Destination-local setup failed before entry into concrete sending. */ + readonly stage: "preparation-failed"; + /** The original local setup failure, without inventing an attempt. */ + readonly failure: { + readonly kind: "local"; + readonly phase: "request-preparation"; + readonly error: unknown; + }; + } + ); + +/** + * The failure classification and available error evidence for a send attempt. + * @since 2.5.0 + */ +export type OutboundAttemptFailure = + | { + readonly kind: "http"; + readonly phase: "response"; + readonly error?: unknown; + } + | { + readonly kind: "transport"; + readonly phase: "validation" | "request"; + readonly error: unknown; + } + | { + readonly kind: "policy"; + readonly phase: "validation"; + readonly error: unknown; + } + | { + readonly kind: "local"; + readonly phase: + | "serialization" + | "request-preparation" + | "cleanup" + | "signing" + | "spec-lookup" + | "post-response"; + readonly error: unknown; + } + | { + readonly kind: "unknown"; + readonly phase: "unknown"; + readonly error: unknown; + }; + +/** Immutable JSON data in a privacy projection. @since 2.5.0 */ +export type OutboundJsonValue = + | null + | boolean + | number + | string + | readonly OutboundJsonValue[] + | { readonly [key: string]: OutboundJsonValue }; + +/** Attempt input evidence, never proof of a particular exchange's wire body. @since 2.5.0 */ +export type OutboundSenderPayload = + & { + /** Concrete sender input, not a producer history or per-exchange wire body. */ + readonly stage: "sender-payload"; + } + & ( + | { + /** A safe independent projection was established without extra processing. */ + readonly type: "available"; + /** Always a projection; no signature/proof or exact-wire claim is retained. */ + readonly representation: "privacy-projection"; + /** At least one blind-recipient field was removed from the copied value. */ + readonly redacted: boolean; + /** Recursively immutable JSON data; transport/signing input is unchanged. */ + readonly value: OutboundJsonValue; + } + | { + /** No document value is exposed, including when export is disabled. */ + readonly type: "unavailable"; + /** Opt-out, pre-snapshot failure, or unsupported privacy/context semantics. */ + readonly reason: + | "disabled" + | "not-materialized" + | "privacy-filter-unavailable"; + } + ); + +/** Result of an enabled response-body capture. Absence means disabled. @since 2.5.0 */ +export type OutboundBodyCaptureSummary = + & { + /** Bytes handed to the application, not bytes durably stored. */ + readonly bytesExposed: number; + } + & ( + | { + /** Selection, verified EOF, byte-cap uncertainty, or local lifetime end. */ + readonly type: + | "filtered" + | "no-body" + | "complete" + | "limit-reached" + | "deadline" + | "callback-returned"; + } + | { + /** Application cancellation ended capture. */ + readonly type: "consumer-stopped"; + /** Original consumer reason, including non-Error values. */ + readonly reason: unknown; + } + | { + /** Capture failed without changing received HTTP evidence. */ + readonly type: "failed"; + /** Source reading versus application callback failure. */ + readonly phase: "body" | "observer"; + /** Original capture error; it does not become the attempt's failure. */ + readonly error: unknown; + } + ); + +/** Bounded, revocable access to one selected response. @since 2.5.0 */ +export type OutboundResponseBodyEvent = + & Omit + & { + /** Immutable response metadata, independently of eventual attempt outcome. */ + readonly response: Extract< + OutboundHttpExchangeReport["outcome"], + { type: "response" } + >["response"]; + /** Pull-driven bytes; access ends when the callback settles or its limits apply. */ + readonly body: ReadableStream; + /** Capture-only lifetime signal; aborting it does not cancel transport. */ + readonly signal: AbortSignal; + }; + +/** Explicit opt-in response-body capture limits and consumer. @since 2.5.0 */ +export interface OutboundResponseBodyCapture { + /** Positive safe-integer exposure limit; exact cap without EOF stays uncertain. */ + readonly maxBytes: number; + /** Positive finite absolute lifetime in milliseconds, rounded up; at most 2^31-1. */ + readonly timeoutMs: number; + /** Actual response selector, defaulting to `all` only when enabled. */ + readonly responses?: "all" | "non-success-http-responses"; + /** + * Invoked after headers, before the terminal exchange report, with current data. + * Awaited within the absolute lifetime, including no-pull/slow-consumer time. + * Return/throw/cancel/cap/deadline revokes access even for retained locked readers. + * Errors cannot veto transport; ignoring the signal cannot be forcibly stopped. + */ + readonly onBody: ( + context: Context, + event: OutboundResponseBodyEvent, + ) => void | Promise; +} + +/** Opt-in policy for one side of an actual HTTP exchange. @since 2.5.0 */ +export interface OutboundHeaderPolicy { + /** Literal HTTP names, normalized to lowercase and deduplicated at registration. */ + readonly allow: readonly string[]; + /** + * Additional names whose selected values are always `[redacted]`. + * Credentials, signatures and cookies remain protected even without this list. + */ + readonly redact?: readonly string[]; + /** + * Positive safe-integer UTF-8 budget for `name: value\r\n` after redaction. + * Whole oversized entries are omitted; later entries remain eligible. + */ + readonly maxBytes: number; +} + +/** Immutable selected header values, using the runtime representation. @since 2.5.0 */ +export interface OutboundHeaderSnapshot { + /** Lowercase names in lexical order; credentials/signatures/cookies are redacted. */ + readonly entries: readonly (readonly [string, string])[]; + /** At least one selected entry was omitted because it exceeded the budget. */ + readonly limitReached: boolean; +} + +/** + * Immutable HTTP response facts captured from a received response. + * @since 2.5.0 + */ +export interface OutboundHttpResponseMetadata { + /** Actual HTTP status returned by fetch. */ + readonly status: number; + /** Copied HTTP status text. */ + readonly statusText: string; + /** Response URL when supplied by fetch. */ + readonly url?: string; +} + +/** + * Response evidence and finality established during a send attempt. + * @since 2.5.0 + */ +export type OutboundAttemptResponse = + | { + /** The HTTP flow selected this response before follow-up work. */ + readonly type: "final"; + /** Actual submission index of the selected response. */ + readonly exchangeIndex: number; + /** Frozen response facts, retained across later local failures. */ + readonly metadata: OutboundHttpResponseMetadata; + } + | { + /** No response headers were received, including zero-fetch attempts. */ + readonly type: "not-observed"; + } + | { + /** Continued past the response, or failed before deciding finality. */ + readonly type: "intermediate-only" | "undetermined"; + /** Latest received response, without a finality claim. */ + readonly lastObserved: { + readonly exchangeIndex: number; + readonly metadata: OutboundHttpResponseMetadata; + }; + }; + +/** + * The terminal result of one concrete send. + * + * A send that fails before submitting HTTP still constitutes an attempt, with + * zero exchanges. Each gateway send and retry gets a fresh attempt identity. + * Final response evidence follows the HTTP flow's selection, not the last + * observed status; later cache/cleanup failures can still fail the execution. + * @since 2.5.0 + */ +export type OutboundAttemptReport = OutboundSourceMetadata & { + /** Fresh opaque identity allocated at concrete sender entry. */ + readonly attemptId: string; + /** The originating invocation, independent of tracing. */ + readonly broadcast: OutboundDeliveryEvent["broadcast"]; + /** Destination obligation shared with producer delivery events. */ + readonly deliveryId: string; + /** Original planned inbox; unavailable on legacy outbox tasks. */ + readonly destinationInbox?: string; + /** Canonical portable inbox when the planned destination represents one. */ + readonly portableInbox?: string; + /** Inbox selected for this invocation of the concrete sender. */ + readonly inboxUrl: string; + /** Actual fetch submissions, including challenges and redirects; may be zero. */ + readonly httpExchangeCount: number; + /** Opt-in independent sender input projection; unavailable states remain explicit. */ + readonly senderPayload: OutboundSenderPayload; + /** Current queue execution; absent on immediate sends. */ + readonly queue?: { + readonly queueMessageId: string; + /** Present when the queue message carries a Fedify retry counter. */ + readonly queueAttempt?: number; + }; + /** Settled execution result, separate from producer preparation. */ + readonly outcome: + | { + readonly type: "succeeded"; + /** Final response selected by the sender. */ + readonly response: Extract; + } + | { + readonly type: "failed"; + /** + * Bounded failure kind and operation phase, captured before broad wrapping. + * Unknown provenance stays unknown; exception names do not establish it. + * Final non-OK HTTP evidence remains primary alongside local diagnostics. + */ + readonly failure: OutboundAttemptFailure; + /** Exact value escaping the sender, possibly different from failure.error. */ + readonly error: unknown; + /** Explicit final or last-observed response evidence, including absence. */ + readonly response: OutboundAttemptResponse; + /** Additional failures, without replacing the primary evidence. */ + readonly diagnostics?: readonly OutboundAttemptFailure[]; + }; +}; + +/** + * One actual fetch submission within a concrete send attempt. + * + * A redirect response remains an observed response even if a later hop fails. + * This report does not establish the attempt's final response. Metadata is + * copied before callback awaiting; no live Request/Response/Headers are exposed. + * Enabled body capture uses a separate revocable callback; reports hold summaries. + * Exchange callbacks run after selected capture ends and may precede attempt settlement. + * @since 2.5.0 + */ +export type OutboundHttpExchangeReport = + & OutboundSourceMetadata + & Pick< + OutboundAttemptReport, + | "attemptId" + | "broadcast" + | "deliveryId" + | "destinationInbox" + | "portableInbox" + | "inboxUrl" + | "queue" + > + & { + /** Zero-based submission order within this attempt. */ + readonly exchangeIndex: number; + /** Enabled capture result; absent when disabled or no response was received. */ + readonly bodyCapture?: OutboundBodyCaptureSummary; + /** Immutable facts of the request actually submitted. */ + readonly request: { + readonly url: string; + readonly method: string; + /** Selected header snapshot; absent when request header capture is disabled. */ + readonly headers?: OutboundHeaderSnapshot; + }; + /** A received response, including non-2xx, or the original fetch rejection. */ + readonly outcome: + | { + readonly type: "response"; + readonly response: OutboundHttpResponseMetadata & { + /** Selected snapshot; absent when response header capture is disabled. */ + readonly headers?: OutboundHeaderSnapshot; + }; + } + | { readonly type: "failed"; readonly error: unknown }; + }; + +/** + * A worker retry or deferral submission, separate from its enqueue outcome. + * @since 2.5.0 + */ +export type OutboundWorkerSchedule = { + readonly action: "retry" | "defer"; + /** Actual continuation task and Fedify retry counter. */ + readonly queue: NonNullable; + /** Actual affected routes, copied independently of the planned delivery. */ + readonly inboxUrls: readonly string[]; + /** ISO 8601 duration passed to enqueue after delay normalization. */ + readonly delay: string; + /** Per-route ISO instants retained by the scheduled continuation, if any. */ + readonly notBefore?: Readonly>; + /** Local preparation failures that contributed to this actual action. */ + readonly preparationFailures?: readonly OutboundWorkerPreparationDiagnostic[]; +}; + +/** + * One local preparation failure associated with a worker route. + * @since 2.5.0 + */ +export type OutboundWorkerPreparationDiagnostic = { + /** Route whose local setup produced the failure. */ + readonly inboxUrl: string; + /** Existing worker preparation classification and original thrown value. */ + readonly failure: Extract["failure"]; +}; + +/** + * A child outbox task submitted by a fanout worker. + * @since 2.5.0 + */ +export type OutboundWorkerChild = { + readonly action: "enqueue"; + /** Actual child task identity and initial Fedify retry counter. */ + readonly queue: NonNullable; + /** Routes submitted with this child task. */ + readonly inboxUrls: readonly string[]; + /** Stable child obligation, independent of its task identity. */ + readonly delivery?: OutboundWorkerReport["delivery"]; +}; + +/** + * An observed worker handoff, scheduling outcome, or local abandonment. + * @since 2.5.0 + */ +export type OutboundWorkerAction = + | ((OutboundWorkerSchedule | OutboundWorkerChild) & { + readonly type: "scheduled"; + }) + | ((OutboundWorkerSchedule | OutboundWorkerChild) & { + readonly type: "action-failed"; + readonly acceptance: "unknown"; + readonly error: unknown; + }) + | { + readonly type: "handoff"; + readonly action: "direct"; + /** Actual concrete sender target; entry does not establish HTTP success. */ + readonly inboxUrls: readonly string[]; + /** Planned obligation retained across gateway fallback targets. */ + readonly delivery?: OutboundWorkerReport["delivery"]; + } + | { + readonly type: "abandoned"; + readonly reason: + | "held-ttl-expired" + | "permanent-failure" + | "retry-policy-declined" + | "queue-unavailable"; + /** Routes locally abandoned; other routes of the delivery may continue. */ + readonly inboxUrls: readonly string[]; + /** Local preparation failures that contributed to this abandonment. */ + readonly preparationFailures?: + readonly OutboundWorkerPreparationDiagnostic[]; + } + | { + readonly type: "action-failed"; + readonly action: "drop"; + readonly inboxUrls: readonly string[]; + readonly error: unknown; + }; + +/** + * One fanout or outbox worker execution after its local processing settles. + * + * Emitted after the local worker path and its instrumentation settle. A normal + * return can include handled send failures or abandonment; completion does not + * establish HTTP success or backend acknowledgment. Actions retain local route + * decisions and actual enqueue settlement, separately from attempt outcomes. + * A native rethrow does not establish a scheduled continuation or future replay. + * @since 2.5.0 + */ +export type OutboundWorkerReport = OutboundSourceMetadata & { + /** Current outgoing queue role; inbox/custom-task workers are separate. */ + readonly role: "fanout" | "outbox"; + /** Actual task and available Fedify retry counter for this execution. */ + readonly queue: NonNullable; + /** Concrete sender entries, independent of enabled attempt/exchange callbacks. */ + readonly attemptsStarted: number; + /** + * Observed enqueue settlement, concrete direct-entry, and local abandonment. + * Accepted actions survive later worker failure; rejected batches leave each + * submitted child uncertain. Empty when no action was observed by settlement. + */ + readonly actions: readonly OutboundWorkerAction[]; + /** + * Caught preparation failures retained when later routes succeed or when + * the worker exits without scheduling a continuation. Omitted when none. + */ + readonly preparationFailures?: readonly OutboundWorkerPreparationDiagnostic[]; + /** + * Outbox obligation, with unavailable legacy relationships omitted. + * Fanout has no single delivery; its child actions carry their own references. + */ + readonly delivery?: Pick< + OutboundAttemptReport, + "broadcast" | "deliveryId" | "destinationInbox" | "portableInbox" + >; + /** + * Actual local settlement, separate from individual attempts and queue actions. + * Preparation failures identify the operation that produced the escaping error. + */ + readonly result: + | { readonly type: "completed" } + | { readonly type: "failed"; readonly error: unknown } + | { + readonly type: "preparation-failed"; + readonly failure: { + readonly kind: "local"; + readonly phase: + | "key-import" + | "activity-parsing" + | "context-preparation" + | "validation" + | "key-resolution" + | "signing" + | "serialization" + | "key-export" + | "request-preparation"; + readonly error: unknown; + }; + /** Exact error escaping the worker preparation operation. */ + readonly error: unknown; + }; +}; + +/** + * Optional observers of outgoing activity processing. + * + * Send, original-payload forwarding, and automatic portable replication producer + * events are observed, including outbox-context operations and fanout + * continuation handoffs. Concrete attempts and actual fetch exchanges include + * queue workers, signature challenges, redirects, and alternate gateways. + * Worker reports describe local fanout/outbox settlement and actual queue + * actions; they do not establish backend acknowledgment or future execution. + * Known IDs and provenance survive queue continuations; callback context data + * belongs to the current execution. Each callback is independently optional. + * Metadata callbacks are awaited without a default timeout. An enabled body + * consumer instead uses its explicit bounded capture lifetime. + * Their errors, including errors logging observer failures, do not replace the + * send's result. + * A metadata callback that never settles can delay execution indefinitely. Detached + * recording must handle its own rejection and shutdown; no persistence is + * guaranteed. Existing portable forwarding deadlines may return while + * background preparation, sending, and observer callbacks continue. + * @template TContextData Application context data. + * @since 2.5.0 + */ +export interface OutboundObservers { + /** + * Opt-in immutable privacy projection of concrete sender input on attempt reports. + * Disabled by default. Unsupported context/data semantics yield no raw fallback; + * this is not exact wire evidence or proof that redirected requests kept a body. + */ + readonly senderPayload?: boolean; + /** + * Independent opt-in body consumer; explicit byte/time limits are required. + * Shares one source reader with separately bounded diagnostics, without cloning + * or whole-body accumulation for the application. Finished reports hold summaries. + */ + readonly responseBodyCapture?: OutboundResponseBodyCapture; + /** + * Opt-in actual request headers for exchange reports; disabled by default. + * Protected credentials, signatures, and cookies remain redacted even if allowed. + */ + readonly requestHeaders?: OutboundHeaderPolicy; + /** Independent opt-in response header policy for exchange reports. */ + readonly responseHeaders?: OutboundHeaderPolicy; + + /** + * Observes terminal outgoing worker processing with current context/data. + * Errors are isolated; results do not replace attempt outcomes or establish + * future backend execution. Actions retain actual accepted/uncertain child + * handoffs and retry/defer/drop facts, separately from attempt outcomes. + */ + readonly onWorker?: ( + context: Context, + report: OutboundWorkerReport, + ) => void | Promise; + + /** + * Observes each actual fetch after response headers or rejection, before + * subsequent challenge/redirect handling. Non-2xx responses are still response + * outcomes. Pre-fetch failures emit no exchange. Errors cannot veto transport. + */ + readonly onHttpExchange?: ( + context: Context, + report: OutboundHttpExchangeReport, + ) => void | Promise; + + /** + * Observes settlement of concrete sends and forwarding, including workers + * and pre-fetch failures. Holds, drops, and failures before sender entry emit + * no attempt. Final HTTP evidence, local diagnostics, and the outward error + * remain distinct. Callback errors cannot replace the sender's result. + */ + readonly onAttempt?: ( + context: Context, + report: OutboundAttemptReport, + ) => void | Promise; + + /** + * Observes invocation entry and the current producer's preparation result. + * The existing context is passed without loading data for observation. + * Returning or throwing cannot veto sending. Earlier immediate sends may + * already be running when preparation finishes; their outcomes are separate. + */ + readonly onBroadcast?: ( + context: Context, + event: OutboundBroadcastEvent, + ) => void | Promise; + /** + * Observes producer-side planning, handoffs, and destination preparation + * failures, including fanout continuation handoffs. Filtered recipients do + * not become planned deliveries. Concrete attempts and exchanges have their + * own callbacks. These events do not report remote success or worker + * scheduling outcomes. + */ + readonly onDelivery?: ( + context: Context, + event: OutboundDeliveryEvent, + ) => void | Promise; +} +/** URL facts captured independently of a mutable context URL. @since 2.5.0 */ +export interface OutboundLocalContext { + /** Scheme and authority of the context URL. */ + readonly origin: string; + /** URL hostname with its optional port. */ + readonly host: string; + /** URL hostname without a port. */ + readonly hostname: string; + /** Configured canonical web origin, falling back to this context's origin. */ + readonly canonicalOrigin: string; +} + +/** The original routed HTTP inbox request that caused outgoing work. @since 2.5.0 */ +export interface OutboundInboxRequest { + /** Fresh server-generated identity, independent of Activity and trace IDs. */ + readonly inboxRequestId: string; + /** Informational logging label; externally controlled and nonunique. */ + readonly requestId: string; + /** Original request URL facts, preserved separately from current context. */ + readonly context: OutboundLocalContext; +} + +/** Explicit historical cause of outgoing work. @since 2.5.0 */ +export interface OutboundProvenance { + /** + * Exact application-owned correlation, up to 1,024 UTF-8 bytes; an empty + * string is valid. Never a Fedify deduplication key. Each new public call + * supplies its own value; queued continuations retain it. + */ + readonly dispatchId?: string; + /** Absent for independent contexts, programmatic routing, and unknown work. */ + readonly inboxRequest?: OutboundInboxRequest; + /** Original logging label for request-derived work without an inbox cause. */ + readonly originatingRequestId?: string; +} + +/** + * Observation time, optional tracing, original provenance, and current context. + * @since 2.5.0 + */ +export interface OutboundSourceMetadata { + /** Unix milliseconds at observation; wall clocks do not ensure global ordering. */ + readonly observedAt: number; + /** Valid IDs of this operation's own span, when one is available. */ + readonly trace?: { + readonly traceId: string; + readonly spanId: string; + }; + /** Immutable historical cause, distinct from this callback's context. */ + readonly provenance: OutboundProvenance; + /** URL facts of the current callback context. */ + readonly local: OutboundLocalContext; +} diff --git a/packages/fedify/src/federation/outbound-workers.test.ts b/packages/fedify/src/federation/outbound-workers.test.ts new file mode 100644 index 000000000..00374654a --- /dev/null +++ b/packages/fedify/src/federation/outbound-workers.test.ts @@ -0,0 +1,58 @@ +import { test } from "@fedify/fixture"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { captureResponseBody } from "./outbound-capture.ts"; +import { projectSenderPayload } from "./outbound-document.ts"; + +test("outbound sender payload privacy on Workers", () => { + const secret = "https://private.example/recipient"; + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + bcc: [secret], + object: { "as:bto": [secret], content: "visible" }, + }; + const result = projectSenderPayload(input); + ok(result.type === "available"); + strictEqual(result.redacted, true); + strictEqual(JSON.stringify(result.value).includes(secret), false); + ok(Object.isFrozen(result.value)); + strictEqual(input.bcc[0], secret); + deepStrictEqual( + projectSenderPayload({ + ...input, + "@context": [input["@context"], { private: "as:bcc" }], + }), + { + stage: "sender-payload", + type: "unavailable", + reason: "privacy-filter-unavailable", + }, + ); +}); + +test("outbound body capture revocation on Workers", async () => { + let cancelled = 0; + let reader!: ReadableStreamDefaultReader; + let signal!: AbortSignal; + const source = new ReadableStream({ + pull() {}, + cancel() { + cancelled++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const result = await captureResponseBody(new Response(source), { + maxBytes: 16, + timeoutMs: 10, + }, async (body, current) => { + signal = current; + reader = body.getReader(); + await rejects(reader.read()); + }); + deepStrictEqual(result, { type: "deadline", bytesExposed: 0 }); + ok(signal.aborted); + strictEqual(cancelled, 1); + strictEqual(source.locked, false); + await rejects(reader.read()); + reader.releaseLock(); +}); diff --git a/packages/fedify/src/federation/outbound-workflow.test.ts b/packages/fedify/src/federation/outbound-workflow.test.ts new file mode 100644 index 000000000..f4ee42445 --- /dev/null +++ b/packages/fedify/src/federation/outbound-workflow.test.ts @@ -0,0 +1,178 @@ +import { test } from "@fedify/fixture"; +import { Create, Person } from "@fedify/vocab"; +import { deepStrictEqual, ok, strictEqual } from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; +import { createFederation } from "./middleware.ts"; +import { MemoryKvStore } from "./kv.ts"; +import type { Message } from "./queue.ts"; +import type { + OutboundAttemptReport, + OutboundHttpExchangeReport, + OutboundWorkerReport, +} from "./outbound-report.ts"; + +const { Deno: deno } = globalThis as typeof globalThis & { + Deno?: { + permissions: { + query(options: { name: "net" }): Promise<{ state: string }>; + }; + }; +}; + +test({ + name: "outbound recorder archives real HTTP exchanges across a queued retry", + // The package Deno suite is offline; focused --allow-all and Node/Bun run it. + ignore: deno != null && + (await deno.permissions.query({ name: "net" })).state !== "granted", + async fn() { + let accepting = false; + const received: { status: number; activity: unknown }[] = []; + const receiverErrors: unknown[] = []; + const server = createServer((request, response) => { + void (async () => { + let body = ""; + for await (const chunk of request) body += chunk.toString(); + const status = accepting ? 202 : 503; + received.push({ status, activity: JSON.parse(body) }); + response.writeHead(status, { + "Content-Type": "text/plain", + "Connection": "close", + }); + response.end(accepting ? "accepted" : "retry later"); + })().catch((error) => { + receiverErrors.push(error); + response.writeHead(500); + response.end(); + }); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const inbox = new URL( + `http://127.0.0.1:${(server.address() as AddressInfo).port}/inbox`, + ); + const queued: Message[] = []; + const attempts: OutboundAttemptReport[] = []; + const exchanges: OutboundHttpExchangeReport[] = []; + const workers: OutboundWorkerReport[] = []; + const archive = new Map(); + const federation = createFederation({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + manuallyStartQueue: true, + circuitBreaker: false, + outboxRetryPolicy: () => Temporal.Duration.from({ milliseconds: 0 }), + queue: { + outbox: { + enqueue(message) { + queued.push(message); + return Promise.resolve(); + }, + listen: () => Promise.resolve(), + }, + }, + }); + federation.setOutboundObservers({ + senderPayload: true, + responseBodyCapture: { + maxBytes: 4096, + timeoutMs: 1000, + async onBody(_ctx, event) { + const key = `${event.attemptId}:${event.exchangeIndex}`; + const chunks: Uint8Array[] = []; + archive.set(key, chunks); + const reader = event.body.getReader(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + chunks.push(chunk.value.slice()); + } + } finally { + reader.releaseLock(); + } + }, + }, + onAttempt(_ctx, report) { + attempts.push(report); + }, + onHttpExchange(_ctx, report) { + exchanges.push(report); + }, + onWorker(_ctx, report) { + workers.push(report); + }, + }); + try { + await federation.createContext(new URL("https://example.com"), undefined) + .sendActivity( + { keyId: rsaPublicKey2.id!, privateKey: rsaPrivateKey2 }, + new Person({ id: new URL(inbox.origin + "/actor"), inbox }), + new Create({ + id: new URL("https://example.com/workflow/activity"), + actor: new URL("https://example.com/person2"), + }), + { fanout: "skip", dispatchId: "saved-workflow-dispatch" }, + ); + strictEqual(received.length, 0); + strictEqual(queued.length, 1); + await federation.processQueuedTask(undefined, queued.shift()!); + strictEqual(attempts.length, 1); + strictEqual(attempts[0].outcome.type, "failed"); + ok(attempts[0].outcome.response.type === "final"); + strictEqual(attempts[0].outcome.response.metadata.status, 503); + strictEqual(queued.length, 1); + const retry = workers[0].actions.find((action) => + action.type === "scheduled" && action.action === "retry" + ); + ok(retry?.type === "scheduled"); + strictEqual(retry.queue.queueAttempt, 1); + accepting = true; + await federation.processQueuedTask(undefined, queued.shift()!); + strictEqual(queued.length, 0); + strictEqual(attempts.length, 2); + strictEqual(attempts[1].outcome.type, "succeeded"); + strictEqual(attempts[0].deliveryId, attempts[1].deliveryId); + ok(attempts[0].attemptId !== attempts[1].attemptId); + deepStrictEqual(attempts[0].broadcast, attempts[1].broadcast); + strictEqual(attempts[1].queue?.queueAttempt, 1); + for (const report of attempts) { + strictEqual(report.provenance.dispatchId, "saved-workflow-dispatch"); + strictEqual(report.senderPayload.type, "available"); + strictEqual( + report.httpExchangeCount, + exchanges.filter((exchange) => + exchange.attemptId === report.attemptId + ) + .length, + ); + } + strictEqual(received.length, exchanges.length); + strictEqual(archive.size, exchanges.length); + for (const [index, exchange] of exchanges.entries()) { + deepStrictEqual(exchange.bodyCapture, { + type: "complete", + bytesExposed: received[index].status === 503 ? 11 : 8, + }); + const chunks = archive.get( + `${exchange.attemptId}:${exchange.exchangeIndex}`, + )!; + const decoder = new TextDecoder(); + strictEqual( + chunks.map((chunk) => decoder.decode(chunk)).join(""), + received[index].status === 503 ? "retry later" : "accepted", + ); + } + deepStrictEqual(workers.map((worker) => worker.attemptsStarted), [1, 1]); + deepStrictEqual(receiverErrors, []); + } finally { + const closed = new Promise((resolve, reject) => + server.close((error) => error ? reject(error) : resolve()) + ); + server.closeAllConnections(); + await closed; + } + }, +}); diff --git a/packages/fedify/src/federation/portable-delivery.test.ts b/packages/fedify/src/federation/portable-delivery.test.ts index 66d5e682e..662ab15de 100644 --- a/packages/fedify/src/federation/portable-delivery.test.ts +++ b/packages/fedify/src/federation/portable-delivery.test.ts @@ -2,11 +2,21 @@ import { mockDocumentLoader, test } from "@fedify/fixture"; import { Create, Endpoints, Note, Person, type Recipient } from "@fedify/vocab"; import { parseIri } from "@fedify/vocab-runtime"; import fetchMock from "fetch-mock"; -import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; +import { + deepStrictEqual, + notStrictEqual, + ok, + rejects, + strictEqual, +} from "node:assert/strict"; import { rsaPrivateKey2, rsaPublicKey2 } from "../testing/keys.ts"; import { MemoryKvStore } from "./kv.ts"; -import { FederationImpl, InboxContextImpl } from "./middleware.ts"; +import { ContextImpl, FederationImpl, InboxContextImpl } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; +import type { + OutboundAttemptReport, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; import { MAX_DELIVERY_GATEWAYS } from "./portable-delivery.ts"; import type { FanoutMessage, Message, OutboxMessage } from "./queue.ts"; import { extractInboxes, SendActivityError } from "./send.ts"; @@ -540,6 +550,122 @@ test("Context.sendActivity() to a portable inbox", async (t) => { fetchMock.hardReset(); }); +test("portable fallback attempts retain one delivery across concrete gateway sends", async (t) => { + for ( + const operation of ["send", "inbox-forward", "outbox-forward"] as const + ) { + for (const accepted of [true, false]) { + await t.step( + `${operation}: ${ + accepted ? "accepted fallback" : "exhausted gateways" + }`, + async () => { + const raw = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: "https://example.com/activities/forwarded", + actor: "https://example.com/person2", + extra: "preserve received payload", + }; + const reports: OutboundAttemptReport[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const submitted: unknown[] = []; + fetchMock.mockGlobal() + .post("begin:https://gw1.example/", (call) => { + submitted.push(JSON.parse(String(call.options.body))); + return 503; + }) + .post("begin:https://gw2.example/", (call) => { + submitted.push(JSON.parse(String(call.options.body))); + return accepted ? 202 : 502; + }); + try { + const federation = new FederationImpl({ + kv: new MemoryKvStore(), + allowPrivateAddress: true, + }); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onAttempt(_ctx, report) { + reports.push(report); + throw new Error( + "attempt observer cannot stop gateway fallback", + ); + }, + }); + const ctx = federation.createContext( + new URL("https://example.com/"), + ); + ok(ctx instanceof ContextImpl); + const target = portableActor({ + gateways: ["https://gw1.example", "https://gw2.example"], + }); + const pending = operation === "send" + ? ctx.sendActivity(sender, target, createActivity(), { + immediate: true, + }) + : (operation === "inbox-forward" + ? ctx.toInboxContext(null, raw, raw.id, Create.typeId.href) + : ctx.toOutboxContext("alice", raw, raw.id, Create.typeId.href)) + .forwardActivity(sender, target, { + immediate: true, + skipIfUnsigned: false, + }); + if (accepted) await pending; + else {await rejects(pending, (error: unknown) => + error instanceof SendActivityError && + error.statusCode === 502);} + strictEqual(reports.length, 2); + deepStrictEqual( + reports.map((report) => report.inboxUrl), + [ + inboxOn("https://gw1.example"), + inboxOn("https://gw2.example"), + ], + ); + notStrictEqual(reports[0].attemptId, reports[1].attemptId); + strictEqual(reports[0].deliveryId, reports[1].deliveryId); + deepStrictEqual(reports[0].broadcast, reports[1].broadcast); + for (const report of reports) { + strictEqual( + report.destinationInbox, + inboxOn("https://gw1.example"), + ); + strictEqual(report.portableInbox, PORTABLE_INBOX); + strictEqual(report.deliveryId, deliveries[0].deliveryId); + } + strictEqual(reports[0].outcome.type, "failed"); + strictEqual( + reports[1].outcome.type, + accepted ? "succeeded" : "failed", + ); + for (const [index, report] of reports.entries()) { + const response = report.outcome.response; + ok(response.type === "final"); + strictEqual( + response.metadata.status, + index === 0 ? 503 : accepted ? 202 : 502, + ); + strictEqual( + report.httpExchangeCount, + index === 1 && accepted ? 1 : 2, + ); + strictEqual(response.exchangeIndex, report.httpExchangeCount - 1); + } + if (operation !== "send") { + for (const payload of submitted) deepStrictEqual(payload, raw); + } + } finally { + fetchMock.hardReset(); + } + }, + ); + } + } +}); + test("InboxContext.forwardActivity() to a portable inbox", async (t) => { fetchMock.spyGlobal(); const activity = { @@ -584,6 +710,12 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { kv: new MemoryKvStore(), allowPrivateAddress: true, }); + const events: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); await createInboxContext(federation).forwardActivity(sender, recipient, { immediate: true, skipIfUnsigned: false, @@ -592,6 +724,15 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { inboxOn("https://gw1.example"), inboxOn("https://gw2.example"), ]); + deepStrictEqual(events.map((event) => event.stage), ["planned", "handoff"]); + strictEqual(events[0].deliveryId, events[1].deliveryId); + for (const event of events) { + strictEqual(event.destinationInbox, inboxOn("https://gw1.example")); + strictEqual(event.portableInbox, PORTABLE_INBOX); + } + const planned = events[0]; + ok(planned.stage === "planned"); + strictEqual(planned.mode, "portable-fallback"); }); await t.step("through the queue", async () => { @@ -604,6 +745,12 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { manuallyStartQueue: true, allowPrivateAddress: true, }); + const events: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + events.push(event); + }, + }); await createInboxContext(federation).forwardActivity(sender, recipient, { orderingKey: "key", skipIfUnsigned: false, @@ -617,6 +764,15 @@ test("InboxContext.forwardActivity() to a portable inbox", async (t) => { inboxOn("https://gw2.example"), ]); strictEqual(queued[0].options?.orderingKey, `key\n${DID}`); + deepStrictEqual(events.map((event) => event.stage), ["planned", "handoff"]); + strictEqual(events[0].deliveryId, events[1].deliveryId); + for (const event of events) { + strictEqual(event.destinationInbox, inboxOn("https://gw1.example")); + strictEqual(event.portableInbox, PORTABLE_INBOX); + } + const planned = events[0]; + ok(planned.stage === "planned"); + strictEqual(planned.mode, "portable-fallback"); }); fetchMock.hardReset(); diff --git a/packages/fedify/src/federation/portable-inbox.test.ts b/packages/fedify/src/federation/portable-inbox.test.ts index a22ef54b6..28b35ddd2 100644 --- a/packages/fedify/src/federation/portable-inbox.test.ts +++ b/packages/fedify/src/federation/portable-inbox.test.ts @@ -1,5 +1,9 @@ import type { InboxRequestReport } from "./inbox-report.ts"; -import { mockDocumentLoader, test } from "@fedify/fixture"; +import { + createTestTracerProvider, + mockDocumentLoader, + test, +} from "@fedify/fixture"; import { type Activity, Create, @@ -16,7 +20,13 @@ import { parseIri, type RemoteDocument, } from "@fedify/vocab-runtime"; -import { assert, assertEquals, assertFalse, assertThrows } from "@std/assert"; +import { + assert, + assertEquals, + assertFalse, + assertStrictEquals, + assertThrows, +} from "@std/assert"; import { encodeBase64 } from "byte-encodings/base64"; import fetchMock from "fetch-mock"; import { signRequest, verifyRequestDetailed } from "../sig/http.ts"; @@ -35,6 +45,12 @@ import { type KvKey, type KvStore, MemoryKvStore } from "./kv.ts"; import type { FederationOptions } from "./federation.ts"; import { createFederation } from "./middleware.ts"; import type { MessageQueue } from "./mq.ts"; +import { observeOutboundBroadcast } from "./outbound-observation.ts"; +import type { + OutboundAttemptReport, + OutboundBroadcastEvent, + OutboundDeliveryEvent, +} from "./outbound-report.ts"; import { coversDelivery, forwardPortableInboxActivity, @@ -2280,3 +2296,553 @@ test("onRequestFinished covers portable ingress and recipient lookup failures", error: failure, }); }); + +function replicationResult(events: OutboundBroadcastEvent[]) { + assertEquals(events.map((event) => event.stage), [ + "started", + "preparation-finished", + ]); + assertEquals(events.map((event) => event.operation), [ + "portable-replication", + "portable-replication", + ]); + assertEquals(events[0].broadcastId, events[1].broadcastId); + const event = events[1]; + assert(event.stage === "preparation-finished"); + return event.result; +} + +test("portable replication observers distinguish disabled, unavailable, and unclaimed work", async (t) => { + for (const scenario of ["disabled", "unavailable", "no-targets"] as const) { + await t.step(scenario, async () => { + const memory = new MemoryKvStore(); + const kv: KvStore = scenario === "unavailable" + ? { + get: (key) => memory.get(key), + set: (key, value, options) => memory.set(key, value, options), + delete: (key) => memory.delete(key), + list: (prefix) => memory.list(prefix), + } + : memory; + const queue = new RecordingQueue(); + const { federation } = setup({ + kv, + queue, + keyPairs: false, + gateways: scenario === "no-targets" ? [LOCAL] : [LOCAL, GATEWAY2], + options: { + portableInboxForwarding: { + maxTargets: scenario === "disabled" ? 0 : 10, + }, + }, + }); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + const contexts: unknown[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast(ctx, event) { + contexts.push(ctx); + events.push(event); + }, + }); + const response = await federation.fetch( + post(inboxUrl(), await signedFollow()), + { contextData: undefined }, + ); + assertEquals(response.status, 202); + assertEquals(replicationResult(events), { + type: "skipped", + reason: scenario === "disabled" + ? "forwarding-disabled" + : scenario === "unavailable" + ? "forwarding-unavailable" + : "no-forwarding-targets", + }); + assertEquals(events[0].activity, {}); + assertEquals(events[1].activity.type, Follow.typeId.href); + assert(events[1].activity.id?.startsWith("ap+ef61://")); + assert(Object.isFrozen(events[1].activity)); + assertStrictEquals(contexts[0], contexts[1]); + assertEquals(deliveries, []); + assertEquals(queue.outbox.length, 0); + }); + } +}); + +test("portable replication observers preserve lazy enqueue, unsigned fallback, and failed claims", async (t) => { + await t.step( + "readiness precedes acceptance without delaying earlier handoffs", + async () => { + const queue = new RecordingQueue(); + let accept!: () => void; + const acceptance = new Promise((resolve) => { + accept = resolve; + }); + queue.enqueue = (message) => { + queue.messages.push(message); + return queue.messages.length === 1 + ? acceptance.then(() => { + throw new Error("acceptance unknown"); + }) + : Promise.resolve(); + }; + const { federation, reports } = setup({ queue, keyPairs: false }); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + let ready!: () => void; + const prepared = new Promise((resolve) => { + ready = resolve; + }); + let release!: () => void; + const observerGate = new Promise((resolve) => { + release = resolve; + }); + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + async onBroadcast(_ctx, event) { + events.push(event); + if (event.stage === "preparation-finished") { + ready(); + await observerGate; + } + throw new Error("recorder failed"); + }, + }); + const json = await signedFollow(); + let settled = false; + const pending = federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + }) + .finally(() => { + settled = true; + }); + await prepared; + assertEquals(queue.outbox.length, 1); + assertFalse(settled); + assertEquals(replicationResult(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 2, + }); + accept(); + release(); + assertEquals((await pending).status, 202); + assertEquals(queue.outbox.length, 2); + for (const message of queue.outbox) { + assertEquals(message.activity, json); + assertEquals(message.keys, []); + assertFalse("broadcastId" in message); + const plan = deliveries.find((event) => + event.stage === "planned" && event.destinationInbox === message.inbox + ); + assert(plan); + assertEquals(message.outbound, { + broadcast: plan.broadcast, + deliveryId: plan.deliveryId, + destinationInbox: plan.destinationInbox, + portableInbox: plan.portableInbox, + }); + assertEquals( + JSON.parse(JSON.stringify(message.outbound)), + message.outbound, + ); + assertEquals(structuredClone(message.outbound), message.outbound); + } + assertEquals( + deliveries.filter((event) => event.stage === "planned").length, + 2, + ); + const handoffs = deliveries.filter((event) => event.stage === "handoff"); + assertEquals(handoffs.length, 2); + assertEquals( + handoffs.filter((event) => + event.stage === "handoff" && event.result.type === "enqueued" + ).length, + 1, + ); + assertEquals( + handoffs.filter((event) => + event.stage === "handoff" && event.result.type === "enqueue-failed" && + event.result.acceptance === "unknown" + ).length, + 1, + ); + for (const event of deliveries) { + assertEquals(event.portableInbox, `ap+ef61://${did}/users/alice/inbox`); + if (event.stage === "planned") { + assertEquals(event.mode, "portable-replication"); + } + } + assertEquals(events.length, 2); + const source = events[0].provenance.inboxRequest; + assert(source); + assertEquals(source.inboxRequestId, reports[0].inboxRequestId); + assertEquals(source.requestId, reports[0].requestId); + assertEquals(source.context.origin, LOCAL); + for (const message of queue.outbox) { + assertEquals(message.provenance?.inboxRequest, source); + } + for (const event of [...events, ...deliveries]) { + assertEquals(event.provenance.inboxRequest, source); + } + // A swallowed enqueue failure keeps its claim; redelivery is not retried. + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + assertEquals( + (await federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + })).status, + 202, + ); + assertEquals(replicationResult(events.slice(2)), { + type: "skipped", + reason: "no-forwarding-targets", + }); + assert(events[0].broadcastId !== events[2].broadcastId); + assertEquals(queue.outbox.length, 2); + }, + ); + await t.step( + "partial claim failure is incomplete planning and preserves ingress acceptance", + async () => { + const memory = new MemoryKvStore(); + const cas = memory.cas.bind(memory); + const failure = { failed: "second claim" }; + let claims = 0; + memory.cas = (...args) => { + if (args[0][1] === "portableInboxForwarding" && ++claims === 2) { + return Promise.reject(failure); + } + return cas(...args); + }; + const queue = new RecordingQueue(); + const { federation } = setup({ kv: memory, queue, keyPairs: false }); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + federation.setOutboundObservers({ + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast(_ctx, event) { + events.push(event); + }, + }); + const response = await federation.fetch( + post(inboxUrl(), await signedFollow()), + { contextData: undefined }, + ); + assertEquals(response.status, 202); + assertEquals(replicationResult(events), { + type: "failed", + failure: { + kind: "local", + phase: "destination-planning", + error: failure, + }, + planning: "incomplete", + plannedDestinationCount: 1, + }); + assertEquals(deliveries.map((event) => event.stage), ["planned"]); + assertEquals(queue.outbox.length, 0); + }, + ); + await t.step( + "a handled key failure prepares unsigned queue messages", + async () => { + const queue = new RecordingQueue(); + const parameters = forwardingParameters({ + outboxQueue: queue, + getKeys: () => Promise.reject(new Error("no keys")), + }); + const ctx = createFederation({ kv: parameters.kv }).createContext( + new URL(LOCAL), + ); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + await observeOutboundBroadcast( + ctx, + { + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast: (_ctx, event) => { + events.push(event); + }, + }, + (observation) => forwardPortableInboxActivity(parameters, observation), + "portable-replication", + ); + assertEquals(replicationResult(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + assertEquals(deliveries.map((event) => event.stage), [ + "planned", + "handoff", + ]); + const accepted = deliveries[1]; + assert( + accepted.stage === "handoff" && accepted.result.type === "enqueued", + ); + assertEquals(queue.outbox[0].keys, []); + assertStrictEquals(queue.outbox[0].activity, parameters.activity); + }, + ); +}); + +test("portable replication attempts settle independently after the forwarding deadline", async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + let failed!: () => void; + const failureReported = new Promise((resolve) => { + failed = resolve; + }); + let succeeded!: () => void; + const successReported = new Promise((resolve) => { + succeeded = resolve; + }); + let ended!: () => void; + const concreteSendsEnded = new Promise((resolve) => { + ended = resolve; + }); + const exportSpans = exporter.export.bind(exporter); + exporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (exporter.getSpans("activitypub.send_activity").length === 2) ended(); + }; + const attempts: OutboundAttemptReport[] = []; + const submitted: unknown[] = []; + fetchMock.spyGlobal() + .post(`begin:${GATEWAY2}/`, (call) => { + submitted.push(JSON.parse(String(call.options.body))); + return 503; + }) + .post(`begin:${GATEWAY3}/`, async (call) => { + submitted.push(JSON.parse(String(call.options.body))); + fetched(); + await gate; + return 202; + }); + try { + const { federation, reports } = setup({ + keyPairs: false, + options: { + allowPrivateAddress: true, + tracerProvider, + portableInboxForwarding: { deadline: { milliseconds: 0 } }, + }, + }); + let replacements = 0; + federation.setOutboundObservers({ + senderPayload: true, + onAttempt(_ctx, report) { + attempts.push(report); + if (report.outcome.type === "failed") failed(); + else succeeded(); + throw new Error("replication observer failure must be isolated"); + }, + }); + const json = await signedFollow(); + const response = await federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + }); + assertEquals(response.status, 202); + await Promise.all([fetching, failureReported]); + // The pending gateway has no synthetic timeout attempt outcome. + assertEquals(attempts.length, 1); + assertEquals(attempts[0].outcome.type, "failed"); + assert(attempts[0].outcome.response.type === "final"); + assertEquals(attempts[0].outcome.response.metadata.status, 503); + assertEquals(attempts[0].httpExchangeCount, 1); + federation.setOutboundObservers({ + onAttempt() { + replacements++; + }, + }); + release(); + await Promise.all([successReported, concreteSendsEnded]); + assertEquals(attempts.length, 2); + assertEquals(attempts[1].outcome.type, "succeeded"); + assert(attempts[1].outcome.response.type === "final"); + assertEquals(attempts[1].outcome.response.metadata.status, 202); + assertEquals(attempts[1].httpExchangeCount, 1); + assertEquals(replacements, 0); + assert(attempts[0].attemptId !== attempts[1].attemptId); + assert(attempts[0].deliveryId !== attempts[1].deliveryId); + assertEquals(attempts[0].broadcast, attempts[1].broadcast); + for (const report of attempts) { + assertEquals(report.senderPayload.stage, "sender-payload"); + if (report.senderPayload.type === "unavailable") { + assertEquals(report.senderPayload.reason, "privacy-filter-unavailable"); + } else { + assertEquals(report.senderPayload.representation, "privacy-projection"); + assert(Object.isFrozen(report.senderPayload.value)); + } + assertEquals(report.destinationInbox, report.inboxUrl); + assertEquals(report.portableInbox, `ap+ef61://${did}/users/alice/inbox`); + assertEquals( + report.provenance.inboxRequest?.inboxRequestId, + reports[0].inboxRequestId, + ); + assert(report.provenance.inboxRequest); + const span = exporter.spans.find((span) => + span.spanContext().spanId === report.trace?.spanId + ); + assert(span); + assertEquals(span.name, "activitypub.send_activity"); + } + assertEquals(submitted, [json, json]); + // Swallowed send failures retain their claims, rather than causing retries. + assertEquals( + (await federation.fetch(post(inboxUrl(), json), { + contextData: undefined, + })).status, + 202, + ); + assertEquals(submitted.length, 2); + assertEquals(replacements, 0); + } finally { + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } +}); + +test("portable replication observers retain preparation after the forwarding deadline", async (t) => { + for (const failed of [false, true]) { + await t.step( + failed ? "late preparation failure" : "late unsigned readiness", + async () => { + let release!: () => void; + const keyGate = new Promise((resolve) => { + release = resolve; + }); + const [tracerProvider, exporter] = createTestTracerProvider(); + let ended!: () => void; + const concreteSendEnded = new Promise((resolve) => { + ended = resolve; + }); + const exportSpans = exporter.export.bind(exporter); + exporter.export = (spans, callback) => { + exportSpans(spans, callback); + if (spans.some((span) => span.name === "activitypub.send_activity")) { + ended(); + } + }; + const failure = { failed: "late local preparation" }; + const parameters = forwardingParameters({ + tracerProvider, + options: resolvePortableInboxForwardingOptions({ + deadline: { milliseconds: 0 }, + }), + getKeys: async () => { + await keyGate; + if (failed) { + Object.defineProperty(parameters.activityId, "href", { + get() { + throw failure; + }, + }); + } + return []; + }, + }); + const ctx = createFederation({ kv: parameters.kv }).createContext( + new URL(LOCAL), + ); + const events: OutboundBroadcastEvent[] = []; + const deliveries: OutboundDeliveryEvent[] = []; + let finished!: () => void; + const preparation = new Promise((resolve) => { + finished = resolve; + }); + let fetched!: () => void; + const fetching = new Promise((resolve) => { + fetched = resolve; + }); + fetchMock.spyGlobal().post(`begin:${GATEWAY2}/`, () => { + fetched(); + return new Response(null, { status: 503 }); + }); + try { + const forwarded = await observeOutboundBroadcast( + ctx, + { + onDelivery(_ctx, event) { + deliveries.push(event); + }, + onBroadcast: (_ctx, event) => { + events.push(event); + if (event.stage === "preparation-finished") finished(); + }, + }, + (observation) => + forwardPortableInboxActivity(parameters, observation), + "portable-replication", + ); + assertEquals(forwarded.length, 1); + assertEquals(events.map((event) => event.stage), ["started"]); + release(); + await preparation; + assertEquals(deliveries.map((event) => event.stage), [ + "planned", + failed ? "preparation-failed" : "handoff", + ]); + assertEquals(deliveries[0].deliveryId, deliveries[1].deliveryId); + if (failed) { + const failedDelivery = deliveries[1]; + assert(failedDelivery.stage === "preparation-failed"); + assertStrictEquals(failedDelivery.failure.error, failure); + assertEquals(replicationResult(events), { + type: "failed", + failure: { + kind: "local", + phase: "request-preparation", + error: failure, + }, + planning: "complete", + plannedDestinationCount: 1, + }); + assertEquals(fetchMock.callHistory.calls().length, 0); + } else { + assertEquals(replicationResult(events), { + type: "prepared", + preparedFor: "delivery", + destinationCount: 1, + }); + await fetching; + await concreteSendEnded; + // Only now drain the late catch/logging continuation. + await new Promise((resolve) => setTimeout(resolve, 0)); + assertEquals(events.length, 2); + // The next request sees the retained claim and cannot forward again. + assertEquals(await forwardPortableInboxActivity(parameters), []); + } + } finally { + release(); + fetchMock.hardReset(); + await tracerProvider.shutdown(); + } + }, + ); + } +}); diff --git a/packages/fedify/src/federation/portable-inbox.ts b/packages/fedify/src/federation/portable-inbox.ts index 04866a11e..5c1c729da 100644 --- a/packages/fedify/src/federation/portable-inbox.ts +++ b/packages/fedify/src/federation/portable-inbox.ts @@ -1,3 +1,5 @@ +import { serializeProvenance } from "./outbound-provenance.ts"; +import type { OutboundProvenance } from "./outbound-report.ts"; import type { Actor, CryptographicKey } from "@fedify/vocab"; import { fromCompatibleEf61Id, @@ -27,6 +29,7 @@ import type { PortableInboxForwardingOptions } from "./federation.ts"; import type { KvKey, KvStore } from "./kv.ts"; import { recordOutboxEnqueue } from "./metrics.ts"; import type { MessageQueue } from "./mq.ts"; +import type { OutboundBroadcastObservation } from "./outbound-observation.ts"; import type { OutboxMessage, SenderKeyJwkPair } from "./queue.ts"; import { sendActivity, type SenderKeyPair } from "./send.ts"; @@ -228,6 +231,8 @@ export function coversDelivery(signature: RequestSignature): boolean { * Parameters for {@link forwardPortableInboxActivity}. */ export interface ForwardPortableInboxActivityParameters { + /** Explicit source carried by queued replication work, independent of observers. */ + readonly provenance?: OutboundProvenance; /** The portable inbox that received the activity. */ readonly recipient: PortableInboxRecipient; /** The received activity, exactly as it was received. */ @@ -291,10 +296,12 @@ export interface ForwardPortableInboxActivityParameters { * other gateways, that gateway has the activity already, so it is claimed * without forwarding the activity to it. See {@link identifySendingGateway}. * @param parameters The parameters. + * @param observation The in-process producer observation, if registered. * @returns The target inbox URLs that the activity was handed off to. */ -export async function forwardPortableInboxActivity( +export async function forwardPortableInboxActivity( parameters: ForwardPortableInboxActivityParameters, + observation?: OutboundBroadcastObservation, ): Promise { const logger = getLogger(["fedify", "federation", "inbox"]); const { @@ -308,10 +315,20 @@ export async function forwardPortableInboxActivity( const options = parameters.options ?? resolvePortableInboxForwardingOptions(); const { maxTargets, ttl } = options; - if (kv.cas == null || maxTargets < 1) return []; + if (kv.cas == null) { + if (observation != null) { + await observation.skipped("forwarding-unavailable"); + } + return []; + } + if (maxTargets < 1) { + if (observation != null) await observation.skipped("forwarding-disabled"); + return []; + } // The deadline covers both identifying the sending gateway and, without // an outbox queue, forwarding: const deadline = Date.now() + options.deadline.total("millisecond"); + observation?.setPhase("destination-planning"); // A compatible activity ID is canonicalized too, so that its equivalent // representations share the same forwarding claims: const canonicalActivityId = getCanonicalPortableId(activityId) ?? @@ -362,9 +379,21 @@ export async function forwardPortableInboxActivity( for (const { gateway, inbox } of targets) { if (await kv.cas(claimKey(gateway), undefined, true, { ttl })) { claimed.push(inbox); + if (observation != null) { + await observation.planDestination(inbox.href, { + actorIds: [recipient.actorId.href], + sharedInbox: false, + portableInbox: recipient.canonicalInboxId, + mode: "portable-replication", + }); + } } } - if (claimed.length < 1) return []; + observation?.planned(claimed.length); + if (claimed.length < 1) { + if (observation != null) await observation.skipped("no-forwarding-targets"); + return []; + } logger.debug( "Forwarding activity {activityId} from the portable inbox {inbox} to " + "other gateways:\n{targets}", @@ -375,9 +404,9 @@ export async function forwardPortableInboxActivity( }, ); if (outboxQueue == null) { - await forwardImmediately(parameters, claimed, deadline); + await forwardImmediately(parameters, claimed, deadline, observation); } else { - await enqueueForwarding(parameters, outboxQueue, claimed); + await enqueueForwarding(parameters, outboxQueue, claimed, observation); } return claimed; } @@ -565,10 +594,11 @@ async function resolveForwardingKey( return key; } -async function forwardImmediately( +async function forwardImmediately( parameters: ForwardPortableInboxActivityParameters, inboxes: readonly URL[], deadlineAt: number, + observation?: OutboundBroadcastObservation, ): Promise { const { activity, @@ -582,27 +612,52 @@ async function forwardImmediately( const logger = getLogger(["fedify", "federation", "inbox"]); const deadline = Math.max(0, deadlineAt - Date.now()); // The deadline also covers resolving the key, which may be slow: - const forwarding = resolveForwardingKey(parameters).then((key) => - Promise.all(inboxes.map((inbox) => - sendActivity({ - activity, - activityId: activityId.href, - activityType, - keys: key == null ? [] : [key], - inbox, - allowPrivateAddress, - specDeterminer, - meterProvider, - tracerProvider, - }).catch((error) => { - logger.error( - "Failed to forward activity {activityId} to {inbox}; it will not " + - "be forwarded to the inbox again:\n{error}", - { activityId: activityId.href, inbox: inbox.href, error }, - ); - }) - )) - ); + observation?.setPhase("key-resolution"); + const forwarding = (async () => { + try { + const key = await resolveForwardingKey(parameters); + observation?.setPhase("request-preparation"); + const sending: Promise[] = []; + for (const inbox of inboxes) { + const delivery = observation?.delivery(inbox.href); + let pending: Promise; + try { + pending = sendActivity({ + observation: delivery, + activity, + activityId: activityId.href, + activityType, + keys: key == null ? [] : [key], + inbox, + allowPrivateAddress, + specDeterminer, + meterProvider, + tracerProvider, + }); + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } + const settled = pending.catch((error) => { + logger.error( + "Failed to forward activity {activityId} to {inbox}; it will not " + + "be forwarded to the inbox again:\n{error}", + { activityId: activityId.href, inbox: inbox.href, error }, + ); + }); + // Callback awaiting must not leave an earlier rejection unhandled. + if (observation != null) settled.catch(() => {}); + sending.push(settled); + if (delivery != null) await delivery.handoff({ type: "direct" }); + } + if (observation != null) await observation.prepared("delivery"); + await Promise.all(sending); + } catch (error) { + // Preparation can settle after the deadline has returned to the caller. + if (observation != null) await observation.failed(error); + throw error; + } + })(); let timer: ReturnType | undefined; const timedOut = await Promise.race([ forwarding.then(() => false), @@ -620,10 +675,11 @@ async function forwardImmediately( } } -async function enqueueForwarding( +async function enqueueForwarding( parameters: ForwardPortableInboxActivityParameters, outboxQueue: MessageQueue, inboxes: readonly URL[], + observation?: OutboundBroadcastObservation, ): Promise { const { activity, @@ -634,6 +690,7 @@ async function enqueueForwarding( meterProvider, } = parameters; const logger = getLogger(["fedify", "federation", "inbox"]); + observation?.setPhase("key-resolution"); const key = await resolveForwardingKey(parameters); const keys: SenderKeyJwkPair[] = []; if (key != null) { @@ -650,28 +707,50 @@ async function enqueueForwarding( ); } } + observation?.setPhase("request-preparation"); startQueue?.(); const started = new Date().toISOString(); const traceContext: Record = {}; propagation.inject(context.active(), traceContext); + let preparedCount = 0; await Promise.all(inboxes.map(async (inbox) => { - const message: OutboxMessage = { - type: "outbox", - id: crypto.randomUUID(), - baseUrl, - keys, - activity, - activityId: activityId.href, - activityType, - inbox: inbox.href, - sharedInbox: false, - started, - attempt: 0, - headers: {}, - traceContext, - }; + const delivery = observation?.delivery(inbox.href); + let message: OutboxMessage; + try { + message = { + type: "outbox", + ...serializeProvenance(parameters.provenance), + id: crypto.randomUUID(), + baseUrl, + keys, + activity, + activityId: activityId.href, + activityType, + inbox: inbox.href, + ...observation?.continuation(inbox.href), + sharedInbox: false, + started, + attempt: 0, + headers: {}, + traceContext, + }; + } catch (error) { + if (delivery != null) await delivery.failed(error); + throw error; + } + if (observation != null && ++preparedCount === inboxes.length) { + // Earlier destinations may already be enqueuing; do not wait for their + // acceptance before reporting the last local message construction. + await observation.prepared("delivery"); + } try { - await outboxQueue.enqueue(message); + await (observation == null + ? outboxQueue.enqueue(message) + : observation.enqueue( + [{ inbox: inbox.href, queueMessageId: message.id }], + "outbox", + () => outboxQueue.enqueue(message), + )); } catch (error) { logger.error( "Failed to enqueue activity {activityId} to forward to {inbox}; it " + diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index 02b9af9f3..2028c24e4 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -1,6 +1,17 @@ +import type { OutboundProvenance } from "./outbound-report.ts"; import type { SerializedInboxAuthentication } from "./inbox-authentication.ts"; import type { SerializedVerifiedActivityView } from "./inbox-view.ts"; +/** Serializable grouping facts, separate from queue task identity. @internal */ +export interface SerializedOutboundDelivery { + readonly broadcast: + | { readonly type: "known"; readonly id: string } + | { readonly type: "unknown" }; + readonly deliveryId: string; + readonly destinationInbox: string; + readonly portableInbox?: string; +} + export interface SenderKeyJwkPair { readonly keyId: string; readonly privateKey: JsonWebKey; @@ -22,6 +33,8 @@ export type Message = | TaskMessage; export interface FanoutMessage { + /** Historical source facts; absent on legacy/independent messages. */ + readonly provenance?: OutboundProvenance; readonly type: "fanout"; readonly id: ReturnType; readonly baseUrl: string; @@ -32,6 +45,8 @@ export interface FanoutMessage { { readonly actorIds: readonly string[]; readonly sharedInbox: boolean; + /** Optional for continuations created before outbound observation. */ + readonly outbound?: SerializedOutboundDelivery; /** * The canonical form of the FEP-ef61 portable inbox that the inbox * stands for. @@ -65,8 +80,12 @@ export interface FanoutMessage { } export interface OutboxMessage { + /** Historical source facts; absent on legacy/independent messages. */ + readonly provenance?: OutboundProvenance; readonly type: "outbox"; readonly id: ReturnType; + /** Original delivery identity; absent on legacy/unobserved producers. */ + readonly outbound?: SerializedOutboundDelivery; readonly baseUrl: string; readonly keys: readonly SenderKeyJwkPair[]; readonly activity: unknown; @@ -134,6 +153,8 @@ export interface TaskMessage { } export interface InboxMessage { + /** Historical source facts; absent on legacy/independent messages. */ + readonly provenance?: OutboundProvenance; readonly type: "inbox"; readonly id: ReturnType; readonly baseUrl: string; diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index ad2efe2a7..ecb32dc67 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -1837,3 +1837,53 @@ function recordHops( }, }; } + +test("sendActivity() bounds a stalled diagnostic preview without changing HTTP failure", async () => { + const inbox = new URL("https://example.com/lifecycle/stalled-error"); + let reads = 0; + let cancels = 0; + let closed = 0; + let submissions = 0; + const body = new ReadableStream({ + pull(controller) { + if (reads++ === 0) { + controller.enqueue(new TextEncoder().encode("partial error")); + } + }, + cancel() { + cancels++; + return new Promise(() => {}); + }, + }, { highWaterMark: 0 }); + const error = await assertRejects(() => + sendActivity({ + activity: lifecycleActivity, + keys: unsignedKeys, + inbox, + }, { + openHop: () => + Promise.resolve({ + url: inbox, + fetch: () => { + submissions++; + return Promise.resolve( + new Response(body, { + status: 503, + headers: { "Retry-After": "1" }, + }), + ); + }, + close: () => { + closed++; + assertFalse(body.locked); + return Promise.resolve(); + }, + }), + }), SendActivityError); + assertEquals(error.statusCode, 503); + assertEquals(error.responseBody, "partial error… (timed out)"); + assertEquals(error.responseHeaders.get("Retry-After"), "1"); + assertEquals(submissions, 1); + assertEquals(cancels, 1); + assertEquals(closed, 1); +}); diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index d8356c498..400a073e1 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -1,3 +1,5 @@ +import { takeCapturedDiagnostic } from "./outbound-capture.ts"; +import { ResponseBodyPump } from "./outbound-body.ts"; import type { Recipient } from "@fedify/vocab"; import { FetchError, UrlError } from "@fedify/vocab-runtime"; import { @@ -8,6 +10,7 @@ import { import { getLogger, withContext } from "@logtape/logtape"; import { type Attributes, + context, type MeterProvider, type Span, SpanKind, @@ -22,6 +25,7 @@ import { type HttpMessageSignaturesSpecDeterminer, } from "../sig/http.ts"; import { isPortableUri } from "../sig/portable-key-id.ts"; +import type { OutboundAttemptObservation } from "./outbound-observation.ts"; import { getDurationMs, getFederationMetrics } from "./metrics.ts"; import { mergeGatewayInboxes, @@ -205,6 +209,13 @@ export interface SenderKeyPair { * Parameters for {@link sendActivity}. */ export interface SendActivityParameters { + /** Count actual sender entry for the current outgoing worker. @internal */ + readonly attemptStarted?: (inboxUrl: string) => void; + /** In-process destination observation; never serialized. @internal */ + readonly observation?: { + startAttempt(inboxUrl: string): OutboundAttemptObservation | undefined; + }; + /** * The activity to send. */ @@ -296,6 +307,9 @@ export function sendActivity( options: SendActivityParameters, deps: SendActivityDependencies = {}, ): Promise { + options.attemptStarted?.(options.inbox.href); + const attempt = options.observation?.startAttempt(options.inbox.href); + const parentSpanContext = trace.getSpanContext(context.active()); const tracerProvider = options.tracerProvider ?? trace.getTracerProvider(); const tracer = tracerProvider.getTracer(metadata.name, metadata.version); return tracer.startActiveSpan( @@ -307,6 +321,7 @@ export function sendActivity( }, }, async (span) => { + attempt?.setSpan(span, parentSpanContext); if (options.activityId != null) { span.setAttribute("activitypub.activity.id", options.activityId); } @@ -318,10 +333,17 @@ export function sendActivity( await withContext( { traceId: spanContext.traceId, spanId: spanContext.spanId }, () => - sendActivityInternal({ ...options, tracerProvider }, span, deps), + sendActivityInternal( + { ...options, tracerProvider }, + span, + deps, + attempt, + ), ); + await attempt?.succeeded(); } catch (e) { span.setStatus({ code: SpanStatusCode.ERROR, message: String(e) }); + await attempt?.failed(e); throw e; } finally { span.end(); @@ -368,6 +390,9 @@ export async function sendActivityThroughGateways( } const MAX_ERROR_RESPONSE_BODY_BYTES = 1024; +// Diagnostic previews must not indefinitely delay an already known HTTP failure. +// This budget is independent of opt-in application response-body capture. +const ERROR_RESPONSE_BODY_TIMEOUT_MS = 1000; function getActivityActorId(activity: unknown): string | undefined { if (!isRecord(activity)) return undefined; @@ -396,18 +421,30 @@ async function readLimitedResponseBody( response: Response, maxBytes: number, ): Promise { + const captured = takeCapturedDiagnostic(response); + if (captured !== undefined) return captured; if (response.body == null) { return ""; } - const reader = response.body.getReader(); + const pump = new ResponseBodyPump( + response.body, + ERROR_RESPONSE_BODY_TIMEOUT_MS, + ); const decoder = new TextDecoder(); const chunks: string[] = []; let totalBytes = 0; let truncated = false; + let timedOut = false; try { while (true) { - const { done, value } = await reader.read(); + const result = await pump.read(); + if (result == null) { + timedOut = true; + break; + } + const { done, value } = result; if (done) break; + if (value.length === 0) continue; if (totalBytes + value.length > maxBytes) { const remaining = maxBytes - totalBytes; if (remaining > 0) { @@ -422,15 +459,13 @@ async function readLimitedResponseBody( totalBytes += value.length; } } finally { - // Start cancelling the rest of a truncated body instead of leaving it - // unread. It is not awaited, since the cancellation of a tee branch - // (e.g., a body cloned by a fetch mock) may never settle: - if (truncated) reader.cancel().catch(() => {}); - reader.releaseLock(); + pump.close(); } let result = chunks.join(""); if (truncated) { result += "… (truncated)"; + } else if (timedOut) { + result += "… (timed out)"; } return result; } @@ -463,6 +498,7 @@ async function sendActivityInternal( }: SendActivityParameters, span: Span, deps: SendActivityDependencies, + attempt?: OutboundAttemptObservation, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); const federationMetrics = getFederationMetrics(meterProvider); @@ -476,17 +512,40 @@ async function sendActivityInternal( session = await openUnvalidatedHop(url); } catch (error) { if (error instanceof UrlError && error.reason === "dns") { + attempt?.failure({ kind: "transport", phase: "validation", error }); logger.error("DNS resolution failed for URL: {url}", { url, error }); const failure = new FetchError(url, error.message); failure.cause = error; throw failure; } if (error instanceof UrlError) { + if (error.reason === "disallowed") { + attempt?.failure({ kind: "policy", phase: "validation", error }); + } logger.error("Disallowed private URL: {url}", { url, error }); } throw error; } - return session; + if (attempt == null) return session; + return { + url: session.url, + async fetch(input, init) { + try { + return await session.fetch(input, init); + } catch (error) { + attempt.failure({ kind: "transport", phase: "request", error }); + throw error; + } + }, + async close() { + try { + await session.close(); + } catch (error) { + attempt.failure({ kind: "local", phase: "cleanup", error }); + throw error; + } + }, + }; } let initialSession: PublicFetchSession; @@ -514,14 +573,33 @@ async function sendActivityInternal( return openHop(url); }; try { - headers = new Headers(headers); - headers.set("Content-Type", "application/activity+json"); - const request = new Request(inbox, { - method: "POST", - headers, - body: JSON.stringify(activity), - redirect: "manual", - }); + let request: Request; + let body: string | undefined; + try { + headers = new Headers(headers); + headers.set("Content-Type", "application/activity+json"); + } catch (error) { + attempt?.failure({ kind: "local", phase: "request-preparation", error }); + throw error; + } + try { + attempt?.snapshotSenderPayload(activity); + body = JSON.stringify(activity); + } catch (error) { + attempt?.failure({ kind: "local", phase: "serialization", error }); + throw error; + } + try { + request = new Request(inbox, { + method: "POST", + headers, + body, + redirect: "manual", + }); + } catch (error) { + attempt?.failure({ kind: "local", phase: "request-preparation", error }); + throw error; + } let rsaKey: SenderKeyPair | null = null; for (const key of keys) { if (key.privateKey.algorithm.name === "RSASSA-PKCS1-v1_5") { @@ -549,22 +627,34 @@ async function sendActivityInternal( }, ); } + const consume = (response: Response) => { + return consumeSendActivityResponse(response, inbox); + }; let outcome: SendActivityOutcome; try { outcome = rsaKey == null ? await fetchWithValidatedRedirects( request, openDeliveryHop, - (response) => consumeSendActivityResponse(response, inbox), + consume, + attempt, ) : await doubleKnockScoped( request, rsaKey, { tracerProvider, specDeterminer }, openDeliveryHop, - (response) => consumeSendActivityResponse(response, inbox), + consume, + attempt == null ? undefined : { + fetch: (request, submit) => attempt.fetch(request, submit), + response: (response, disposition) => + attempt.response(response, disposition), + error: (phase, error) => + attempt.failure({ kind: "local", phase, error }), + }, ); } catch (error) { + attempt?.unknown(error); // A refused redirect remains a policy error rather than a transport // failure, just as a refused initial inbox does. const failure = error instanceof FetchError || error instanceof UrlError @@ -590,7 +680,7 @@ async function sendActivityInternal( "{statusText}):\n{error}", { activityId, inbox: inbox.href, status, statusText, error }, ); - throw new SendActivityError( + const failure = new SendActivityError( inbox, status, `Failed to send activity ${activityId} to ${inbox.href} ` + @@ -598,6 +688,8 @@ async function sendActivityInternal( error, responseHeaders, ); + attempt?.failure({ kind: "http", phase: "response", error: failure }); + throw failure; } deliverySuccess = true; @@ -654,7 +746,7 @@ function cancelResponseBody(response: Response, url: string): void { /** * Reads what is needed from the final response of a delivery while its * session is open: the body of a successful response is cancelled, and at - * most 1 KiB of an error response body is read. + * most 1 KiB of an error response body is read within its diagnostic budget. */ async function consumeSendActivityResponse( response: Response, @@ -691,12 +783,14 @@ async function consumeSendActivityResponse( * @param request The request to send. * @param openHop The function to open a session for each hop. * @param consume The function to consume the final response. + * @param attempt The in-process attempt observation, if registered. * @returns The value returned by `consume`. */ export async function fetchWithValidatedRedirects( request: Request, openHop: DoubleKnockOpenHop, consume: (response: Response) => Promise, + attempt?: OutboundAttemptObservation, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); let body: string | undefined = await request.clone().text(); @@ -705,7 +799,10 @@ export async function fetchWithValidatedRedirects( let response: Response; try { // Bun also needs the redirect option on fetch() itself. - response = await session.fetch(request, { redirect: "manual" }); + response = await (attempt == null + ? session.fetch(request, { redirect: "manual" }) + : attempt.fetch(request, () => + session.fetch(request, { redirect: "manual" }))); } catch (error) { await closeAfterError(session, logger); throw error; @@ -714,6 +811,7 @@ export async function fetchWithValidatedRedirects( if ( ![301, 302, 303, 307, 308].includes(response.status) || location == null ) { + attempt?.response(response, "final"); try { return await consume(response); } catch (error) { @@ -728,6 +826,7 @@ export async function fetchWithValidatedRedirects( } } // The current hop is closed before the next one is validated and opened: + attempt?.response(response, "intermediate"); cancelResponseBody(response, request.url); await session.close(); if (redirects >= 20) throw new TypeError("Too many redirects"); @@ -796,7 +895,9 @@ export class SendActivityError extends Error { * The response body from the inbox, if any. Note that this may be * truncated to a maximum of 1 KiB to prevent excessive memory consumption * when remote servers return large error pages (e.g., Cloudflare error pages). - * If truncated, the string will end with `"… (truncated)"`. + * If truncated, the string will end with `"… (truncated)"`. Diagnostic reading + * has a separate one-second budget; a deadline retains any partial preview and + * appends `"… (timed out)"`. This is independent of application body capture. */ readonly responseBody: string; diff --git a/packages/fedify/src/sig/http.test.ts b/packages/fedify/src/sig/http.test.ts index 1ef7cfd82..d84fa379e 100644 --- a/packages/fedify/src/sig/http.test.ts +++ b/packages/fedify/src/sig/http.test.ts @@ -3280,6 +3280,59 @@ const lifecycleIdentity = { privateKey: rsaPrivateKey2, }; +test("doubleKnockScoped() observes each actual GET and HEAD transport retry", async (t) => { + for (const method of ["GET", "HEAD"]) { + await t.step(method, async () => { + const url = "https://example.com/transport-observation"; + const observed: Request[] = []; + let submissions = 0; + let closes = 0; + const result = await doubleKnockScoped( + new Request(url, { method }), + { + keyId: rsaPublicKey2.id!, + privateKey: rsaPrivateKey2, + }, + {}, + () => + Promise.resolve({ + url: new URL(url), + fetch() { + submissions++; + return submissions === 1 + ? Promise.reject(new TypeError("temporary failure")) + : Promise.resolve(new Response(null, { status: 202 })); + }, + close() { + closes++; + return Promise.resolve(); + }, + }), + (response) => Promise.resolve(response.status), + { + fetch(request, submit) { + observed.push(request); + return submit(); + }, + response() {}, + error() { + throw new Error("transport retry must not be a local failure"); + }, + }, + ); + assertEquals(result, 202); + assertEquals(submissions, 2); + assertEquals(observed.length, 2); + assert( + observed[0] === observed[1], + "transport retry reuses the signed request", + ); + assertEquals(observed[0].method, method); + assertEquals(closes, 1); + }); + } +}); + test("doubleKnockScoped() keeps challenge retries and fallback in the validated hop", async () => { const url = "https://example.com/lifecycle/challenge-fallback"; const hops = recordHops((_, index): RecordedResponseInit => diff --git a/packages/fedify/src/sig/http.ts b/packages/fedify/src/sig/http.ts index ff825fbff..94d1ec412 100644 --- a/packages/fedify/src/sig/http.ts +++ b/packages/fedify/src/sig/http.ts @@ -2110,24 +2110,68 @@ async function createRedirectRequest( }); } +/** In-process hooks for scoped outbound delivery, outside the public options. */ +export interface DoubleKnockObservation { + fetch(request: Request, submit: () => Promise): Promise; + response(response: Response, disposition: "final" | "intermediate"): void; + error( + phase: "signing" | "spec-lookup" | "post-response", + error: unknown, + ): void; +} + +function observeDoubleKnockOperation( + observation: DoubleKnockObservation | undefined, + phase: "signing" | "spec-lookup" | "post-response", + operation: () => T | Promise, +): T | Promise { + if (observation == null) return operation(); + try { + return Promise.resolve(operation()).catch((error) => { + observation.error(phase, error); + throw error; + }); + } catch (error) { + observation.error(phase, error); + throw error; + } +} + +function submitDoubleKnockRequest( + session: PublicFetchSession, + request: Request, + signal: AbortSignal | undefined, + observation: DoubleKnockObservation | undefined, +): Promise { + const submit = () => + session.fetch(request, { + // Since Bun has a bug that ignores the `Request.redirect` option, + // to work around it we specify `redirect: "manual"` here too: + // https://github.com/oven-sh/bun/issues/10754 + redirect: "manual", + signal, + }); + return observation == null ? submit() : observation.fetch(request, submit); +} + async function fetchDoubleKnockRequest( session: PublicFetchSession, request: Request, signedRequest: Request, - signal?: AbortSignal, + signal: AbortSignal | undefined, + observation?: DoubleKnockObservation, ): Promise { const maxAttempts = request.method === "GET" || request.method === "HEAD" ? 2 : 1; for (let attempt = 1;; attempt++) { try { - return await session.fetch(signedRequest, { - // Since Bun has a bug that ignores the `Request.redirect` option, - // to work around it we specify `redirect: "manual"` here too: - // https://github.com/oven-sh/bun/issues/10754 - redirect: "manual", + return await submitDoubleKnockRequest( + session, + signedRequest, signal, - }); + observation, + ); } catch (error) { const abortedSignal = getAbortedSignal( signal, @@ -2261,6 +2305,7 @@ export async function doubleKnock( * @param openHop The function to open a session for each hop. The session of * a hop is closed before the next hop is opened. * @param consume The function to consume the final response. + * @param observation The in-process transport/finality hooks, if registered. * @returns The value returned by `consume`. */ export async function doubleKnockScoped( @@ -2269,12 +2314,14 @@ export async function doubleKnockScoped( options: DoubleKnockOptions, openHop: DoubleKnockOpenHop, consume: (response: Response) => Promise, + observation?: DoubleKnockObservation, ): Promise { const { response, session } = await doubleKnockCore( request, identity, options, openHop, + observation, ); try { return await consume(response); @@ -2332,6 +2379,7 @@ async function doubleKnockCore( identity: { keyId: URL; privateKey: CryptoKey }, options: DoubleKnockOptions, openHop: DoubleKnockOpenHop, + observation?: DoubleKnockObservation, ): Promise { const { specDeterminer, log, tracerProvider, signal } = options; signal?.throwIfAborted(); @@ -2352,13 +2400,22 @@ async function doubleKnockCore( const origin = new URL(request.url).origin; const firstTrySpec: HttpMessageSignaturesSpec = specDeterminer == null ? "rfc9421" - : await specDeterminer.determineSpec(origin); + : await observeDoubleKnockOperation( + observation, + "spec-lookup", + () => specDeterminer.determineSpec(origin), + ); signal?.throwIfAborted(); - let signedRequest = await signRequest( - request, - identity.privateKey, - identity.keyId, - { spec: firstTrySpec, tracerProvider, body }, + let signedRequest = await observeDoubleKnockOperation( + observation, + "signing", + () => + signRequest( + request, + identity.privateKey, + identity.keyId, + { spec: firstTrySpec, tracerProvider, body }, + ), ); signal?.throwIfAborted(); log?.(signedRequest); @@ -2367,6 +2424,7 @@ async function doubleKnockCore( request, signedRequest, signal, + observation, ); const firstRedirected = isRedirectResponse(response); if ( @@ -2431,22 +2489,33 @@ async function doubleKnockCore( } } if (fulfilled && challengeRequest != null) { + observation?.response(response, "intermediate"); signal?.throwIfAborted(); // The challenge retry stays on the current validated hop. cancelResponseBody(response, request.url); signedRequest = challengeRequest; log?.(signedRequest); - response = await session.fetch(signedRequest, { - redirect: "manual", + response = await submitDoubleKnockRequest( + session, + signedRequest, signal, - }); + observation, + ); } if (fulfilled && !isRedirectResponse(response)) { if (response.status < 300) { - await specDeterminer?.rememberSpec(origin, "rfc9421"); + observation?.response(response, "final"); + await observeDoubleKnockOperation( + observation, + "post-response", + () => + specDeterminer?.rememberSpec(origin, "rfc9421") ?? + Promise.resolve(), + ); return { response, session }; } if (response.status !== 400 && response.status !== 401) { + observation?.response(response, "final"); return { response, session }; } } @@ -2454,6 +2523,7 @@ async function doubleKnockCore( } if (!isRedirectResponse(response)) { + observation?.response(response, "intermediate"); // verification failed; retry with the other spec of HTTP Signatures // (double-knocking; see https://swicg.github.io/activitypub-http-signature/#how-to-upgrade-supported-versions) const spec = firstTrySpec === "draft-cavage-http-signatures-12" @@ -2471,11 +2541,16 @@ async function doubleKnockCore( // The retry goes to the same URL, so it uses the same session: cancelResponseBody(response, request.url); signal?.throwIfAborted(); - signedRequest = await signRequest( - request, - identity.privateKey, - identity.keyId, - { spec, tracerProvider, body }, + signedRequest = await observeDoubleKnockOperation( + observation, + "signing", + () => + signRequest( + request, + identity.privateKey, + identity.keyId, + { spec, tracerProvider, body }, + ), ); signal?.throwIfAborted(); log?.(signedRequest); @@ -2484,21 +2559,39 @@ async function doubleKnockCore( request, signedRequest, signal, + observation, ); if ( !isRedirectResponse(response) && response.status !== 400 && response.status !== 401 ) { - await specDeterminer?.rememberSpec(origin, spec); + observation?.response(response, "final"); + await observeDoubleKnockOperation( + observation, + "post-response", + () => + specDeterminer?.rememberSpec(origin, spec) ?? Promise.resolve(), + ); } } } else if (!firstRedirected) { - await specDeterminer?.rememberSpec(origin, firstTrySpec); + observation?.response(response, "final"); + await observeDoubleKnockOperation( + observation, + "post-response", + () => + specDeterminer?.rememberSpec(origin, firstTrySpec) ?? + Promise.resolve(), + ); + } + if (!isRedirectResponse(response)) { + observation?.response(response, "final"); + return { response, session }; } - if (!isRedirectResponse(response)) return { response, session }; // Follow redirects manually to get the final URL. The current hop is // closed before the next one is validated and opened: + observation?.response(response, "intermediate"); cancelResponseBody(response, request.url); await session.close(); if (redirected >= maximumRedirection) { diff --git a/packages/testing/README.md b/packages/testing/README.md index a352a5479..6b32b3431 100644 --- a/packages/testing/README.md +++ b/packages/testing/README.md @@ -87,6 +87,12 @@ const sent = context.getSentActivities(); console.log(sent[0].activity); ~~~~ +Both `context.getSentActivities()` and `federation.sentActivities` retain the +optional `dispatchId` supplied to `sendActivity()`. Reusing a dispatch ID +records another send, so tests can check correlation across application-managed +retries. Mock federations also accept `setOutboundObservers()` registration, +but mock sends do not invoke those callbacks. + ### Helper functions The package also exports helper functions for creating various context types: diff --git a/packages/testing/src/mock.test.ts b/packages/testing/src/mock.test.ts index 799d451a0..fde5aadc6 100644 --- a/packages/testing/src/mock.test.ts +++ b/packages/testing/src/mock.test.ts @@ -177,6 +177,101 @@ test("getSentActivities returns sent activities", async () => { deepStrictEqual(mockFederation.sentActivities[0].sentOrder, 1); }); +test("sent activity records preserve dispatch IDs", async () => { + const federation = createFederation(); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sender = { identifier: "alice" }; + const recipient = new Person({ + id: new URL("https://example.com/users/bob"), + }); + const activity = new Create({ + id: new URL("https://example.com/activities/dispatch-id"), + actor: new URL("https://example.com/users/alice"), + }); + + await context.sendActivity(sender, recipient, activity, { + dispatchId: "same-id", + }); + await context.sendActivity(sender, recipient, activity, { + dispatchId: "same-id", + }); + await context.sendActivity(sender, recipient, activity); + await context.sendActivity(sender, recipient, activity, { dispatchId: "" }); + + const contextRecords = context.getSentActivities(); + const federationRecords = federation.sentActivities; + deepStrictEqual(contextRecords.length, 4); + deepStrictEqual(federationRecords.length, 4); + strictEqual(contextRecords[0].dispatchId, "same-id"); + strictEqual(contextRecords[1].dispatchId, "same-id"); + strictEqual(federationRecords[0].dispatchId, "same-id"); + strictEqual(federationRecords[1].dispatchId, "same-id"); + strictEqual(contextRecords[3].dispatchId, ""); + strictEqual(federationRecords[3].dispatchId, ""); + strictEqual(Object.hasOwn(contextRecords[2], "dispatchId"), false); + strictEqual(Object.hasOwn(federationRecords[2], "dispatchId"), false); +}); + +test("sendActivity validates dispatch IDs before recording", async () => { + const federation = createFederation(); + const context = federation.createContext( + new URL("https://example.com"), + undefined, + ); + const sender = { identifier: "alice" }; + const recipient = new Person({ + id: new URL("https://example.com/users/bob"), + }); + const activity = new Create({ + id: new URL("https://example.com/activities/invalid-dispatch-id"), + actor: new URL("https://example.com/users/alice"), + }); + + for (const dispatchId of [null, 1, {}]) { + await rejects( + context.sendActivity(sender, recipient, activity, { + dispatchId, + } as never), + { + name: "TypeError", + message: "dispatchId must be a string.", + }, + ); + } + for (const dispatchId of ["x".repeat(1025), "한".repeat(342)]) { + await rejects( + context.sendActivity(sender, recipient, activity, { dispatchId }), + { + name: "RangeError", + message: "dispatchId must not exceed 1,024 UTF-8 bytes.", + }, + ); + } + + strictEqual(context.getSentActivities().length, 0); + strictEqual(federation.sentActivities.length, 0); + + for (const dispatchId of ["x".repeat(1024), "한".repeat(341) + "a"]) { + await context.sendActivity(sender, recipient, activity, { dispatchId }); + } + deepStrictEqual( + federation.sentActivities.map((record) => record.sentOrder), + [1, 2], + ); + const expectedDispatchIds = ["x".repeat(1024), "한".repeat(341) + "a"]; + deepStrictEqual( + context.getSentActivities().map((record) => record.dispatchId), + expectedDispatchIds, + ); + deepStrictEqual( + federation.sentActivities.map((record) => record.dispatchId), + expectedDispatchIds, + ); +}); + test("reset clears sent activities", async () => { const mockFederation = createFederation(); const context = mockFederation.createContext( @@ -207,6 +302,15 @@ test("reset clears sent activities", async () => { deepStrictEqual(mockFederation.sentActivities.length, 0); }); +test("MockFederation accepts outbound observer registration", () => { + const federation = createFederation(); + federation.setOutboundObservers({ + onBroadcast: () => {}, + onDelivery: () => {}, + }); + federation.setOutboundObservers({}); +}); + test("MockFederation accepts inbox request observation registration", () => { const federation = createFederation(); const setters = federation.setInboxListeners("/users/{identifier}/inbox"); diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index fbcbe71c1..b1161c54c 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -7,6 +7,7 @@ import type { FederationStartQueueOptions, GetObjectOptions, Message, + OutboundObservers, ParseUriOptions, ParseUriResult, PortableRequest, @@ -274,6 +275,8 @@ interface SentActivity { activity: Activity; /** The raw forwarded payload, if preserved by the caller. */ rawActivity?: unknown; + /** Application-provided correlation identifier, if present. */ + dispatchId?: string; /** The order in which the activity was sent (auto-incrementing counter). */ sentOrder: number; } @@ -308,6 +311,7 @@ interface TestContext recipients: any; activity: Activity; rawActivity?: unknown; + dispatchId?: string; }>; reset(): void; } @@ -592,6 +596,10 @@ class MockFederation implements Federation { return setters; } + setOutboundObservers(_observers: OutboundObservers): void { + // Registration compatibility; mock sends do not run transport observers. + } + setInboxListeners(inboxPath: any, sharedInboxPath?: string): any { this.inboxPath = inboxPath; this.sharedInboxPath = sharedInboxPath; @@ -1205,6 +1213,7 @@ class MockContext implements Context { recipients: any; activity: Activity; rawActivity?: unknown; + dispatchId?: string; }> = []; constructor( @@ -1723,11 +1732,25 @@ class MockContext implements Context { activity: Activity, options?: any, ): Promise { + const dispatchId: unknown = options?.dispatchId; + if (dispatchId !== undefined && typeof dispatchId !== "string") { + return Promise.reject(new TypeError("dispatchId must be a string.")); + } + if ( + dispatchId !== undefined && + (dispatchId.length > 1024 || + new TextEncoder().encode(dispatchId).byteLength > 1024) + ) { + return Promise.reject( + new RangeError("dispatchId must not exceed 1,024 UTF-8 bytes."), + ); + } this.sentActivities.push({ sender, recipients, activity, rawActivity: options?.rawActivity, + ...(dispatchId === undefined ? {} : { dispatchId }), }); // If this is a MockFederation, also record it there @@ -1738,6 +1761,7 @@ class MockContext implements Context { queue: queued ? "outbox" : undefined, activity, rawActivity: options?.rawActivity, + ...(dispatchId === undefined ? {} : { dispatchId }), sentOrder: ++this.federation.sentCounter, }); } @@ -1765,6 +1789,7 @@ class MockContext implements Context { recipients: any; activity: Activity; rawActivity?: unknown; + dispatchId?: string; }> { return [...this.sentActivities]; }