From e123cb1efb932e4df1f2e3f7d8abbc1edc185eaa Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 11:59:56 -0700 Subject: [PATCH 01/13] fix(durable): dispatch effects from Start, Tick and RunService Only Fire dispatched domain effects. Tick and RunService now record a step through the same helper as Fire, then dispatch its effects as Fire does, after the append and before any checkpoint. A failed dispatch fails the Handle before its checkpoint, so a later checkpoint can't compact the effect away, and Recover retries it. Start applies the initial entry's effects before the baseline, so a handler failure leaves no instance. EffectInstance gives a handler its instance id, and state.ServiceRunner gains WithResultTransform so a service's result reaches the machine in the form a replay supplies. Effects from a parent transition an actor drives are still not dispatched. Relates to #214. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 22 ++- durable/CHANGELOG.md | 22 ++- durable/README.md | 22 ++- durable/doc.go | 9 +- durable/effect.go | 57 ++++++- durable/effect_test.go | 179 ++++++++++++++++++++++ durable/errors.go | 7 +- durable/runner.go | 176 ++++++++++++++------- durable/runner_options.go | 13 +- durable/service.go | 103 ++++++++++--- durable/service_test.go | 55 +++++++ durable/store.go | 2 +- examples/fooddelivery/fooddelivery.go | 13 +- state/CHANGELOG.md | 2 + state/runner.go | 23 ++- 15 files changed, 586 insertions(+), 119 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 8a0c6af0..2460056f 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -30,10 +30,12 @@ implement the `Store` interface out of tree. - **Deterministic replay.** A recovered instance is byte-identical to one that never crashed, because recovery replays the recorded driving events and nondeterministic results rather than re-executing their sources. -- **Exactly-once effects.** A domain effect is applied exactly once over the +- **Exactly-once effects.** A domain effect from a step (not an initial entry, which is at-least-once, or a parent transition an actor drives, which is not dispatched) is applied exactly once over the instance's lifetime (the live run plus any number of recoveries), even though - the replay loop is at-least-once. Each effect carries a deterministic - `EffectID` and is deduplicated through the `Store`'s dispatch set. + the replay loop is at-least-once. The exception is a crash or failed mark after the + handler returned, or a handoff between owners, where it can apply twice, so keep + handlers idempotent on `EffectInstance` plus the `EffectID`. Each effect carries a + deterministic `EffectID` and is deduplicated through the `Store`'s dispatch set. - **Durability across restart.** Every `Fire` step is write-ahead appended to the `Store` before it is acknowledged, so a crash after a successful `Fire` never loses the step. Periodic checkpoints bound the tail that recovery replays. @@ -74,7 +76,7 @@ the first time, and replayed verbatim on recovery: | **Clock** (timers) | `WithRunnerClock` | `JournalClockRead`, replayed so timers fire at the same recorded instants, wall-clock-independent; armed deadlines survive checkpoint compaction | | **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; a recorded result replays through the kernel's settle seam without re-running the service, and an unrecorded one (a context canceled during the service or its settle, a crash, or a failed `Append` whose write didn't land) runs again when the host calls `RunService` | | **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; recovery re-fires a recorded parent transition; a settled or stopped actor doesn't re-run; a running `InvokeActor` actor restarts from its initial state on any `Recover`, and a `Spawn` actor doesn't return | -| **Domain effects** | `WithEffectHandler` | dispatch set, applied exactly once via deterministic `EffectID` dedup | +| **Domain effects** | `WithEffectHandler` | dispatch set, a step's effects applied exactly once via deterministic `EffectID` dedup, with the exceptions under Exactly-once effects | Use `WithCheckpointEvery(n)` to tune how often a full snapshot is written: a shorter interval bounds recovery replay, a longer one cuts checkpoint cost. @@ -153,15 +155,19 @@ failed the Handle returns `ErrHandleFailed` saying it was recorded; `Recover` se its effects. To move a healthy Handle's instance to a new Handle, stop calling the old one and let its calls return first: a step can start after `Drain` returns. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer -owner holds the instance, so stop. +owner holds the instance, so stop. `Recover` retries an unapplied effect, and returns +`ErrEffectDispatch` while the handler still fails. ## A note on serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` -observes the JSON-decoded shape on a replayed `onDone`. A `WithEventData` payload is -decoded from its JSON form live as well as on replay, so a reducer gets the same value -both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per +observes the JSON-decoded shape of actor done-data on a replayed `onDone`. A service's +result is JSON-decoded live as well as on replay, so a reducer gets the same value both +ways: a struct is a map and an integer a `float64`, which loses precision above 2^53, and +an error an error with the same message, which `errors.Is` no longer matches. A +`WithEventData` payload is decoded from its JSON form live as well as on replay, so a +reducer gets the same value both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own form, so `Handle.Fire` refuses a struct or an integer rather than hand the reducer a map or a `float64`. An error stays an error with the same message, with any invalid UTF-8 diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 60405754..f0ae203d 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -25,6 +25,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. makes `Recover` spawn ended actors again, and replay arm nothing for a Record without its `Format`. MemStore and FileStore keep all three. +- **`EffectInstance`** reads the instance from the ctx an `EffectHandler` receives, because + effect ids repeat across instances. - **Fencing.** `WithFence` on `Store.Append` and `WithCheckpointFence` on `Store.Checkpoint` carry a caller-issued token, and a Store refuses a token lower than the highest it has seen for the instance with `ErrFenced`, writing nothing. @@ -98,7 +100,17 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A failed effect dispatch fails the Handle**, and so does an effect handler that panics, which now returns `ErrEffectDispatch` instead of panicking out of `Fire`. The effect stays unmarked in the journal, so no later checkpoint compacts it away, and - `Recover` dispatches it again. + `Recover` dispatches it again, failing with `ErrEffectDispatch` while the handler still + does. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect + again. The handler's error now matches through `errors.Is`. +- **A `KindedEffect` that can't marshal fails `Tick` and `RunService`,** as it already + failed `Fire`, even with no handler wired. +- **A service result reaches `Assign` decoded, live as well as on replay,** so a retried + effect can't differ from the first attempt: a struct arrives as a map, an integer as a + `float64`, which loses precision above 2^53, and an error as an error with the same + message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` + method that panics settles with a message saying so. The journal records this decoded + value. - **A step that may not have been recorded fails the Handle.** That call and every later one return `ErrHandleFailed`, wrapping the cause. A step that was recorded while another goroutine failed the Handle also returns `ErrHandleFailed`, saying the step was @@ -168,6 +180,14 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. the step that tried it. A retry the Store refuses as not advancing means the first write landed, and counts as written. Any other checkpoint failure is returned, and after that only the next step the policy makes due tries again. +- **Effects reach the handler from `Start`, `Tick` and `RunService`.** They were only + dispatched from `Fire`. `Start` applies the initial entry's effects after encoding the + baseline and before writing it, and only if the entry succeeded. Those effects are + at-least-once: a handler failure, a failed baseline write or overlapping `Start` calls + re-apply them, so key any dedupe on `EffectInstance` plus the effect id. Effects from a + parent transition an actor drives, including during `Fire`, are still not dispatched. + The first `Recover` of an older journal applies effects from uncompacted `Tick` and + service steps. - **A Tick, RunService or DeliverToActor whose context ended partway isn't journaled.** A canceled `RunService`, `Tick` or `DeliverToActor` used to record a result the live instance never applied, which recovery then committed, or for an actor, failed on. Each now returns `ErrHandleFailed` diff --git a/durable/README.md b/durable/README.md index 643d40cc..ee5cc513 100644 --- a/durable/README.md +++ b/durable/README.md @@ -31,10 +31,12 @@ backends implement the `Store` interface out of tree. - **Deterministic replay**: a recovered instance is byte-identical to one that never crashed, because recovery replays the recorded driving events and nondeterministic results rather than re-executing their sources. -- **Exactly-once effects**: a domain effect is applied exactly once over the +- **Exactly-once effects**: a domain effect from a step (not an initial entry, which is at-least-once, or a parent transition an actor drives, which is not dispatched) is applied exactly once over the instance's lifetime (the live run plus any number of recoveries), even though - the replay loop is at-least-once. Each effect carries a deterministic - `EffectID` and is deduplicated through the `Store`'s dispatch set. + the replay loop is at-least-once. The exception is a crash or failed mark after the + handler returned, or a handoff between owners, where it can apply twice, so keep + handlers idempotent on `EffectInstance` plus the `EffectID`. Each effect carries a + deterministic `EffectID` and is deduplicated through the `Store`'s dispatch set. - **Durability across restart**: every `Fire` step is write-ahead appended to the `Store` before it is acknowledged, so a crash after a successful `Fire` never loses the step. Periodic checkpoints bound the tail recovery replays. @@ -84,7 +86,7 @@ the first time, and replayed verbatim on recovery: | **Clock** (timers) | `WithRunnerClock` | `JournalClockRead`, replayed so timers fire at the same recorded instants, wall-clock-independent; armed deadlines survive checkpoint compaction | | **Invoked services** | `WithServiceRegistry` + `Handle.RunService` | `JournalServiceResult`; a recorded result replays through the kernel's settle seam without re-running the service, and an unrecorded one (a context canceled during the service or its settle, a crash, or a failed `Append` whose write didn't land) runs again when the host calls `RunService` | | **Child-machine actors** | `WithActorPalette` + `Handle.DeliverToActor` | `JournalActorMessage`; recovery re-fires a recorded parent transition; a settled or stopped actor doesn't re-run; a running `InvokeActor` actor restarts from its initial state on any `Recover`, and a `Spawn` actor doesn't return | -| **Domain effects** | `WithEffectHandler` | dispatch set, applied exactly once via deterministic `EffectID` dedup | +| **Domain effects** | `WithEffectHandler` | dispatch set, a step's effects applied exactly once via deterministic `EffectID` dedup, with the exceptions under Exactly-once effects | Use `WithCheckpointEvery(n)` to tune how often a full snapshot is written (a shorter interval bounds recovery replay; a longer one cuts checkpoint cost). @@ -164,15 +166,19 @@ failed the Handle returns `ErrHandleFailed` saying it was recorded; `Recover` se its effects. To move a healthy Handle's instance to a new Handle, stop calling the old one and let its calls return first: a step can start after `Drain` returns. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer -owner holds the instance, so stop. +owner holds the instance, so stop. `Recover` retries an unapplied effect, and returns +`ErrEffectDispatch` while the handler still fails. ## Caveat: serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` -observes the JSON-decoded shape on a replayed `onDone`. A `WithEventData` payload is -decoded from its JSON form live as well as on replay, so a reducer gets the same value -both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per +observes the JSON-decoded shape of actor done-data on a replayed `onDone`. A service's +result is JSON-decoded live as well as on replay, so a reducer gets the same value both +ways: a struct is a map and an integer a `float64`, which loses precision above 2^53, and +an error an error with the same message, which `errors.Is` no longer matches. A +`WithEventData` payload is decoded from its JSON form live as well as on replay, so a +reducer gets the same value both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own form, so `Handle.Fire` refuses a struct or an integer rather than hand the reducer a map or a `float64`. An error stays an error with the same message, with any invalid UTF-8 diff --git a/durable/doc.go b/durable/doc.go index 559ca3b4..87985aeb 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -17,9 +17,12 @@ // recovery replays the exact recorded driving events and nondeterministic results // through the same pure transition function. // -// Exactly-once effects: a domain effect emitted by a transition is applied +// Exactly-once effects: a domain effect emitted by a step (Runner.Start's initial entry +// is at-least-once, and a parent transition an actor drives isn't dispatched) is applied // exactly once over the instance's lifetime — the live run plus any number of -// recoveries — even though the runtime's replay loop is at-least-once. Each +// recoveries — even though the runtime's replay loop is at-least-once. The exception is +// a crash or failed mark after the handler returned, or a handoff between owners, where +// an effect can apply twice, so keep handlers idempotent on EffectInstance plus the EffectID. Each // effect is stamped with a deterministic EffectID (step, ordinal, kind) and // deduplicated through the Store's dispatch set. // @@ -120,7 +123,7 @@ // publishing a message — as an Effect value the kernel routes out through // FireResult.Effects without applying. The Runner applies it through the caller- // supplied EffectHandler (WithEffectHandler), exactly once over the instance's -// lifetime. +// lifetime, with the exceptions under Exactly-once effects above. // // Each emitted effect is stamped with a deterministic EffectID derived from its // step, its emission ordinal within that step, and its kind — every component a diff --git a/durable/effect.go b/durable/effect.go index 71da5ff3..11341ce1 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -34,8 +34,8 @@ import ( // Each dispatchable effect is stamped with an EffectID that is a pure function of // the step it was emitted at, its ordinal among the dispatchable effects of that // step, and its kind: "##". Every component is deterministic -// — the step is the Fire ordinal, emission order is fixed by the machine -// definition, and the kind is the effect's stable discriminant — so the same +// — the step is its journal ordinal (-1 for Start's entry effects), emission order is +// fixed by the machine definition, and the kind is the effect's stable discriminant — so the same // effect is stamped with the identical id on the live run and on every recovery // replay, which re-fires the same events and re-emits the same effects in the // same order. That stable identity is what makes the dedup set meaningful across @@ -49,6 +49,8 @@ import ( // (write-ahead: the intent to dispatch is durable before any dispatch). // 2. Dispatch each effect whose id is not already in the dispatched set, calling // the handler, and mark each id dispatched in the Store as it succeeds. +// 3. Checkpoint, if one is due. This comes last so a checkpoint never compacts an +// effect that hasn't landed. // // This yields exactly-once effect application over the at-least-once delivery the // runtime is built on: @@ -59,6 +61,10 @@ import ( // once) and, because it is then marked, never again. // - Crash AFTER dispatch (id marked): recovery re-emits the effect but finds its // id already marked and skips it — so it is not redispatched (exactly once). +// - Crash or failed mark AFTER the handler returned: the effect was applied but its +// id isn't marked, so recovery applies it again. The handler should be idempotent +// on EffectInstance plus the id for this window, and for a handoff, when two owners +// can run it at once. // - Handler error: the effect is left un-marked, and the Handle fails with // ErrHandleFailed wrapping ErrEffectDispatch, so Recover retries it // (at-least-once until it succeeds; exactly-once thereafter). @@ -68,12 +74,31 @@ import ( // already-applied, since-compacted effect is still recognized and skipped. // EffectHandler applies one emitted domain effect, identified by its stamped -// EffectID. The Runner calls it exactly once per EffectID over the lifetime of an -// instance (see WithEffectHandler). A non-nil error leaves the effect un-marked -// for Recover to retry, and fails the Handle with ErrHandleFailed wrapping -// ErrEffectDispatch and the error. +// EffectID. The Runner calls it once per EffectID over the lifetime of an instance +// (see WithEffectHandler), with three exceptions: a repeated Start can apply the initial +// entry's effects again (see Runner.Start), an effect applies again if the handler +// returned but the mark didn't land, and two owners can run it at once during a handoff. +// An id is unique only within its instance: EffectInstance reads the instance from ctx. +// A non-nil error leaves the effect un-marked for Recover to retry, and fails the Handle +// with ErrHandleFailed wrapping ErrEffectDispatch and the error. type EffectHandler func(ctx context.Context, effectID string, effect state.Effect) error +// effectInstanceKey is the ctx key for the instance an EffectHandler call belongs to. +type effectInstanceKey struct{} + +// withEffectInstance returns ctx carrying id for the handler. +func withEffectInstance(ctx context.Context, id InstanceID) context.Context { + return context.WithValue(ctx, effectInstanceKey{}, id) +} + +// EffectInstance returns the instance an EffectHandler call belongs to, from the ctx the +// handler received. A handler that keeps its own dedupe record must key on this plus the +// effect id, because effect ids repeat across instances. +func EffectInstance(ctx context.Context) (InstanceID, bool) { + id, ok := ctx.Value(effectInstanceKey{}).(InstanceID) + return id, ok +} + // DispatchStore is the dedup seam for idempotent effect dispatch: the Store // tracks which effect ids an instance has applied, so a (re)dispatch skips the // ones that already landed. It is an additive, optional capability — the core @@ -194,6 +219,24 @@ func (h *Handle[S, E, C]) dispatchEffects(ctx context.Context, des []dispatchabl return h.runner.dispatch(ctx, h.id, des) } +// applyStartEffects invokes the handler for the initial entry's domain effects. Like +// dispatch it is a no-op without a handler or a DispatchStore. It marks nothing, because +// the instance doesn't exist yet and nothing replays these effects. +func (r *Runner[S, E, C]) applyStartEffects(ctx context.Context, id InstanceID, des []dispatchableEffect) error { + if len(des) == 0 || r.cfg.effectHandler == nil { + return nil + } + if _, ok := r.store.(DispatchStore); !ok { + return nil + } + for _, de := range des { + if err := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); err != nil { + return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) + } + } + return nil +} + // dispatchReplayEffects re-dispatches a replayed step's domain effects during // recovery: it re-stamps the re-emitted effects with the same deterministic ids // (step and emission order reproduce exactly) and applies only those not already @@ -226,7 +269,7 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis if dispatched[de.id] { continue } - if err := r.cfg.effectHandler(ctx, de.id, de.effect); err != nil { + if err := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); err != nil { return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) } if err := ds.MarkDispatched(ctx, id, de.id); err != nil { diff --git a/durable/effect_test.go b/durable/effect_test.go index bb065564..1be259b5 100644 --- a/durable/effect_test.go +++ b/durable/effect_test.go @@ -507,3 +507,182 @@ func TestMemStore_DispatchSet_MarkAndQuery(t *testing.T) { t.Fatalf("unmarked id reported dispatched") } } + +// entryNotifyMachine emits a host effect on entering the state an instance starts +// in, so the effect originates in Start rather than Fire. +func entryNotifyMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("entryNotify"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("ready").OnEntry("notify"). + Initial("ready"). + Quench() +} + +// TestRunner_EffectDispatch_Start_AppliesEntryEffect proves a domain effect emitted +// on entering the initial state reaches the handler. +func TestRunner_EffectDispatch_Start_AppliesEntryEffect(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + r := durable.NewRunner(entryNotifyMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + if n := rec.totalApplies(); n != 1 { + t.Fatalf("handler applied %d effects, want 1 (the initial state's entry notify)", n) + } +} + +// TestRunner_EffectDispatch_TimerTransition_AppliesEffect proves a domain effect +// emitted by a timer-driven transition reaches the handler. +func TestRunner_EffectDispatch_TimerTransition_AppliesEffect(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + clk := state.NewFakeClock(epoch) + r := durable.NewRunner(timerNotifyMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](rec.handler()), + durable.WithRunnerClock[string, string, *runCtx](clk)) + h, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + if n := rec.totalApplies(); n != 2 { + t.Fatalf("handler applied %d effects, want 2 (the timer transition's notify)", n) + } +} + +// TestRunner_EffectDispatch_ServiceSettle_AppliesEffect proves a domain effect +// emitted by the transition a service result drives reaches the handler. +func TestRunner_EffectDispatch_ServiceSettle_AppliesEffect(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + var calls int64 + fn := counterService(&calls) + r := durable.NewRunner(fetchThenWaitMachine(fn), durable.NewMemStore(), + durable.WithEffectHandler[string, string, svcCtx](rec.handler()), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))) + h, err := r.Start(ctx, "i1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire start: %v", err) + } + mustRunService(t, h, state.InvokeID("fetchThenWait", "loading", 0)) + if n := rec.totalApplies(); n != 1 { + t.Fatalf("handler applied %d effects, want 1 (the settle transition's notify)", n) + } +} + +// TestRunner_EffectDispatch_FailedThenCheckpointed_AppliedOnce proves an effect +// whose dispatch failed is still applied exactly once, live or on recovery, after +// a later checkpoint compacts its step. +func TestRunner_EffectDispatch_FailedThenCheckpointed_AppliedOnce(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + live := newEffectRecorder() + live.failKinds["demo.notify"] = errors.New("downstream unavailable") + opts := []durable.Option[string, string, *runCtx]{ + durable.WithEffectHandler[string, string, *runCtx](live.handler()), + durable.WithCheckpointEvery[string, string, *runCtx](2), + } + h, err := durable.NewRunner(effectMachine(), st, opts...).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err == nil { // step 0: its notify fails + t.Fatalf("Fire go: want the handler error") + } + delete(live.failKinds, "demo.notify") + if _, err = h.Fire(ctx, "bill"); err != nil { // step 1: checkpoint compacts step 0 + // A Handle that refuses Fires after a failed dispatch is resumed by Recover. + if h, err = durable.Recover(ctx, effectMachine(), st, "i1", opts...); err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = h.Fire(ctx, "bill"); err != nil { + t.Fatalf("Fire bill: %v", err) + } + } + + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](healthy.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + // go emits notify; bill emits charge and notify. Each applies exactly once. + ids := append(live.ids(), healthy.ids()...) + if len(ids) != 3 { + t.Fatalf("applied %d effect ids across live and recovery, want 3: %v", len(ids), ids) + } + for _, id := range ids { + if n := live.count(id) + healthy.count(id); n != 1 { + t.Fatalf("effect %q applied %d times, want once", id, n) + } + } +} + +// failingEntryMachine emits a host effect on entry, then fails the entry. +func failingEntryMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("failingEntry"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + Action("boom", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return nil, errors.New("boom") + }). + State("ready").OnEntry("notify").OnEntry("boom"). + Initial("ready"). + Quench() +} + +// TestRunner_EffectDispatch_Start_FailedEntryAppliesNothing proves an effect emitted +// before a failed entry action doesn't reach the handler, as with a failed Fire. +func TestRunner_EffectDispatch_Start_FailedEntryAppliesNothing(t *testing.T) { + ctx := context.Background() + rec := newEffectRecorder() + r := durable.NewRunner(failingEntryMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + if n := rec.totalApplies(); n != 0 { + t.Fatalf("handler applied %d effects from a failed entry, want 0", n) + } +} + +// TestEffectInstance_ReachesTheHandler proves a handler can tell instances apart, on +// Start and on a step. +func TestEffectInstance_ReachesTheHandler(t *testing.T) { + ctx := context.Background() + var seen []durable.InstanceID + handler := func(ctx context.Context, _ string, _ state.Effect) error { + id, ok := durable.EffectInstance(ctx) + if !ok { + t.Error("no instance on the handler's ctx") + } + seen = append(seen, id) + return nil + } + r := durable.NewRunner(entryNotifyMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](handler)) + for _, id := range []durable.InstanceID{"a", "b"} { + if _, err := r.Start(ctx, id, &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + } + if len(seen) != 2 || seen[0] != "a" || seen[1] != "b" { + t.Fatalf("handler saw %v, want [a b]", seen) + } + if _, ok := durable.EffectInstance(ctx); ok { + t.Fatal("instance on a ctx the runner never touched") + } +} diff --git a/durable/errors.go b/durable/errors.go index f24f15e3..69061aca 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -90,8 +90,9 @@ var ( // ErrEffectDispatch wraps a failure returned by the caller-supplied effect // handler (WithEffectHandler) while applying an emitted effect, or a panic in it. The effect was // persisted (write-ahead) but not marked dispatched, so the Handle fails with - // ErrHandleFailed wrapping it, and Recover dispatches it again. Callers match - // this with errors.Is to tell a dispatch failure from a kernel transition or - // persistence error. + // ErrHandleFailed wrapping it, and Recover dispatches it again; a failure in + // Runner.Start leaves no instance. Recover fails with this error while the handler + // still does. Callers match this with errors.Is to tell a dispatch failure from a + // kernel transition or persistence error. ErrEffectDispatch = errors.New(errorPrefix + errorEffectDispatch) ) diff --git a/durable/runner.go b/durable/runner.go index 7af0b366..73b30c1d 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -119,7 +119,8 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a -// newer owner holds the instance: stop, because recovering won't help. +// newer owner holds the instance: stop, because recovering won't help. A failed effect +// dispatch fails the Handle the same way, and Recover retries the effect. type Handle[S comparable, E comparable, C any] struct { runner *Runner[S, E, C] id InstanceID @@ -345,10 +346,9 @@ func (h *Handle[S, E, C]) drainClock() []state.JournalEntry { // durable seams compose: the scheduler arms or cancels delayed (`after`) timers (and // the Handle's timer table mirrors their deadlines for checkpointing), the service // runner starts or stops invoked services, and the actor system spawns or stops -// child-machine actors. It is the shared absorb the live seam methods (Fire, -// RunService, DeliverToActor, Tick) call for the effects they produced, so a state a -// settle or delivery enters arms whatever it declares — a service whose onDone spawns -// an actor, an actor whose onDone arms a timer, and so on. Absorbing a driver's +// child-machine actors. It is the shared absorb Fire and RunService call for the +// effects they produced, so a state a settle enters arms whatever it declares, such as +// a service whose onDone spawns an actor. Absorbing a driver's // effects is a no-op when that driver is not wired or the effects name none of its // kind. func (h *Handle[S, E, C]) absorbDrivers(ctx context.Context, effects []state.Effect) { @@ -498,6 +498,14 @@ func (h *Handle[S, E, C]) ID() InstanceID { return h.id } // wrapping the context's error and no Handle. The baseline is recorded by then, so // Recover the instance rather than Start it again. A ctx that ends with no Fire refused // doesn't fail Start. +// +// Start applies the initial entry's domain effects before the baseline write, and only +// if the entry succeeded: a failed entry still returns a Handle and a nil error, and +// Instance().InitialErr reports it. They're at-least-once, unlike a step's: a handler failure or a +// failed baseline write leaves no instance, so calling Start again re-applies every +// effect, as does a crash before the baseline. Overlapping Starts on one id, including a +// stale owner's, each apply them before the Store refuses all but one. To dedupe, key on +// EffectInstance plus the effect id. A handler wired only at Recover never sees them. func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opts ...state.CastOption[S]) (*Handle[S, E, C], error) { if _, _, err := r.store.Load(ctx, id); err == nil { return nil, fmt.Errorf("%w: %q", ErrInstanceExists, id) @@ -518,10 +526,10 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt castOpts := append([]state.CastOption[S]{state.WithClock[S](recClock), state.WithUnboundedHistory[S]()}, opts...) inst := r.machine.Cast(input, castOpts...) - // Persist a baseline checkpoint at baselineStep (below the first fired step) - // so the instance is reconstructable from the Store before any event is fired. - // The baseline is the cast snapshot; the tail then accumulates fired steps on - // top of it. + // Encode a baseline checkpoint at baselineStep (below the first fired step), so the + // instance is reconstructable from the Store before any event is fired. The baseline + // is the cast snapshot; the tail then accumulates fired steps on top of it. Encoding + // comes before the effects, so an instance that can't be encoded applies none. snap, err := state.MarshalSnapshot(inst.Snapshot()) if err != nil { return nil, fmt.Errorf("durable: marshaling start snapshot for %q: %w", id, err) @@ -532,6 +540,16 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt if err != nil { return nil, fmt.Errorf("durable: marshaling start checkpoint envelope for %q: %w", id, err) } + // Apply the initial entry's domain effects before the instance exists: a handler + // failure then leaves nothing behind, so the caller can Start again. Nothing can + // re-derive these effects after the baseline, so they're at-least-once. An entry that + // failed applies none, as a failed Fire dispatches none. + if inst.InitialErr() == nil { + if err := r.applyStartEffects(ctx, id, dispatchableEffects(baselineStep, inst.InitialEffects())); err != nil { + return nil, err + } + } + if err := r.store.Checkpoint(ctx, id, baseEnv, baselineStep, WithCheckpointFence(r.cfg.fence)); err != nil { return nil, fmt.Errorf("durable: checkpointing start baseline for %q: %w", id, err) } @@ -605,7 +623,7 @@ func (r *Runner[S, E, C]) newServiceRunner(inst *state.Instance[S, E, C]) *state if r.cfg.serviceReg == nil { return nil } - return state.NewServiceRunner(inst, r.cfg.serviceReg) + return state.NewServiceRunner(inst, r.cfg.serviceReg).WithResultTransform(replayableOutcome) } // newActorSystem builds the kernel host driver for child-machine actors bound to @@ -792,7 +810,8 @@ func parentFireErrs[S comparable](fires []state.ParentFire[S]) []error { // Fire drives one event through a durable instance identified by id, loading and // replaying it from the Store first, then recording the step. Handle.Fire describes -// how a state.WithEventData payload is handled, and when it's refused. It is the +// how a state.WithEventData payload is handled, and when it's refused. Loading can +// return ErrEffectDispatch before the event fires, as Recover can. It is the // stateless entry point (no Handle required); for a hot path that fires many // events in sequence, hold a Handle from Start or Recover and use Handle.Fire to // avoid reloading between steps. Calls on one id aren't serialized, and must not overlap @@ -815,7 +834,9 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // policy makes due checkpoints. A checkpoint this step wrote for an earlier one, which // failed because ctx ended, isn't reported, since this step succeeded. If another // goroutine failed the Handle while this step was recording, Fire returns -// ErrHandleFailed saying the step was recorded; Recover sends its effects. +// ErrHandleFailed saying the step was recorded; Recover sends its effects. It applies +// the step's domain effects after the append and before any checkpoint; a failed +// dispatch fails the Handle with ErrHandleFailed. // // A Fire the kernel refuses, because ctx ended or for any other reason, returns that // error, records nothing and leaves the Handle usable. When the event applies, an actor @@ -905,18 +926,30 @@ func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.Fi return res, nil, nil, h.fail(fmt.Errorf("durable: recording %q's parent Fires: %w", h.id, err)) } - step := h.nextStep - // Stamp this step's domain effects with their deterministic ids and carry them - // in the Record so the write-ahead append durably names every effect that must - // be dispatched before any dispatch happens. - des := dispatchableEffects(step, res.Effects) + rec := &Record{Step: h.nextStep, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: append(h.drainClock(), driven...)} + des, err := h.recordStep(ctx, rec, res.Effects) + if err != nil { + return res, nil, nil, err + } + return res, rec, des, nil +} + +// recordStep makes a step durable, in write-ahead order, under the step lock. It stamps +// the step's domain effects with their deterministic ids and carries them in rec, so the +// append durably names every effect that must be dispatched before any dispatch +// happens, then appends rec and advances the next step. A step with effects is marked +// dispatching, so no checkpoint compacts it until finishStep, which the caller runs +// after releasing the step lock, has dispatched them. The kernel has already applied the +// step, so a failed encode or append fails the Handle. +func (h *Handle[S, E, C]) recordStep(ctx context.Context, rec *Record, effects []state.Effect) ([]dispatchableEffect, error) { + des := dispatchableEffects(rec.Step, effects) effEnvs, err := recordEffects(des) if err != nil { - return res, nil, nil, h.fail(err) + return nil, h.fail(err) } - rec := &Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: append(h.drainClock(), driven...), Effects: effEnvs} - if err := h.appendStep(ctx, step, rec); err != nil { - return res, nil, nil, h.fail(err) + rec.Effects = effEnvs + if err := h.appendStep(ctx, rec.Step, rec); err != nil { + return nil, h.fail(err) } h.nextStep = rec.followingStep() h.out.mu.Lock() @@ -924,16 +957,16 @@ func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.Fi // Another goroutine's dispatch may have failed the Handle while this step held the // lock, and its caller may already be recovering. Leave this step's effects unmarked // in the journal for that Recover to send, rather than race it. - if err := h.recordedOnFailedLocked(step); err != nil { - return res, nil, nil, err + if err := h.recordedOnFailedLocked(rec.Step); err != nil { + return nil, err } if len(des) > 0 { if len(h.out.dispatching) == 0 { h.out.idle = make(chan struct{}) } - h.out.dispatching[step] = struct{}{} + h.out.dispatching[rec.Step] = struct{}{} } - return res, rec, des, nil + return des, nil } // finishStep dispatches a recorded step's domain effects, outside the step lock, then @@ -987,15 +1020,35 @@ func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []d // Handle with ErrHandleFailed and records nothing, even for timers that applied: the // results it returns weren't recorded, so don't act on their effects; a recovered // Handle re-arms those timers and fires them again. A timer's Fire refused for another -// reason, such as a failing guard, is recorded, and replay meets the same refusal. +// reason, such as a failing guard, is recorded, and replay meets the same refusal. As +// Fire does, it dispatches the fired timers' domain effects once the tick is recorded; a +// failed dispatch fails the Handle with ErrHandleFailed, and Recover retries the effect. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { if err := h.acquire(ctx); err != nil { return nil, err } + results, rec, des, err := h.lockedTickStep(ctx) + if err != nil || rec == nil { + return results, err + } + // Dispatch after releasing the step lock, as Fire does. + return results, h.finishStep(ctx, rec, des) +} + +// lockedTickStep runs tickStep and releases the step lock however it returns. A panic +// in host code fails the Handle first, since the kernel may have fired a timer. +func (h *Handle[S, E, C]) lockedTickStep(ctx context.Context) ([]state.FireResult[S], *Record, []dispatchableEffect, error) { defer h.release() defer h.failOnPanic() + return h.tickStep(ctx) +} + +// tickStep is the part of Tick that holds the step lock: it fires the due timers and +// records the tick, naming their domain effects, before any is dispatched. It returns a +// nil Record when the tick recorded nothing. +func (h *Handle[S, E, C]) tickStep(ctx context.Context) ([]state.FireResult[S], *Record, []dispatchableEffect, error) { if err := h.refuse(ctx); err != nil { - return nil, err + return nil, nil, nil, err } results := h.sched.Tick(h.stepContext(ctx)) // A context that ended partway can make the kernel refuse a due timer's Fire after @@ -1004,11 +1057,10 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro // A tick that fired no timer consumed nothing, so an ended ctx can't have cost it // anything: it records as usual. if err := consumedCanceled(ctx, len(results) > 0, append(resultErrs(results), parentFireErrs(h.actorFires.take())...)...); err != nil { - return results, h.fail(fmt.Errorf("durable: ticking %q: %w", h.id, err)) + return results, nil, nil, h.fail(fmt.Errorf("durable: ticking %q: %w", h.id, err)) } - step := h.nextStep - rec := Record{Step: step, Tick: true, TickSteps: len(results), Entries: h.drainClock()} + rec := &Record{Step: h.nextStep, Tick: true, TickSteps: len(results), Entries: h.drainClock()} // Reconcile the Handle's timer table with the tick: arm any timer a chained // transition scheduled, then drop the ones the tick fired, by id, so a checkpoint // after the tick persists exactly the still-pending deadlines. A chained timer's @@ -1029,12 +1081,17 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro // A tick that read no clock and fired nothing records nothing: there is no // nondeterminism to journal and no step was produced. if len(rec.Entries) == 0 && rec.TickSteps == 0 { - return results, nil + return results, nil, nil, nil } - if err := h.persistStep(ctx, step, &rec); err != nil { - return results, err + var effects []state.Effect + for i := range results { + effects = append(effects, results[i].Effects...) } - return results, nil + des, err := h.recordStep(ctx, rec, effects) + if err != nil { + return results, nil, nil, err + } + return results, rec, des, nil } // persistStep write-ahead appends rec for a step with no domain effects to dispatch, @@ -1349,7 +1406,9 @@ func respawn[S comparable, E comparable, C any](ctx context.Context, actors *sta // for an instance that was never started, and the context's error if ctx ends before // the journal is replayed, or a context's ending refuses a parent Fire an actor it // spawned drove. A ctx that ends once the actors are spawned, with no Fire refused, -// doesn't fail it, as for Start: their behaviors already ran. +// doesn't fail it, as for Start: their behaviors already ran. It applies any recorded +// effect not yet marked dispatched, so it returns ErrEffectDispatch and no Handle while +// the effect handler keeps failing. func Recover[S comparable, E comparable, C any](ctx context.Context, m *state.Machine[S, E, C], st Store, id InstanceID, opts ...Option[S, E, C]) (*Handle[S, E, C], error) { r := NewRunner(m, st, opts...) return r.recover(ctx, id) @@ -1449,7 +1508,7 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S if err := checkFormat(env.Format, rec); err != nil { return nil, fmt.Errorf("durable: recovering %q: %w", id, err) } - fired, err := replayRecord(ctx, inst, &r.cfg, sched, svc, rec, mirror, armsActorFires(rec)) + stepEffects, err := replayRecord(ctx, inst, &r.cfg, sched, svc, rec, mirror, armsActorFires(rec)) if err != nil { return nil, fmt.Errorf("durable: recovering %q: %w", id, err) } @@ -1459,14 +1518,12 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S if rec.UnstartedActors != nil { unstarted = rec.UnstartedActors } - if fired != nil { - // Re-dispatch this step's domain effects, deduped by the Store's dispatched - // set: an effect already marked (it landed before the crash) is skipped; one - // recorded-but-un-marked (the crash fell between append and dispatch) is - // applied now — exactly-once across the crash boundary. - if err := r.dispatchReplayEffects(ctx, id, rec.Step, fired.Effects); err != nil { - return nil, err - } + // Re-dispatch this step's domain effects, a Fire's, a settlement's or a tick's, + // deduped by the Store's dispatched set: an effect already marked (it landed + // before the crash) is skipped; one recorded-but-un-marked (the crash fell between + // append and dispatch) is applied now — exactly-once across the crash boundary. + if err := r.dispatchReplayEffects(ctx, id, rec.Step, stepEffects); err != nil { + return nil, err } } @@ -1845,7 +1902,9 @@ func resultErrs[S comparable](results []state.FireResult[S]) []error { // replayRecord re-applies one journaled Record to inst the way the live step applied // it, running no service, actor or effect handler. Recover and StateAt both walk the // journal with it; Recover adds its deadline table, which mirror tracks, and effect -// re-dispatch on top. A nil mirror tracks no deadlines. +// re-dispatch on top. A nil mirror tracks no deadlines. It returns the effects of the +// step's own work, a plain event's, a settlement's or a tick's timers', so Recover can +// re-dispatch their domain effects; nil for any other Record. // // - Parent Fires a Start's or Recover's spawns drove, recorded with the next step, // re-fire first, before the step's own work. @@ -1873,48 +1932,53 @@ func replayRecord[S comparable, E comparable, C any]( rec *Record, mirror *armMirror, actorArms bool, -) (*state.FireResult[S], error) { - var fired *state.FireResult[S] +) ([]state.Effect, error) { + var stepEffects []state.Effect before, rest, err := splitActorEntries(rec) if err != nil { - return fired, fmt.Errorf("at step %d: %w", rec.Step, err) + return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) } // Parent Fires a Start's or Recover's spawns drove ran before the step's own work. if err := replayActors(ctx, inst, cfg, sched, svc, rec, before, mirror, actorArms); err != nil { - return fired, err + return stepEffects, err } switch { case len(rec.Event) > 0: event, err := cfg.eventCodec.Decode(rec.Event) if err != nil { - return fired, fmt.Errorf("decoding recorded event at step %d: %w", rec.Step, err) + return stepEffects, fmt.Errorf("decoding recorded event at step %d: %w", rec.Step, err) } fireOpts, err := replayFireOptions(rec, event, cfg.eventDataDecoder) if err != nil { - return fired, fmt.Errorf("decoding recorded event data at step %d: %w", rec.Step, err) + return stepEffects, fmt.Errorf("decoding recorded event data at step %d: %w", rec.Step, err) } res := inst.Fire(ctx, event, fireOpts...) if res.Err != nil { - return fired, fmt.Errorf("replaying step %d: %w", rec.Step, res.Err) + return stepEffects, fmt.Errorf("replaying step %d: %w", rec.Step, res.Err) } replayArm(ctx, sched, svc, res.Effects, mirror) - fired = &res + stepEffects = res.Effects case hasServiceEntry(rec): for _, entry := range serviceEntries(rec) { effects, err := replayService(ctx, svc, entry) if err != nil { - return fired, fmt.Errorf("at step %d: %w", rec.Step, err) + return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) } mirror.settle(entry.CorrelationID) // The settle armed its own services; arm its timers, as RunService did. replayArm(ctx, sched, nil, effects, mirror) + stepEffects = append(stepEffects, effects...) } case rec.Tick: - mirrorTick(mirror, sched, sched.Tick(ctx)) + ticked := sched.Tick(ctx) + mirrorTick(mirror, sched, ticked) + for i := range ticked { + stepEffects = append(stepEffects, ticked[i].Effects...) + } } // The rest are a delivery's parent Fires, or the ones a Fire's or settle's absorb // drove after it, such as a spawned actor's onError. - return fired, replayActors(ctx, inst, cfg, sched, svc, rec, rest, mirror, actorArms) + return stepEffects, replayActors(ctx, inst, cfg, sched, svc, rec, rest, mirror, actorArms) } // replayActors re-fires recorded parent transitions, in order. When actorArms, the diff --git a/durable/runner_options.go b/durable/runner_options.go index b653c5e1..12203e63 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -127,10 +127,14 @@ func WithActorPalette[S comparable, E comparable, C any](palette map[string]stat } // WithEffectHandler binds the seam the Runner calls to apply each emitted domain -// effect — an email, a charge, a published message: any at-most-once side effect -// a transition emits as a domain Effect value. A durable effect is applied +// effect — an email, a charge, a published message: any side effect +// a transition emits as a domain Effect value. Effects from a parent transition an actor +// drives aren't dispatched, and a Store that isn't a DispatchStore applies none. A +// durable effect from a step is applied // exactly once over the whole lifetime of an instance (the live run plus any -// number of recoveries), deduped by a deterministic EffectID: the Runner stamps +// number of recoveries), except that it can apply twice after a crash or failed mark +// once the handler returned, or during a handoff, so keep the handler idempotent on +// EffectInstance plus the id. It is deduped by a deterministic EffectID: the Runner stamps // each emitted effect, write-ahead persists the step Record carrying those ids, // then invokes the handler for every id not already marked dispatched, marking // each as it succeeds. A handler error or panic fails the Handle with ErrHandleFailed, @@ -187,7 +191,8 @@ func WithEventDataDecoder[S comparable, E comparable, C any](d EventDataDecoder[ // Checkpoint, so a Handle that has lost the instance to a newer owner gets ErrFenced // instead of committing. Pass the token the host's lease issued; a retry by the same // owner keeps the same token. A non-positive token is the same as no token, which a -// fenced instance refuses. MarkDispatched is not fenced. +// fenced instance refuses. MarkDispatched is not fenced, so during a handoff an effect +// handler can run concurrently for one effect id. // // The token applies to every instance the Runner drives, so a host with a token per // instance passes it to Recover, or builds a Runner per instance. No fence token is written diff --git a/durable/service.go b/durable/service.go index 66dde8cd..a5f8215c 100644 --- a/durable/service.go +++ b/durable/service.go @@ -29,8 +29,8 @@ import ( // The Runner wraps the kernel's reusable host driver, state.ServiceRunner, which // turns StartService / StopService effects into real ServiceFn executions and // re-fires each result through the invocation's onDone / onError event. On the live -// path RunService delegates to ServiceRunner.Run (resolve + run + settle), reads -// the raw outcome the service produced, and journals it as a JournalServiceResult +// path RunService delegates to ServiceRunner.Tick (resolve + run + settle), reads +// the outcome in its JSON-decoded form, and journals it as a JournalServiceResult // correlated by the invocation id. On recovery the recorded outcomes are replayed // in order through ServiceRunner.SettleDone / SettleError — the same settle methods // the live run drove — so the kernel re-fires the identical onDone / onError event @@ -55,7 +55,7 @@ type serviceResultPayload struct { // Result is the JSON-encoded successful return of the ServiceFn, present for a // SettleDone outcome. It is decoded to a Go value and re-fired through onDone so // the transition's Assign reads it from AssignCtx.Event exactly as on the live - // run. + // run, which settles with the same decoded form (see replayableOutcome). Result json.RawMessage `json:"result,omitempty"` // Error is the failure message for a SettleError outcome. It is reconstructed // into an error and re-fired through onError so the transition's Assign reads it @@ -80,46 +80,69 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // records its outcome, and settles it — firing the invocation's onDone (on // success) or onError (on failure) event through the durable instance and // recording the produced step. The service is resolved and executed against the -// registry supplied with WithServiceRegistry; its raw result or error is journaled +// registry supplied with WithServiceRegistry; its result or error is journaled // as a JournalServiceResult correlated by id so recovery replays it without -// re-invoking the service. +// re-invoking the service. The outcome reaches the machine JSON-decoded, live as on +// replay, and the journal holds that form: a struct is a map, an integer a float64 that +// loses precision above 2^53, and an error an error with the same message, which +// errors.Is no longer matches. // // It returns the routed FireResult and true, or the zero result and false when id // names no in-flight service (already settled, stopped, or never started). It returns // false and an error when no registry is wired, the Handle has failed, or ctx ended // before the service started. // -// Once the service has run, it returns true with any error. A result that won't encode, -// a ctx that ended during the service or while its settle's effects were absorbed (even -// after a clean settle), or a Fire the settle drove refused because a context ended (ctx, -// or one middleware derived from it) returns ErrHandleFailed and records nothing: a -// recovered Handle has the service in flight again, and RunService runs it a second time. +// Once the service has run, it returns true with any error. A result or KindedEffect +// that won't encode, a ctx that ended during the service or while its settle's effects +// were absorbed (even after a clean settle), or a Fire the settle drove refused because a +// context ended (ctx, or one middleware derived from it) returns ErrHandleFailed and +// records nothing: a recovered Handle has the service in flight again, and RunService +// runs it a second time. // A failed Append also returns ErrHandleFailed, but the write may have landed anyway: // Recover finds out, and the service runs again only if the result isn't in the // journal. If another goroutine failed the Handle while this step was recording, it // returns the result, true and ErrHandleFailed saying the step was recorded. Every // ErrHandleFailed also matches its cause, so test for ErrHandleFailed first, then // ErrFenced, as the Handle doc says. A failed Checkpoint returns the plain error, and -// the result is recorded. +// the result is recorded. A failed effect dispatch also returns ErrHandleFailed, but the +// result is recorded: Recover retries the effect and the service doesn't run again. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { if err := h.acquire(ctx); err != nil { return state.FireResult[S]{}, false, err } + res, ran, rec, des, err := h.lockedServiceStep(ctx, id) + if err != nil || rec == nil { + return res, ran, err + } + // Dispatch after releasing the step lock, as Fire does. + return res, true, h.finishStep(ctx, rec, des) +} + +// lockedServiceStep runs serviceStep and releases the step lock however it returns. A +// panic in host code fails the Handle first, since the settle may have applied. +func (h *Handle[S, E, C]) lockedServiceStep(ctx context.Context, id string) (state.FireResult[S], bool, *Record, []dispatchableEffect, error) { defer h.release() defer h.failOnPanic() + return h.serviceStep(ctx, id) +} + +// serviceStep is the part of RunService that holds the step lock: it runs and settles +// the service and records the step, naming its domain effects, before any is +// dispatched. It returns a nil Record when nothing was recorded. +func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.FireResult[S], bool, *Record, []dispatchableEffect, error) { if err := h.refuse(ctx); err != nil { - return state.FireResult[S]{}, false, err + return state.FireResult[S]{}, false, nil, nil, err } if h.svc == nil { - return state.FireResult[S]{}, false, fmt.Errorf("durable: no service registry wired for %q (supply WithServiceRegistry)", h.id) + return state.FireResult[S]{}, false, nil, nil, fmt.Errorf("durable: no service registry wired for %q (supply WithServiceRegistry)", h.id) } if !h.svc.HasPending(id) { - return state.FireResult[S]{}, false, nil + return state.FireResult[S]{}, false, nil, nil, nil } results := h.svc.Tick(h.stepContext(ctx), id) if len(results) == 0 { - return state.FireResult[S]{}, false, nil + return state.FireResult[S]{}, false, nil, nil, nil } res := results[0] // The service settled: unless the settle's own Fire leaves or re-enters the state @@ -139,12 +162,12 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire // Recover re-arms it. fires := h.actorFires.take() if err := stepCanceled(ctx, append([]error{res.Err}, parentFireErrs(fires)...)...); err != nil { - return res, true, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, err)) + return res, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, err)) } entry, err := recordServiceOutcome(id, h.svc.LastResult, h.svc.LastError()) if err != nil { - return res, true, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) + return res, true, nil, nil, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) } if h.svcBuf != nil { *h.svcBuf = append(*h.svcBuf, entry) @@ -155,17 +178,17 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire // them, and replay re-fires them after it. driven, err := recordParentFires(fires, false) if err != nil { - return res, true, h.fail(fmt.Errorf("durable: recording %q's parent Fires: %w", h.id, err)) + return res, true, nil, nil, h.fail(fmt.Errorf("durable: recording %q's parent Fires: %w", h.id, err)) } - step := h.nextStep // Arming a timer the settle's state declares reads the clock, so the step carries // its reads. - rec := Record{Step: step, Entries: append(append(h.drainClock(), h.drainServices()...), driven...)} - if err := h.persistStep(ctx, step, &rec); err != nil { - return res, true, err + rec := &Record{Step: h.nextStep, Entries: append(append(h.drainClock(), h.drainServices()...), driven...)} + des, err := h.recordStep(ctx, rec, res.Effects) + if err != nil { + return res, true, nil, nil, err } - return res, true, nil + return res, true, rec, des, nil } // noteSettled records that the service id settled in its state's current activation. @@ -176,6 +199,38 @@ func (h *Handle[S, E, C]) noteSettled(id string) { h.settled[id] = true } +// replayableOutcome gives a service's outcome the form a replay supplies, so the live +// settle and a replay can't differ: a result in its JSON-decoded form, an error as an +// error with the same message once any invalid UTF-8 is replaced, which is what the +// round trip through JSON does. An Error method that panics settles with a message +// saying so. A result with no JSON form is left as it is, and fails +// the Handle when recording it fails. +func replayableOutcome(result any, err error) (any, error) { + if err != nil { + msg, merr := errorMessage(err) + if merr != nil { + return nil, merr + } + b, jerr := marshalPayload(msg) + if jerr != nil { + return nil, errors.New(msg) + } + if jerr = json.Unmarshal(b, &msg); jerr != nil { + return nil, errors.New(msg) + } + return nil, errors.New(msg) + } + b, jerr := marshalPayload(result) + if jerr != nil { + return result, nil + } + var decoded any + if jerr = json.Unmarshal(b, &decoded); jerr != nil { + return result, nil + } + return decoded, nil +} + // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service. It reads the success result through lastResult (valid only during the // settle's synchronous window) and the error through lastErr, encoding whichever @@ -186,7 +241,7 @@ func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr erro msg := lastErr.Error() sp.Error = &msg } else if result, ok := lastResult(); ok { - raw, err := json.Marshal(result) + raw, err := marshalPayload(result) if err != nil { return state.JournalEntry{}, fmt.Errorf("marshaling service result: %w", err) } diff --git a/durable/service_test.go b/durable/service_test.go index 4d29a5d8..60f01872 100644 --- a/durable/service_test.go +++ b/durable/service_test.go @@ -472,6 +472,25 @@ func TestService_Determinism(t *testing.T) { } } +// fetchThenWaitMachine routes its service result into a state that arms a timer, +// emitting a host effect on the way, so both originate in RunService. +func fetchThenWaitMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("fetchThenWait"). + Service("fetch", fn). + Action("notify", func(state.ActionCtx[svcCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("idle"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("ok")). + State("waiting"). + State("done").Final(). + Initial("idle"). + Transition("idle").On("start").GoTo("loading"). + Transition("loading").On("ok").GoTo("waiting").Do("notify"). + Transition("waiting").After(5 * time.Second).On("elapsed").GoTo("done"). + Quench() +} + // TestService_ResultFromCanceledTurn_NotCommittedOnRecover proves a service result // whose settle is canceled with the turn's context, and so never reaches the live // instance, is not committed by recovery either. @@ -993,3 +1012,39 @@ func TestRecover_StartsAServiceThatNeverSettled(t *testing.T) { } } } + +// TestRunService_AssignSeesTheDecodedResult proves a typed result reaches Assign in the +// form a replay supplies, so the two can't differ. +func TestRunService_AssignSeesTheDecodedResult(t *testing.T) { + type user struct{ Email string } + ctx := context.Background() + var seen []any + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return user{Email: "a@b.c"}, nil } + m := state.ForgeFor[svcCtx]("decoded"). + Service("fetch", fn). + Reducer("note", func(in state.AssignCtx[svcCtx]) svcCtx { + seen = append(seen, in.Event) + return in.Entity + }). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready").Assign("note"). + Transition("loading").On("fail").GoTo("failed"). + Quench() + store := durable.NewMemStore() + id := durable.InstanceID("d-1") + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + mustRunService(t, h, state.InvokeID("decoded", "loading", 0)) + if _, err = durable.Recover(ctx, m, store, id, opts...); err != nil { + t.Fatalf("Recover: %v", err) + } + if len(seen) != 2 || fmt.Sprintf("%T:%v", seen[0], seen[0]) != fmt.Sprintf("%T:%v", seen[1], seen[1]) { + t.Fatalf("live and replay saw %#v", seen) + } +} diff --git a/durable/store.go b/durable/store.go index 0ba8ca0d..44885977 100644 --- a/durable/store.go +++ b/durable/store.go @@ -59,7 +59,7 @@ type Record struct { // Effects are the effects the step emitted, each with its deterministically // stamped EffectID, persisted before dispatch so a crash between persist and - // dispatch is recoverable: Resume re-dispatches, deduped by EffectID. + // dispatch is recoverable: Recover re-dispatches, deduped by EffectID. Effects []state.EffectEnvelope // Snapshot, when non-empty, is a full marshaled state.Snapshot captured at diff --git a/examples/fooddelivery/fooddelivery.go b/examples/fooddelivery/fooddelivery.go index f7b50b8b..9c005ae0 100644 --- a/examples/fooddelivery/fooddelivery.go +++ b/examples/fooddelivery/fooddelivery.go @@ -388,10 +388,17 @@ func settleReducer(in state.AssignCtx[Order]) Order { // onDone event's payload — the saga's compensation made observable. func recordRefund(in state.AssignCtx[Order]) Order { c := in.Entity - if amount, ok := in.Event.(int64); ok { - c.Refund = amount - c.Log = append(c.Log, fmt.Sprintf("refunded:%d", amount)) + var amount int64 + switch v := in.Event.(type) { + case int64: + amount = v + case float64: // a durable Runner delivers a service result JSON-decoded + amount = int64(v) + default: + return c } + c.Refund = amount + c.Log = append(c.Log, fmt.Sprintf("refunded:%d", amount)) return c } diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index 6617e6a6..2d8dabcf 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -36,6 +36,8 @@ counts as an additive (minor) versus breaking (major) change. Use the onError doesn't escalate again, since its start already escalated it; a respawned actor's own failure with no onError does. The durable runner's `Recover` uses it for the actors that were running. It's additive, within v1.1.0. +- **`ServiceRunner.WithResultTransform`** rewrites a service's outcome before `Tick` + settles it, so a host that records outcomes can give the live run what a replay will see. - **`FireEventData`** reports the payload a `WithEventData` option carries, so a host that records a Fire can persist it. It's additive, so the next release is v1.1.0. - **Callbacks get the step's context.** `GuardCtx`, `ActionCtx` and `AssignCtx` have a new diff --git a/state/runner.go b/state/runner.go index 699ebeb7..1d8d8a29 100644 --- a/state/runner.go +++ b/state/runner.go @@ -97,6 +97,9 @@ type ServiceRunner[S comparable, E comparable, C any] struct { // transition's action. It is atomic so a concurrent reader sees a consistent // snapshot. lastResult atomic.Value // serviceOutcome + + // transform, when set, rewrites a ServiceFn's outcome before Tick settles it. + transform func(any, error) (any, error) } // serviceOutcome is the result/error pair a settled service exposes through @@ -217,6 +220,17 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a return res, true } +// WithResultTransform sets a function that rewrites a ServiceFn's outcome before Tick +// settles it, and returns the runner. A host that records outcomes uses it to give the +// live run the form a replay will see. LastResult and LastError then report the rewritten +// outcome. It also runs on an UnboundRefError, where only the returned error is used. It +// doesn't apply to SettleDone or SettleError, which take the outcome as given. Set it +// before the first Tick: the field isn't guarded. +func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(result any, err error) (any, error)) *ServiceRunner[S, E, C] { + r.transform = f + return r +} + // Tick resolves and runs the in-flight service id against the bound registry, // settling it with the ServiceFn's result or error. It is the ServiceRunner's // advance verb — the host-driver counterpart of Scheduler.Tick and @@ -237,10 +251,17 @@ func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResu } fn := r.resolve(rs.src.Name) if fn == nil { - res, _ := r.SettleError(ctx, id, &UnboundRefError{Kind: "service", Name: rs.src.Name}) + var err error = &UnboundRefError{Kind: "service", Name: rs.src.Name} + if r.transform != nil { + _, err = r.transform(nil, err) + } + res, _ := r.SettleError(ctx, id, err) return []FireResult[S]{res} } out, err := fn(ctx, ServiceCtx[C]{Entity: r.inst.entity, Params: rs.src.Params, Input: rs.input}) + if r.transform != nil { + out, err = r.transform(out, err) + } if err != nil { res, _ := r.SettleError(ctx, id, err) return []FireResult[S]{res} From 73f67dffea8ccbf893a92aeeaba099468d9e78c5 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 12:54:37 -0700 Subject: [PATCH 02/13] fix(durable): guard Start's effect handler and settle service outcomes a replay can read Review follow-ups for the effect dispatch change, each pinned by a test. - One helper, applyEffect, calls the effect handler for Start, a step and Recover, and turns a handler panic into ErrEffectDispatch. Start and Recover used to let the panic out, for a Fire's effects as well as a tick's or a service settle's. - A service result whose JSON form won't decode, such as a number beyond float64, now fails RunService with ErrHandleFailed and records nothing, as a result that won't encode does. It used to be journaled, and every Recover then failed to decode it. - A service error whose Error method panics settles with "service error panicked: ...", its invalid UTF-8 replaced, so live and replay see the same message. encodeEventData and replayableOutcome share normalizedErrorMessage, which drops replayableOutcome's dead fallbacks. - Recover re-dispatches a step's effects only when its Record names them, so it doesn't send effects from an older version's Tick or service steps, which never dispatched them. - state: a WithResultTransform that returns no error for an unbound service no longer turns it into a success; the service still settles through onError. - Tests that Recover re-dispatches a tick's, a multi-timer tick's and a service settle's unmarked effects once, under the ids the live run stamped. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 14 +- durable/effect.go | 23 +- durable/effect_recover_test.go | 407 +++++++++++++++++++++++++ durable/error_message_internal_test.go | 44 +++ durable/runner.go | 52 +++- durable/service.go | 37 +-- durable/service_outcome_test.go | 100 ++++++ state/CHANGELOG.md | 2 + state/result_transform_test.go | 74 +++++ state/runner.go | 9 +- 10 files changed, 719 insertions(+), 43 deletions(-) create mode 100644 durable/effect_recover_test.go create mode 100644 durable/error_message_internal_test.go create mode 100644 durable/service_outcome_test.go create mode 100644 state/result_transform_test.go diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index f0ae203d..dd839493 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -101,7 +101,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. panics, which now returns `ErrEffectDispatch` instead of panicking out of `Fire`. The effect stays unmarked in the journal, so no later checkpoint compacts it away, and `Recover` dispatches it again, failing with `ErrEffectDispatch` while the handler still - does. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect + does. A handler that panics in `Start` or `Recover` returns `ErrEffectDispatch` too. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect again. The handler's error now matches through `errors.Is`. - **A `KindedEffect` that can't marshal fails `Tick` and `RunService`,** as it already failed `Fire`, even with no handler wired. @@ -109,8 +109,11 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. effect can't differ from the first attempt: a struct arrives as a map, an integer as a `float64`, which loses precision above 2^53, and an error as an error with the same message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` - method that panics settles with a message saying so. The journal records this decoded - value. + method that panics settles with a message saying the service error panicked, its + invalid UTF-8 replaced too. The journal records this decoded value. A result whose JSON + form won't decode, such as a number beyond `float64`, fails `RunService` with + `ErrHandleFailed` and records nothing, as a result that won't encode does, so + `Recover` doesn't fail on it. - **A step that may not have been recorded fails the Handle.** That call and every later one return `ErrHandleFailed`, wrapping the cause. A step that was recorded while another goroutine failed the Handle also returns `ErrHandleFailed`, saying the step was @@ -186,8 +189,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. at-least-once: a handler failure, a failed baseline write or overlapping `Start` calls re-apply them, so key any dedupe on `EffectInstance` plus the effect id. Effects from a parent transition an actor drives, including during `Fire`, are still not dispatched. - The first `Recover` of an older journal applies effects from uncompacted `Tick` and - service steps. + `Recover` re-dispatches a step's effects only when its Record names them, so it + doesn't send effects from an older version's `Tick` and service steps, which never + dispatched them. - **A Tick, RunService or DeliverToActor whose context ended partway isn't journaled.** A canceled `RunService`, `Tick` or `DeliverToActor` used to record a result the live instance never applied, which recovery then committed, or for an actor, failed on. Each now returns `ErrHandleFailed` diff --git a/durable/effect.go b/durable/effect.go index 11341ce1..a89bf95d 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -230,13 +230,28 @@ func (r *Runner[S, E, C]) applyStartEffects(ctx context.Context, id InstanceID, return nil } for _, de := range des { - if err := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); err != nil { - return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) + if err := r.applyEffect(ctx, id, de); err != nil { + return err } } return nil } +// applyEffect calls the effect handler for de, with the instance on its ctx. It wraps a +// handler error, or a panic in the handler, in ErrEffectDispatch, so Start, a step and +// Recover all fail the same way, rather than let a panic out. +func (r *Runner[S, E, C]) applyEffect(ctx context.Context, id InstanceID, de dispatchableEffect) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("%w: effect %q for %q: handler panicked: %v", ErrEffectDispatch, de.id, id, p) + } + }() + if herr := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); herr != nil { + return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, herr) + } + return nil +} + // dispatchReplayEffects re-dispatches a replayed step's domain effects during // recovery: it re-stamps the re-emitted effects with the same deterministic ids // (step and emission order reproduce exactly) and applies only those not already @@ -269,8 +284,8 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis if dispatched[de.id] { continue } - if err := r.cfg.effectHandler(withEffectInstance(ctx, id), de.id, de.effect); err != nil { - return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) + if err := r.applyEffect(ctx, id, de); err != nil { + return err } if err := ds.MarkDispatched(ctx, id, de.id); err != nil { return fmt.Errorf("durable: marking effect %q dispatched for %q: %w", de.id, id, err) diff --git a/durable/effect_recover_test.go b/durable/effect_recover_test.go new file mode 100644 index 00000000..d17f8eae --- /dev/null +++ b/durable/effect_recover_test.go @@ -0,0 +1,407 @@ +package durable_test + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// stepIDHandler applies every effect except those of step failStep, for which it +// returns an error, and records every id it was called with, failed or not. +type stepIDHandler struct { + failStep string + tried []string + applied []string +} + +func (s *stepIDHandler) handle(_ context.Context, id string, _ state.Effect) error { + s.tried = append(s.tried, id) + if strings.HasPrefix(id, s.failStep+"#") { + return errors.New("downstream unavailable") + } + s.applied = append(s.applied, id) + return nil +} + +// panicHandler panics on every effect. +func panicHandler(context.Context, string, state.Effect) error { + panic("handler exploded") +} + +// noPanic runs f and turns a panic out of it into a test failure, so a test of a +// panic guard reports instead of crashing the test binary. +func noPanic(t *testing.T, what string, f func() error) (err error) { + t.Helper() + defer func() { + if p := recover(); p != nil { + t.Fatalf("%s panicked: %v", what, p) + } + }() + return f() +} + +// pairNotifyMachine has two parallel regions whose timers both fall due one second +// after "go", each emitting a different host effect, so one Tick fires both. +func pairNotifyMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("pairNotify"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + Action("charge", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return chargeEffect{Cents: 500}, nil + }). + State("idle"). + SuperState("both"). + Region("left"). + Initial("l1"). + SubState("l1"). + SubState("l2"). + EndRegion(). + Region("right"). + Initial("r1"). + SubState("r1"). + SubState("r2"). + EndRegion(). + EndSuperState(). + Initial("idle"). + Transition("idle").On("go").GoTo("both"). + Transition("l1").After(time.Second).On("toL2").GoTo("l2").Do("notify"). + Transition("r1").After(time.Second).On("toR2").GoTo("r2").Do("charge"). + Quench() +} + +// assertAppliedOnceAsLive checks the recovered handler applied exactly the ids the +// live run tried and failed, each once, and nothing the live run already applied. +func assertAppliedOnceAsLive(t *testing.T, live *stepIDHandler, healthy *effectRecorder, wantFailed int) { + t.Helper() + var failed []string + for _, id := range live.tried { + if strings.HasPrefix(id, live.failStep+"#") { + failed = append(failed, id) + } + } + if len(failed) != wantFailed { + t.Fatalf("live run tried %d effects of step %s (%v), want %d", len(failed), live.failStep, live.tried, wantFailed) + } + for _, id := range failed { + if n := healthy.count(id); n != 1 { + t.Fatalf("Recover applied the live run's failed effect %q %d times, want once (applied %v)", id, n, healthy.ids()) + } + } + for _, id := range live.applied { + if n := healthy.count(id); n != 0 { + t.Fatalf("Recover applied %q again, which the live run already applied", id) + } + } + if n := healthy.totalApplies(); n != len(failed) { + t.Fatalf("Recover applied %d effects (%v), want the %d the live run failed", n, healthy.ids(), len(failed)) + } +} + +// TestRecover_RedispatchesAFailedTimerEffect proves an effect a timer transition +// emitted, whose dispatch failed live, is applied exactly once by Recover, under the +// EffectID the live attempt saw. +func TestRecover_RedispatchesAFailedTimerEffect(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + clk := state.NewFakeClock(epoch) + live := &stepIDHandler{failStep: "1"} // step 0 is "go", step 1 the tick + h, err := durable.NewRunner(timerNotifyMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](live.handle), + durable.WithRunnerClock[string, string, *runCtx](clk)).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Tick: %v, want ErrHandleFailed wrapping ErrEffectDispatch", err) + } + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, timerNotifyMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](healthy.handler()), + durable.WithRunnerClock[string, string, *runCtx](clk)); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 1) +} + +// TestRecover_RedispatchesAFailedServiceSettleEffect proves the same for an effect +// the transition a service result drives emitted. +func TestRecover_RedispatchesAFailedServiceSettleEffect(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + var calls int64 + fn := counterService(&calls) + live := &stepIDHandler{failStep: "1"} // step 0 is "start", step 1 the settle + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(fetchThenWaitMachine(fn), st, + append(opts, durable.WithEffectHandler[string, string, svcCtx](live.handle))...).Start(ctx, "i1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire start: %v", err) + } + if _, _, err = h.RunService(ctx, state.InvokeID("fetchThenWait", "loading", 0)); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("RunService: %v, want ErrEffectDispatch", err) + } + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, fetchThenWaitMachine(fn), st, "i1", + append(opts, durable.WithEffectHandler[string, string, svcCtx](healthy.handler()))...); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 1) + if calls != 1 { + t.Fatalf("the service ran %d times, want once: Recover replays its recorded result", calls) + } +} + +// TestRecover_RedispatchesAMultiTimerTicksEffects proves a tick that fires two timers, +// each emitting an effect, re-dispatches both on Recover under the ids the live +// attempt saw. +func TestRecover_RedispatchesAMultiTimerTicksEffects(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + clk := state.NewFakeClock(epoch) + live := &stepIDHandler{failStep: "1"} + h, err := durable.NewRunner(pairNotifyMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](live.handle), + durable.WithRunnerClock[string, string, *runCtx](clk)).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + fired, err := h.Tick(ctx) + if len(fired) != 2 || !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Tick fired %d timers, err %v; want both, and ErrEffectDispatch", len(fired), err) + } + // The handler stops at the first failure, so the live run tried only the first + // effect; the tick's Record names both ids it stamped. + _, recs, err := st.Load(ctx, "i1") + if err != nil { + t.Fatalf("Load: %v", err) + } + var stamped []string + for _, r := range recs { + if r.Tick { + for _, env := range r.Effects { + stamped = append(stamped, env.EffectID) + } + } + } + if len(stamped) != 2 || len(live.tried) != 1 || live.tried[0] != stamped[0] { + t.Fatalf("the tick stamped %v and the live run tried %v; want two ids, the first tried live", stamped, live.tried) + } + live.tried = stamped + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, pairNotifyMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](healthy.handler()), + durable.WithRunnerClock[string, string, *runCtx](clk)); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 2) +} + +// TestStart_HandlerPanic_ReturnsEffectDispatchAndLeavesNoInstance proves a handler +// that panics on the initial entry's effect fails Start with ErrEffectDispatch, as a +// handler error does, and leaves no instance behind. +func TestStart_HandlerPanic_ReturnsEffectDispatchAndLeavesNoInstance(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + r := durable.NewRunner(entryNotifyMachine(), st, durable.WithEffectHandler[string, string, *runCtx](panicHandler)) + err := noPanic(t, "Start", func() error { + _, serr := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")) + return serr + }) + if !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Start: %v, want ErrEffectDispatch", err) + } + if _, _, err = st.Load(ctx, "i1"); !errors.Is(err, durable.ErrInstanceNotFound) { + t.Fatalf("Load after a failed Start: %v, want ErrInstanceNotFound", err) + } +} + +// TestRecover_HandlerPanic_ReturnsEffectDispatch proves a handler that panics while +// Recover re-dispatches a step's unmarked effect, a Fire's, a tick's or a service +// settle's, fails Recover with ErrEffectDispatch rather than panicking out of it. +func TestRecover_HandlerPanic_ReturnsEffectDispatch(t *testing.T) { + ctx := context.Background() + t.Run("fire", func(t *testing.T) { + st := durable.NewMemStore() + live := &stepIDHandler{failStep: "0"} + h, err := durable.NewRunner(effectMachine(), st, durable.WithEffectHandler[string, string, *runCtx](live.handle)). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire go: %v, want ErrEffectDispatch", err) + } + err = noPanic(t, "Recover", func() error { + _, rerr := durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](panicHandler)) + return rerr + }) + if !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Recover: %v, want ErrEffectDispatch", err) + } + }) + t.Run("tick", func(t *testing.T) { + st := durable.NewMemStore() + clk := state.NewFakeClock(epoch) + live := &stepIDHandler{failStep: "1"} + h, err := durable.NewRunner(timerNotifyMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](live.handle), + durable.WithRunnerClock[string, string, *runCtx](clk)).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Tick: %v, want ErrEffectDispatch", err) + } + err = noPanic(t, "Recover", func() error { + _, rerr := durable.Recover(ctx, timerNotifyMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](panicHandler), + durable.WithRunnerClock[string, string, *runCtx](clk)) + return rerr + }) + if !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Recover: %v, want ErrEffectDispatch", err) + } + }) + t.Run("service", func(t *testing.T) { + st := durable.NewMemStore() + var calls int64 + fn := counterService(&calls) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + live := &stepIDHandler{failStep: "1"} + h, err := durable.NewRunner(fetchThenWaitMachine(fn), st, reg, + durable.WithEffectHandler[string, string, svcCtx](live.handle)).Start(ctx, "i1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire start: %v", err) + } + if _, _, err = h.RunService(ctx, state.InvokeID("fetchThenWait", "loading", 0)); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("RunService: %v, want ErrEffectDispatch", err) + } + err = noPanic(t, "Recover", func() error { + _, rerr := durable.Recover(ctx, fetchThenWaitMachine(fn), st, "i1", reg, + durable.WithEffectHandler[string, string, svcCtx](panicHandler)) + return rerr + }) + if !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Recover: %v, want ErrEffectDispatch", err) + } + }) +} + +// olderVersionJournal copies src's journal for id into a fresh MemStore as a version +// before tick and service effects were dispatched would have left it: their Records +// name no effects, and only the Fire effects are marked dispatched. +func olderVersionJournal(t *testing.T, src *durable.MemStore, id durable.InstanceID) *durable.MemStore { + t.Helper() + ctx := context.Background() + snap, recs, err := src.Load(ctx, id) + if err != nil { + t.Fatalf("Load: %v", err) + } + dst := durable.NewMemStore() + if err = dst.Checkpoint(ctx, id, snap, -1); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + stripped := 0 + for _, rec := range recs { + if len(rec.Event) == 0 { + if len(rec.Effects) > 0 { + stripped++ + } + rec.Effects = nil + } else { + for _, env := range rec.Effects { + if err = dst.MarkDispatched(ctx, id, env.EffectID); err != nil { + t.Fatalf("MarkDispatched: %v", err) + } + } + } + if _, err = dst.Append(ctx, id, rec); err != nil { + t.Fatalf("Append: %v", err) + } + } + if stripped == 0 { + t.Fatal("no tick or service Record named an effect to strip") + } + return dst +} + +// TestRecover_DoesNotSendAnOlderVersionsTickOrServiceEffects proves Recover doesn't +// dispatch an effect a tick or service settle emitted under a version that never +// dispatched them: their Records name no effects, so there is nothing write-ahead to +// retry, and sending it now would apply an effect the run never applied. +func TestRecover_DoesNotSendAnOlderVersionsTickOrServiceEffects(t *testing.T) { + ctx := context.Background() + t.Run("tick", func(t *testing.T) { + st := durable.NewMemStore() + clk := state.NewFakeClock(epoch) + h, err := durable.NewRunner(timerNotifyMachine(), st, durable.WithRunnerClock[string, string, *runCtx](clk)). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + old := olderVersionJournal(t, st, "i1") + rec := newEffectRecorder() + if _, err = durable.Recover(ctx, timerNotifyMachine(), old, "i1", + durable.WithEffectHandler[string, string, *runCtx](rec.handler()), + durable.WithRunnerClock[string, string, *runCtx](clk)); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.totalApplies(); n != 0 { + t.Fatalf("Recover sent %d effects from an older version's tick (%v), want 0", n, rec.ids()) + } + }) + t.Run("service", func(t *testing.T) { + st := durable.NewMemStore() + var calls int64 + fn := counterService(&calls) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + h, err := durable.NewRunner(fetchThenWaitMachine(fn), st, reg).Start(ctx, "i1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire start: %v", err) + } + mustRunService(t, h, state.InvokeID("fetchThenWait", "loading", 0)) + old := olderVersionJournal(t, st, "i1") + rec := newEffectRecorder() + if _, err = durable.Recover(ctx, fetchThenWaitMachine(fn), old, "i1", reg, + durable.WithEffectHandler[string, string, svcCtx](rec.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.totalApplies(); n != 0 { + t.Fatalf("Recover sent %d effects from an older version's service settle (%v), want 0", n, rec.ids()) + } + }) +} diff --git a/durable/error_message_internal_test.go b/durable/error_message_internal_test.go new file mode 100644 index 00000000..59a36441 --- /dev/null +++ b/durable/error_message_internal_test.go @@ -0,0 +1,44 @@ +package durable + +import ( + "encoding/json" + "errors" + "testing" +) + +// TestValidUTF8_MatchesAJSONRoundTrip pins validUTF8 to what a Store keeps: the string +// a JSON round trip gives back, byte for byte, for valid text, a lone bad byte, a +// truncated multi-byte sequence and an encoded surrogate. +func TestValidUTF8_MatchesAJSONRoundTrip(t *testing.T) { + for _, in := range []string{"plain", "café 
 <&>", "bad \xff byte", "cut \xe2\x82", "\xed\xa0\x80 surrogate", "� kept"} { + b, err := json.Marshal(in) + if err != nil { + t.Fatalf("Marshal(%q): %v", in, err) + } + var want string + if err = json.Unmarshal(b, &want); err != nil { + t.Fatalf("Unmarshal(%s): %v", b, err) + } + if got := validUTF8(in); got != want { + t.Fatalf("validUTF8(%q) = %q, want the JSON round trip's %q", in, got, want) + } + } +} + +// panicError's Error method panics. +type panicError struct{} + +func (panicError) Error() string { panic("boom") } + +// TestNormalizedErrorMessage pins the shared helper encodeEventData and +// replayableOutcome use: a normalized message, or errorMessage's error naming what +// panicked. +func TestNormalizedErrorMessage(t *testing.T) { + msg, err := normalizedErrorMessage(errors.New("bad \xff"), "error payload") + if err != nil || msg != "bad �" { + t.Fatalf("got %q, %v; want the normalized message", msg, err) + } + if _, err = normalizedErrorMessage(panicError{}, "service error"); err == nil || err.Error() != "service error panicked: boom" { + t.Fatalf("got %v, want the panic named as a service error's", err) + } +} diff --git a/durable/runner.go b/durable/runner.go index 73b30c1d..85ffd8cd 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -9,6 +9,7 @@ import ( "slices" "sync" "time" + "unicode/utf8" "github.com/stablekernel/crucible/state" ) @@ -994,8 +995,8 @@ func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dis return h.offerCheckpointLocked(ctx, rec) } -// dispatchGuarded dispatches des, turning a panic in the effect handler, or in the -// Store's dispatch tracking, into an ErrEffectDispatch error. +// dispatchGuarded dispatches des, turning a panic in the Store's dispatch tracking into +// an ErrEffectDispatch error. applyEffect turns one in the effect handler into the same. func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []dispatchableEffect) (err error) { defer func() { if p := recover(); p != nil { @@ -1522,8 +1523,13 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // deduped by the Store's dispatched set: an effect already marked (it landed // before the crash) is skipped; one recorded-but-un-marked (the crash fell between // append and dispatch) is applied now — exactly-once across the crash boundary. - if err := r.dispatchReplayEffects(ctx, id, rec.Step, stepEffects); err != nil { - return nil, err + // A Record that names no effects owes none: a tick's or a settlement's from a + // version that didn't dispatch them never applied them, so sending them now + // would apply effects that run never did. + if len(rec.Effects) > 0 { + if err := r.dispatchReplayEffects(ctx, id, rec.Step, stepEffects); err != nil { + return nil, err + } } } @@ -1687,10 +1693,11 @@ func encodeEventData[E comparable](event E, opts []state.FireOption, maxBytes in } var payload eventPayload encoded := data + var msg string e, isErr := data.(error) if isErr { - msg, err := errorMessage(e) - if err != nil { + var err error + if msg, err = normalizedErrorMessage(e, "error payload"); err != nil { return payload, nil, fmt.Errorf("%w: %w", ErrEventDataEncoding, err) } encoded = msg @@ -1703,12 +1710,6 @@ func encodeEventData[E comparable](event E, opts []state.FireOption, maxBytes in return payload, nil, eventDataTooLarge(len(b), maxBytes) } if isErr { - // Take the message back from its JSON form, which is what a Store keeps: JSON - // replaces invalid UTF-8, so the raw message could differ from a replay's. - var msg string - if err = json.Unmarshal(b, &msg); err != nil { - return payload, nil, fmt.Errorf("%w: %w", ErrEventDataEncoding, err) - } payload.errMsg = &msg } else { payload.data = b @@ -1727,16 +1728,37 @@ func encodeEventData[E comparable](event E, opts []state.FireOption, maxBytes in return payload, append(append(live, opts...), state.WithEventData(value)), nil } -// errorMessage returns e.Error(), or an error if it panics, as a typed nil can. -func errorMessage(e error) (msg string, err error) { +// errorMessage returns e.Error(), or an error if it panics, as a typed nil can. what +// names e in that error, such as "error payload". +func errorMessage(e error, what string) (msg string, err error) { defer func() { if p := recover(); p != nil { - err = fmt.Errorf("error payload panicked: %v", p) + err = fmt.Errorf("%s panicked: %v", what, p) } }() return e.Error(), nil } +// normalizedErrorMessage returns e's message in the form a Store keeps it, so the live +// run and a replay can't differ: JSON replaces each byte of invalid UTF-8 with U+FFFD. +// It returns errorMessage's error if e's Error method panics. +func normalizedErrorMessage(e error, what string) (string, error) { + msg, err := errorMessage(e, what) + if err != nil { + return "", err + } + return validUTF8(msg), nil +} + +// validUTF8 returns s with each byte of invalid UTF-8 replaced by U+FFFD, as a round +// trip through JSON does. Converting to runes makes that replacement. +func validUTF8(s string) string { + if utf8.ValidString(s) { + return s + } + return string([]rune(s)) +} + // marshalPayload returns v's JSON form, or an error if a custom marshaler panics, so // a caller's payload can't panic out of Fire. func marshalPayload(v any) (b []byte, err error) { diff --git a/durable/service.go b/durable/service.go index a5f8215c..36f5e057 100644 --- a/durable/service.go +++ b/durable/service.go @@ -93,7 +93,8 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // before the service started. // // Once the service has run, it returns true with any error. A result or KindedEffect -// that won't encode, a ctx that ended during the service or while its settle's effects +// that won't encode, a result whose JSON form won't decode, such as a number beyond +// float64, a ctx that ended during the service or while its settle's effects // were absorbed (even after a clean settle), or a Fire the settle drove refused because a // context ended (ctx, or one middleware derived from it) returns ErrHandleFailed and // records nothing: a recovered Handle has the service in flight again, and RunService @@ -200,23 +201,17 @@ func (h *Handle[S, E, C]) noteSettled(id string) { } // replayableOutcome gives a service's outcome the form a replay supplies, so the live -// settle and a replay can't differ: a result in its JSON-decoded form, an error as an -// error with the same message once any invalid UTF-8 is replaced, which is what the -// round trip through JSON does. An Error method that panics settles with a message -// saying so. A result with no JSON form is left as it is, and fails -// the Handle when recording it fails. +// settle and a replay can't differ: a result in its JSON-decoded form, and an error as +// an error with its message as a Store keeps it (see normalizedErrorMessage). An Error +// method that panics settles with a message saying so, normalized the same way. A +// result with no JSON form is left as it is, and one whose JSON form won't decode is +// marked undecodableResult; recording either fails the Handle, so the step records +// nothing and a recovered Handle runs the service again. func replayableOutcome(result any, err error) (any, error) { if err != nil { - msg, merr := errorMessage(err) - if merr != nil { - return nil, merr - } - b, jerr := marshalPayload(msg) - if jerr != nil { - return nil, errors.New(msg) - } - if jerr = json.Unmarshal(b, &msg); jerr != nil { - return nil, errors.New(msg) + msg, perr := normalizedErrorMessage(err, "service error") + if perr != nil { + msg = validUTF8(perr.Error()) } return nil, errors.New(msg) } @@ -226,11 +221,16 @@ func replayableOutcome(result any, err error) (any, error) { } var decoded any if jerr = json.Unmarshal(b, &decoded); jerr != nil { - return result, nil + return undecodableResult{err: jerr}, nil } return decoded, nil } +// undecodableResult stands in for a service result whose JSON form won't decode, such +// as a number beyond float64, so recordServiceOutcome refuses it: a replay couldn't +// decode the journaled form, and every Recover would fail. +type undecodableResult struct{ err error } + // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service. It reads the success result through lastResult (valid only during the // settle's synchronous window) and the error through lastErr, encoding whichever @@ -241,6 +241,9 @@ func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr erro msg := lastErr.Error() sp.Error = &msg } else if result, ok := lastResult(); ok { + if u, bad := result.(undecodableResult); bad { + return state.JournalEntry{}, fmt.Errorf("service result won't decode from its JSON form: %w", u.err) + } raw, err := marshalPayload(result) if err != nil { return state.JournalEntry{}, fmt.Errorf("marshaling service result: %w", err) diff --git a/durable/service_outcome_test.go b/durable/service_outcome_test.go new file mode 100644 index 00000000..b6b205e6 --- /dev/null +++ b/durable/service_outcome_test.go @@ -0,0 +1,100 @@ +package durable_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// outcomeMachine invokes fn on entering "loading" and records what the onDone or +// onError Assign sees in *seen, live and on replay. +func outcomeMachine(fn state.ServiceFn[svcCtx], seen *[]string) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("outcome"). + Service("fetch", fn). + Reducer("noteDone", func(in state.AssignCtx[svcCtx]) svcCtx { + *seen = append(*seen, "done") + return in.Entity + }). + Reducer("noteError", func(in state.AssignCtx[svcCtx]) svcCtx { + if e, ok := in.Event.(error); ok { + *seen = append(*seen, e.Error()) + } + return in.Entity + }). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready").Assign("noteDone"). + Transition("loading").On("fail").GoTo("failed").Assign("noteError"). + Quench() +} + +// TestRunService_ResultThatEncodesButWontDecode_FailsTheStep proves a result whose +// JSON form a replay can't decode, such as a number beyond float64, fails RunService +// with ErrHandleFailed and records nothing, as a result that won't encode does, so the +// instance stays recoverable with the service in flight again. +func TestRunService_ResultThatEncodesButWontDecode_FailsTheStep(t *testing.T) { + ctx := context.Background() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return json.RawMessage("1e400"), nil } + var seen []string + m := outcomeMachine(fn, &seen) + st := durable.NewMemStore() + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, st, opts...).Start(ctx, "i1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + id := state.InvokeID("outcome", "loading", 0) + if _, _, err = h.RunService(ctx, id); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("RunService: %v, want ErrHandleFailed", err) + } + if _, recs, lerr := st.Load(ctx, "i1"); lerr != nil || len(recs) != 0 { + t.Fatalf("journal holds %d Records (err %v), want none", len(recs), lerr) + } + recovered, err := durable.Recover(ctx, m, st, "i1", opts...) + if err != nil { + t.Fatalf("Recover: %v, want the instance recoverable", err) + } + if got := recovered.Instance().Current(); got != "loading" { + t.Fatalf("recovered in %q, want loading with the service in flight again", got) + } +} + +// panickyError is a service error whose Error method panics with a message holding +// invalid UTF-8. +type panickyError struct{} + +func (panickyError) Error() string { panic("bad \xff byte") } + +// TestRunService_PanickingErrorMessage_LiveMatchesReplay proves an error whose Error +// method panics settles onError with the same message live and on replay, though the +// panic's text holds invalid UTF-8, and that the message doesn't call it an error +// payload. +func TestRunService_PanickingErrorMessage_LiveMatchesReplay(t *testing.T) { + ctx := context.Background() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return nil, panickyError{} } + var seen []string + m := outcomeMachine(fn, &seen) + st := durable.NewMemStore() + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + h, err := durable.NewRunner(m, st, opts...).Start(ctx, "i1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + mustRunService(t, h, state.InvokeID("outcome", "loading", 0)) + if _, err = durable.Recover(ctx, m, st, "i1", opts...); err != nil { + t.Fatalf("Recover: %v", err) + } + if len(seen) != 2 || seen[0] != seen[1] { + t.Fatalf("live and replay saw %q, want the same message", seen) + } + if strings.Contains(seen[0], "error payload") || !strings.Contains(seen[0], "panicked") { + t.Fatalf("onError saw %q, want a message saying the service error panicked", seen[0]) + } +} diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index 2d8dabcf..632deef1 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -38,6 +38,8 @@ counts as an additive (minor) versus breaking (major) change. Use the the actors that were running. It's additive, within v1.1.0. - **`ServiceRunner.WithResultTransform`** rewrites a service's outcome before `Tick` settles it, so a host that records outcomes can give the live run what a replay will see. + An unbound service still settles through onError: a transform that returns no error for + its `UnboundRefError` doesn't make it succeed. - **`FireEventData`** reports the payload a `WithEventData` option carries, so a host that records a Fire can persist it. It's additive, so the next release is v1.1.0. - **Callbacks get the step's context.** `GuardCtx`, `ActionCtx` and `AssignCtx` have a new diff --git a/state/result_transform_test.go b/state/result_transform_test.go new file mode 100644 index 00000000..71d81fa5 --- /dev/null +++ b/state/result_transform_test.go @@ -0,0 +1,74 @@ +package state_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/stablekernel/crucible/state" +) + +// TestWithResultTransform_RewritesTheOutcomeTickSettles proves Tick settles a bound +// service with the transform's outcome, a result or an error, and that LastResult and +// LastError report the rewritten form. +func TestWithResultTransform_RewritesTheOutcomeTickSettles(t *testing.T) { + ctx := context.Background() + upper := func(result any, err error) (any, error) { + if err != nil { + return nil, errors.New("wrapped: " + err.Error()) + } + return strings.ToUpper(result.(string)), nil + } + cases := []struct { + name string + fn state.ServiceFn[*trec] + wantState string + wantNotes string + }{ + {"result", func(context.Context, state.ServiceCtx[*trec]) (any, error) { return "payload", nil }, "ready", "result:PAYLOAD"}, + {"error", func(context.Context, state.ServiceCtx[*trec]) (any, error) { return nil, errors.New("boom") }, "errored", "error:wrapped: boom"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var run *state.ServiceRunner[string, string, *trec] + m := invokeMachine(&run) + entity := &trec{} + inst := m.Cast(entity, state.WithInitialState("idle")) + run = state.NewServiceRunner(inst, state.NewRegistry[*trec]().Service("fetch", tc.fn)).WithResultTransform(upper) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + res := run.Tick(ctx, state.InvokeID("loader", "loading", 0)) + if len(res) != 1 || res[0].NewState != tc.wantState { + t.Fatalf("Tick settled to %+v, want %q", res, tc.wantState) + } + if got := join(entity.notes); got != tc.wantNotes { + t.Fatalf("notes = %q, want %q", got, tc.wantNotes) + } + }) + } +} + +// TestWithResultTransform_UnboundServiceStillRoutesOnError proves a transform can't +// turn an unbound service into a success: a transform that returns a result and no +// error for the UnboundRefError still settles the service through onError, with the +// UnboundRefError. +func TestWithResultTransform_UnboundServiceStillRoutesOnError(t *testing.T) { + ctx := context.Background() + var run *state.ServiceRunner[string, string, *trec] + m := invokeMachine(&run) + entity := &trec{} + inst := m.Cast(entity, state.WithInitialState("idle")) + // A nil registry leaves "fetch" unbound. + run = state.NewServiceRunner(inst, nil).WithResultTransform(func(any, error) (any, error) { + return "fallback", nil + }) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + res := run.Tick(ctx, state.InvokeID("loader", "loading", 0)) + if len(res) != 1 || res[0].NewState != "errored" { + t.Fatalf("an unbound service settled to %+v, want onError's errored", res) + } + var unbound *state.UnboundRefError + if !errors.As(run.LastError(), &unbound) { + t.Fatalf("LastError = %v, want the UnboundRefError", run.LastError()) + } +} diff --git a/state/runner.go b/state/runner.go index 1d8d8a29..504c078f 100644 --- a/state/runner.go +++ b/state/runner.go @@ -223,7 +223,8 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a // WithResultTransform sets a function that rewrites a ServiceFn's outcome before Tick // settles it, and returns the runner. A host that records outcomes uses it to give the // live run the form a replay will see. LastResult and LastError then report the rewritten -// outcome. It also runs on an UnboundRefError, where only the returned error is used. It +// outcome. It also runs on an UnboundRefError, where only a non-nil returned error is +// used: a nil one leaves the UnboundRefError, so the service still settles through onError. It // doesn't apply to SettleDone or SettleError, which take the outcome as given. Set it // before the first Tick: the field isn't guarded. func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(result any, err error) (any, error)) *ServiceRunner[S, E, C] { @@ -253,7 +254,11 @@ func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResu if fn == nil { var err error = &UnboundRefError{Kind: "service", Name: rs.src.Name} if r.transform != nil { - _, err = r.transform(nil, err) + // An unbound service always settles as an error: a transform that returns + // no error can't turn it into a success. + if _, terr := r.transform(nil, err); terr != nil { + err = terr + } } res, _ := r.SettleError(ctx, id, err) return []FireResult[S]{res} From d7c6548fb92abffb3b0d26c071b0a366bba37384 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 13:14:27 -0700 Subject: [PATCH 03/13] feat(durable): decode service results through WithEventDataDecoder; pin Start's at-least-once effects A service result now decodes through the Runner's EventDataDecoder for the invocation's onDone event, live and on replay, as a WithEventData payload does. A decoder for that event hands Assign its type; without one the result arrives in JSON's generic form. A service error stays message-only. - state: ServiceRunner.WithResultTransform's function takes the event that will carry the outcome, func(event E, result any, err error) (any, error), and runs on every settle: Tick, SettleDone and SettleError. It runs before any event fires. An UnboundRefError still settles through onError when the transform returns no error. - A decoder that fails on a result, or returns a type other than the service returned, fails RunService with ErrHandleFailed wrapping ErrEventDataEncoding before any onDone or onError Assign runs, and records nothing. Recover and StateAt fail with ErrEventDataEncoding on a recorded result their decoder can't decode, also running no Assign. This also covers a result whose JSON form won't decode, such as a number beyond float64. - The docs say to register the decoder before results are recorded: replay decodes the journal with the recovering Runner's decoder. - Tests pin the decoding scenarios, that a decode failure runs no Assign, and the documented hazard of a decoder added after recording. - Tests pin Start's initial effects as at-least-once: applied before the baseline write and never marked, so a handler failure, a failed baseline write or overlapping Starts re-apply them, and Recover never applies them. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 15 +- durable/CHANGELOG.md | 23 +- durable/README.md | 15 +- durable/doc.go | 10 + durable/errors.go | 5 +- durable/result_decoding_scenarios_test.go | 373 ++++++++++++++++++++++ durable/runner.go | 2 +- durable/runner_options.go | 12 + durable/service.go | 186 ++++++++--- durable/service_outcome_test.go | 11 +- durable/start_effects_contract_test.go | 307 ++++++++++++++++++ durable/timetravel.go | 2 +- state/CHANGELOG.md | 10 +- state/result_transform_test.go | 105 +++++- state/runner.go | 57 ++-- 15 files changed, 1024 insertions(+), 109 deletions(-) create mode 100644 durable/result_decoding_scenarios_test.go create mode 100644 durable/start_effects_contract_test.go diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 2460056f..8ec1f633 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -163,9 +163,18 @@ owner holds the instance, so stop. `Recover` retries an unapplied effect, and re Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` observes the JSON-decoded shape of actor done-data on a replayed `onDone`. A service's -result is JSON-decoded live as well as on replay, so a reducer gets the same value both -ways: a struct is a map and an integer a `float64`, which loses precision above 2^53, and -an error an error with the same message, which `errors.Is` no longer matches. A +result is decoded from its JSON form live as well as on replay, so a reducer gets the same +value both ways. When `WithEventDataDecoder` covers the invocation's `onDone` event, the +result arrives as the decoder's type, for example `durable.DecodeAs[User]`. Otherwise it +arrives in JSON's generic form: a struct as a map and an integer as a `float64`, which +loses precision above 2^53. A decoder that fails, or that returns a type other than the +one the service returned, fails `RunService` loudly with `ErrHandleFailed` wrapping +`ErrEventDataEncoding`: no `onDone` or `onError` reducer runs and nothing is recorded. A +service error stays message-only, an error with the same message, which `errors.Is` no +longer matches. Register the decoder before you record results: replay decodes the journal +with the recovering Runner's decoder, so a decoder added or changed after results were +recorded changes what replay hands the reducer, and one that can't decode a recorded result +fails `Recover` and `StateAt` with `ErrEventDataEncoding`. A `WithEventData` payload is decoded from its JSON form live as well as on replay, so a reducer gets the same value both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index dd839493..ee46f2d2 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -106,14 +106,21 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A `KindedEffect` that can't marshal fails `Tick` and `RunService`,** as it already failed `Fire`, even with no handler wired. - **A service result reaches `Assign` decoded, live as well as on replay,** so a retried - effect can't differ from the first attempt: a struct arrives as a map, an integer as a - `float64`, which loses precision above 2^53, and an error as an error with the same - message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` - method that panics settles with a message saying the service error panicked, its - invalid UTF-8 replaced too. The journal records this decoded value. A result whose JSON - form won't decode, such as a number beyond `float64`, fails `RunService` with - `ErrHandleFailed` and records nothing, as a result that won't encode does, so - `Recover` doesn't fail on it. + effect can't differ from the first attempt. The result is decoded through the + `WithEventDataDecoder` decoder for the invocation's onDone event, as a `WithEventData` + payload is: a decoder for that event, such as `DecodeAs[Order]`, hands `Assign` that + type, and without one a struct arrives as a map and an integer as a `float64`, which + loses precision above 2^53. A decoder that fails, or returns a type other than the + service returned, fails `RunService` with `ErrHandleFailed` wrapping + `ErrEventDataEncoding` before any onDone or onError `Assign` runs, and records nothing; + so does a result whose JSON form won't decode, such as a number beyond `float64`. + `Recover` and `StateAt` fail with `ErrEventDataEncoding`, running no `Assign`, on a + recorded result their decoder can't decode. Replay decodes the journal with the + recovering Runner's decoder, so register a decoder before results are recorded: one + added or changed later changes what replay hands `Assign`. An error stays message-only: + an error with the same message, any invalid UTF-8 replaced, which `errors.Is` no longer + matches. An `Error` method that panics settles with a message saying the service error + panicked, its invalid UTF-8 replaced too. The journal records the decoded value. - **A step that may not have been recorded fails the Handle.** That call and every later one return `ErrHandleFailed`, wrapping the cause. A step that was recorded while another goroutine failed the Handle also returns `ErrHandleFailed`, saying the step was diff --git a/durable/README.md b/durable/README.md index ee5cc513..25e9d36c 100644 --- a/durable/README.md +++ b/durable/README.md @@ -174,9 +174,18 @@ owner holds the instance, so stop. `Recover` retries an unapplied effect, and re Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` observes the JSON-decoded shape of actor done-data on a replayed `onDone`. A service's -result is JSON-decoded live as well as on replay, so a reducer gets the same value both -ways: a struct is a map and an integer a `float64`, which loses precision above 2^53, and -an error an error with the same message, which `errors.Is` no longer matches. A +result is decoded from its JSON form live as well as on replay, so a reducer gets the same +value both ways. When `WithEventDataDecoder` covers the invocation's `onDone` event, the +result arrives as the decoder's type, for example `durable.DecodeAs[User]`. Otherwise it +arrives in JSON's generic form: a struct as a map and an integer as a `float64`, which +loses precision above 2^53. A decoder that fails, or that returns a type other than the +one the service returned, fails `RunService` loudly with `ErrHandleFailed` wrapping +`ErrEventDataEncoding`: no `onDone` or `onError` reducer runs and nothing is recorded. A +service error stays message-only, an error with the same message, which `errors.Is` no +longer matches. Register the decoder before you record results: replay decodes the journal +with the recovering Runner's decoder, so a decoder added or changed after results were +recorded changes what replay hands the reducer, and one that can't decode a recorded result +fails `Recover` and `StateAt` with `ErrEventDataEncoding`. A `WithEventData` payload is decoded from its JSON form live as well as on replay, so a reducer gets the same value both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own diff --git a/durable/doc.go b/durable/doc.go index 87985aeb..9f3bb8bb 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -41,6 +41,16 @@ // well as on replay, so a reducer gets the same value both ways; WithEventDataDecoder // decodes it as the type the caller passed. // +// A service result is decoded the same way, live and on replay. When +// WithEventDataDecoder covers the invocation's onDone event, the result arrives as the +// decoder's type; otherwise it arrives in JSON's generic form, a struct as a +// map[string]any and a number as a float64. A decoder that fails, or returns a type +// other than the service's result, fails RunService with ErrHandleFailed wrapping +// ErrEventDataEncoding, and no onDone or onError reducer runs. A service error stays +// message-only. Register the decoder before recording results: replay decodes the +// journal with the recovering Runner's decoder, so one added or changed after results +// were recorded changes what replay produces. +// // The kernel stays pure and dependency-free. The durable module is host-side: // database drivers, cloud SDKs, and I/O libraries live in the caller or in out- // of-tree Store backends, never in this module. The in-tree MemStore and diff --git a/durable/errors.go b/durable/errors.go index 69061aca..21eaadea 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -46,7 +46,10 @@ var ( // its state.WithEventData payload has no JSON form that decodes again, decodes as a // different type than the caller passed (see WithEventDataDecoder), panics while // encoding (such as an error that's a typed nil), or exceeds WithMaxEventDataBytes. - // Nothing applied, so the Handle stays usable. + // Nothing applied, so the Handle stays usable. RunService wraps it in ErrHandleFailed + // for a service result that won't decode for its onDone event, or decodes as another + // type than the service returned; Recover and StateAt return it for a recorded result + // that won't decode. No onDone or onError Assign runs for that result. ErrEventDataEncoding = errors.New(errorPrefix + errorEventDataEncoding) // ErrRecordTooLarge is reported by FileStore.Append for a Record whose journal line diff --git a/durable/result_decoding_scenarios_test.go b/durable/result_decoding_scenarios_test.go new file mode 100644 index 00000000..64d267f0 --- /dev/null +++ b/durable/result_decoding_scenarios_test.go @@ -0,0 +1,373 @@ +package durable_test + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These scenario tests pin what a host's Assign sees for a service result, live and +// after Recover: a result whose onDone event WithEventDataDecoder covers arrives as the +// decoder's type, any other arrives in JSON's generic form, an error is message-only, +// and a decoder that fails or returns the wrong type fails the step before any Assign +// runs. + +type scnUser struct { + ID int64 + Name string +} + +type scnAdmin struct { + ID int64 + Name string + Level int +} + +var errScnNotFound = errors.New("user not found") + +// scnAssigns records what the onDone and onError Assigns saw, in order, and how many +// times each ran. +type scnAssigns struct { + seen []any + done, fails int +} + +// scnMachine invokes fn on entering "loading": onDone "done" goes to ready, onError +// "fail" to failed, each through an Assign that notes what it saw in a. +func scnMachine(fn state.ServiceFn[svcCtx], a *scnAssigns) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("scn"). + Service("fetch", fn). + Reducer("noteDone", func(in state.AssignCtx[svcCtx]) svcCtx { + a.done++ + a.seen = append(a.seen, in.Event) + return in.Entity + }). + Reducer("noteFail", func(in state.AssignCtx[svcCtx]) svcCtx { + a.fails++ + a.seen = append(a.seen, in.Event) + return in.Entity + }). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready").Assign("noteDone"). + Transition("loading").On("fail").GoTo("failed").Assign("noteFail"). + Quench() +} + +const scnInvoke = "scn" // the machine id InvokeID takes + +// scnStart starts an instance of m in "loading" and returns its Handle. +func scnStart(t *testing.T, m *state.Machine[string, string, svcCtx], store durable.Store, opts ...durable.Option[string, string, svcCtx]) *durable.Handle[string, string, svcCtx] { + t.Helper() + h, err := durable.NewRunner(m, store, opts...).Start(context.Background(), "scn-1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + return h +} + +// scnRun starts an instance in a state that invokes fn, runs the service live, then +// Recovers it, and returns what onDone's or onError's Assign saw each time (live +// first), with the RunService and Recover errors. +func scnRun(t *testing.T, fn state.ServiceFn[svcCtx], opts ...durable.Option[string, string, svcCtx]) (a *scnAssigns, runErr, recoverErr error) { + t.Helper() + ctx := context.Background() + a = &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + opts = append([]durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))}, opts...) + h := scnStart(t, m, store, opts...) + _, _, runErr = h.RunService(ctx, state.InvokeID(scnInvoke, "loading", 0)) + _, recoverErr = durable.Recover(ctx, m, store, "scn-1", opts...) + return a, runErr, recoverErr +} + +func describe(v any) string { return fmt.Sprintf("%T %+v", v, v) } + +// decoderFor decodes event's payload with d and every other event's as JSON's own form. +func decoderFor(event string, d func([]byte) (any, error)) durable.Option[string, string, svcCtx] { + return durable.WithEventDataDecoder[string, string, svcCtx](func(e string, data []byte) (any, error) { + if e == event { + return d(data) + } + return durable.DecodeAs[any](data) + }) +} + +func returning(v any, err error) state.ServiceFn[svcCtx] { + return func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return v, err } +} + +// mustTwo fails unless the Assign ran exactly live and once on replay, and saw the +// same value both times. +func mustTwo(t *testing.T, a *scnAssigns, runErr, recErr error) { + t.Helper() + if runErr != nil || recErr != nil { + t.Fatalf("RunService: %v, Recover: %v", runErr, recErr) + } + if len(a.seen) != 2 { + t.Fatalf("Assign ran %d times, want 2 (live, replay): %v", len(a.seen), a.seen) + } + if describe(a.seen[0]) != describe(a.seen[1]) { + t.Fatalf("live saw %s, replay %s; want the same", describe(a.seen[0]), describe(a.seen[1])) + } +} + +// TestServiceResult_DecoderForOnDone_ArrivesAsTheDecodersType: a struct result whose +// onDone event has a decoder reaches the Assign as that struct, live and on replay. +func TestServiceResult_DecoderForOnDone_ArrivesAsTheDecodersType(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(scnUser{ID: 7, Name: "ann"}, nil), + decoderFor("done", durable.DecodeAs[scnUser])) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + if u, ok := v.(scnUser); !ok || u != (scnUser{ID: 7, Name: "ann"}) { + t.Errorf("seen[%d] = %s, want the scnUser", i, describe(v)) + } + } +} + +// TestServiceResult_Int64Above2to53_ExactWithADecoder: an int64 beyond float64's exact +// range survives when the decoder decodes it as an int64. +func TestServiceResult_Int64Above2to53_ExactWithADecoder(t *testing.T) { + const want = int64(1<<53 + 1) + a, runErr, recErr := scnRun(t, returning(want, nil), decoderFor("done", durable.DecodeAs[int64])) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + if n, ok := v.(int64); !ok || n != want { + t.Errorf("seen[%d] = %s, want int64 %d", i, describe(v), want) + } + } +} + +// TestServiceResult_StructWithInt64Above2to53_ExactWithADecoder: the same id inside a +// struct survives too. +func TestServiceResult_StructWithInt64Above2to53_ExactWithADecoder(t *testing.T) { + const want = int64(1<<53 + 1) + a, runErr, recErr := scnRun(t, returning(scnUser{ID: want, Name: "big"}, nil), + decoderFor("done", durable.DecodeAs[scnUser])) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + if u, ok := v.(scnUser); !ok || u.ID != want { + t.Errorf("seen[%d] = %s, want ID %d", i, describe(v), want) + } + } +} + +// TestServiceResult_SentinelError_ErrorsIs would have errors.Is match a wrapped +// sentinel in onError. Errors are message-only by design, so it can't. +func TestServiceResult_SentinelError_ErrorsIs(t *testing.T) { + t.Skip("service errors are message-only by design: onError gets an error with the same message, which errors.Is doesn't match") + a, runErr, recErr := scnRun(t, returning(nil, fmt.Errorf("lookup 7: %w", errScnNotFound))) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + if e, ok := v.(error); !ok || !errors.Is(e, errScnNotFound) { + t.Errorf("seen[%d] = %s, want an error matching errScnNotFound", i, describe(v)) + } + } +} + +// TestServiceResult_ErrorIsMessageOnly: a service error reaches onError as an error +// with the same message, live and on replay, and doesn't match the sentinel it wraps. +func TestServiceResult_ErrorIsMessageOnly(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(nil, fmt.Errorf("lookup 7: %w", errScnNotFound)), + decoderFor("fail", func([]byte) (any, error) { return nil, errors.New("an error must not reach the decoder") })) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + e, ok := v.(error) + if !ok || e.Error() != "lookup 7: user not found" || errors.Is(e, errScnNotFound) { + t.Errorf("seen[%d] = %s, want an error with the message only", i, describe(v)) + } + } +} + +// TestServiceResult_NoDecoder_StructIsAMap: with no decoder for onDone a struct +// arrives in JSON's generic form. +func TestServiceResult_NoDecoder_StructIsAMap(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(scnUser{ID: 7, Name: "ann"}, nil)) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + if _, ok := v.(map[string]any); !ok { + t.Errorf("seen[%d] = %T, want map[string]any", i, v) + } + } +} + +// TestServiceResult_NoDecoder_Int64IsFloat64: with no decoder for onDone an int64 +// arrives as a float64. +func TestServiceResult_NoDecoder_Int64IsFloat64(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(int64(42), nil)) + mustTwo(t, a, runErr, recErr) + for i, v := range a.seen { + if f, ok := v.(float64); !ok || f != 42 { + t.Errorf("seen[%d] = %s, want float64 42", i, describe(v)) + } + } +} + +// TestServiceResult_NoJSONForm_FailsTheHandle: a result with no JSON form fails the +// Handle and records nothing, so the recovered instance has the service in flight +// again. The Assign ran live only: the result settled before recording found it had no +// JSON form. +func TestServiceResult_NoJSONForm_FailsTheHandle(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(make(chan int), nil), decoderFor("done", durable.DecodeAs[scnUser])) + if !errors.Is(runErr, durable.ErrHandleFailed) { + t.Errorf("RunService err = %v, want ErrHandleFailed", runErr) + } + if recErr != nil { + t.Errorf("Recover: %v", recErr) + } + if len(a.seen) != 1 { + t.Errorf("Assign ran %d times, want 1 (live only; nothing recorded)", len(a.seen)) + } +} + +// mustFailLoudly fails unless RunService failed the Handle with ErrEventDataEncoding, +// no Assign ran, and Recover found the service in flight again. +func mustFailLoudly(t *testing.T, a *scnAssigns, runErr, recErr error) { + t.Helper() + if !errors.Is(runErr, durable.ErrHandleFailed) || !errors.Is(runErr, durable.ErrEventDataEncoding) { + t.Errorf("RunService err = %v, want ErrHandleFailed wrapping ErrEventDataEncoding", runErr) + } + if recErr != nil { + t.Errorf("Recover: %v, want the instance recoverable", recErr) + } + if a.done != 0 || a.fails != 0 { + t.Errorf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } +} + +// TestServiceResult_WrongDecoderType_Compatible_FailsLoudly: a decoder for the wrong +// type (scnAdmin for a scnUser result), whose JSON still decodes, is refused live. +func TestServiceResult_WrongDecoderType_Compatible_FailsLoudly(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(scnUser{ID: 7, Name: "ann"}, nil), + decoderFor("done", durable.DecodeAs[scnAdmin])) + mustFailLoudly(t, a, runErr, recErr) +} + +// TestServiceResult_WrongDecoderType_Incompatible_FailsLoudly: a decoder for a type +// the JSON won't decode as (int64 for an object) fails the step too. +func TestServiceResult_WrongDecoderType_Incompatible_FailsLoudly(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(scnUser{ID: 7, Name: "ann"}, nil), + decoderFor("done", durable.DecodeAs[int64])) + mustFailLoudly(t, a, runErr, recErr) +} + +// TestServiceResult_NilResult_ArrivesNil: a nil result arrives as nil both ways, +// whatever the decoder. +func TestServiceResult_NilResult_ArrivesNil(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(nil, nil), decoderFor("done", durable.DecodeAs[scnUser])) + mustTwo(t, a, runErr, recErr) + if a.seen[0] != nil { + t.Errorf("live saw %T, want nil", a.seen[0]) + } +} + +// TestServiceResult_DecoderAddedAfterRecording_ChangesReplay pins the documented hazard +// that makes the docs say to register the decoder before recording: a result recorded +// with no decoder, then recovered by a Runner that has one for onDone, reaches the +// replayed Assign as the decoder's type, though live saw JSON's generic form. The +// journal is the same JSON either way; replay decodes it with the decoder it has. +func TestServiceResult_DecoderAddedAfterRecording_ChangesReplay(t *testing.T) { + ctx := context.Background() + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + h := scnStart(t, m, store, reg) + mustRunService(t, h, state.InvokeID(scnInvoke, "loading", 0)) + if _, err := durable.Recover(ctx, m, store, "scn-1", reg, decoderFor("done", durable.DecodeAs[scnUser])); err != nil { + t.Fatalf("Recover: %v", err) + } + if len(a.seen) != 2 { + t.Fatalf("Assign ran %d times, want 2 (live, replay)", len(a.seen)) + } + if _, ok := a.seen[0].(map[string]any); !ok { + t.Errorf("live saw %s, want map[string]any: no decoder was registered", describe(a.seen[0])) + } + // The hazard: replay hands the Assign a different type than live did. + if _, ok := a.seen[1].(scnUser); !ok { + t.Errorf("replay saw %s, want the scnUser the added decoder gives", describe(a.seen[1])) + } +} + +// failingDecoder fails for onDone and decodes every other event as JSON's own form. +func failingDecoder() durable.Option[string, string, svcCtx] { + return decoderFor("done", func([]byte) (any, error) { return nil, errors.New("decoder refused") }) +} + +// TestServiceResult_DecodeFailure_RunsNoOnErrorAssign proves a result that fails to +// decode, or decodes live as another type than the service returned, runs no Assign +// at all: neither onDone's nor onError's, not even once in memory before the Handle +// fails. It fails the Handle with ErrEventDataEncoding, records nothing and leaves the +// instance where it was. +func TestServiceResult_DecodeFailure_RunsNoOnErrorAssign(t *testing.T) { + cases := []struct { + name string + opt durable.Option[string, string, svcCtx] + }{ + {"decoder fails", failingDecoder()}, + {"type mismatch", decoderFor("done", durable.DecodeAs[scnAdmin])}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), tc.opt} + h := scnStart(t, m, store, opts...) + _, ran, err := h.RunService(ctx, state.InvokeID(scnInvoke, "loading", 0)) + if !ran || !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("RunService = %v, %v; want ran and ErrHandleFailed wrapping ErrEventDataEncoding", ran, err) + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } + if got := h.Instance().Current(); got != "loading" { + t.Fatalf("the live instance is in %q, want loading", got) + } + if _, recs, lerr := store.Load(ctx, "scn-1"); lerr != nil || len(recs) != 0 { + t.Fatalf("journal holds %d Records (err %v), want none", len(recs), lerr) + } + }) + } +} + +// TestServiceResult_RecordedResultThatWontDecode_RunsNoAssignOnReplay proves Recover and +// StateAt, replaying a recorded result their decoder can't decode, fail with +// ErrEventDataEncoding before any Assign runs. +func TestServiceResult_RecordedResultThatWontDecode_RunsNoAssignOnReplay(t *testing.T) { + ctx := context.Background() + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + h := scnStart(t, m, store, reg) + mustRunService(t, h, state.InvokeID(scnInvoke, "loading", 0)) + if a.done != 1 || a.fails != 0 { + t.Fatalf("live: onDone ran %d times and onError %d, want 1 and 0", a.done, a.fails) + } + + if _, err := durable.Recover(ctx, m, store, "scn-1", reg, failingDecoder()); !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Recover: %v, want ErrEventDataEncoding", err) + } + if a.done != 1 || a.fails != 0 { + t.Fatalf("after Recover: onDone ran %d times and onError %d, want no more than live's 1 and 0", a.done, a.fails) + } + + if _, err := durable.StateAt(ctx, m, store, "scn-1", 0, reg, failingDecoder()); !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("StateAt: %v, want ErrEventDataEncoding", err) + } + if a.done != 1 || a.fails != 0 { + t.Fatalf("after StateAt: onDone ran %d times and onError %d, want no more than live's 1 and 0", a.done, a.fails) + } +} diff --git a/durable/runner.go b/durable/runner.go index 85ffd8cd..f749f821 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -624,7 +624,7 @@ func (r *Runner[S, E, C]) newServiceRunner(inst *state.Instance[S, E, C]) *state if r.cfg.serviceReg == nil { return nil } - return state.NewServiceRunner(inst, r.cfg.serviceReg).WithResultTransform(replayableOutcome) + return state.NewServiceRunner(inst, r.cfg.serviceReg).WithResultTransform(replayableOutcome(r.cfg.eventDataDecoder)) } // newActorSystem builds the kernel host driver for child-machine actors bound to diff --git a/durable/runner_options.go b/durable/runner_options.go index 12203e63..22e24f57 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -179,6 +179,18 @@ func WithMaxEventDataBytes[S comparable, E comparable, C any](n int) Option[S, E // float64, map[string]any or []any) is accepted; a struct, an int or a typed slice is // refused with ErrEventDataEncoding until the decoder covers its event. A nil decoder is // ignored. +// +// The decoder also decodes a service's result, for the invocation's onDone event, live +// in RunService and on every replay. A result whose onDone event the decoder covers +// arrives as the decoder's type; otherwise it arrives in JSON's generic form, a struct +// as a map[string]any and a number as a float64. A decoder that fails on a result, or +// returns a type other than the service returned, fails RunService with ErrHandleFailed +// wrapping ErrEventDataEncoding before any onDone or onError Assign runs, and records +// nothing. A service error stays message-only and doesn't reach the decoder. Register +// the decoder before results are recorded: replay decodes the journal with the +// recovering Runner's decoder, so one added or changed after results were recorded +// changes what replay produces, and one that can't decode a recorded result fails +// Recover and StateAt with ErrEventDataEncoding. func WithEventDataDecoder[S comparable, E comparable, C any](d EventDataDecoder[E]) Option[S, E, C] { return func(c *runnerConfig[S, E, C]) { if d != nil { diff --git a/durable/service.go b/durable/service.go index 36f5e057..65302176 100644 --- a/durable/service.go +++ b/durable/service.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "github.com/stablekernel/crucible/state" ) @@ -29,13 +30,13 @@ import ( // The Runner wraps the kernel's reusable host driver, state.ServiceRunner, which // turns StartService / StopService effects into real ServiceFn executions and // re-fires each result through the invocation's onDone / onError event. On the live -// path RunService delegates to ServiceRunner.Tick (resolve + run + settle), reads -// the outcome in its JSON-decoded form, and journals it as a JournalServiceResult -// correlated by the invocation id. On recovery the recorded outcomes are replayed -// in order through ServiceRunner.SettleDone / SettleError — the same settle methods -// the live run drove — so the kernel re-fires the identical onDone / onError event -// with the identical data and re-derives byte-identical context, having run no -// service. +// path RunService delegates to ServiceRunner.Tick (resolve + run + settle), reads the +// outcome in its decoded form (see replayableOutcome), and journals it as a +// JournalServiceResult correlated by the invocation id. On recovery the recorded +// outcomes are replayed in order through ServiceRunner.SettleDone / SettleError — the +// same settle methods the live run drove — so the kernel re-fires the identical onDone +// / onError event with the identical data and re-derives byte-identical context, +// having run no service. // // # Correlation and ordering // @@ -53,9 +54,10 @@ import ( // whichever is present and settles the invocation through the matching seam. type serviceResultPayload struct { // Result is the JSON-encoded successful return of the ServiceFn, present for a - // SettleDone outcome. It is decoded to a Go value and re-fired through onDone so - // the transition's Assign reads it from AssignCtx.Event exactly as on the live - // run, which settles with the same decoded form (see replayableOutcome). + // SettleDone outcome. It is decoded through the Runner's EventDataDecoder for the + // onDone event and re-fired through onDone, so the transition's Assign reads it + // from AssignCtx.Event exactly as on the live run, which settles with the same + // decoded form (see replayableOutcome). Result json.RawMessage `json:"result,omitempty"` // Error is the failure message for a SettleError outcome. It is reconstructed // into an error and re-fired through onError so the transition's Assign reads it @@ -82,22 +84,27 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // recording the produced step. The service is resolved and executed against the // registry supplied with WithServiceRegistry; its result or error is journaled // as a JournalServiceResult correlated by id so recovery replays it without -// re-invoking the service. The outcome reaches the machine JSON-decoded, live as on -// replay, and the journal holds that form: a struct is a map, an integer a float64 that -// loses precision above 2^53, and an error an error with the same message, which -// errors.Is no longer matches. +// re-invoking the service. The outcome reaches the machine decoded from its JSON form, +// live as on replay, and the journal holds that form. A result whose onDone event +// WithEventDataDecoder covers arrives as the decoder's type; otherwise it arrives in +// JSON's generic form: a struct is a map, an integer a float64 that loses precision +// above 2^53. An error is message-only: an error with the same message, which +// errors.Is no longer matches. Register the decoder before results are recorded: a +// decoder added or changed later changes what replay produces. // // It returns the routed FireResult and true, or the zero result and false when id // names no in-flight service (already settled, stopped, or never started). It returns // false and an error when no registry is wired, the Handle has failed, or ctx ended // before the service started. // -// Once the service has run, it returns true with any error. A result or KindedEffect -// that won't encode, a result whose JSON form won't decode, such as a number beyond -// float64, a ctx that ended during the service or while its settle's effects -// were absorbed (even after a clean settle), or a Fire the settle drove refused because a -// context ended (ctx, or one middleware derived from it) returns ErrHandleFailed and -// records nothing: a recovered Handle has the service in flight again, and RunService +// Once the service has run, it returns true with any error. A result the decoder for +// its onDone event can't decode, such as a number beyond float64, or decodes as a type +// other than the service returned, returns ErrHandleFailed wrapping +// ErrEventDataEncoding before any onDone or onError Assign runs. That, a result or +// KindedEffect that won't encode, a ctx that ended during the service or while its +// settle's effects were absorbed (even after a clean settle), or a Fire the settle +// drove refused because a context ended (ctx, or one middleware derived from it) +// returns ErrHandleFailed and records nothing: a recovered Handle has the service in flight again, and RunService // runs it a second time. // A failed Append also returns ErrHandleFailed, but the write may have landed anyway: // Recover finds out, and the service runs again only if the result isn't in the @@ -141,7 +148,13 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return state.FireResult[S]{}, false, nil, nil, nil } - results := h.svc.Tick(h.stepContext(ctx), id) + results, derr := tickService(h.stepContext(ctx), h.svc, id) + if derr != nil { + // The result didn't decode for its onDone event, so nothing fired: no Assign ran + // and nothing is recorded. The runner has already dropped the service, so this + // Handle can't run it again; Recover re-arms it. + return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, derr)) + } if len(results) == 0 { return state.FireResult[S]{}, false, nil, nil, nil } @@ -200,36 +213,95 @@ func (h *Handle[S, E, C]) noteSettled(id string) { h.settled[id] = true } -// replayableOutcome gives a service's outcome the form a replay supplies, so the live -// settle and a replay can't differ: a result in its JSON-decoded form, and an error as -// an error with its message as a Store keeps it (see normalizedErrorMessage). An Error -// method that panics settles with a message saying so, normalized the same way. A -// result with no JSON form is left as it is, and one whose JSON form won't decode is -// marked undecodableResult; recording either fails the Handle, so the step records -// nothing and a recovered Handle runs the service again. -func replayableOutcome(result any, err error) (any, error) { - if err != nil { - msg, perr := normalizedErrorMessage(err, "service error") - if perr != nil { - msg = validUTF8(perr.Error()) +// replayableOutcome returns the transform a ServiceRunner applies to every outcome as +// it settles, so the live settle and a replay can't differ. A result is encoded to +// JSON and decoded through decode for the invocation's onDone event, as a +// WithEventData payload is: the live run passes the ServiceFn's value, a replay the +// recorded JSON as a recordedResult, and both get decode's value. With the default +// decoder that is JSON's generic form. Live, a decoder that returns a concrete type +// other than the result's is refused, though JSON's generic form is always accepted, +// so the default keeps working. +// +// A result that won't decode, or is refused, panics with a resultDecodeFailure before +// the settle fires anything, so no onDone or onError Assign runs; tickService and +// replayService recover it and fail the step. A result with no JSON form is left as it +// is, and fails the Handle when recordServiceOutcome can't encode it. +// +// An error becomes an error with its message as a Store keeps it (see +// normalizedErrorMessage), live and on replay. An Error method that panics settles +// with a message saying so, normalized the same way. +func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, error) (any, error) { + return func(event E, result any, err error) (any, error) { + if err != nil { + msg, perr := normalizedErrorMessage(err, "service error") + if perr != nil { + msg = validUTF8(perr.Error()) + } + return nil, errors.New(msg) } - return nil, errors.New(msg) - } - b, jerr := marshalPayload(result) - if jerr != nil { - return result, nil + recorded, replay := result.(recordedResult) + b := []byte(recorded) + if !replay { + var jerr error + if b, jerr = marshalPayload(result); jerr != nil { + return result, nil + } + } + value, derr := decodePayload(eventPayload{data: b}, event, decode) + if derr != nil { + panic(resultDecodeFailure{err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)}) + } + if !replay && value != nil && !genericJSON(value) { + if got, want := reflect.TypeOf(value), reflect.TypeOf(result); got != want { + panic(resultDecodeFailure{err: fmt.Errorf("%w: a %v service result decodes as %v; decode it as %v with WithEventDataDecoder", + ErrEventDataEncoding, want, got, want)}) + } + } + return value, nil } - var decoded any - if jerr = json.Unmarshal(b, &decoded); jerr != nil { - return undecodableResult{err: jerr}, nil +} + +// recordedResult is a journaled service result's JSON, which replayService settles so +// the transform decodes it as the live run decoded the ServiceFn's value. +type recordedResult json.RawMessage + +// resultDecodeFailure is the panic value replayableOutcome raises for a result that +// won't decode for its onDone event, or decodes live as another type than the +// ServiceFn returned. The transform runs before the settle fires any event, so the +// panic stops the settle with no Assign run; tickService and replayService recover it +// and fail the step with its err, which wraps ErrEventDataEncoding. +type resultDecodeFailure struct{ err error } + +// genericJSON reports whether v is a type JSON decodes into an any: what the default +// decoder gives. +func genericJSON(v any) bool { + switch v.(type) { + case map[string]any, []any, float64, string, bool: + return true } - return decoded, nil + return false +} + +// tickService runs svc.Tick and returns the error of a resultDecodeFailure the +// transform raised, in which case no event fired. Any other panic goes on out. +func tickService[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], id string) (results []state.FireResult[S], err error) { + defer recoverDecodeFailure(&err) + return svc.Tick(ctx, id), nil } -// undecodableResult stands in for a service result whose JSON form won't decode, such -// as a number beyond float64, so recordServiceOutcome refuses it: a replay couldn't -// decode the journaled form, and every Recover would fail. -type undecodableResult struct{ err error } +// recoverDecodeFailure, deferred, sets *err to a resultDecodeFailure's error and stops +// its panic. It re-panics anything else. +func recoverDecodeFailure(err *error) { + p := recover() + if p == nil { + return + } + f, ok := p.(resultDecodeFailure) + if !ok { + panic(p) + } + *err = f.err +} // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service. It reads the success result through lastResult (valid only during the @@ -241,9 +313,6 @@ func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr erro msg := lastErr.Error() sp.Error = &msg } else if result, ok := lastResult(); ok { - if u, bad := result.(undecodableResult); bad { - return state.JournalEntry{}, fmt.Errorf("service result won't decode from its JSON form: %w", u.err) - } raw, err := marshalPayload(result) if err != nil { return state.JournalEntry{}, fmt.Errorf("marshaling service result: %w", err) @@ -284,19 +353,30 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * } return res.Effects, nil } + // The transform decodes the recorded JSON for the onDone event, as it did live. One + // that won't decode stops the settle before it fires anything. var result any if len(sp.Result) > 0 { - if err := json.Unmarshal(sp.Result, &result); err != nil { - return nil, fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, err) - } + result = recordedResult(sp.Result) + } + res, ok, err := settleRecorded(ctx, svc, entry.CorrelationID, result) + if err != nil { + return nil, fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, err) } - res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) if !ok { return nil, fmt.Errorf("durable: replaying service %q: no in-flight service to settle", entry.CorrelationID) } return res.Effects, nil } +// settleRecorded runs svc.SettleDone and returns the error of a resultDecodeFailure +// the transform raised, in which case no event fired. +func settleRecorded[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], id string, result any) (res state.FireResult[S], ok bool, err error) { + defer recoverDecodeFailure(&err) + res, ok = svc.SettleDone(ctx, id, result) + return res, ok, nil +} + // serviceEntries returns the recorded service outcomes carried by a single // Record, in recorded (settle) order, so replay re-settles them in the order the // live run produced them. diff --git a/durable/service_outcome_test.go b/durable/service_outcome_test.go index b6b205e6..72d5054b 100644 --- a/durable/service_outcome_test.go +++ b/durable/service_outcome_test.go @@ -37,8 +37,8 @@ func outcomeMachine(fn state.ServiceFn[svcCtx], seen *[]string) *state.Machine[s // TestRunService_ResultThatEncodesButWontDecode_FailsTheStep proves a result whose // JSON form a replay can't decode, such as a number beyond float64, fails RunService -// with ErrHandleFailed and records nothing, as a result that won't encode does, so the -// instance stays recoverable with the service in flight again. +// with ErrHandleFailed and records nothing, before any onDone or onError Assign runs, +// so the instance stays recoverable with the service in flight again. func TestRunService_ResultThatEncodesButWontDecode_FailsTheStep(t *testing.T) { ctx := context.Background() fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return json.RawMessage("1e400"), nil } @@ -51,8 +51,11 @@ func TestRunService_ResultThatEncodesButWontDecode_FailsTheStep(t *testing.T) { t.Fatalf("Start: %v", err) } id := state.InvokeID("outcome", "loading", 0) - if _, _, err = h.RunService(ctx, id); !errors.Is(err, durable.ErrHandleFailed) { - t.Fatalf("RunService: %v, want ErrHandleFailed", err) + if _, _, err = h.RunService(ctx, id); !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("RunService: %v, want ErrHandleFailed wrapping ErrEventDataEncoding", err) + } + if len(seen) != 0 { + t.Fatalf("an Assign saw %q, want none to run for a result that won't decode", seen) } if _, recs, lerr := st.Load(ctx, "i1"); lerr != nil || len(recs) != 0 { t.Fatalf("journal holds %d Records (err %v), want none", len(recs), lerr) diff --git a/durable/start_effects_contract_test.go b/durable/start_effects_contract_test.go new file mode 100644 index 00000000..bcf92072 --- /dev/null +++ b/durable/start_effects_contract_test.go @@ -0,0 +1,307 @@ +package durable_test + +// These tests pin Start's documented contract for the initial entry's domain effects: +// they are at-least-once. Start applies them before it writes the baseline and never +// marks them, and nothing replays them, so a handler failure, a failed baseline write +// or overlapping Starts re-apply them, and Recover never applies them. + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +const ( + startNotifyID = "-1#0#demo.notify" + startChargeID = "-1#1#demo.charge" +) + +// entryTwoEffectsMachine emits two host effects on entering its initial state. +func entryTwoEffectsMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("entryTwo"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + Action("charge", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return chargeEffect{Cents: 500}, nil + }). + State("ready").OnEntry("notify").OnEntry("charge"). + State("done").Final(). + Initial("ready"). + Transition("ready").On("finish").GoTo("done"). + Quench() +} + +// startContractStore wraps a MemStore. It can fail Checkpoint once, and report an +// instance not found on its first lieLoads probes, as two Starts whose probes both ran +// before either wrote see it. It counts every call. +type startContractStore struct { + *durable.MemStore + mu sync.Mutex + failCheckpoint int + lieLoads int + calls map[string]int +} + +func newStartContractStore() *startContractStore { + return &startContractStore{MemStore: durable.NewMemStore(), calls: map[string]int{}} +} + +func (s *startContractStore) count(op string) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls[op]++ +} + +func (s *startContractStore) callCount(op string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls[op] +} + +func (s *startContractStore) Load(ctx context.Context, id durable.InstanceID) ([]byte, []durable.Record, error) { + s.count("Load") + s.mu.Lock() + lie := s.lieLoads > 0 + if lie { + s.lieLoads-- + } + s.mu.Unlock() + if lie { + return nil, nil, fmt.Errorf("%w: %q", durable.ErrInstanceNotFound, id) + } + return s.MemStore.Load(ctx, id) +} + +func (s *startContractStore) Checkpoint(ctx context.Context, id durable.InstanceID, snap []byte, through int, opts ...durable.CheckpointOption) error { + s.count("Checkpoint") + s.mu.Lock() + fail := s.failCheckpoint > 0 + if fail { + s.failCheckpoint-- + } + s.mu.Unlock() + if fail { + return errors.New("checkpoint unavailable") + } + return s.MemStore.Checkpoint(ctx, id, snap, through, opts...) +} + +func (s *startContractStore) MarkDispatched(ctx context.Context, id durable.InstanceID, ids ...string) error { + s.count("MarkDispatched") + return s.MemStore.MarkDispatched(ctx, id, ids...) +} + +func handlerOpt(rec *effectRecorder) durable.Option[string, string, *runCtx] { + return durable.WithEffectHandler[string, string, *runCtx](rec.handler()) +} + +// mustNotExist fails unless the Store has no instance id. +func mustNotExist(t *testing.T, st durable.Store, id durable.InstanceID) { + t.Helper() + if _, _, err := st.Load(context.Background(), id); !errors.Is(err, durable.ErrInstanceNotFound) { + t.Fatalf("Load(%q) = %v, want ErrInstanceNotFound: a failed Start leaves no instance", id, err) + } +} + +// TestStart_HandlerFails_LeavesNoInstance_StartAgainReappliesEffects: a handler that +// fails on the second initial effect fails Start with ErrEffectDispatch and leaves no +// instance, so the caller can Start again, which applies the first effect a second +// time. Recover applies neither. +func TestStart_HandlerFails_LeavesNoInstance_StartAgainReappliesEffects(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + rec := newEffectRecorder() + rec.failKinds["demo.charge"] = errors.New("downstream unavailable") + r := durable.NewRunner(entryTwoEffectsMachine(), st, handlerOpt(rec)) + + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("first Start = %v, want ErrEffectDispatch", err) + } + mustNotExist(t, st, "i1") + if n := rec.count(startNotifyID); n != 1 { + t.Fatalf("notify applied %d times by the failed Start, want 1", n) + } + + delete(rec.failKinds, "demo.charge") + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("second Start = %v, want it to create the instance", err) + } + if _, err := durable.Recover(ctx, entryTwoEffectsMachine(), st, "i1", handlerOpt(rec)); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.count(startNotifyID); n != 2 { + t.Errorf("notify applied %d times, want 2: Start again re-applies it", n) + } + if n := rec.count(startChargeID); n != 1 { + t.Errorf("charge applied %d times, want 1: the retry's", n) + } +} + +// TestStart_BaselineCheckpointFails_RetryReappliesEffects: the effects apply before the +// baseline write, so a failed baseline Checkpoint leaves no instance, and the retried +// Start applies them again. +func TestStart_BaselineCheckpointFails_RetryReappliesEffects(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + st.failCheckpoint = 1 + rec := newEffectRecorder() + r := durable.NewRunner(entryNotifyMachine(), st, handlerOpt(rec)) + + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err == nil { + t.Fatal("first Start = nil, want the Checkpoint failure") + } + if n := rec.count(startNotifyID); n != 1 { + t.Fatalf("notify applied %d times before the failed baseline write, want 1", n) + } + mustNotExist(t, st, "i1") + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("retried Start: %v", err) + } + if n := rec.count(startNotifyID); n != 2 { + t.Errorf("notify applied %d times across the retry, want 2", n) + } +} + +// TestStart_OverlappingStarts_EachAppliesBeforeTheStoreRefusesTheLoser: two Starts on +// one id whose probes both ran before either wrote each apply the effects; the Store +// then refuses the second baseline. +func TestStart_OverlappingStarts_EachAppliesBeforeTheStoreRefusesTheLoser(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + st.lieLoads = 2 + rec := newEffectRecorder() + r := durable.NewRunner(entryNotifyMachine(), st, handlerOpt(rec)) + + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("first Start: %v", err) + } + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); !errors.Is(err, durable.ErrCheckpointNotAdvancing) { + t.Fatalf("second Start = %v, want ErrCheckpointNotAdvancing", err) + } + if n := rec.count(startNotifyID); n != 2 { + t.Errorf("notify applied %d times across overlapping Starts, want 2", n) + } +} + +// TestStart_OverlappingFencedOwners_EachAppliesBeforeTheStoreRefusesTheLoser: a stale +// owner (token 1) and a new one (token 2) whose probes both ran before either wrote +// each apply the effects, in either order, before the Store refuses the loser's +// baseline. The new owner's Recover applies none. +func TestStart_OverlappingFencedOwners_EachAppliesBeforeTheStoreRefusesTheLoser(t *testing.T) { + cases := []struct { + first, second int64 + wantLoserErr error + }{ + {2, 1, durable.ErrFenced}, + {1, 2, durable.ErrCheckpointNotAdvancing}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("token%d_then_token%d", tc.first, tc.second), func(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + st.lieLoads = 2 + rec := newEffectRecorder() + runner := func(tok int64) *durable.Runner[string, string, *runCtx] { + return durable.NewRunner(entryTwoEffectsMachine(), st, handlerOpt(rec), + durable.WithOwnerFence[string, string, *runCtx](tok)) + } + if _, err := runner(tc.first).Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start(token %d): %v", tc.first, err) + } + if _, err := runner(tc.second).Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); !errors.Is(err, tc.wantLoserErr) { + t.Fatalf("Start(token %d) = %v, want %v", tc.second, err, tc.wantLoserErr) + } + if n := rec.count(startNotifyID); n != 2 { + t.Errorf("notify applied %d times across both owners, want 2", n) + } + if _, err := durable.Recover(ctx, entryTwoEffectsMachine(), st, "i1", handlerOpt(rec), + durable.WithOwnerFence[string, string, *runCtx](2)); err != nil { + t.Fatalf("Recover(token 2): %v", err) + } + if n := rec.count(startNotifyID); n != 2 { + t.Errorf("notify applied %d times after the takeover, want still 2: Recover applies none", n) + } + }) + } +} + +// TestStart_InitialErr_AppliesNoEffects: an initial entry that failed applies no +// effects, at Start or at Recover. +func TestStart_InitialErr_AppliesNoEffects(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + rec := newEffectRecorder() + h, err := durable.NewRunner(failingEntryMachine(), st, handlerOpt(rec)).Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if h.Instance().InitialErr() == nil { + t.Fatal("InitialErr = nil, want the entry's error") + } + if _, err = durable.Recover(ctx, failingEntryMachine(), st, "i1", handlerOpt(rec)); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.totalApplies(); n != 0 { + t.Errorf("applied %d effects from a failed entry, want 0", n) + } +} + +// TestStart_HandlerWiredOnlyAtRecover_NeverSeesStartEffects: Start's effects are applied +// by Start or not at all. A handler first wired at Recover doesn't see them. +func TestStart_HandlerWiredOnlyAtRecover_NeverSeesStartEffects(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + if _, err := durable.NewRunner(entryTwoEffectsMachine(), st).Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + rec := newEffectRecorder() + if _, err := durable.Recover(ctx, entryTwoEffectsMachine(), st, "i1", handlerOpt(rec)); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.totalApplies(); n != 0 { + t.Errorf("Recover's handler applied %d of Start's effects, want 0", n) + } +} + +// TestStart_HandlerSeesBaselineEffectIDsAndInstance_MarksNothing: at Start the handler +// gets each initial effect under its baseline id, "-1##", with the +// instance on its ctx, and Start marks none of them dispatched. +func TestStart_HandlerSeesBaselineEffectIDsAndInstance_MarksNothing(t *testing.T) { + ctx := context.Background() + st := newStartContractStore() + type call struct { + inst durable.InstanceID + ok bool + id string + } + var calls []call + handler := func(ctx context.Context, id string, _ state.Effect) error { + inst, ok := durable.EffectInstance(ctx) + calls = append(calls, call{inst, ok, id}) + return nil + } + if _, err := durable.NewRunner(entryTwoEffectsMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](handler)).Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")); err != nil { + t.Fatalf("Start: %v", err) + } + want := []call{{"i1", true, startNotifyID}, {"i1", true, startChargeID}} + if len(calls) != len(want) || calls[0] != want[0] || calls[1] != want[1] { + t.Fatalf("handler saw %+v, want %+v", calls, want) + } + if n := st.callCount("MarkDispatched"); n != 0 { + t.Errorf("Start called MarkDispatched %d times, want none", n) + } + dispatched, err := st.Dispatched(ctx, "i1") + if err != nil { + t.Fatalf("Dispatched: %v", err) + } + if len(dispatched) != 0 { + t.Errorf("Start marked %v dispatched, want none", dispatched) + } +} diff --git a/durable/timetravel.go b/durable/timetravel.go index 82518afc..9645e45d 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -283,5 +283,5 @@ func newReplayServiceRunner[S comparable, E comparable, C any]( if cfg.serviceReg == nil { return nil } - return state.NewServiceRunner(inst, cfg.serviceReg) + return state.NewServiceRunner(inst, cfg.serviceReg).WithResultTransform(replayableOutcome(cfg.eventDataDecoder)) } diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index 632deef1..4213d4d3 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -36,9 +36,13 @@ counts as an additive (minor) versus breaking (major) change. Use the onError doesn't escalate again, since its start already escalated it; a respawned actor's own failure with no onError does. The durable runner's `Recover` uses it for the actors that were running. It's additive, within v1.1.0. -- **`ServiceRunner.WithResultTransform`** rewrites a service's outcome before `Tick` - settles it, so a host that records outcomes can give the live run what a replay will see. - An unbound service still settles through onError: a transform that returns no error for +- **`ServiceRunner.WithResultTransform`** rewrites every service outcome before it + settles, whether `Tick`, `SettleDone` or `SettleError` settles it, so a host that records + outcomes can give the live run and a replay the same value. The transform takes the + event that will carry the outcome, the invocation's onDone for a result and its onError + for an error: `func(event E, result any, err error) (any, error)`. It runs before any + event fires, so a transform that panics leaves the settle with no event fired. An + unbound service still settles through onError: a transform that returns no error for its `UnboundRefError` doesn't make it succeed. - **`FireEventData`** reports the payload a `WithEventData` option carries, so a host that records a Fire can persist it. It's additive, so the next release is v1.1.0. diff --git a/state/result_transform_test.go b/state/result_transform_test.go index 71d81fa5..3d04ca4c 100644 --- a/state/result_transform_test.go +++ b/state/result_transform_test.go @@ -9,33 +9,42 @@ import ( "github.com/stablekernel/crucible/state" ) -// TestWithResultTransform_RewritesTheOutcomeTickSettles proves Tick settles a bound -// service with the transform's outcome, a result or an error, and that LastResult and -// LastError report the rewritten form. -func TestWithResultTransform_RewritesTheOutcomeTickSettles(t *testing.T) { - ctx := context.Background() - upper := func(result any, err error) (any, error) { +// upperTransform uppercases a string result and wraps an error's message, and notes +// the event it was given for each outcome in *events. +func upperTransform(events *[]string) func(string, any, error) (any, error) { + return func(event string, result any, err error) (any, error) { + *events = append(*events, event) if err != nil { return nil, errors.New("wrapped: " + err.Error()) } return strings.ToUpper(result.(string)), nil } +} + +// TestWithResultTransform_RewritesTheOutcomeTickSettles proves Tick settles a bound +// service with the transform's outcome, a result or an error, that LastResult and +// LastError report the rewritten form, and that the transform gets onDone's event for +// a result and onError's for an error. +func TestWithResultTransform_RewritesTheOutcomeTickSettles(t *testing.T) { + ctx := context.Background() cases := []struct { name string fn state.ServiceFn[*trec] wantState string wantNotes string + wantEvent string }{ - {"result", func(context.Context, state.ServiceCtx[*trec]) (any, error) { return "payload", nil }, "ready", "result:PAYLOAD"}, - {"error", func(context.Context, state.ServiceCtx[*trec]) (any, error) { return nil, errors.New("boom") }, "errored", "error:wrapped: boom"}, + {"result", func(context.Context, state.ServiceCtx[*trec]) (any, error) { return "payload", nil }, "ready", "result:PAYLOAD", "ok"}, + {"error", func(context.Context, state.ServiceCtx[*trec]) (any, error) { return nil, errors.New("boom") }, "errored", "error:wrapped: boom", "fail"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var run *state.ServiceRunner[string, string, *trec] + var events []string m := invokeMachine(&run) entity := &trec{} inst := m.Cast(entity, state.WithInitialState("idle")) - run = state.NewServiceRunner(inst, state.NewRegistry[*trec]().Service("fetch", tc.fn)).WithResultTransform(upper) + run = state.NewServiceRunner(inst, state.NewRegistry[*trec]().Service("fetch", tc.fn)).WithResultTransform(upperTransform(&events)) run.Absorb(ctx, inst.Fire(ctx, "start").Effects) res := run.Tick(ctx, state.InvokeID("loader", "loading", 0)) if len(res) != 1 || res[0].NewState != tc.wantState { @@ -44,10 +53,86 @@ func TestWithResultTransform_RewritesTheOutcomeTickSettles(t *testing.T) { if got := join(entity.notes); got != tc.wantNotes { t.Fatalf("notes = %q, want %q", got, tc.wantNotes) } + if len(events) != 1 || events[0] != tc.wantEvent { + t.Fatalf("the transform got events %q, want [%q]", events, tc.wantEvent) + } + }) + } +} + +// TestWithResultTransform_AppliesToSettleDoneAndSettleError proves the transform runs +// on an outcome a host settles directly, as a replay does, with the event that carries +// it. +func TestWithResultTransform_AppliesToSettleDoneAndSettleError(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("loader", "loading", 0) + cases := []struct { + name string + settle func(*state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) + wantState string + wantNotes string + wantEvent string + }{ + {"SettleDone", func(r *state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) { + return r.SettleDone(ctx, id, "payload") + }, "ready", "result:PAYLOAD", "ok"}, + {"SettleError", func(r *state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) { + return r.SettleError(ctx, id, errors.New("boom")) + }, "errored", "error:wrapped: boom", "fail"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var run *state.ServiceRunner[string, string, *trec] + var events []string + m := invokeMachine(&run) + entity := &trec{} + inst := m.Cast(entity, state.WithInitialState("idle")) + run = state.NewServiceRunner(inst, nil).WithResultTransform(upperTransform(&events)) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + res, ok := tc.settle(run) + if !ok || res.NewState != tc.wantState { + t.Fatalf("settled to %+v (%v), want %q", res, ok, tc.wantState) + } + if got := join(entity.notes); got != tc.wantNotes { + t.Fatalf("notes = %q, want %q", got, tc.wantNotes) + } + if len(events) != 1 || events[0] != tc.wantEvent { + t.Fatalf("the transform got events %q, want [%q]", events, tc.wantEvent) + } }) } } +// TestWithResultTransform_PanicFiresNoEvent proves a transform that panics leaves the +// settle before any event fires: the instance stays in the invoking state, and the +// service is no longer in flight. +func TestWithResultTransform_PanicFiresNoEvent(t *testing.T) { + ctx := context.Background() + var run *state.ServiceRunner[string, string, *trec] + m := invokeMachine(&run) + entity := &trec{} + inst := m.Cast(entity, state.WithInitialState("idle")) + run = state.NewServiceRunner(inst, nil).WithResultTransform(func(string, any, error) (any, error) { + panic("refused") + }) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + id := state.InvokeID("loader", "loading", 0) + func() { + defer func() { + if p := recover(); p != "refused" { + t.Fatalf("recovered %v, want the transform's panic", p) + } + }() + run.SettleDone(ctx, id, "payload") + }() + if got := inst.Current(); got != "loading" || len(entity.notes) != 0 { + t.Fatalf("after the panic the instance is in %q with notes %q, want loading and none", got, entity.notes) + } + if run.HasPending(id) { + t.Fatalf("the service is still in flight, want it dropped") + } +} + // TestWithResultTransform_UnboundServiceStillRoutesOnError proves a transform can't // turn an unbound service into a success: a transform that returns a result and no // error for the UnboundRefError still settles the service through onError, with the @@ -59,7 +144,7 @@ func TestWithResultTransform_UnboundServiceStillRoutesOnError(t *testing.T) { entity := &trec{} inst := m.Cast(entity, state.WithInitialState("idle")) // A nil registry leaves "fetch" unbound. - run = state.NewServiceRunner(inst, nil).WithResultTransform(func(any, error) (any, error) { + run = state.NewServiceRunner(inst, nil).WithResultTransform(func(string, any, error) (any, error) { return "fallback", nil }) run.Absorb(ctx, inst.Fire(ctx, "start").Effects) diff --git a/state/runner.go b/state/runner.go index 504c078f..ee519b03 100644 --- a/state/runner.go +++ b/state/runner.go @@ -2,6 +2,7 @@ package state import ( "context" + "errors" "sort" "sync" "sync/atomic" @@ -98,8 +99,9 @@ type ServiceRunner[S comparable, E comparable, C any] struct { // snapshot. lastResult atomic.Value // serviceOutcome - // transform, when set, rewrites a ServiceFn's outcome before Tick settles it. - transform func(any, error) (any, error) + // transform, when set, rewrites every outcome before it settles, given the event + // that will carry it. + transform func(event E, result any, err error) (any, error) } // serviceOutcome is the result/error pair a settled service exposes through @@ -201,6 +203,9 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a if !ok { return FireResult[S]{}, false } + if r.transform != nil { + result, err = r.applyTransform(rs, result, err) + } r.lastResult.Store(serviceOutcome{result: result, err: err}) var ev E var payload any @@ -220,18 +225,37 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a return res, true } -// WithResultTransform sets a function that rewrites a ServiceFn's outcome before Tick -// settles it, and returns the runner. A host that records outcomes uses it to give the -// live run the form a replay will see. LastResult and LastError then report the rewritten -// outcome. It also runs on an UnboundRefError, where only a non-nil returned error is -// used: a nil one leaves the UnboundRefError, so the service still settles through onError. It -// doesn't apply to SettleDone or SettleError, which take the outcome as given. Set it -// before the first Tick: the field isn't guarded. -func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(result any, err error) (any, error)) *ServiceRunner[S, E, C] { +// WithResultTransform sets a function that rewrites every outcome before it settles, +// whether Tick, SettleDone or SettleError settles it, and returns the runner. f gets +// the event that will carry the outcome: the invocation's onDone for a result, its +// onError for an error. A host that records outcomes uses it to give the live run and +// a replay the same value, decoded for that event. A non-nil returned error settles +// through onError, and a nil one through onDone, except for an UnboundRefError: a nil +// returned error leaves the UnboundRefError, so an unbound service still settles +// through onError. LastResult and LastError report the rewritten outcome. f runs +// after the service is dropped and before any event fires, so if f panics, the panic +// leaves the settle with no event fired. Set it before the first settle: the field +// isn't guarded. +func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(event E, result any, err error) (any, error)) *ServiceRunner[S, E, C] { r.transform = f return r } +// applyTransform runs the transform on an outcome of rs. An UnboundRefError stays an +// error: a transform that returns no error for it can't turn it into a success. +func (r *ServiceRunner[S, E, C]) applyTransform(rs running[E], result any, err error) (any, error) { + ev := rs.onDone + if err != nil { + ev = rs.onError + } + out, terr := r.transform(ev, result, err) + var unbound *UnboundRefError + if terr == nil && errors.As(err, &unbound) { + return nil, err + } + return out, terr +} + // Tick resolves and runs the in-flight service id against the bound registry, // settling it with the ServiceFn's result or error. It is the ServiceRunner's // advance verb — the host-driver counterpart of Scheduler.Tick and @@ -252,21 +276,10 @@ func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResu } fn := r.resolve(rs.src.Name) if fn == nil { - var err error = &UnboundRefError{Kind: "service", Name: rs.src.Name} - if r.transform != nil { - // An unbound service always settles as an error: a transform that returns - // no error can't turn it into a success. - if _, terr := r.transform(nil, err); terr != nil { - err = terr - } - } - res, _ := r.SettleError(ctx, id, err) + res, _ := r.SettleError(ctx, id, &UnboundRefError{Kind: "service", Name: rs.src.Name}) return []FireResult[S]{res} } out, err := fn(ctx, ServiceCtx[C]{Entity: r.inst.entity, Params: rs.src.Params, Input: rs.input}) - if r.transform != nil { - out, err = r.transform(out, err) - } if err != nil { res, _ := r.SettleError(ctx, id, err) return []FireResult[S]{res} From a088c46693c09e04e2871b66f8bd58698c2e5480 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 13:53:10 -0700 Subject: [PATCH 04/13] refactor(state): let a result transform abort a settle instead of panicking ServiceRunner.WithResultTransform's function now returns an explicit abort: func(event E, result any, err error) (out any, outErr error, abort error). - A non-nil abort stops the settle: no event fires, so no Assign, middleware, inspector or history entry. SettleDone and SettleError return false with the abort in FireResult.Err; Tick returns that FireResult as its one element. LastResult and LastError keep the previous settle. - The transform runs before the service is dropped, so an aborted settle, or a transform that panics, leaves the service in flight. A panic in the transform goes on out of the settle, as any host panic does. - durable's transform returns an abort for a result that won't decode for its onDone event, in place of the resultDecodeFailure panic. RunService and replay check the returned abort; tickService, settleRecorded and their recover() are gone. Live, the step still fails the Handle and records nothing, and Recover has the service in flight again; replay still fails with ErrEventDataEncoding. - TestWithResultTransform_AbortFiresNoEvent replaces the panic test, covering Tick, SettleDone and SettleError. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 2 + durable/service.go | 92 ++++++++++----------------- state/CHANGELOG.md | 10 +-- state/result_transform_test.go | 110 ++++++++++++++++++++++++++++----- state/runner.go | 76 ++++++++++++++++------- 5 files changed, 191 insertions(+), 99 deletions(-) diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index ee46f2d2..8b82a7ce 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -114,6 +114,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. service returned, fails `RunService` with `ErrHandleFailed` wrapping `ErrEventDataEncoding` before any onDone or onError `Assign` runs, and records nothing; so does a result whose JSON form won't decode, such as a number beyond `float64`. + The decode aborts the settle through `ServiceRunner.WithResultTransform`'s abort + return, so no event fires, and a recovered Handle has the service in flight again. `Recover` and `StateAt` fail with `ErrEventDataEncoding`, running no `Assign`, on a recorded result their decoder can't decode. Replay decodes the journal with the recovering Runner's decoder, so register a decoder before results are recorded: one diff --git a/durable/service.go b/durable/service.go index 65302176..9607c4e7 100644 --- a/durable/service.go +++ b/durable/service.go @@ -148,17 +148,18 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return state.FireResult[S]{}, false, nil, nil, nil } - results, derr := tickService(h.stepContext(ctx), h.svc, id) - if derr != nil { - // The result didn't decode for its onDone event, so nothing fired: no Assign ran - // and nothing is recorded. The runner has already dropped the service, so this - // Handle can't run it again; Recover re-arms it. - return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, derr)) - } + results := h.svc.Tick(h.stepContext(ctx), id) if len(results) == 0 { return state.FireResult[S]{}, false, nil, nil, nil } res := results[0] + if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { + // The result didn't decode for its onDone event, so the transform aborted the + // settle: nothing fired, no Assign ran, and nothing is recorded. The service is + // still in flight, but the Handle has failed, so it can't run it again; Recover + // re-arms it. + return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, derr)) + } // The service settled: unless the settle's own Fire leaves or re-enters the state // that declares it, which the absorb below notes, a checkpoint records it as settled. h.noteSettled(id) @@ -222,42 +223,43 @@ func (h *Handle[S, E, C]) noteSettled(id string) { // other than the result's is refused, though JSON's generic form is always accepted, // so the default keeps working. // -// A result that won't decode, or is refused, panics with a resultDecodeFailure before -// the settle fires anything, so no onDone or onError Assign runs; tickService and -// replayService recover it and fail the step. A result with no JSON form is left as it -// is, and fails the Handle when recordServiceOutcome can't encode it. +// A result that won't decode, or is refused, aborts the settle with an +// *undecodableResult before it fires anything, so no onDone or onError Assign runs; +// serviceStep and replayService see it in the settle's FireResult.Err and fail the +// step. A result with no JSON form is left as it is, and fails the Handle when +// recordServiceOutcome can't encode it. // // An error becomes an error with its message as a Store keeps it (see // normalizedErrorMessage), live and on replay. An Error method that panics settles // with a message saying so, normalized the same way. -func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, error) (any, error) { - return func(event E, result any, err error) (any, error) { +func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, error) (any, error, error) { + return func(event E, result any, err error) (any, error, error) { if err != nil { msg, perr := normalizedErrorMessage(err, "service error") if perr != nil { msg = validUTF8(perr.Error()) } - return nil, errors.New(msg) + return nil, errors.New(msg), nil } recorded, replay := result.(recordedResult) b := []byte(recorded) if !replay { var jerr error if b, jerr = marshalPayload(result); jerr != nil { - return result, nil + return result, nil, nil } } value, derr := decodePayload(eventPayload{data: b}, event, decode) if derr != nil { - panic(resultDecodeFailure{err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)}) + return nil, nil, &undecodableResult{err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} } if !replay && value != nil && !genericJSON(value) { if got, want := reflect.TypeOf(value), reflect.TypeOf(result); got != want { - panic(resultDecodeFailure{err: fmt.Errorf("%w: a %v service result decodes as %v; decode it as %v with WithEventDataDecoder", - ErrEventDataEncoding, want, got, want)}) + return nil, nil, &undecodableResult{err: fmt.Errorf("%w: a %v service result decodes as %v; decode it as %v with WithEventDataDecoder", + ErrEventDataEncoding, want, got, want)} } } - return value, nil + return value, nil, nil } } @@ -265,12 +267,15 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, er // the transform decodes it as the live run decoded the ServiceFn's value. type recordedResult json.RawMessage -// resultDecodeFailure is the panic value replayableOutcome raises for a result that -// won't decode for its onDone event, or decodes live as another type than the -// ServiceFn returned. The transform runs before the settle fires any event, so the -// panic stops the settle with no Assign run; tickService and replayService recover it -// and fail the step with its err, which wraps ErrEventDataEncoding. -type resultDecodeFailure struct{ err error } +// undecodableResult is the abort replayableOutcome returns for a result that won't +// decode for its onDone event, or decodes live as another type than the ServiceFn +// returned. The ServiceRunner fires no event for it and hands it back in the settle's +// FireResult.Err, where serviceStep and replayService find it and fail the step. err +// wraps ErrEventDataEncoding. +type undecodableResult struct{ err error } + +func (u *undecodableResult) Error() string { return u.err.Error() } +func (u *undecodableResult) Unwrap() error { return u.err } // genericJSON reports whether v is a type JSON decodes into an any: what the default // decoder gives. @@ -282,27 +287,6 @@ func genericJSON(v any) bool { return false } -// tickService runs svc.Tick and returns the error of a resultDecodeFailure the -// transform raised, in which case no event fired. Any other panic goes on out. -func tickService[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], id string) (results []state.FireResult[S], err error) { - defer recoverDecodeFailure(&err) - return svc.Tick(ctx, id), nil -} - -// recoverDecodeFailure, deferred, sets *err to a resultDecodeFailure's error and stops -// its panic. It re-panics anything else. -func recoverDecodeFailure(err *error) { - p := recover() - if p == nil { - return - } - f, ok := p.(resultDecodeFailure) - if !ok { - panic(p) - } - *err = f.err -} - // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service. It reads the success result through lastResult (valid only during the // settle's synchronous window) and the error through lastErr, encoding whichever @@ -354,14 +338,14 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * return res.Effects, nil } // The transform decodes the recorded JSON for the onDone event, as it did live. One - // that won't decode stops the settle before it fires anything. + // that won't decode aborts the settle before it fires anything. var result any if len(sp.Result) > 0 { result = recordedResult(sp.Result) } - res, ok, err := settleRecorded(ctx, svc, entry.CorrelationID, result) - if err != nil { - return nil, fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, err) + res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) + if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { + return nil, fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, derr) } if !ok { return nil, fmt.Errorf("durable: replaying service %q: no in-flight service to settle", entry.CorrelationID) @@ -369,14 +353,6 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * return res.Effects, nil } -// settleRecorded runs svc.SettleDone and returns the error of a resultDecodeFailure -// the transform raised, in which case no event fired. -func settleRecorded[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], id string, result any) (res state.FireResult[S], ok bool, err error) { - defer recoverDecodeFailure(&err) - res, ok = svc.SettleDone(ctx, id, result) - return res, ok, nil -} - // serviceEntries returns the recorded service outcomes carried by a single // Record, in recorded (settle) order, so replay re-settles them in the order the // live run produced them. diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index 4213d4d3..b60e3b23 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -40,10 +40,12 @@ counts as an additive (minor) versus breaking (major) change. Use the settles, whether `Tick`, `SettleDone` or `SettleError` settles it, so a host that records outcomes can give the live run and a replay the same value. The transform takes the event that will carry the outcome, the invocation's onDone for a result and its onError - for an error: `func(event E, result any, err error) (any, error)`. It runs before any - event fires, so a transform that panics leaves the settle with no event fired. An - unbound service still settles through onError: a transform that returns no error for - its `UnboundRefError` doesn't make it succeed. + for an error: `func(event E, result any, err error) (out any, outErr error, abort error)`. + It runs before any event fires and before the service is dropped. A non-nil `abort` + stops the settle: no event fires, the service stays in flight, `SettleDone` and + `SettleError` return false with `abort` in `FireResult.Err`, and `Tick` returns that + `FireResult` as its one element. An unbound service still settles through onError: a + transform that returns no error for its `UnboundRefError` doesn't make it succeed. - **`FireEventData`** reports the payload a `WithEventData` option carries, so a host that records a Fire can persist it. It's additive, so the next release is v1.1.0. - **Callbacks get the step's context.** `GuardCtx`, `ActionCtx` and `AssignCtx` have a new diff --git a/state/result_transform_test.go b/state/result_transform_test.go index 3d04ca4c..d9dc9334 100644 --- a/state/result_transform_test.go +++ b/state/result_transform_test.go @@ -11,13 +11,13 @@ import ( // upperTransform uppercases a string result and wraps an error's message, and notes // the event it was given for each outcome in *events. -func upperTransform(events *[]string) func(string, any, error) (any, error) { - return func(event string, result any, err error) (any, error) { +func upperTransform(events *[]string) func(string, any, error) (any, error, error) { + return func(event string, result any, err error) (any, error, error) { *events = append(*events, event) if err != nil { - return nil, errors.New("wrapped: " + err.Error()) + return nil, errors.New("wrapped: " + err.Error()), nil } - return strings.ToUpper(result.(string)), nil + return strings.ToUpper(result.(string)), nil, nil } } @@ -103,23 +103,103 @@ func TestWithResultTransform_AppliesToSettleDoneAndSettleError(t *testing.T) { } } -// TestWithResultTransform_PanicFiresNoEvent proves a transform that panics leaves the -// settle before any event fires: the instance stays in the invoking state, and the -// service is no longer in flight. -func TestWithResultTransform_PanicFiresNoEvent(t *testing.T) { +// TestWithResultTransform_AbortFiresNoEvent proves a transform that aborts stops the +// settle, whether Tick, SettleDone or SettleError drives it: no event fires, so no +// Assign, inspector or history entry; the caller gets the abort in FireResult.Err, with +// false from SettleDone and SettleError and as Tick's one element; LastResult and +// LastError keep the previous settle; and the service stays in flight, so a later +// settle the transform lets through routes it. +func TestWithResultTransform_AbortFiresNoEvent(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("loader", "loading", 0) + refused := errors.New("refused") + fetch := func(context.Context, state.ServiceCtx[*trec]) (any, error) { return "payload", nil } + cases := []struct { + name string + settle func(*state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) + }{ + {"Tick", func(r *state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) { + res := r.Tick(ctx, id) + if len(res) != 1 { + t.Fatalf("Tick returned %d results, want 1", len(res)) + } + return res[0], false + }}, + {"SettleDone", func(r *state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) { + return r.SettleDone(ctx, id, "payload") + }}, + {"SettleError", func(r *state.ServiceRunner[string, string, *trec]) (state.FireResult[string], bool) { + return r.SettleError(ctx, id, errors.New("boom")) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var run *state.ServiceRunner[string, string, *trec] + m := invokeMachine(&run) + entity := &trec{} + var inspected int + inst := m.Cast(entity, state.WithInitialState("idle"), state.WithUnboundedHistory[string](), + state.WithInspector[string](state.InspectorFunc(func(state.InspectionEvent) { inspected++ }))) + abort := true + run = state.NewServiceRunner(inst, state.NewRegistry[*trec]().Service("fetch", fetch)). + WithResultTransform(func(_ string, result any, err error) (any, error, error) { + if abort { + return "ignored", nil, refused + } + return result, err, nil + }) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + history, events := len(inst.History()), inspected + + res, ok := tc.settle(run) + if ok || !errors.Is(res.Err, refused) { + t.Fatalf("an aborted settle returned %+v, %v; want false and the abort in Err", res, ok) + } + if res.NewState != "" || len(res.Effects) != 0 { + t.Fatalf("an aborted settle returned %+v, want no state or effects", res) + } + if got := inst.Current(); got != "loading" || len(entity.notes) != 0 { + t.Fatalf("after the abort the instance is in %q with notes %q, want loading and none", got, entity.notes) + } + if len(inst.History()) != history || inspected != events { + t.Fatalf("the abort added %d history entries and %d inspection events, want none", + len(inst.History())-history, inspected-events) + } + if _, ok := run.LastResult(); ok || run.LastError() != nil { + t.Fatalf("the abort set LastResult or LastError (%v)", run.LastError()) + } + if !run.HasPending(id) { + t.Fatalf("the aborted service isn't in flight, want it kept") + } + + // The service is still in flight, so a settle the transform lets through + // routes it. + abort = false + res, _ = tc.settle(run) + if res.Err != nil || res.NewState == "" || run.HasPending(id) { + t.Fatalf("settling after the abort gave %+v, pending %v; want it routed", res, run.HasPending(id)) + } + }) + } +} + +// TestWithResultTransform_PanicLeavesServiceInFlight proves a transform that panics +// panics out of the settle, as any host panic does, before any event fires and +// before the service is dropped. +func TestWithResultTransform_PanicLeavesServiceInFlight(t *testing.T) { ctx := context.Background() var run *state.ServiceRunner[string, string, *trec] m := invokeMachine(&run) entity := &trec{} inst := m.Cast(entity, state.WithInitialState("idle")) - run = state.NewServiceRunner(inst, nil).WithResultTransform(func(string, any, error) (any, error) { - panic("refused") + run = state.NewServiceRunner(inst, nil).WithResultTransform(func(string, any, error) (any, error, error) { + panic("boom") }) run.Absorb(ctx, inst.Fire(ctx, "start").Effects) id := state.InvokeID("loader", "loading", 0) func() { defer func() { - if p := recover(); p != "refused" { + if p := recover(); p != "boom" { t.Fatalf("recovered %v, want the transform's panic", p) } }() @@ -128,8 +208,8 @@ func TestWithResultTransform_PanicFiresNoEvent(t *testing.T) { if got := inst.Current(); got != "loading" || len(entity.notes) != 0 { t.Fatalf("after the panic the instance is in %q with notes %q, want loading and none", got, entity.notes) } - if run.HasPending(id) { - t.Fatalf("the service is still in flight, want it dropped") + if !run.HasPending(id) { + t.Fatalf("the service isn't in flight after the panic, want it kept") } } @@ -144,8 +224,8 @@ func TestWithResultTransform_UnboundServiceStillRoutesOnError(t *testing.T) { entity := &trec{} inst := m.Cast(entity, state.WithInitialState("idle")) // A nil registry leaves "fetch" unbound. - run = state.NewServiceRunner(inst, nil).WithResultTransform(func(string, any, error) (any, error) { - return "fallback", nil + run = state.NewServiceRunner(inst, nil).WithResultTransform(func(string, any, error) (any, error, error) { + return "fallback", nil, nil }) run.Absorb(ctx, inst.Fire(ctx, "start").Effects) res := run.Tick(ctx, state.InvokeID("loader", "loading", 0)) diff --git a/state/runner.go b/state/runner.go index ee519b03..e9f7ea33 100644 --- a/state/runner.go +++ b/state/runner.go @@ -100,8 +100,8 @@ type ServiceRunner[S comparable, E comparable, C any] struct { lastResult atomic.Value // serviceOutcome // transform, when set, rewrites every outcome before it settles, given the event - // that will carry it. - transform func(event E, result any, err error) (any, error) + // that will carry it, or aborts the settle. + transform func(event E, result any, err error) (out any, outErr error, abort error) } // serviceOutcome is the result/error pair a settled service exposes through @@ -176,7 +176,9 @@ func (r *ServiceRunner[S, E, C]) HasPending(id string) bool { // and fires its OnDone event (carrying result) through the instance, then absorbs // the resulting effects so a chained invoke arms its successor. It returns the // FireResult and true, or the zero result and false when id names no in-flight -// service (already stopped or settled). result is delivered to the onDone +// service (already stopped or settled). When a WithResultTransform function aborts +// the settle, it returns false and a FireResult whose Err is the abort error, fires +// nothing, and leaves the service in flight. result is delivered to the onDone // transition's effects through the instance entity by the host's actions — the // kernel routes the event; the action reads the result. func (r *ServiceRunner[S, E, C]) SettleDone(ctx context.Context, id string, result any) (FireResult[S], bool) { @@ -186,26 +188,40 @@ func (r *ServiceRunner[S, E, C]) SettleDone(ctx context.Context, id string, resu // SettleError fails the in-flight service id: it drops the service and fires its // OnError event (carrying err) through the instance, then absorbs the resulting // effects. It returns the FireResult and true, or the zero result and false when -// id names no in-flight service. +// id names no in-flight service. When a WithResultTransform function aborts the +// settle, it returns false and a FireResult whose Err is the abort error, fires +// nothing, and leaves the service in flight. func (r *ServiceRunner[S, E, C]) SettleError(ctx context.Context, id string, err error) (FireResult[S], bool) { return r.settle(ctx, id, nil, err) } -// settle is the shared body of SettleDone / SettleError: drop the running service -// under lock, then fire its routing event and absorb the follow-on effects. +// settle is the shared body of SettleDone / SettleError: run the transform on the +// outcome, drop the running service under lock, then fire its routing event and absorb +// the follow-on effects. The transform runs outside the lock before the service is +// dropped, so one that aborts, or panics, leaves the service in flight. func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result any, err error) (FireResult[S], bool) { r.mu.Lock() rs, ok := r.running[id] + r.mu.Unlock() + if !ok { + return FireResult[S]{}, false + } + if r.transform != nil { + var abort error + if result, err, abort = r.applyTransform(rs, result, err); abort != nil { + return FireResult[S]{Err: abort}, false + } + } + r.mu.Lock() + _, ok = r.running[id] if ok { delete(r.running, id) } r.mu.Unlock() if !ok { + // Settled or stopped while the transform ran. return FireResult[S]{}, false } - if r.transform != nil { - result, err = r.applyTransform(rs, result, err) - } r.lastResult.Store(serviceOutcome{result: result, err: err}) var ev E var payload any @@ -229,31 +245,45 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a // whether Tick, SettleDone or SettleError settles it, and returns the runner. f gets // the event that will carry the outcome: the invocation's onDone for a result, its // onError for an error. A host that records outcomes uses it to give the live run and -// a replay the same value, decoded for that event. A non-nil returned error settles -// through onError, and a nil one through onDone, except for an UnboundRefError: a nil -// returned error leaves the UnboundRefError, so an unbound service still settles -// through onError. LastResult and LastError report the rewritten outcome. f runs -// after the service is dropped and before any event fires, so if f panics, the panic -// leaves the settle with no event fired. Set it before the first settle: the field -// isn't guarded. -func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(event E, result any, err error) (any, error)) *ServiceRunner[S, E, C] { +// a replay the same value, decoded for that event. +// +// f returns the outcome to settle with, out and outErr. A non-nil outErr settles +// through onError, and a nil one through onDone with out, except for an +// UnboundRefError: a nil outErr leaves the UnboundRefError, so an unbound service +// still settles through onError. LastResult and LastError report the rewritten +// outcome. +// +// A non-nil abort stops the settle instead: no event fires, so no Assign, middleware +// or inspector runs, and the service stays in flight, so it can be settled again. +// SettleDone and SettleError return false with abort, as given, in FireResult.Err; +// Tick returns a one-element slice holding that FireResult. LastResult and LastError +// still report the previous settle. +// +// f runs outside the runner's lock before the service is dropped, so it may run for a +// settle that then finds the service stopped or settled by another goroutine, and +// returns false. A panic in f goes on out of the settle, leaving the service in +// flight. Set f before the first settle: the field isn't guarded. +func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(event E, result any, err error) (out any, outErr error, abort error)) *ServiceRunner[S, E, C] { r.transform = f return r } // applyTransform runs the transform on an outcome of rs. An UnboundRefError stays an // error: a transform that returns no error for it can't turn it into a success. -func (r *ServiceRunner[S, E, C]) applyTransform(rs running[E], result any, err error) (any, error) { +func (r *ServiceRunner[S, E, C]) applyTransform(rs running[E], result any, err error) (any, error, error) { ev := rs.onDone if err != nil { ev = rs.onError } - out, terr := r.transform(ev, result, err) + out, terr, abort := r.transform(ev, result, err) + if abort != nil { + return nil, nil, abort + } var unbound *UnboundRefError if terr == nil && errors.As(err, &unbound) { - return nil, err + return nil, err, nil } - return out, terr + return out, terr, nil } // Tick resolves and runs the in-flight service id against the bound registry, @@ -266,7 +296,9 @@ func (r *ServiceRunner[S, E, C]) applyTransform(rs running[E], result any, err e // Scheduler.Tick and ActorSystem.Tick, or an empty slice when id is not in flight. // A missing registry / unresolved ServiceFn still settles the service as an error // (so the machine routes onError rather than hanging) and that routed result is -// returned in the slice. +// returned in the slice. When a WithResultTransform function aborts the settle, the +// slice holds one FireResult whose Err is the abort error, no event fires, and the +// service stays in flight. func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResult[S] { r.mu.Lock() rs, ok := r.running[id] From b79f2caf0b56ca421e579ea072759df8f548da83 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 14:17:44 -0700 Subject: [PATCH 05/13] fix(durable,state): review follow-ups for effect dispatch and service results Each finding gets a test that fails if it's real, committed whether it proved or disproved the finding. - state: a settle can't settle a later activation of its service. Each activation carries a generation; a SettleDone or SettleError whose transform ran, or a Tick whose ServiceFn ran, while the service was stopped and armed again under the same invoke id now returns false and leaves the new activation in flight. - durable: a DispatchStore whose Dispatched or MarkDispatched panics fails Recover and the stateless Runner.Fire with ErrEffectDispatch, as a step already did. The guard moves into the shared dispatch path, which applyStartEffects now shares its handler and DispatchStore checks with. - durable: a decoder may return json.Number, JSON's generic form under UseNumber, for any service result. The service and Fire paths share one type check; a Fire payload stays strict. - durable: DeliverToActor records through recordStep and finishStep; persistStep is gone. - Pins: an undecodable service result reruns the ServiceFn on every Recover; an unmarshalable KindedEffect fails Tick and RunService with no handler wired; Recover loads the dispatched set once per effect Record, as main did for Fire Records; the journal holds the decoded result's encoding, which differs from the first encoding for a generic struct or an int64 above 2^53. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 8 +- durable/actor.go | 11 +- durable/dispatch_store_test.go | 172 ++++++++++++++++++++++++ durable/effect.go | 32 +++-- durable/result_type_check_test.go | 98 ++++++++++++++ durable/runner.go | 46 ++----- durable/service.go | 31 +++-- durable/service_encode_internal_test.go | 84 ++++++++++++ durable/undecodable_result_loop_test.go | 70 ++++++++++ durable/unmarshalable_effect_test.go | 154 +++++++++++++++++++++ state/CHANGELOG.md | 6 + state/runner.go | 44 ++++-- state/settle_stale_test.go | 118 ++++++++++++++++ 13 files changed, 800 insertions(+), 74 deletions(-) create mode 100644 durable/dispatch_store_test.go create mode 100644 durable/result_type_check_test.go create mode 100644 durable/service_encode_internal_test.go create mode 100644 durable/undecodable_result_loop_test.go create mode 100644 durable/unmarshalable_effect_test.go create mode 100644 state/settle_stale_test.go diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 8b82a7ce..1dda2bbf 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -101,7 +101,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. panics, which now returns `ErrEffectDispatch` instead of panicking out of `Fire`. The effect stays unmarked in the journal, so no later checkpoint compacts it away, and `Recover` dispatches it again, failing with `ErrEffectDispatch` while the handler still - does. A handler that panics in `Start` or `Recover` returns `ErrEffectDispatch` too. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect + does. A handler that panics in `Start` or `Recover` returns `ErrEffectDispatch` too, and + so does a `DispatchStore` whose `Dispatched` or `MarkDispatched` panics, in a step, + `Recover` or the stateless `Runner.Fire`. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect again. The handler's error now matches through `errors.Is`. - **A `KindedEffect` that can't marshal fails `Tick` and `RunService`,** as it already failed `Fire`, even with no handler wired. @@ -110,7 +112,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. `WithEventDataDecoder` decoder for the invocation's onDone event, as a `WithEventData` payload is: a decoder for that event, such as `DecodeAs[Order]`, hands `Assign` that type, and without one a struct arrives as a map and an integer as a `float64`, which - loses precision above 2^53. A decoder that fails, or returns a type other than the + loses precision above 2^53. A decoder may always return JSON's generic form, including + a `json.Number` from one that decodes with `UseNumber`; a `WithEventData` payload + must still come back as the type its caller passed. A decoder that fails, or returns a type other than the service returned, fails `RunService` with `ErrHandleFailed` wrapping `ErrEventDataEncoding` before any onDone or onError `Assign` runs, and records nothing; so does a result whose JSON form won't decode, such as a number beyond `float64`. diff --git a/durable/actor.go b/durable/actor.go index 03ffcc87..b95cf43c 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -158,13 +158,14 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef return true, h.fail(fmt.Errorf("durable: recording actor %q result for %q: %w", ref.ID, h.id, err)) } - step := h.nextStep - // A parent Fire the delivery drove can arm a timer, which reads the clock. - rec := Record{Step: step, Entries: append(h.drainClock(), entries...)} - if err := h.persistStep(ctx, step, &rec); err != nil { + // A parent Fire the delivery drove can arm a timer, which reads the clock. Its effects + // aren't dispatched, so the step names none, and finishStep only offers its + // checkpoint. + rec := &Record{Step: h.nextStep, Entries: append(h.drainClock(), entries...)} + if _, err := h.recordStep(ctx, rec, nil); err != nil { return true, err } - return true, nil + return true, h.finishStep(ctx, rec, nil) } // DeliverToActorByID is DeliverToActor keyed by raw actor id, for a host that diff --git a/durable/dispatch_store_test.go b/durable/dispatch_store_test.go new file mode 100644 index 00000000..12c9e865 --- /dev/null +++ b/durable/dispatch_store_test.go @@ -0,0 +1,172 @@ +package durable_test + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// dispatchProbeStore wraps a MemStore's dispatch tracking: it counts each call to +// Dispatched and MarkDispatched, and panics in either when told to. +type dispatchProbeStore struct { + *durable.MemStore + mu sync.Mutex + dispatched, marks int + panicDispatched, panicMk bool +} + +func newDispatchProbeStore(st *durable.MemStore) *dispatchProbeStore { + return &dispatchProbeStore{MemStore: st} +} + +func (s *dispatchProbeStore) Dispatched(ctx context.Context, id durable.InstanceID) (map[string]bool, error) { + s.mu.Lock() + s.dispatched++ + p := s.panicDispatched + s.mu.Unlock() + if p { + panic("Dispatched exploded") + } + return s.MemStore.Dispatched(ctx, id) +} + +func (s *dispatchProbeStore) MarkDispatched(ctx context.Context, id durable.InstanceID, ids ...string) error { + s.mu.Lock() + s.marks++ + p := s.panicMk + s.mu.Unlock() + if p { + panic("MarkDispatched exploded") + } + return s.MemStore.MarkDispatched(ctx, id, ids...) +} + +func (s *dispatchProbeStore) counts() (dispatched, marks int) { + s.mu.Lock() + defer s.mu.Unlock() + return s.dispatched, s.marks +} + +// unmarkedFireEffect starts i1 on effectMachine and fires "go" with a handler that +// fails, so the journal holds step 0's effect unmarked for the next Recover to send. +func unmarkedFireEffect(t *testing.T) *durable.MemStore { + t.Helper() + ctx := context.Background() + st := durable.NewMemStore() + live := &stepIDHandler{failStep: "0"} + h, err := durable.NewRunner(effectMachine(), st, durable.WithEffectHandler[string, string, *runCtx](live.handle)). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire go: %v, want ErrEffectDispatch", err) + } + return st +} + +// TestDispatchStorePanic_ReturnsEffectDispatch proves a DispatchStore whose Dispatched +// or MarkDispatched panics fails Recover and the stateless Runner.Fire with +// ErrEffectDispatch, as a live step's dispatch and a panicking handler already do, +// rather than letting the panic out. +func TestDispatchStorePanic_ReturnsEffectDispatch(t *testing.T) { + ctx := context.Background() + healthy := durable.WithEffectHandler[string, string, *runCtx](newEffectRecorder().handler()) + cases := []struct { + name string + arm func(*dispatchProbeStore) + }{ + {"MarkDispatched", func(s *dispatchProbeStore) { s.panicMk = true }}, + {"Dispatched", func(s *dispatchProbeStore) { s.panicDispatched = true }}, + } + for _, tc := range cases { + t.Run(tc.name+"/Recover", func(t *testing.T) { + st := newDispatchProbeStore(unmarkedFireEffect(t)) + tc.arm(st) + err := noPanic(t, "Recover", func() error { + _, rerr := durable.Recover(ctx, effectMachine(), st, "i1", healthy) + return rerr + }) + if !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Recover: %v, want ErrEffectDispatch", err) + } + }) + t.Run(tc.name+"/Runner.Fire", func(t *testing.T) { + st := newDispatchProbeStore(unmarkedFireEffect(t)) + tc.arm(st) + err := noPanic(t, "Runner.Fire", func() error { + _, ferr := durable.NewRunner(effectMachine(), st, healthy).Fire(ctx, "i1", "bill") + return ferr + }) + if !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Runner.Fire: %v, want ErrEffectDispatch", err) + } + }) + } + // Start applies its effects without the dispatched set, so a panicking + // MarkDispatched can't reach it; a step's live dispatch already turns it into + // ErrEffectDispatch. Pin both, so the three paths stay alike. + t.Run("MarkDispatched/live step", func(t *testing.T) { + st := newDispatchProbeStore(durable.NewMemStore()) + h, err := durable.NewRunner(effectMachine(), st, healthy).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + st.panicMk = true + err = noPanic(t, "Fire", func() error { + _, ferr := h.Fire(ctx, "go") + return ferr + }) + if !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire: %v, want ErrHandleFailed wrapping ErrEffectDispatch", err) + } + }) +} + +// pingMachine emits a notify on every "ping", so each Fire records one effect. +func pingMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("ping"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("idle"). + Initial("idle"). + Transition("idle").On("ping").GoTo("idle").Do("notify"). + Quench() +} + +// TestRecover_LoadsTheDispatchedSetOncePerEffectRecord pins what Recover costs the +// Store: it loads the instance's whole dispatched set once for each tail Record that +// names effects, and marks nothing when every effect already landed. main did the same +// for a Fire's Records; this change adds a tick's and a service settle's at the same +// rate. Loading the set once per Recover would make this 1. +func TestRecover_LoadsTheDispatchedSetOncePerEffectRecord(t *testing.T) { + ctx := context.Background() + const n = 25 + st := newDispatchProbeStore(durable.NewMemStore()) + handler := durable.WithEffectHandler[string, string, *runCtx](newEffectRecorder().handler()) + h, err := durable.NewRunner(pingMachine(), st, handler).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for i := 0; i < n; i++ { + if _, err = h.Fire(ctx, "ping"); err != nil { + t.Fatalf("Fire ping %d: %v", i, err) + } + } + liveLoads, liveMarks := st.counts() + if _, err = durable.Recover(ctx, pingMachine(), st, "i1", handler); err != nil { + t.Fatalf("Recover: %v", err) + } + loads, marks := st.counts() + if got := loads - liveLoads; got != n { + t.Fatalf("Recover over %d effect Records loaded the dispatched set %d times, want %d", n, got, n) + } + if got := marks - liveMarks; got != 0 { + t.Fatalf("Recover marked %d effects, want none: every one landed live", got) + } +} diff --git a/durable/effect.go b/durable/effect.go index a89bf95d..2bbdaf8f 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -219,14 +219,22 @@ func (h *Handle[S, E, C]) dispatchEffects(ctx context.Context, des []dispatchabl return h.runner.dispatch(ctx, h.id, des) } +// dispatchStore returns the Store's dispatch tracking when des has effects to apply: a +// handler is wired and the Store is a DispatchStore. Otherwise dispatch is a no-op, and +// it returns false. +func (r *Runner[S, E, C]) dispatchStore(des []dispatchableEffect) (DispatchStore, bool) { + if len(des) == 0 || r.cfg.effectHandler == nil { + return nil, false + } + ds, ok := r.store.(DispatchStore) + return ds, ok +} + // applyStartEffects invokes the handler for the initial entry's domain effects. Like // dispatch it is a no-op without a handler or a DispatchStore. It marks nothing, because // the instance doesn't exist yet and nothing replays these effects. func (r *Runner[S, E, C]) applyStartEffects(ctx context.Context, id InstanceID, des []dispatchableEffect) error { - if len(des) == 0 || r.cfg.effectHandler == nil { - return nil - } - if _, ok := r.store.(DispatchStore); !ok { + if _, ok := r.dispatchStore(des); !ok { return nil } for _, de := range des { @@ -265,15 +273,19 @@ func (r *Runner[S, E, C]) dispatchReplayEffects(ctx context.Context, id Instance // dispatch is the shared exactly-once apply loop used by the live path and by // recovery replay: it reads the instance's dispatched set, invokes the handler // for each not-yet-dispatched effect, and marks each id as it succeeds. It is a -// no-op without a handler or a DispatchStore. -func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dispatchableEffect) error { - if len(des) == 0 || r.cfg.effectHandler == nil { - return nil - } - ds, ok := r.store.(DispatchStore) +// no-op without a handler or a DispatchStore. A panic in the Store's Dispatched or +// MarkDispatched becomes an ErrEffectDispatch error, as applyEffect makes one in the +// handler, so a step and Recover fail the same way. +func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dispatchableEffect) (err error) { + ds, ok := r.dispatchStore(des) if !ok { return nil } + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("%w: effect dispatch for %q panicked: %v", ErrEffectDispatch, id, p) + } + }() dispatched, err := ds.Dispatched(ctx, id) if err != nil { diff --git a/durable/result_type_check_test.go b/durable/result_type_check_test.go new file mode 100644 index 00000000..563dded7 --- /dev/null +++ b/durable/result_type_check_test.go @@ -0,0 +1,98 @@ +package durable_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These tests compare the type check a live service result passes with the one a +// Fire's WithEventData payload passes. Both decode through the Runner's +// EventDataDecoder; they differ in what they accept back. + +type tcOrder struct { + ID int64 + Items []string +} + +// useNumber decodes a payload as JSON's generic form, keeping numbers as json.Number +// so an integer keeps its precision. +func useNumber(data []byte) (any, error) { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + return v, nil +} + +// fireDone fires "done" on a fresh scn instance with payload, and returns the error. +func fireDone(t *testing.T, payload any, opts ...durable.Option[string, string, svcCtx]) error { + t.Helper() + fn := returning(nil, nil) + m := scnMachine(fn, &scnAssigns{}) + opts = append([]durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))}, opts...) + h := scnStart(t, m, durable.NewMemStore(), opts...) + _, err := h.Fire(context.Background(), "done", state.WithEventData(payload)) + return err +} + +// TestTypeCheck_NoDecoder_ServiceResultIsAMap_FirePayloadIsRefused pins the documented +// asymmetry: with no decoder, a struct service result reaches Assign as JSON's generic +// form, a map, while a Fire carrying the same struct is refused. A Fire is refused +// before the kernel runs, so its caller can add a decoder and fire again; a service has +// already run, so refusing its result would only fail the step and run it again. +func TestTypeCheck_NoDecoder_ServiceResultIsAMap_FirePayloadIsRefused(t *testing.T) { + order := tcOrder{ID: 7, Items: []string{"tea"}} + a, runErr, recErr := scnRun(t, returning(order, nil)) + mustTwo(t, a, runErr, recErr) + if _, ok := a.seen[0].(map[string]any); !ok { + t.Fatalf("the service result reached Assign as %s, want map[string]any", describe(a.seen[0])) + } + if err := fireDone(t, order); !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Fire with the same struct: %v, want ErrEventDataEncoding", err) + } +} + +// TestTypeCheck_UseNumberDecoder_ServiceResult proves a decoder that keeps numbers as +// json.Number, a generic form, is accepted for a service result whatever its shape: an +// int arrives as a json.Number, as the same decoder's struct result arrives as a map +// holding json.Numbers. Before the fix the int was refused and the struct accepted. +func TestTypeCheck_UseNumberDecoder_ServiceResult(t *testing.T) { + t.Run("int", func(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(42, nil), decoderFor("done", useNumber)) + mustTwo(t, a, runErr, recErr) + if n, ok := a.seen[0].(json.Number); !ok || n.String() != "42" { + t.Fatalf("Assign saw %s, want json.Number 42", describe(a.seen[0])) + } + }) + t.Run("struct", func(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(tcOrder{ID: 7}, nil), decoderFor("done", useNumber)) + mustTwo(t, a, runErr, recErr) + m, ok := a.seen[0].(map[string]any) + if !ok { + t.Fatalf("Assign saw %s, want map[string]any", describe(a.seen[0])) + } + if _, ok := m["ID"].(json.Number); !ok { + t.Fatalf("the map's ID is %s, want json.Number", describe(m["ID"])) + } + }) +} + +// TestTypeCheck_UseNumberDecoder_FirePayloadIsRefused pins that a Fire stays strict: an +// int payload decoded as a json.Number isn't the type its caller passed, so the Fire is +// refused, as one decoded as a map is. +func TestTypeCheck_UseNumberDecoder_FirePayloadIsRefused(t *testing.T) { + if err := fireDone(t, 42, decoderFor("done", useNumber)); !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Fire with an int and a json.Number decoder: %v, want ErrEventDataEncoding", err) + } + if err := fireDone(t, tcOrder{ID: 7}, decoderFor("done", useNumber)); !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Fire with a struct and a json.Number decoder: %v, want ErrEventDataEncoding", err) + } +} diff --git a/durable/runner.go b/durable/runner.go index f749f821..b5db888d 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "reflect" "slices" "sync" "time" @@ -940,8 +939,12 @@ func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.Fi // append durably names every effect that must be dispatched before any dispatch // happens, then appends rec and advances the next step. A step with effects is marked // dispatching, so no checkpoint compacts it until finishStep, which the caller runs -// after releasing the step lock, has dispatched them. The kernel has already applied the -// step, so a failed encode or append fails the Handle. +// after releasing the step lock, has dispatched them; a step with no effects to dispatch +// may run finishStep under the lock, as DeliverToActor does. The kernel has already +// applied the step, so a failed encode or append fails the Handle; a failed checkpoint +// doesn't, because the Record is durable. This relies on the Append running inline, +// under the step lock, before the step returns: an Append deferred off this goroutine +// would acknowledge a step whose outcome isn't known, and the Handle couldn't fail. func (h *Handle[S, E, C]) recordStep(ctx context.Context, rec *Record, effects []state.Effect) ([]dispatchableEffect, error) { des := dispatchableEffects(rec.Step, effects) effEnvs, err := recordEffects(des) @@ -978,7 +981,7 @@ func (h *Handle[S, E, C]) recordStep(ctx context.Context, rec *Record, effects [ func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dispatchableEffect) error { var dispatchErr error if len(des) > 0 { - dispatchErr = h.dispatchGuarded(h.dispatchContext(ctx), rec.Step, des) + dispatchErr = h.dispatchEffects(h.dispatchContext(ctx), des) } h.out.mu.Lock() defer h.out.mu.Unlock() @@ -995,17 +998,6 @@ func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dis return h.offerCheckpointLocked(ctx, rec) } -// dispatchGuarded dispatches des, turning a panic in the Store's dispatch tracking into -// an ErrEffectDispatch error. applyEffect turns one in the effect handler into the same. -func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []dispatchableEffect) (err error) { - defer func() { - if p := recover(); p != nil { - err = fmt.Errorf("%w: step %d for %q: effect dispatch panicked: %v", ErrEffectDispatch, step, h.id, p) - } - }() - return h.dispatchEffects(ctx, des) -} - // Tick advances the Handle's delayed-transition scheduler: it fires every timer // whose recorded deadline is at or before the recording clock's current time, in // due order, re-firing the delayed events through the durable instance and @@ -1095,25 +1087,6 @@ func (h *Handle[S, E, C]) tickStep(ctx context.Context) ([]state.FireResult[S], return results, rec, des, nil } -// persistStep write-ahead appends rec for a step with no domain effects to dispatch, -// advances the next step, then offers its checkpoint. A failed append fails the -// Handle, because the kernel already applied the step; a failed checkpoint doesn't, -// because the Record is durable. This relies on the Append running inline, under the -// step lock, before the step returns: an Append deferred off this goroutine would -// acknowledge a step whose outcome isn't known, and the Handle couldn't fail. -func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { - if err := h.appendStep(ctx, step, rec); err != nil { - return h.fail(err) - } - h.nextStep = rec.followingStep() - h.out.mu.Lock() - defer h.out.mu.Unlock() - if err := h.recordedOnFailedLocked(step); err != nil { - return err - } - return h.offerCheckpointLocked(ctx, rec) -} - // recordedOnFailedLocked reports a step that was recorded while another goroutine failed // the Handle: ErrHandleFailed, saying the step was recorded and Recover sends any of its // effects. It returns nil on a healthy Handle. h.out.mu must be held. @@ -1719,9 +1692,8 @@ func encodeEventData[E comparable](event E, opts []state.FireOption, maxBytes in return payload, nil, fmt.Errorf("%w: %w", ErrEventDataEncoding, err) } if !isErr { - if got, want := reflect.TypeOf(value), reflect.TypeOf(data); got != want { - return payload, nil, fmt.Errorf("%w: a %v payload decodes as %v; decode it as %v with WithEventDataDecoder", - ErrEventDataEncoding, want, got, want) + if err := decodedAsPassed(value, data, "payload", false); err != nil { + return payload, nil, err } } live := make([]state.FireOption, 0, len(opts)+1) diff --git a/durable/service.go b/durable/service.go index 9607c4e7..96fcc144 100644 --- a/durable/service.go +++ b/durable/service.go @@ -220,8 +220,8 @@ func (h *Handle[S, E, C]) noteSettled(id string) { // WithEventData payload is: the live run passes the ServiceFn's value, a replay the // recorded JSON as a recordedResult, and both get decode's value. With the default // decoder that is JSON's generic form. Live, a decoder that returns a concrete type -// other than the result's is refused, though JSON's generic form is always accepted, -// so the default keeps working. +// other than the result's is refused, though JSON's generic form, including a +// json.Number, is always accepted, so the default keeps working. // // A result that won't decode, or is refused, aborts the settle with an // *undecodableResult before it fires anything, so no onDone or onError Assign runs; @@ -253,10 +253,9 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, er if derr != nil { return nil, nil, &undecodableResult{err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} } - if !replay && value != nil && !genericJSON(value) { - if got, want := reflect.TypeOf(value), reflect.TypeOf(result); got != want { - return nil, nil, &undecodableResult{err: fmt.Errorf("%w: a %v service result decodes as %v; decode it as %v with WithEventDataDecoder", - ErrEventDataEncoding, want, got, want)} + if !replay { + if terr := decodedAsPassed(value, result, "service result", true); terr != nil { + return nil, nil, &undecodableResult{err: terr} } } return value, nil, nil @@ -278,15 +277,31 @@ func (u *undecodableResult) Error() string { return u.err.Error() } func (u *undecodableResult) Unwrap() error { return u.err } // genericJSON reports whether v is a type JSON decodes into an any: what the default -// decoder gives. +// decoder gives, or a json.Number from a decoder that keeps numbers with UseNumber. func genericJSON(v any) bool { switch v.(type) { - case map[string]any, []any, float64, string, bool: + case map[string]any, []any, float64, json.Number, string, bool: return true } return false } +// decodedAsPassed reports, as an ErrEventDataEncoding error, a value the decoder +// returned as another type than passed, the value the host handed over; what names it. +// A Fire's payload must come back as the caller's type. A service result may also come +// back as nil or in JSON's generic form (generic), so the default decoder serves every +// result: the service has already run, and refusing its result would only run it again. +func decodedAsPassed(value, passed any, what string, generic bool) error { + if generic && (value == nil || genericJSON(value)) { + return nil + } + if got, want := reflect.TypeOf(value), reflect.TypeOf(passed); got != want { + return fmt.Errorf("%w: a %v %s decodes as %v; decode it as %v with WithEventDataDecoder", + ErrEventDataEncoding, want, what, got, want) + } + return nil +} + // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service. It reads the success result through lastResult (valid only during the // settle's synchronous window) and the error through lastErr, encoding whichever diff --git a/durable/service_encode_internal_test.go b/durable/service_encode_internal_test.go new file mode 100644 index 00000000..f794cfce --- /dev/null +++ b/durable/service_encode_internal_test.go @@ -0,0 +1,84 @@ +package durable + +import ( + "encoding/json" + "testing" +) + +// f8Order's fields aren't in alphabetical order, so its JSON and the JSON of the map +// it decodes to by default list them in a different order. +type f8Order struct { + Zone string + ID int64 +} + +// TestServiceResult_JournalBytesAreTheDecodedValuesEncoding pins why a live service +// result is encoded twice. replayableOutcome encodes the ServiceFn's value to decode +// it; recordServiceOutcome then journals the decoded value's own encoding. Journaling +// the first encoding instead would record different bytes for a struct decoded as a map +// (the keys come out sorted) and for an integer beyond 2^53 decoded as a float64 (the +// journal would keep digits the machine never saw). Replay decodes either to the same +// value, but the journal would no longer hold the form the machine saw, as RunService's +// doc says it does, so the second encode stays. +func TestServiceResult_JournalBytesAreTheDecodedValuesEncoding(t *testing.T) { + asOrder := func(_ string, data []byte) (any, error) { return DecodeAs[f8Order](data) } + asInt64 := func(_ string, data []byte) (any, error) { return DecodeAs[int64](data) } + cases := []struct { + name string + result any + decode EventDataDecoder[string] + same bool // whether the two encodings match + recorded string // what the journal records + }{ + {"string", "ok", decodeAny[string], true, `"ok"`}, + {"small int, generic", 42, decodeAny[string], true, `42`}, + {"int64 above 2^53, generic", int64(1<<53 + 1), decodeAny[string], false, `9007199254740992`}, + {"int64 above 2^53, typed", int64(1<<53 + 1), asInt64, true, `9007199254740993`}, + {"struct, generic", f8Order{Zone: "w", ID: 7}, decodeAny[string], false, `{"ID":7,"Zone":"w"}`}, + {"struct, typed", f8Order{Zone: "w", ID: 7}, asOrder, true, `{"Zone":"w","ID":7}`}, + {"map, generic", map[string]any{"b": 1.0, "a": "x"}, decodeAny[string], true, `{"a":"x","b":1}`}, + {"slice, generic", []any{"x", 2.5}, decodeAny[string], true, `["x",2.5]`}, + {"nil", nil, decodeAny[string], true, `null`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + first, err := marshalPayload(tc.result) + if err != nil { + t.Fatalf("encoding the result: %v", err) + } + value, outErr, abort := replayableOutcome(tc.decode)("done", tc.result, nil) + if outErr != nil || abort != nil { + t.Fatalf("the transform returned %v, %v", outErr, abort) + } + entry, err := recordServiceOutcome("svc", func() (any, bool) { return value, true }, nil) + if err != nil { + t.Fatalf("recordServiceOutcome: %v", err) + } + var sp serviceResultPayload + if err = json.Unmarshal(entry.Payload, &sp); err != nil { + t.Fatalf("decoding the entry: %v", err) + } + if got := string(sp.Result); got != tc.recorded { + t.Fatalf("the journal records %s, want %s", got, tc.recorded) + } + if same := string(first) == string(sp.Result); same != tc.same { + t.Fatalf("first encoding %s, journal %s: same = %v, want %v", first, sp.Result, same, tc.same) + } + // Replay decodes either encoding to the value live saw. + for _, b := range [][]byte{first, sp.Result} { + replayed, _, rabort := replayableOutcome(tc.decode)("done", recordedResult(b), nil) + if rabort != nil { + t.Fatalf("replaying %s: %v", b, rabort) + } + if describeValue(replayed) != describeValue(value) { + t.Fatalf("replaying %s gives %s, live saw %s", b, describeValue(replayed), describeValue(value)) + } + } + }) + } +} + +func describeValue(v any) string { + b, _ := json.Marshal(v) + return string(b) +} diff --git a/durable/undecodable_result_loop_test.go b/durable/undecodable_result_loop_test.go new file mode 100644 index 00000000..0046218f --- /dev/null +++ b/durable/undecodable_result_loop_test.go @@ -0,0 +1,70 @@ +package durable_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// countingFetch returns a scnUser and counts its runs, standing in for a ServiceFn +// with a side effect. +func countingFetch(runs *int64) state.ServiceFn[svcCtx] { + return func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(runs, 1) + return scnUser{ID: 7, Name: "ann"}, nil + } +} + +// TestServiceResult_UndecodableResult_RerunsTheServiceOnEveryRecover pins the current +// behavior for a result the decoder for its onDone event refuses: the step fails, +// nothing is recorded and the service stays in flight, so every Recover re-arms it and +// every RunService runs the ServiceFn again, repeating its side effect, and fails +// again, until the decoder is fixed. No Assign runs. This is a known loop the review +// raised (an undecodable service result loops forever); the options for ending it are +// compared on the exp/f2-* branches, and this test changes when one is chosen. +func TestServiceResult_UndecodableResult_RerunsTheServiceOnEveryRecover(t *testing.T) { + for _, tc := range []struct { + name string + opt durable.Option[string, string, svcCtx] + }{ + {"decoder fails", failingDecoder()}, + {"type mismatch", decoderFor("done", durable.DecodeAs[scnAdmin])}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + var runs int64 + fn := countingFetch(&runs) + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), tc.opt} + id := state.InvokeID(scnInvoke, "loading", 0) + h := scnStart(t, m, store, opts...) + if _, ran, err := h.RunService(ctx, id); !ran || !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("RunService = %v, %v; want ran and ErrEventDataEncoding", ran, err) + } + for i := 1; i <= 3; i++ { + rh, err := durable.Recover(ctx, m, store, "scn-1", opts...) + if err != nil { + t.Fatalf("Recover %d: %v", i, err) + } + if _, ran, err := rh.RunService(ctx, id); !ran || !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("RunService after Recover %d = %v, %v; want ran and ErrHandleFailed wrapping ErrEventDataEncoding", i, ran, err) + } + } + if got := atomic.LoadInt64(&runs); got != 4 { + t.Fatalf("the ServiceFn ran %d times, want 4: once live and once after each of 3 Recovers", got) + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } + if _, recs, err := store.Load(ctx, "scn-1"); err != nil || len(recs) != 0 { + t.Fatalf("the journal holds %d Records (err %v), want none", len(recs), err) + } + }) + } +} diff --git a/durable/unmarshalable_effect_test.go b/durable/unmarshalable_effect_test.go new file mode 100644 index 00000000..b05f473e --- /dev/null +++ b/durable/unmarshalable_effect_test.go @@ -0,0 +1,154 @@ +package durable_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// badEffect is a KindedEffect with no JSON form: encoding/json can't encode a chan. +type badEffect struct { + Done chan struct{} `json:"done"` +} + +func (badEffect) Kind() string { return "demo.bad" } + +func emitBad[C any](state.ActionCtx[C]) (state.Effect, error) { + return badEffect{Done: make(chan struct{})}, nil +} + +// badEffectMachine emits a badEffect on "go", on the timer that follows, and on the +// settle of the service "loading" invokes, so each kind of step can emit one. +func badEffectMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("bad"). + Service("fetch", fn). + Action("bad", emitBad[svcCtx]). + State("idle"). + State("waiting"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("ok")). + State("done").Final(). + Initial("idle"). + Transition("idle").On("go").GoTo("done").Do("bad"). + Transition("idle").On("wait").GoTo("waiting"). + Transition("waiting").After(time.Second).GoTo("done").Do("bad"). + Transition("idle").On("load").GoTo("loading"). + Transition("loading").On("ok").GoTo("done").Do("bad"). + Quench() +} + +// badRun is a Runner set up for badEffectMachine with no effect handler, and the count +// of ServiceFn runs. +type badRun struct { + m *state.Machine[string, string, svcCtx] + store *durable.MemStore + clk *state.FakeClock + opts []durable.Option[string, string, svcCtx] + runs *int64 +} + +func newBadRun() *badRun { + runs := new(int64) + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(runs, 1) + return "ok", nil + } + clk := state.NewFakeClock(epoch) + return &badRun{ + m: badEffectMachine(fn), + store: durable.NewMemStore(), + clk: clk, + runs: runs, + opts: []durable.Option[string, string, svcCtx]{ + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + durable.WithRunnerClock[string, string, svcCtx](clk), + }, + } +} + +func (b *badRun) start(t *testing.T) *durable.Handle[string, string, svcCtx] { + t.Helper() + h, err := durable.NewRunner(b.m, b.store, b.opts...).Start(context.Background(), "bad-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + return h +} + +func (b *badRun) recover(t *testing.T) *durable.Handle[string, string, svcCtx] { + t.Helper() + h, err := durable.Recover(context.Background(), b.m, b.store, "bad-1", b.opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + return h +} + +func mustHandleFailed(t *testing.T, what string, err error) { + t.Helper() + if !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("%s: %v, want ErrHandleFailed", what, err) + } +} + +// TestUnmarshalableEffect_NoHandler_FailsEveryStep pins the current behavior for a +// KindedEffect with no JSON form when no WithEffectHandler is wired: every step records +// its effects, so the step fails the Handle and records nothing. A Fire failed this way +// before this change; a Tick and a RunService didn't, since they recorded no effects. +// Now they fail and loop: Recover re-arms the timer, and re-arms the service, whose +// ServiceFn runs again. The review raised this (an unmarshalable KindedEffect with no +// handler); the options are compared on the exp/f3-* branches, and this test changes +// when one is chosen. +func TestUnmarshalableEffect_NoHandler_FailsEveryStep(t *testing.T) { + ctx := context.Background() + t.Run("Fire", func(t *testing.T) { + b := newBadRun() + _, err := b.start(t).Fire(ctx, "go") + mustHandleFailed(t, "Fire", err) + if got := b.recover(t).Instance().Current(); got != "idle" { + t.Fatalf("recovered in %q, want idle: the Fire wasn't recorded", got) + } + }) + t.Run("Tick", func(t *testing.T) { + b := newBadRun() + if _, err := b.start(t).Fire(ctx, "wait"); err != nil { + t.Fatalf("Fire wait: %v", err) + } + b.clk.Advance(2 * time.Second) + for i := 0; i < 3; i++ { + rh := b.recover(t) + _, err := rh.Tick(ctx) + mustHandleFailed(t, "Tick", err) + if got := rh.Instance().Current(); got != "done" { + t.Fatalf("Tick %d left the live instance in %q, want done (applied, then failed)", i, got) + } + } + if got := b.recover(t).Instance().Current(); got != "waiting" { + t.Fatalf("recovered in %q, want waiting: no Tick was recorded", got) + } + }) + t.Run("RunService", func(t *testing.T) { + b := newBadRun() + if _, err := b.start(t).Fire(ctx, "load"); err != nil { + t.Fatalf("Fire load: %v", err) + } + id := state.InvokeID("bad", "loading", 0) + for i := 0; i < 3; i++ { + _, ran, err := b.recover(t).RunService(ctx, id) + if !ran { + t.Fatalf("RunService %d didn't find the service in flight", i) + } + mustHandleFailed(t, "RunService", err) + } + if got := atomic.LoadInt64(b.runs); got != 3 { + t.Fatalf("the ServiceFn ran %d times, want 3: once per Recover", got) + } + if got := b.recover(t).Instance().Current(); got != "loading" { + t.Fatalf("recovered in %q, want loading: no settle was recorded", got) + } + }) +} diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index b60e3b23..a54c6ba3 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -82,6 +82,12 @@ counts as an additive (minor) versus breaking (major) change. Use the whole system, so a message another goroutine's delivery sent the parent while it ran was dropped. The quiet now rides the call chain that spawns each actor, and ends when `AbsorbRespawn` returns. +- **A `ServiceRunner` settle can't settle a later activation of its service.** A + `SettleDone` or `SettleError` whose result transform was running, or a `Tick` whose + `ServiceFn` was running, while the service was stopped and armed again under the same + invoke id, settled the new activation with the old outcome. It now returns false, or + one zero `FireResult` from `Tick`, fires nothing, and leaves the new activation in + flight. ## [1.0.0] — 2026-06-13 diff --git a/state/runner.go b/state/runner.go index e9f7ea33..71eb2938 100644 --- a/state/runner.go +++ b/state/runner.go @@ -70,6 +70,10 @@ type running[E comparable] struct { onDone E onError E state string + // gen identifies this activation: a service stopped and armed again under the same + // invoke id gets a new one, so a settle that captured the old activation can't + // settle the new one. + gen uint64 } // ServiceRunner is the reusable host-driver that turns the kernel's StartService / @@ -90,6 +94,7 @@ type ServiceRunner[S comparable, E comparable, C any] struct { mu sync.Mutex running map[string]running[E] + gen uint64 // the latest activation's gen, under mu // lastResult holds the most recently settled service outcome so the host // action bound to the onDone / onError transition can read the result or error @@ -142,12 +147,14 @@ func (r *ServiceRunner[S, E, C]) Absorb(ctx context.Context, effects []Effect) { if !ok { continue } + r.gen++ r.running[e.ID] = running[E]{ src: e.Src, input: e.Input, onDone: done, onError: erev, state: e.State, + gen: r.gen, } case StopService: delete(r.running, e.ID) @@ -195,10 +202,8 @@ func (r *ServiceRunner[S, E, C]) SettleError(ctx context.Context, id string, err return r.settle(ctx, id, nil, err) } -// settle is the shared body of SettleDone / SettleError: run the transform on the -// outcome, drop the running service under lock, then fire its routing event and absorb -// the follow-on effects. The transform runs outside the lock before the service is -// dropped, so one that aborts, or panics, leaves the service in flight. +// settle is the shared body of SettleDone / SettleError: it settles the activation +// of id in flight now. func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result any, err error) (FireResult[S], bool) { r.mu.Lock() rs, ok := r.running[id] @@ -206,6 +211,17 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a if !ok { return FireResult[S]{}, false } + return r.settleActivation(ctx, id, rs, result, err) +} + +// settleActivation settles rs, one activation of id: run the transform on the +// outcome, drop the activation under lock, then fire its routing event and absorb the +// follow-on effects. The transform runs outside the lock before the activation is +// dropped, so one that aborts, or panics, leaves the service in flight. It returns +// false and fires nothing when rs is no longer in flight, settled or stopped meanwhile, +// even if the service was armed again under the same id: that new activation stays in +// flight. +func (r *ServiceRunner[S, E, C]) settleActivation(ctx context.Context, id string, rs running[E], result any, err error) (FireResult[S], bool) { if r.transform != nil { var abort error if result, err, abort = r.applyTransform(rs, result, err); abort != nil { @@ -213,13 +229,14 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a } } r.mu.Lock() - _, ok = r.running[id] + cur, ok := r.running[id] + ok = ok && cur.gen == rs.gen if ok { delete(r.running, id) } r.mu.Unlock() if !ok { - // Settled or stopped while the transform ran. + // Settled or stopped while the transform ran, and perhaps armed again. return FireResult[S]{}, false } r.lastResult.Store(serviceOutcome{result: result, err: err}) @@ -261,7 +278,8 @@ func (r *ServiceRunner[S, E, C]) settle(ctx context.Context, id string, result a // // f runs outside the runner's lock before the service is dropped, so it may run for a // settle that then finds the service stopped or settled by another goroutine, and -// returns false. A panic in f goes on out of the settle, leaving the service in +// returns false. That holds even if the service was armed again under the same invoke +// id meanwhile: the new activation stays in flight. A panic in f goes on out of the settle, leaving the service in // flight. Set f before the first settle: the field isn't guarded. func (r *ServiceRunner[S, E, C]) WithResultTransform(f func(event E, result any, err error) (out any, outErr error, abort error)) *ServiceRunner[S, E, C] { r.transform = f @@ -298,7 +316,9 @@ func (r *ServiceRunner[S, E, C]) applyTransform(rs running[E], result any, err e // (so the machine routes onError rather than hanging) and that routed result is // returned in the slice. When a WithResultTransform function aborts the settle, the // slice holds one FireResult whose Err is the abort error, no event fires, and the -// service stays in flight. +// service stays in flight. A service stopped while its ServiceFn ran settles nothing, +// even if it was armed again under the same id: the slice holds one zero FireResult, +// and the new activation stays in flight. func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResult[S] { r.mu.Lock() rs, ok := r.running[id] @@ -306,17 +326,17 @@ func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResu if !ok { return nil } + // Settle the activation the ServiceFn runs for, not one armed under id meanwhile. fn := r.resolve(rs.src.Name) if fn == nil { - res, _ := r.SettleError(ctx, id, &UnboundRefError{Kind: "service", Name: rs.src.Name}) + res, _ := r.settleActivation(ctx, id, rs, nil, &UnboundRefError{Kind: "service", Name: rs.src.Name}) return []FireResult[S]{res} } out, err := fn(ctx, ServiceCtx[C]{Entity: r.inst.entity, Params: rs.src.Params, Input: rs.input}) if err != nil { - res, _ := r.SettleError(ctx, id, err) - return []FireResult[S]{res} + out = nil } - res, _ := r.SettleDone(ctx, id, out) + res, _ := r.settleActivation(ctx, id, rs, out, err) return []FireResult[S]{res} } diff --git a/state/settle_stale_test.go b/state/settle_stale_test.go new file mode 100644 index 00000000..523e9d00 --- /dev/null +++ b/state/settle_stale_test.go @@ -0,0 +1,118 @@ +package state_test + +import ( + "context" + "testing" + + "github.com/stablekernel/crucible/state" +) + +// rearm stops the in-flight "loading" service by leaving the state, then enters it +// again, which arms a new activation under the same InvokeID. +func rearm(ctx context.Context, t *testing.T, inst *state.Instance[string, string, *trec], run *state.ServiceRunner[string, string, *trec], id string) { + t.Helper() + run.Absorb(ctx, inst.Fire(ctx, "cancel").Effects) + if run.HasPending(id) { + t.Fatalf("leaving loading left %q in flight", id) + } + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + if !run.HasPending(id) || inst.Current() != "loading" { + t.Fatalf("re-entering loading didn't arm %q again", id) + } +} + +// TestSettle_StaleTransformDoesNotSettleARearmedActivation proves a settle whose +// transform ran for one activation of a service can't settle a later one. The +// transform blocks while the test stops the service and re-arms it under the same +// InvokeID; once released, the old settle must return false, fire nothing, and leave +// the new activation in flight. +func TestSettle_StaleTransformDoesNotSettleARearmedActivation(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("loader", "loading", 0) + var run *state.ServiceRunner[string, string, *trec] + m := invokeMachine(&run) + entity := &trec{} + inst := m.Cast(entity, state.WithInitialState("idle")) + entered, release := make(chan struct{}), make(chan struct{}) + block := true + run = state.NewServiceRunner(inst, nil).WithResultTransform(func(_ string, result any, err error) (any, error, error) { + if block { + close(entered) + <-release + } + return result, err, nil + }) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + + type settled struct { + res state.FireResult[string] + ok bool + } + done := make(chan settled) + go func() { + res, ok := run.SettleDone(ctx, id, "old") + done <- settled{res, ok} + }() + <-entered + rearm(ctx, t, inst, run, id) + block = false + close(release) + got := <-done + + if got.ok || got.res.NewState != "" { + t.Fatalf("the stale settle returned %+v, %v; want false and nothing fired", got.res, got.ok) + } + if inst.Current() != "loading" || len(entity.notes) != 0 { + t.Fatalf("the stale settle moved the instance to %q with notes %q, want loading and none", inst.Current(), entity.notes) + } + if !run.HasPending(id) { + t.Fatalf("the stale settle dropped the re-armed activation, want it in flight") + } + // The new activation still settles with its own outcome. + if res, ok := run.SettleDone(ctx, id, "new"); !ok || res.NewState != "ready" || join(entity.notes) != "result:new" { + t.Fatalf("settling the new activation gave %+v, %v, notes %q; want ready with result:new", res, ok, entity.notes) + } +} + +// TestTick_StaleServiceRunDoesNotSettleARearmedActivation proves the same for the +// ServiceFn's own run: a Tick whose ServiceFn ran for one activation can't settle a +// later one armed under the same InvokeID while it ran. +func TestTick_StaleServiceRunDoesNotSettleARearmedActivation(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("loader", "loading", 0) + var run *state.ServiceRunner[string, string, *trec] + m := invokeMachine(&run) + entity := &trec{} + inst := m.Cast(entity, state.WithInitialState("idle")) + entered, release := make(chan struct{}), make(chan struct{}) + block := true + reg := state.NewRegistry[*trec]().Service("fetch", func(context.Context, state.ServiceCtx[*trec]) (any, error) { + if block { + close(entered) + <-release + } + return "payload", nil + }) + run = state.NewServiceRunner(inst, reg) + run.Absorb(ctx, inst.Fire(ctx, "start").Effects) + + done := make(chan []state.FireResult[string]) + go func() { done <- run.Tick(ctx, id) }() + <-entered + rearm(ctx, t, inst, run, id) + block = false + close(release) + got := <-done + + for _, res := range got { + if res.NewState != "" { + t.Fatalf("the stale Tick fired %+v, want nothing fired", got) + } + } + if inst.Current() != "loading" || len(entity.notes) != 0 { + t.Fatalf("the stale Tick moved the instance to %q with notes %q, want loading and none", inst.Current(), entity.notes) + } + if !run.HasPending(id) { + t.Fatalf("the stale Tick dropped the re-armed activation, want it in flight") + } +} From 943fc4d407d1c65f2fe9ccc550e2a2a30aa0ab31 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 15:47:50 -0700 Subject: [PATCH 06/13] fix(durable): park a service result that won't decode; record an effect that won't marshal by id and kind - durable: a service result the decoder for its onDone event refuses, or decodes as another type than the service returned, is parked. RunService returns ErrEventDataEncoding, not ErrHandleFailed, before any Assign runs, and records the result with its Go type in a Record that settles nothing. The Handle stays healthy and the service stays in flight, but its ServiceFn never runs again: replay parks the result again, and a RunService once the decoder decodes it settles the parked result, held to the type the service returned. The park step offers its checkpoint as any step does, and the checkpoint envelope carries parked results in a new parkedServices field, so compaction can't lose one. Leaving the state that declares the service discards the parked result; entering it again runs the ServiceFn for the new activation. StateAt replays a parked Record as a step that changed nothing. state's ServiceRunner is unchanged: a parked service is in flight there. - durable: a KindedEffect whose payload won't marshal is recorded by id and kind with no payload, as an effect that isn't a KindedEffect is, with the marshal error's message in Meta under crucible.marshalError. No step fails on it, Fire included, and it dispatches as any effect does. - The pins for both now assert the new behavior: the ServiceFn runs once, and every kind of step records the effect. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 14 +- durable/CHANGELOG.md | 38 +- durable/README.md | 14 +- durable/checkpoint.go | 15 + durable/doc.go | 9 +- durable/effect.go | 22 +- durable/errors.go | 9 +- durable/parked_result_test.go | 484 ++++++++++++++++++++++ durable/result_decoding_scenarios_test.go | 26 +- durable/runner.go | 85 +++- durable/runner_options.go | 16 +- durable/service.go | 187 +++++++-- durable/service_encode_internal_test.go | 2 +- durable/service_outcome_test.go | 34 +- durable/store.go | 6 +- durable/undecodable_result_loop_test.go | 70 ---- durable/unmarshalable_effect_test.go | 207 +++++++-- 17 files changed, 1026 insertions(+), 212 deletions(-) create mode 100644 durable/parked_result_test.go delete mode 100644 durable/undecodable_result_loop_test.go diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 8ec1f633..427f515f 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -168,13 +168,17 @@ value both ways. When `WithEventDataDecoder` covers the invocation's `onDone` ev result arrives as the decoder's type, for example `durable.DecodeAs[User]`. Otherwise it arrives in JSON's generic form: a struct as a map and an integer as a `float64`, which loses precision above 2^53. A decoder that fails, or that returns a type other than the -one the service returned, fails `RunService` loudly with `ErrHandleFailed` wrapping -`ErrEventDataEncoding`: no `onDone` or `onError` reducer runs and nothing is recorded. A -service error stays message-only, an error with the same message, which `errors.Is` no +one the service returned, parks the result: `RunService` returns `ErrEventDataEncoding` +and no `onDone` or `onError` reducer runs, but the journal records the result as the +service returned it, and the Handle stays healthy. The service stays in flight and never +runs again: fix the decoder and call `RunService` again, on the same Handle or after +`Recover`, and it settles the parked result. Until then `RunService` keeps returning +`ErrEventDataEncoding`. A checkpoint keeps a parked result, and leaving the state that +declares the service discards it. A service error stays message-only, an error with the same message, which `errors.Is` no longer matches. Register the decoder before you record results: replay decodes the journal with the recovering Runner's decoder, so a decoder added or changed after results were -recorded changes what replay hands the reducer, and one that can't decode a recorded result -fails `Recover` and `StateAt` with `ErrEventDataEncoding`. A +recorded changes what replay hands the reducer, and one that can't decode a settled result +fails `Recover` and `StateAt` with `ErrEventDataEncoding`; a parked one doesn't fail them. A `WithEventData` payload is decoded from its JSON form live as well as on replay, so a reducer gets the same value both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 1dda2bbf..87a26673 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -105,8 +105,14 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. so does a `DispatchStore` whose `Dispatched` or `MarkDispatched` panics, in a step, `Recover` or the stateless `Runner.Fire`. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect again. The handler's error now matches through `errors.Is`. -- **A `KindedEffect` that can't marshal fails `Tick` and `RunService`,** as it already - failed `Fire`, even with no handler wired. +- **A `KindedEffect` that can't marshal no longer fails a step.** A step records it by + its id and kind with no payload, as it records an effect that isn't a `KindedEffect`, + and its `EffectEnvelope.Meta` holds the marshal error's message under + `crucible.marshalError`. Replay re-derives the effect's value, so it dispatches as any + effect does: once live with a handler wired, once on the `Recover` that retries a + failed dispatch, and once on a `Recover` whose handler was wired only then. This + relaxes `Fire`, and the stateless `Runner.Fire`, which failed with the marshal error + before; `Tick` and `RunService` now record such an effect too. - **A service result reaches `Assign` decoded, live as well as on replay,** so a retried effect can't differ from the first attempt. The result is decoded through the `WithEventDataDecoder` decoder for the invocation's onDone event, as a `WithEventData` @@ -115,18 +121,32 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. loses precision above 2^53. A decoder may always return JSON's generic form, including a `json.Number` from one that decodes with `UseNumber`; a `WithEventData` payload must still come back as the type its caller passed. A decoder that fails, or returns a type other than the - service returned, fails `RunService` with `ErrHandleFailed` wrapping - `ErrEventDataEncoding` before any onDone or onError `Assign` runs, and records nothing; - so does a result whose JSON form won't decode, such as a number beyond `float64`. - The decode aborts the settle through `ServiceRunner.WithResultTransform`'s abort - return, so no event fires, and a recovered Handle has the service in flight again. - `Recover` and `StateAt` fail with `ErrEventDataEncoding`, running no `Assign`, on a - recorded result their decoder can't decode. Replay decodes the journal with the + service returned, or a result whose JSON form won't decode, such as a number beyond + `float64`, parks the result (see below). `Recover` and `StateAt` fail with + `ErrEventDataEncoding`, running no `Assign`, on a settled result their decoder can't + decode. Replay decodes the journal with the recovering Runner's decoder, so register a decoder before results are recorded: one added or changed later changes what replay hands `Assign`. An error stays message-only: an error with the same message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` method that panics settles with a message saying the service error panicked, its invalid UTF-8 replaced too. The journal records the decoded value. +- **A service result that won't decode is parked.** `RunService` returns + `ErrEventDataEncoding`, not `ErrHandleFailed`, before any onDone or onError `Assign` + runs. The decode aborts the settle through `ServiceRunner.WithResultTransform`'s abort + return, so no event fires. The journal records the result as the service returned it, + with its Go type, in a Record of its own that settles nothing. The Handle stays + healthy and the service stays in flight, but its `ServiceFn` never runs again: fix the + decoder and call `RunService` again, on a Handle recovered with the fixed decoder, and + it settles the parked result, which the decoder must return as the type the service + returned, or in JSON's generic form, as live. Until then each `RunService` returns + `ErrEventDataEncoding` and records nothing more. `Recover` and `StateAt` replay a + parked result as parked, so they don't fail on it, and `StateAt` at that step shows + the instance as it was before. A checkpoint carries the parked results in a new + `parkedServices` field of its envelope, so compacting the Record that parked one + doesn't lose it. A Fire or timer that leaves the state that declares the service stops + it and discards the parked result; entering the state again runs the `ServiceFn` for + the new activation. A parked Record or checkpoint is new to this version: an older + `durable` replays a parked Record as a settled result. - **A step that may not have been recorded fails the Handle.** That call and every later one return `ErrHandleFailed`, wrapping the cause. A step that was recorded while another goroutine failed the Handle also returns `ErrHandleFailed`, saying the step was diff --git a/durable/README.md b/durable/README.md index 25e9d36c..544e94b4 100644 --- a/durable/README.md +++ b/durable/README.md @@ -179,13 +179,17 @@ value both ways. When `WithEventDataDecoder` covers the invocation's `onDone` ev result arrives as the decoder's type, for example `durable.DecodeAs[User]`. Otherwise it arrives in JSON's generic form: a struct as a map and an integer as a `float64`, which loses precision above 2^53. A decoder that fails, or that returns a type other than the -one the service returned, fails `RunService` loudly with `ErrHandleFailed` wrapping -`ErrEventDataEncoding`: no `onDone` or `onError` reducer runs and nothing is recorded. A -service error stays message-only, an error with the same message, which `errors.Is` no +one the service returned, parks the result: `RunService` returns `ErrEventDataEncoding` +and no `onDone` or `onError` reducer runs, but the journal records the result as the +service returned it, and the Handle stays healthy. The service stays in flight and never +runs again: fix the decoder and call `RunService` again, on the same Handle or after +`Recover`, and it settles the parked result. Until then `RunService` keeps returning +`ErrEventDataEncoding`. A checkpoint keeps a parked result, and leaving the state that +declares the service discards it. A service error stays message-only, an error with the same message, which `errors.Is` no longer matches. Register the decoder before you record results: replay decodes the journal with the recovering Runner's decoder, so a decoder added or changed after results were -recorded changes what replay hands the reducer, and one that can't decode a recorded result -fails `Recover` and `StateAt` with `ErrEventDataEncoding`. A +recorded changes what replay hands the reducer, and one that can't decode a settled result +fails `Recover` and `StateAt` with `ErrEventDataEncoding`; a parked one doesn't fail them. A `WithEventData` payload is decoded from its JSON form live as well as on replay, so a reducer gets the same value both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own diff --git a/durable/checkpoint.go b/durable/checkpoint.go index 1239204d..6c71c37d 100644 --- a/durable/checkpoint.go +++ b/durable/checkpoint.go @@ -60,6 +60,15 @@ type pendingTimerWire struct { Event json.RawMessage `json:"event,omitempty"` } +// parkedServiceWire is a parked service result as a checkpoint envelope carries it: the +// service id, the result's JSON and its Go type, as the parked journal entry records +// them. +type parkedServiceWire struct { + ID string `json:"id"` + Result json.RawMessage `json:"result"` + Type string `json:"type,omitempty"` +} + // checkpointEnvelope is the durable layer's wrapper around the bytes a Store keeps // for a checkpoint: the opaque state.Snapshot bytes, the pending-timer table armed at // the checkpoint instant, and the next step ordinal. The durable layer fully owns the checkpoint @@ -98,6 +107,12 @@ type checkpointEnvelope struct { // settled isn't one. None, as in a Start baseline or an older checkpoint, means // Recover starts every declared service. EndedServices []string `json:"endedServices,omitempty"` + // ParkedServices are the results parked at the checkpoint, by service id, sorted: + // each didn't decode for its onDone event, so its service is in flight and not + // settled, and a later RunService settles the recorded result rather than run the + // ServiceFn again. A checkpoint compacts the Record that parked it, so it carries + // them. None, as in a Start baseline or an older checkpoint, means none is parked. + ParkedServices []parkedServiceWire `json:"parkedServices,omitempty"` // Format is the journal format of the version that wrote the checkpoint, as // Record.Format is. Zero means a checkpoint from before this field. A Record after // it with a lower Format means the Store lost the field, and replay fails. diff --git a/durable/doc.go b/durable/doc.go index 9f3bb8bb..642c82a2 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -45,9 +45,12 @@ // WithEventDataDecoder covers the invocation's onDone event, the result arrives as the // decoder's type; otherwise it arrives in JSON's generic form, a struct as a // map[string]any and a number as a float64. A decoder that fails, or returns a type -// other than the service's result, fails RunService with ErrHandleFailed wrapping -// ErrEventDataEncoding, and no onDone or onError reducer runs. A service error stays -// message-only. Register the decoder before recording results: replay decodes the +// other than the service's result, parks the result: RunService returns +// ErrEventDataEncoding and no onDone or onError reducer runs, but the result is +// recorded and the Handle stays healthy, so the service never runs again. Fix the +// decoder and call RunService again, on this Handle or a recovered one, and it settles +// the parked result. Leaving the state that declares the service discards it. A +// service error stays message-only. Register the decoder before recording results: replay decodes the // journal with the recovering Runner's decoder, so one added or changed after results // were recorded changes what replay produces. // diff --git a/durable/effect.go b/durable/effect.go index 2bbdaf8f..07220dae 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -180,16 +180,24 @@ func dispatchableEffects(step int, effects []state.Effect) []dispatchableEffect return out } +// metaMarshalError is the EffectEnvelope.Meta key under which recordEffects records +// why a KindedEffect's payload is missing: the message of the error its marshal +// returned. It follows the kernel's "crucible." namespace for reserved names. +const metaMarshalError = "crucible.marshalError" + // recordEffects returns the persisted form of a step's dispatchable effects: an // EffectEnvelope per effect carrying its stamped EffectID, so the write-ahead // Record durably names every effect that must be dispatched. A KindedEffect // marshals into a full envelope (kind + payload); a bare domain effect is named // by id and kind alone (its value is re-derived by replay, not reconstructed from -// the envelope). A marshal failure is reported so a non-serializable effect never -// silently drops from the durable record. -func recordEffects(des []dispatchableEffect) ([]state.EffectEnvelope, error) { +// the envelope). A KindedEffect whose payload won't marshal, such as one holding a +// chan, is recorded as a bare effect is, by id and kind with no payload, and its +// envelope's Meta holds the marshal error's message under "crucible.marshalError" +// (metaMarshalError). Replay re-derives its value, so it dispatches, live and on +// Recover, as any effect does, and the step doesn't fail. +func recordEffects(des []dispatchableEffect) []state.EffectEnvelope { if len(des) == 0 { - return nil, nil + return nil } out := make([]state.EffectEnvelope, 0, len(des)) for _, de := range des { @@ -197,14 +205,16 @@ func recordEffects(des []dispatchableEffect) ([]state.EffectEnvelope, error) { if ke, ok := de.effect.(state.KindedEffect); ok { marshaled, err := state.MarshalEffect(ke) if err != nil { - return nil, fmt.Errorf("durable: recording effect %q: %w", de.id, err) + env.Meta = map[string]any{metaMarshalError: validUTF8(err.Error())} + out = append(out, env) + continue } marshaled.EffectID = de.id env = marshaled } out = append(out, env) } - return out, nil + return out } // dispatchEffects applies a step's domain effects exactly once. It is called diff --git a/durable/errors.go b/durable/errors.go index 21eaadea..62ef87ae 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -46,10 +46,11 @@ var ( // its state.WithEventData payload has no JSON form that decodes again, decodes as a // different type than the caller passed (see WithEventDataDecoder), panics while // encoding (such as an error that's a typed nil), or exceeds WithMaxEventDataBytes. - // Nothing applied, so the Handle stays usable. RunService wraps it in ErrHandleFailed - // for a service result that won't decode for its onDone event, or decodes as another - // type than the service returned; Recover and StateAt return it for a recorded result - // that won't decode. No onDone or onError Assign runs for that result. + // Nothing applied, so the Handle stays usable. RunService returns it, with the Handle + // healthy, for a service result that won't decode for its onDone event, or decodes as + // another type than the service returned: the result is parked, and a RunService + // once the decoder decodes it settles it. Recover and StateAt return it for a settled + // result that won't decode. No onDone or onError Assign runs for that result. ErrEventDataEncoding = errors.New(errorPrefix + errorEventDataEncoding) // ErrRecordTooLarge is reported by FileStore.Append for a Record whose journal line diff --git a/durable/parked_result_test.go b/durable/parked_result_test.go new file mode 100644 index 00000000..594a8660 --- /dev/null +++ b/durable/parked_result_test.go @@ -0,0 +1,484 @@ +package durable_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These tests pin what a host sees for a service result the decoder for its onDone +// event refuses: RunService returns ErrEventDataEncoding, runs no Assign, and parks the +// result in the journal, with the Handle healthy and the service in flight; the +// ServiceFn never runs again, and a RunService after the decoder is fixed settles the +// parked result. + +// countingFetch returns a scnUser and counts its runs, standing in for a ServiceFn +// with a side effect. +func countingFetch(runs *int64) state.ServiceFn[svcCtx] { + return func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(runs, 1) + return scnUser{ID: 7, Name: "ann"}, nil + } +} + +// mustPark fails unless err says a RunService parked its result: ErrEventDataEncoding, +// not ErrHandleFailed, with the service found in flight. +func mustPark(t *testing.T, what string, ran bool, err error) { + t.Helper() + if !ran || !errors.Is(err, durable.ErrEventDataEncoding) || errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("%s = %v, %v; want ran and ErrEventDataEncoding without ErrHandleFailed", what, ran, err) + } +} + +// TestServiceResult_UndecodableResult_RunsTheServiceOnce: a result the decoder refuses, +// by failing or by returning another type, is parked. RunService returns +// ErrEventDataEncoding and the Handle stays healthy. Every later RunService, on the +// same Handle or after any number of Recovers, returns ErrEventDataEncoding again +// without running the ServiceFn, and records nothing more. No Assign runs. +func TestServiceResult_UndecodableResult_RunsTheServiceOnce(t *testing.T) { + for _, tc := range []struct { + name string + opt durable.Option[string, string, svcCtx] + }{ + {"decoder fails", failingDecoder()}, + {"type mismatch", decoderFor("done", durable.DecodeAs[scnAdmin])}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + var runs int64 + fn := countingFetch(&runs) + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), tc.opt} + id := state.InvokeID(scnInvoke, "loading", 0) + h := scnStart(t, m, store, opts...) + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) + _, ran, err = h.RunService(ctx, id) + mustPark(t, "RunService again on the same Handle", ran, err) + if verr := h.View(ctx, func(*state.Instance[string, string, svcCtx]) {}); verr != nil { + t.Fatalf("View after the park: %v, want the Handle healthy", verr) + } + for i := 1; i <= 3; i++ { + rh, rerr := durable.Recover(ctx, m, store, "scn-1", opts...) + if rerr != nil { + t.Fatalf("Recover %d: %v", i, rerr) + } + _, ran, err = rh.RunService(ctx, id) + mustPark(t, "RunService after Recover", ran, err) + if got := rh.Instance().Current(); got != "loading" { + t.Fatalf("Recover %d: in %q, want loading", i, got) + } + } + if got := atomic.LoadInt64(&runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1", got) + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } + if _, recs, lerr := store.Load(ctx, "scn-1"); lerr != nil || len(recs) != 1 { + t.Fatalf("the journal holds %d Records (err %v), want the one that parked the result", len(recs), lerr) + } + }) + } +} + +// parkRun starts scn-1 in loading, with the ServiceFn counting its runs, and runs the +// service once under a decoder that refuses the result. +type parkRun struct { + m *state.Machine[string, string, svcCtx] + a *scnAssigns + store *durable.MemStore + runs *int64 + reg durable.Option[string, string, svcCtx] + err error // the first RunService's error +} + +func newParkRun(t *testing.T, extra ...durable.Option[string, string, svcCtx]) *parkRun { + t.Helper() + runs := new(int64) + fn := countingFetch(runs) + r := &parkRun{ + a: &scnAssigns{}, + store: durable.NewMemStore(durable.WithHistory()), + runs: runs, + reg: durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + r.m = scnMachine(fn, r.a) + h := scnStart(t, r.m, r.store, append([]durable.Option[string, string, svcCtx]{r.reg, failingDecoder()}, extra...)...) + _, _, r.err = h.RunService(context.Background(), state.InvokeID(scnInvoke, "loading", 0)) + return r +} + +// recoverAndRun Recovers with opts and, if that works, runs the service. It returns +// the Handle, the RunService error and the Recover error. +func (r *parkRun) recoverAndRun(opts ...durable.Option[string, string, svcCtx]) (*durable.Handle[string, string, svcCtx], error, error) { + ctx := context.Background() + h, err := durable.Recover(ctx, r.m, r.store, "scn-1", append([]durable.Option[string, string, svcCtx]{r.reg}, opts...)...) + if err != nil { + return nil, nil, err + } + _, _, runErr := h.RunService(ctx, state.InvokeID(scnInvoke, "loading", 0)) + return h, runErr, nil +} + +// The ServiceFn, and its side effect, runs once however often the instance is +// recovered and the service run while the decoder still refuses the result. +func TestParkedResult_ServiceRunsOnce(t *testing.T) { + r := newParkRun(t) + for i := 0; i < 3; i++ { + _, _, _ = r.recoverAndRun(failingDecoder()) + } + if got := atomic.LoadInt64(r.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1", got) + } +} + +// A decode failure runs no Assign, onDone's or onError's, live, on replay, or while +// the result is parked. +func TestParkedResult_NoAssignOnDecodeFailure(t *testing.T) { + r := newParkRun(t) + _, _, _ = r.recoverAndRun(failingDecoder()) + _, _, _ = r.recoverAndRun(decoderFor("done", durable.DecodeAs[scnAdmin])) + if r.a.done != 0 || r.a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", r.a.done, r.a.fails) + } +} + +// Once the decoder is fixed, a RunService settles the result the service returned the +// first time, through onDone, without running the ServiceFn again; and a later +// Recover replays that settle. +func TestParkedResult_FixedDecoderSettlesTheOriginalResult(t *testing.T) { + r := newParkRun(t) + fixed := decoderFor("done", durable.DecodeAs[scnUser]) + h, runErr, err := r.recoverAndRun(fixed) + if err != nil || runErr != nil { + t.Fatalf("Recover: %v, RunService: %v, with the fixed decoder", err, runErr) + } + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("the instance is in %q, want ready", got) + } + if got := atomic.LoadInt64(r.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1: the fix should settle the recorded result", got) + } + if r.a.done != 1 || describe(r.a.seen[0]) != describe(scnUser{ID: 7, Name: "ann"}) { + t.Fatalf("onDone's Assign ran %d times and saw %v, want once with the scnUser", r.a.done, r.a.seen) + } + rh, err := durable.Recover(context.Background(), r.m, r.store, "scn-1", r.reg, fixed) + if err != nil { + t.Fatalf("Recover after the settle: %v", err) + } + if got := rh.Instance().Current(); got != "ready" { + t.Fatalf("recovered in %q, want ready", got) + } + if r.a.done != 2 || describe(r.a.seen[1]) != describe(r.a.seen[0]) { + t.Fatalf("replay: onDone's Assign ran %d times in all, saw %v; want twice with the same value", r.a.done, r.a.seen) + } + if got := atomic.LoadInt64(r.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1", got) + } +} + +// A parked result settles only as the type the ServiceFn returned, as live: a decoder +// fixed to another type is refused with ErrEventDataEncoding, and no Assign runs. +// JSON's generic form is accepted, as live. +func TestParkedResult_SettlesOnlyAsTheReturnedType(t *testing.T) { + r := newParkRun(t) + _, runErr, err := r.recoverAndRun(decoderFor("done", durable.DecodeAs[scnAdmin])) + if err != nil || !errors.Is(runErr, durable.ErrEventDataEncoding) { + t.Fatalf("Recover: %v, RunService: %v; want ErrEventDataEncoding for the wrong type", err, runErr) + } + if r.a.done != 0 || r.a.fails != 0 { + t.Fatalf("an Assign ran: onDone %d, onError %d", r.a.done, r.a.fails) + } + h, runErr, err := r.recoverAndRun() // the default decoder: JSON's generic form + if err != nil || runErr != nil { + t.Fatalf("Recover: %v, RunService: %v, with the default decoder", err, runErr) + } + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("in %q, want ready", got) + } + if _, ok := r.a.seen[0].(map[string]any); !ok { + t.Fatalf("onDone's Assign saw %s, want JSON's generic form", describe(r.a.seen[0])) + } +} + +// The instance can still be recovered while the decoder refuses the result. +func TestParkedResult_RecoverableWhileBroken(t *testing.T) { + r := newParkRun(t) + h, err := durable.Recover(context.Background(), r.m, r.store, "scn-1", r.reg, failingDecoder()) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := h.Instance().Current(); got != "loading" { + t.Fatalf("recovered in %q, want loading", got) + } +} + +// The host is told: the RunService that met the failure returns ErrEventDataEncoding, +// and says the result is parked, not ErrHandleFailed. +func TestParkedResult_HostIsTold(t *testing.T) { + r := newParkRun(t) + mustPark(t, "RunService", true, r.err) + if !strings.Contains(r.err.Error(), "parked") { + t.Fatalf("RunService: %v, want it to say the result is parked", r.err) + } +} + +// The machine routing the failure itself, reaching its onError state with no host +// action, isn't what parking does: the owner chose to have the host fix the decoder. +func TestParkedResult_MachineRoutesTheFailure(t *testing.T) { + t.Skip("not this design: a parked result waits for the host to fix the decoder and call RunService; the machine doesn't route the failure through onError") +} + +// A checkpoint that compacts the Record that parked the result carries the parked +// result: the step that parks offers its checkpoint as any step does, and a Recover +// from it knows the service is parked, not in flight to run. The ServiceFn doesn't run +// again, and the fixed decoder gets the parked result. +func TestParkedResult_CheckpointKeepsIt(t *testing.T) { + ctx := context.Background() + r := newParkRun(t, durable.WithCheckpointEvery[string, string, svcCtx](1)) + mustPark(t, "RunService", true, r.err) + snap, recs, err := r.store.Load(ctx, "scn-1") + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(recs) != 0 { + t.Fatalf("the tail holds %d Records, want none: the park step should checkpoint", len(recs)) + } + if !strings.Contains(string(snap), `"parkedServices"`) { + t.Fatalf("the checkpoint carries no parked result: %s", snap) + } + // Still broken: the service is parked, not run. + if _, runErr, rerr := r.recoverAndRun(failingDecoder()); rerr != nil || !errors.Is(runErr, durable.ErrEventDataEncoding) { + t.Fatalf("Recover: %v, RunService: %v; want ErrEventDataEncoding", rerr, runErr) + } + if got := atomic.LoadInt64(r.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times after the checkpoint, want 1", got) + } + h, runErr, err := r.recoverAndRun(decoderFor("done", durable.DecodeAs[scnUser])) + if err != nil || runErr != nil { + t.Fatalf("Recover: %v, RunService: %v, with the fixed decoder", err, runErr) + } + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("in %q, want ready", got) + } + if got := atomic.LoadInt64(r.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1", got) + } + if r.a.done != 1 || describe(r.a.seen[0]) != describe(scnUser{ID: 7, Name: "ann"}) { + t.Fatalf("onDone's Assign ran %d times and saw %v, want the parked scnUser", r.a.done, r.a.seen) + } +} + +// StateAt replays a parked Record as the live run left it: nothing settled, so the +// view at the park step is the instance before it, whatever the decoder, and runs no +// Assign. A later settle replays as any recorded result. +func TestParkedResult_StateAt(t *testing.T) { + ctx := context.Background() + r := newParkRun(t) + fixed := decoderFor("done", durable.DecodeAs[scnUser]) + if _, runErr, err := r.recoverAndRun(fixed); err != nil || runErr != nil { + t.Fatalf("Recover: %v, RunService: %v", err, runErr) + } + steps, err := durable.Steps(ctx, r.store, "scn-1") + if err != nil || len(steps) != 2 { + t.Fatalf("Steps = %v, %v; want the park and the settle", steps, err) + } + doneBefore := r.a.done + for _, dec := range []durable.Option[string, string, svcCtx]{failingDecoder(), fixed} { + v, serr := durable.StateAt(ctx, r.m, r.store, "scn-1", steps[0], r.reg, dec) + if serr != nil { + t.Fatalf("StateAt the park: %v", serr) + } + if got := v.Instance().Current(); got != "loading" { + t.Fatalf("StateAt the park: in %q, want loading", got) + } + } + if r.a.done != doneBefore || r.a.fails != 0 { + t.Fatalf("StateAt the park ran an Assign: onDone %d (was %d), onError %d", r.a.done, doneBefore, r.a.fails) + } + v, err := durable.StateAt(ctx, r.m, r.store, "scn-1", steps[1], r.reg, fixed) + if err != nil { + t.Fatalf("StateAt the settle: %v", err) + } + if got := v.Instance().Current(); got != "ready" { + t.Fatalf("StateAt the settle: in %q, want ready", got) + } +} + +// cancelMachine invokes fn in "loading", which "cancel" leaves for idle and "retry" +// re-enters, so a parked service can be stopped and started again under the same +// invoke id. +func cancelMachine(fn state.ServiceFn[svcCtx], a *scnAssigns) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("park"). + Service("fetch", fn). + Reducer("noteDone", func(in state.AssignCtx[svcCtx]) svcCtx { + a.done++ + a.seen = append(a.seen, in.Event) + return in.Entity + }). + State("idle"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done")). + State("ready").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready").Assign("noteDone"). + Transition("loading").On("cancel").GoTo("idle"). + Transition("idle").On("retry").GoTo("loading"). + Quench() +} + +// countingUsers returns a scnUser whose ID counts the runs. +func countingUsers(runs *int64) state.ServiceFn[svcCtx] { + return func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + return scnUser{ID: atomic.AddInt64(runs, 1), Name: "ann"}, nil + } +} + +// Leaving the state that declares a parked service stops the service and discards the +// parked result, live and on replay, with or without a checkpoint: RunService finds +// nothing in flight, and re-entering the state starts a new activation whose ServiceFn +// runs, rather than settling the discarded result. +func TestParkedResult_LeavingTheStateDiscardsIt(t *testing.T) { + for _, every := range []int{0, 1} { + name := "no checkpoint" + if every > 0 { + name = "checkpoint every step" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + runs := new(int64) + fn := countingUsers(runs) + a := &scnAssigns{} + m := cancelMachine(fn, a) + store := durable.NewMemStore() + base := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))} + if every > 0 { + base = append(base, durable.WithCheckpointEvery[string, string, svcCtx](every)) + } + broken := append(append([]durable.Option[string, string, svcCtx]{}, base...), failingDecoder()) + fixed := append(append([]durable.Option[string, string, svcCtx]{}, base...), decoderFor("done", durable.DecodeAs[scnUser])) + id := state.InvokeID("park", "loading", 0) + + h, err := durable.NewRunner(m, store, broken...).Start(ctx, "p-1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) + if _, err = h.Fire(ctx, "cancel"); err != nil { + t.Fatalf("Fire cancel: %v", err) + } + if _, ran, err = h.RunService(ctx, id); ran || err != nil { + t.Fatalf("RunService after cancel = %v, %v; want nothing in flight", ran, err) + } + rh, err := durable.Recover(ctx, m, store, "p-1", fixed...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, ran, err = rh.RunService(ctx, id); ran || err != nil { + t.Fatalf("RunService after Recover = %v, %v; want nothing in flight", ran, err) + } + if got := rh.Instance().Current(); got != "idle" || a.done != 0 { + t.Fatalf("recovered in %q with onDone run %d times, want idle and none", got, a.done) + } + + // Re-entering starts a new activation: its ServiceFn runs, and the discarded + // result never settles. + if _, err = rh.Fire(ctx, "retry"); err != nil { + t.Fatalf("Fire retry: %v", err) + } + mustRunService(t, rh, id) + if got := atomic.LoadInt64(runs); got != 2 { + t.Fatalf("the ServiceFn ran %d times, want 2: once per activation", got) + } + if a.done != 1 || describe(a.seen[0]) != describe(scnUser{ID: 2, Name: "ann"}) { + t.Fatalf("onDone saw %v, want the second activation's scnUser", a.seen) + } + }) + } +} + +// A service parked, stopped and started again on the same Handle runs its ServiceFn +// for the new activation, rather than settle the first one's parked result, even +// before any Recover. +func TestParkedResult_ReenteredStateRunsTheServiceAgain(t *testing.T) { + ctx := context.Background() + runs := new(int64) + fn := countingUsers(runs) + a := &scnAssigns{} + m := cancelMachine(fn, a) + store := durable.NewMemStore() + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + id := state.InvokeID("park", "loading", 0) + h, err := durable.NewRunner(m, store, reg, failingDecoder()).Start(ctx, "p-1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) + for _, ev := range []string{"cancel", "retry"} { + if _, err = h.Fire(ctx, ev); err != nil { + t.Fatalf("Fire %s: %v", ev, err) + } + } + _, ran, err = h.RunService(ctx, id) + mustPark(t, "RunService of the new activation", ran, err) + if got := atomic.LoadInt64(runs); got != 2 { + t.Fatalf("the ServiceFn ran %d times, want 2: the new activation runs it", got) + } + // The fixed decoder settles the new activation's parked result, not the first's. + rh, err := durable.Recover(ctx, m, store, "p-1", reg, decoderFor("done", durable.DecodeAs[scnUser])) + if err != nil { + t.Fatalf("Recover: %v", err) + } + mustRunService(t, rh, id) + if a.done != 1 || describe(a.seen[0]) != describe(scnUser{ID: 2, Name: "ann"}) { + t.Fatalf("onDone saw %v, want the second activation's scnUser", a.seen) + } + if got := atomic.LoadInt64(runs); got != 2 { + t.Fatalf("the ServiceFn ran %d times, want 2", got) + } +} + +// A Recover whose journal parks a result, then stops and starts its service again, +// replays the new activation as unparked: RunService runs the ServiceFn for it rather +// than settle the discarded result. +func TestParkedResult_ReplayedReentryRunsTheServiceAgain(t *testing.T) { + ctx := context.Background() + runs := new(int64) + fn := countingUsers(runs) + a := &scnAssigns{} + m := cancelMachine(fn, a) + store := durable.NewMemStore() + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + id := state.InvokeID("park", "loading", 0) + h, err := durable.NewRunner(m, store, reg, failingDecoder()).Start(ctx, "p-1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) + for _, ev := range []string{"cancel", "retry"} { + if _, err = h.Fire(ctx, ev); err != nil { + t.Fatalf("Fire %s: %v", ev, err) + } + } + rh, err := durable.Recover(ctx, m, store, "p-1", reg, decoderFor("done", durable.DecodeAs[scnUser])) + if err != nil { + t.Fatalf("Recover: %v", err) + } + mustRunService(t, rh, id) + if got := atomic.LoadInt64(runs); got != 2 { + t.Fatalf("the ServiceFn ran %d times, want 2: the new activation runs it", got) + } + if a.done != 1 || describe(a.seen[0]) != describe(scnUser{ID: 2, Name: "ann"}) { + t.Fatalf("onDone saw %v, want the second activation's scnUser", a.seen) + } +} diff --git a/durable/result_decoding_scenarios_test.go b/durable/result_decoding_scenarios_test.go index 64d267f0..70937acd 100644 --- a/durable/result_decoding_scenarios_test.go +++ b/durable/result_decoding_scenarios_test.go @@ -13,8 +13,8 @@ import ( // These scenario tests pin what a host's Assign sees for a service result, live and // after Recover: a result whose onDone event WithEventDataDecoder covers arrives as the // decoder's type, any other arrives in JSON's generic form, an error is message-only, -// and a decoder that fails or returns the wrong type fails the step before any Assign -// runs. +// and a decoder that fails or returns the wrong type parks the result before any +// Assign runs. type scnUser struct { ID int64 @@ -227,12 +227,12 @@ func TestServiceResult_NoJSONForm_FailsTheHandle(t *testing.T) { } } -// mustFailLoudly fails unless RunService failed the Handle with ErrEventDataEncoding, -// no Assign ran, and Recover found the service in flight again. +// mustFailLoudly fails unless RunService returned ErrEventDataEncoding, parking the +// result with the Handle healthy, no Assign ran, and Recover found the service parked. func mustFailLoudly(t *testing.T, a *scnAssigns, runErr, recErr error) { t.Helper() - if !errors.Is(runErr, durable.ErrHandleFailed) || !errors.Is(runErr, durable.ErrEventDataEncoding) { - t.Errorf("RunService err = %v, want ErrHandleFailed wrapping ErrEventDataEncoding", runErr) + if errors.Is(runErr, durable.ErrHandleFailed) || !errors.Is(runErr, durable.ErrEventDataEncoding) { + t.Errorf("RunService err = %v, want ErrEventDataEncoding without ErrHandleFailed", runErr) } if recErr != nil { t.Errorf("Recover: %v, want the instance recoverable", recErr) @@ -304,9 +304,9 @@ func failingDecoder() durable.Option[string, string, svcCtx] { // TestServiceResult_DecodeFailure_RunsNoOnErrorAssign proves a result that fails to // decode, or decodes live as another type than the service returned, runs no Assign -// at all: neither onDone's nor onError's, not even once in memory before the Handle -// fails. It fails the Handle with ErrEventDataEncoding, records nothing and leaves the -// instance where it was. +// at all: neither onDone's nor onError's, not even once in memory. RunService returns +// ErrEventDataEncoding, the Handle stays healthy, the journal holds one Record that +// parks the result, and the instance stays where it was. func TestServiceResult_DecodeFailure_RunsNoOnErrorAssign(t *testing.T) { cases := []struct { name string @@ -325,17 +325,15 @@ func TestServiceResult_DecodeFailure_RunsNoOnErrorAssign(t *testing.T) { opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), tc.opt} h := scnStart(t, m, store, opts...) _, ran, err := h.RunService(ctx, state.InvokeID(scnInvoke, "loading", 0)) - if !ran || !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEventDataEncoding) { - t.Fatalf("RunService = %v, %v; want ran and ErrHandleFailed wrapping ErrEventDataEncoding", ran, err) - } + mustPark(t, "RunService", ran, err) if a.done != 0 || a.fails != 0 { t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) } if got := h.Instance().Current(); got != "loading" { t.Fatalf("the live instance is in %q, want loading", got) } - if _, recs, lerr := store.Load(ctx, "scn-1"); lerr != nil || len(recs) != 0 { - t.Fatalf("journal holds %d Records (err %v), want none", len(recs), lerr) + if _, recs, lerr := store.Load(ctx, "scn-1"); lerr != nil || len(recs) != 1 { + t.Fatalf("journal holds %d Records (err %v), want the one that parks the result", len(recs), lerr) } }) } diff --git a/durable/runner.go b/durable/runner.go index b5db888d..9cadbe12 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "slices" + "strings" "sync" "time" "unicode/utf8" @@ -140,6 +141,10 @@ type Handle[S comparable, E comparable, C any] struct { // settled holds the services that settled in the current activation of the state // that declares them, so a checkpoint records them and Recover doesn't start them. settled map[string]bool + // parked holds the results recorded parked, by service id: each didn't decode for + // its onDone event, so the service is still in flight, and RunService settles the + // result rather than run the ServiceFn again. A checkpoint records them. + parked map[string]recordedResult actors *state.ActorSystem[S, E, C] // host driver for child-machine actors, nil when none wired ended recordedSet // the ended actors the journal last recorded @@ -353,6 +358,7 @@ func (h *Handle[S, E, C]) drainClock() []state.JournalEntry { // kind. func (h *Handle[S, E, C]) absorbDrivers(ctx context.Context, effects []state.Effect) { forgetSettled(h.settled, effects) + forgetSettled(h.parked, effects) // Arm or cancel timers only when the effects actually schedule or cancel one: the // kernel Scheduler reads the clock unconditionally on Absorb, so skipping the call // for a timer-free transition keeps a non-timer machine free of clock reads. @@ -376,6 +382,7 @@ func (h *Handle[S, E, C]) absorbDrivers(ctx context.Context, effects []state.Eff // is the context of the step that drove the system. func (h *Handle[S, E, C]) absorbParentFire(ctx context.Context, effects []state.Effect) { forgetSettled(h.settled, effects) + forgetSettled(h.parked, effects) if hasTimerEffect(effects) { h.absorbTimers(ctx, effects) } @@ -443,8 +450,10 @@ func dropUnscheduled[S comparable, E comparable, C any](timers map[string]pendin } // forgetSettled removes from settled each service effects start or stop: a state that -// declares it was entered or exited, so the service is new or gone. -func forgetSettled(settled map[string]bool, effects []state.Effect) { +// declares it was entered or exited, so the service is new or gone. It forgets parked +// results the same way: a stopped service's parked result is discarded, and a service +// started again runs its ServiceFn. +func forgetSettled[V any](settled map[string]V, effects []state.Effect) { if len(settled) == 0 { return } @@ -947,11 +956,7 @@ func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.Fi // would acknowledge a step whose outcome isn't known, and the Handle couldn't fail. func (h *Handle[S, E, C]) recordStep(ctx context.Context, rec *Record, effects []state.Effect) ([]dispatchableEffect, error) { des := dispatchableEffects(rec.Step, effects) - effEnvs, err := recordEffects(des) - if err != nil { - return nil, h.fail(err) - } - rec.Effects = effEnvs + rec.Effects = recordEffects(des) if err := h.appendStep(ctx, rec.Step, rec); err != nil { return nil, h.fail(err) } @@ -1064,6 +1069,7 @@ func (h *Handle[S, E, C]) tickStep(ctx context.Context) ([]state.FireResult[S], } mirrorScheduled(h.timers, h.sched, results[i].Effects) forgetSettled(h.settled, results[i].Effects) + forgetSettled(h.parked, results[i].Effects) } if len(results) > 0 { // Drop from the table the timers the tick fired, the ones the scheduler no longer @@ -1180,6 +1186,7 @@ func (h *Handle[S, E, C]) appendStep(ctx context.Context, step int, rec *Record) EndedActors: ended, UnstartedActors: unstarted, EndedServices: h.endedServices(), + ParkedServices: h.parkedServices(), Format: journalFormat, }, h.timers) if err != nil { @@ -1320,6 +1327,26 @@ func (h *Handle[S, E, C]) endedServices() []string { return slices.Compact(ids) } +// parkedServices returns the parked results of the services the active states declare, +// sorted by service id, for a checkpoint. It is nil with none parked. +func (h *Handle[S, E, C]) parkedServices() []parkedServiceWire { + if h.svc == nil || len(h.parked) == 0 { + return nil + } + var out []parkedServiceWire + for _, eff := range h.inst.StartEffects() { + sv, ok := eff.(state.StartService) + if !ok { + continue + } + if p, ok := h.parked[sv.ID]; ok && !slices.ContainsFunc(out, func(w parkedServiceWire) bool { return w.ID == sv.ID }) { + out = append(out, parkedServiceWire{ID: sv.ID, Result: p.raw, Type: p.typ}) + } + } + slices.SortFunc(out, func(a, b parkedServiceWire) int { return strings.Compare(a.ID, b.ID) }) + return out +} + // declaredActors returns the ids of the actors effects spawn. func declaredActors(effects []state.Effect) []string { var ids []string @@ -1460,6 +1487,11 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S for _, sv := range endedServices { mirror.settle(sv) } + // A result parked at the checkpoint is still parked: its service is armed above, in + // flight, and RunService settles the result rather than run the ServiceFn again. + for _, p := range env.ParkedServices { + mirror.park(p.ID, recordedResult{raw: p.Result, typ: p.Type}) + } // ended is the latest recorded set of the declared actors the live run had ended: // the checkpoint's, then each tail Record's that records one. Nil means none was @@ -1584,6 +1616,7 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S svc: svc, svcBuf: &svcBuf, settled: mirror.settled, + parked: mirror.parked, actors: actors, actorFires: actorFires, lock: make(chan struct{}, 1), @@ -1911,7 +1944,8 @@ func resultErrs[S comparable](results []state.FireResult[S]) []error { // - A service settlement re-settles each recorded outcome through the same settle // seam, in settle order; the settle absorbs its own follow-on StartService // effects, so a chained invoke arms its successor for the next entry, and its -// timers are re-armed as RunService armed them. +// timers are re-armed as RunService armed them. A parked result settles nothing: +// its service stays in flight, and mirror keeps the result for RunService. // - A tick barrier ticks the replay clock, which returns the recorded readings, so // the same deadlines come due and the same timers fire at their recorded instants. // - A plain event re-fires it with its recorded event data, then re-arms its timers @@ -1954,10 +1988,20 @@ func replayRecord[S comparable, E comparable, C any]( stepEffects = res.Effects case hasServiceEntry(rec): for _, entry := range serviceEntries(rec) { + if p, ok := parkedEntry(entry); ok { + // Live, the result didn't decode, so nothing settled: park it again, for + // RunService to settle. Its service stays in flight. + if err := replayPark(svc, entry); err != nil { + return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) + } + mirror.park(entry.CorrelationID, p) + continue + } effects, err := replayService(ctx, svc, entry) if err != nil { return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) } + mirror.unpark(entry.CorrelationID) mirror.settle(entry.CorrelationID) // The settle armed its own services; arm its timers, as RunService did. replayArm(ctx, sched, nil, effects, mirror) @@ -2051,6 +2095,26 @@ type armMirror struct { armed map[string]pendingTimer clock *replayClock settled map[string]bool + parked map[string]recordedResult // the results parked and not yet settled, by service id +} + +// park notes that the replayed journal parked the result of service id. +func (m *armMirror) park(id string, r recordedResult) { + if m == nil { + return + } + if m.parked == nil { + m.parked = map[string]recordedResult{} + } + m.parked[id] = r +} + +// unpark notes that a replayed settle settled service id, so no result of it is parked. +func (m *armMirror) unpark(id string) { + if m == nil { + return + } + delete(m.parked, id) } // settle notes that a replayed settle settled the service id. @@ -2064,13 +2128,14 @@ func (m *armMirror) settle(id string) { m.settled[id] = true } -// forget drops from the settled services each one a replayed Fire's effects start or -// stop, as the live step did. +// forget drops from the settled services and the parked results each one a replayed +// Fire's effects start or stop, as the live step did. func (m *armMirror) forget(effects []state.Effect) { if m == nil { return } forgetSettled(m.settled, effects) + forgetSettled(m.parked, effects) } // arm folds a replayed absorb's timer effects into the table, at the reading the diff --git a/durable/runner_options.go b/durable/runner_options.go index 22e24f57..aa2e1d71 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -184,13 +184,15 @@ func WithMaxEventDataBytes[S comparable, E comparable, C any](n int) Option[S, E // in RunService and on every replay. A result whose onDone event the decoder covers // arrives as the decoder's type; otherwise it arrives in JSON's generic form, a struct // as a map[string]any and a number as a float64. A decoder that fails on a result, or -// returns a type other than the service returned, fails RunService with ErrHandleFailed -// wrapping ErrEventDataEncoding before any onDone or onError Assign runs, and records -// nothing. A service error stays message-only and doesn't reach the decoder. Register -// the decoder before results are recorded: replay decodes the journal with the -// recovering Runner's decoder, so one added or changed after results were recorded -// changes what replay produces, and one that can't decode a recorded result fails -// Recover and StateAt with ErrEventDataEncoding. +// returns a type other than the service returned, parks the result: RunService returns +// ErrEventDataEncoding before any onDone or onError Assign runs, records the result +// without settling it, and keeps the Handle healthy. Fix the decoder and call +// RunService again, on a recovered Handle, to settle the parked result without running +// the service again (see Handle.RunService). A service error stays message-only and +// doesn't reach the decoder. Register the decoder before results are recorded: replay +// decodes the journal with the recovering Runner's decoder, so one added or changed +// after results were recorded changes what replay produces, and one that can't decode +// a settled result fails Recover and StateAt with ErrEventDataEncoding. func WithEventDataDecoder[S comparable, E comparable, C any](d EventDataDecoder[E]) Option[S, E, C] { return func(c *runnerConfig[S, E, C]) { if d != nil { diff --git a/durable/service.go b/durable/service.go index 96fcc144..d6e41af7 100644 --- a/durable/service.go +++ b/durable/service.go @@ -25,7 +25,10 @@ import ( // replay against the kernel rather than a re-execution of the host's services. A result // that wasn't recorded, after a context canceled during the service, a crash, or a failed // Append whose write didn't land, has nothing to replay, so it stays in flight, runs again -// when the host calls RunService, and must tolerate a second run. +// when the host calls RunService, and must tolerate a second run. A result that won't +// decode for its onDone event is recorded parked: nothing settles, replay parks it +// again, and a later RunService settles it, once the decoder decodes it, without +// running the service again. // // The Runner wraps the kernel's reusable host driver, state.ServiceRunner, which // turns StartService / StopService effects into real ServiceFn executions and @@ -63,6 +66,24 @@ type serviceResultPayload struct { // into an error and re-fired through onError so the transition's Assign reads it // from AssignCtx.Event. Error *string `json:"error,omitempty"` + // Parked marks a result that didn't decode for its onDone event: it is recorded so + // the ServiceFn never runs again, but it didn't settle. Replay parks it again rather + // than settling it, and a later RunService settles it once the decoder decodes it. + // An older Record never sets it. + Parked bool `json:"parked,omitempty"` + // Type is a parked result's Go type, as reflect names it, so the RunService that + // settles it holds the decoder to the type the ServiceFn returned, as the live + // settle did. + Type string `json:"type,omitempty"` +} + +// parkedEntry returns the result a parked entry recorded, or false for any other entry. +func parkedEntry(e state.JournalEntry) (recordedResult, bool) { + var sp serviceResultPayload + if json.Unmarshal(e.Payload, &sp) != nil || !sp.Parked { + return recordedResult{}, false + } + return recordedResult{raw: sp.Result, typ: sp.Type}, true } // drainServices returns and clears the service outcomes recorded since the last @@ -99,31 +120,51 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // // Once the service has run, it returns true with any error. A result the decoder for // its onDone event can't decode, such as a number beyond float64, or decodes as a type -// other than the service returned, returns ErrHandleFailed wrapping -// ErrEventDataEncoding before any onDone or onError Assign runs. That, a result or -// KindedEffect that won't encode, a ctx that ended during the service or while its +// other than the service returned, is parked: RunService records the result as the +// ServiceFn returned it, with its Go type, settles nothing, and returns +// ErrEventDataEncoding, not ErrHandleFailed, before any onDone or onError Assign runs. +// The Handle stays healthy and the service stays in flight, on this Handle and on a +// recovered one, but its ServiceFn never runs again: fix the decoder and call +// RunService again, and it settles the parked result. Until then each RunService +// returns ErrEventDataEncoding and records nothing more. A Fire or timer that leaves +// the state that declares the service stops it and discards the parked result. A +// checkpoint keeps it, and StateAt shows the instance as it was before the park. +// +// A result that won't encode, a ctx that ended during the service or while its // settle's effects were absorbed (even after a clean settle), or a Fire the settle // drove refused because a context ended (ctx, or one middleware derived from it) -// returns ErrHandleFailed and records nothing: a recovered Handle has the service in flight again, and RunService -// runs it a second time. +// returns ErrHandleFailed and records nothing: a recovered Handle has the service in +// flight again, and RunService runs it a second time. A KindedEffect the settle emits +// whose payload won't marshal doesn't fail it: it is recorded by id and kind, as +// recordEffects says, and dispatches as any effect does. // A failed Append also returns ErrHandleFailed, but the write may have landed anyway: // Recover finds out, and the service runs again only if the result isn't in the // journal. If another goroutine failed the Handle while this step was recording, it // returns the result, true and ErrHandleFailed saying the step was recorded. Every // ErrHandleFailed also matches its cause, so test for ErrHandleFailed first, then // ErrFenced, as the Handle doc says. A failed Checkpoint returns the plain error, and -// the result is recorded. A failed effect dispatch also returns ErrHandleFailed, but the +// the result is recorded; on a step that parked its result, joined with the +// ErrEventDataEncoding. A failed effect dispatch also returns ErrHandleFailed, but the // result is recorded: Recover retries the effect and the service doesn't run again. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { if err := h.acquire(ctx); err != nil { return state.FireResult[S]{}, false, err } res, ran, rec, des, err := h.lockedServiceStep(ctx, id) - if err != nil || rec == nil { + if rec == nil { return res, ran, err } - // Dispatch after releasing the step lock, as Fire does. - return res, true, h.finishStep(ctx, rec, des) + // Dispatch after releasing the step lock, as Fire does. A step that parked its + // result is recorded and returns why it parked; it offers its checkpoint as any + // step does. + ferr := h.finishStep(ctx, rec, des) + if err == nil { + return res, true, ferr + } + if ferr != nil { + return res, true, errors.Join(err, ferr) + } + return res, true, err } // lockedServiceStep runs serviceStep and releases the step lock however it returns. A @@ -148,21 +189,29 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return state.FireResult[S]{}, false, nil, nil, nil } - results := h.svc.Tick(h.stepContext(ctx), id) + // A parked result settles in place of a run: the ServiceFn ran once, and the journal + // holds what it returned. + parked, isParked := h.parked[id] + var results []state.FireResult[S] + if isParked { + res, _ := h.svc.SettleDone(h.stepContext(ctx), id, parked) + results = []state.FireResult[S]{res} + } else { + results = h.svc.Tick(h.stepContext(ctx), id) + } if len(results) == 0 { return state.FireResult[S]{}, false, nil, nil, nil } res := results[0] if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { // The result didn't decode for its onDone event, so the transform aborted the - // settle: nothing fired, no Assign ran, and nothing is recorded. The service is - // still in flight, but the Handle has failed, so it can't run it again; Recover - // re-arms it. - return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, derr)) + // settle: nothing fired and no Assign ran. Park it. + return h.parkResult(ctx, id, derr, isParked) } // The service settled: unless the settle's own Fire leaves or re-enters the state // that declares it, which the absorb below notes, a checkpoint records it as settled. h.noteSettled(id) + delete(h.parked, id) // The settle re-fired the invocation's onDone / onError, which may enter a state // that arms a timer, starts another service, or spawns an actor. Absorb those @@ -206,6 +255,42 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return res, true, rec, des, nil } +// parkResult parks the result of service id that derr says didn't decode for its +// onDone event: it records the result's JSON and Go type in a Record of its own, so the +// ServiceFn never runs again, and keeps it for the next RunService of id to settle. The +// service stays in flight in the ServiceRunner and the Handle stays healthy. It returns +// the Record, which the caller finishes as any step's, and an error wrapping +// ErrEventDataEncoding that says the result is parked. A result already parked +// (already) is in the journal, so it records nothing. A result with no JSON to record +// fails the Handle, which leaves the service in flight for a recovered Handle to run +// again. +func (h *Handle[S, E, C]) parkResult(ctx context.Context, id string, derr *undecodableResult, already bool) (state.FireResult[S], bool, *Record, []dispatchableEffect, error) { + parkErr := fmt.Errorf("durable: settling service %q for %q: its result is parked until RunService can decode it: %w", id, h.id, derr) + if already { + return state.FireResult[S]{}, true, nil, nil, parkErr + } + if derr.raw == nil { + return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, derr)) + } + payload, err := json.Marshal(serviceResultPayload{Result: derr.raw, Parked: true, Type: derr.typ}) + if err != nil { + return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) + } + // Keep it before the Record is appended, so a checkpoint the step makes due carries + // it. A failed append fails the Handle, so a parked result it didn't record is never + // settled. + if h.parked == nil { + h.parked = map[string]recordedResult{} + } + h.parked[id] = recordedResult{raw: derr.raw, typ: derr.typ} + entry := state.JournalEntry{Kind: state.JournalServiceResult, CorrelationID: id, Payload: payload} + rec := &Record{Step: h.nextStep, Entries: append(h.drainClock(), entry)} + if _, err := h.recordStep(ctx, rec, nil); err != nil { + return state.FireResult[S]{}, true, nil, nil, err + } + return state.FireResult[S]{}, true, rec, nil, parkErr +} + // noteSettled records that the service id settled in its state's current activation. func (h *Handle[S, E, C]) noteSettled(id string) { if h.settled == nil { @@ -225,8 +310,10 @@ func (h *Handle[S, E, C]) noteSettled(id string) { // // A result that won't decode, or is refused, aborts the settle with an // *undecodableResult before it fires anything, so no onDone or onError Assign runs; -// serviceStep and replayService see it in the settle's FireResult.Err and fail the -// step. A result with no JSON form is left as it is, and fails the Handle when +// serviceStep sees it in the settle's FireResult.Err and parks the result, and +// replayService fails the replay. A parked result settles as a recordedResult that +// carries its Go type, and the decoder must return that type, or JSON's generic form, +// as live. A result with no JSON form is left as it is, and fails the Handle when // recordServiceOutcome can't encode it. // // An error becomes an error with its message as a Store keeps it (see @@ -242,36 +329,67 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, er return nil, errors.New(msg), nil } recorded, replay := result.(recordedResult) - b := []byte(recorded) + b, typ := []byte(recorded.raw), recorded.typ if !replay { var jerr error if b, jerr = marshalPayload(result); jerr != nil { return result, nil, nil } + typ = typeName(result) } value, derr := decodePayload(eventPayload{data: b}, event, decode) if derr != nil { - return nil, nil, &undecodableResult{err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} + return nil, nil, &undecodableResult{raw: b, typ: typ, err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} } if !replay { if terr := decodedAsPassed(value, result, "service result", true); terr != nil { - return nil, nil, &undecodableResult{err: terr} + return nil, nil, &undecodableResult{raw: b, typ: typ, err: terr} } + } else if terr := decodedAsRecorded(value, recorded.typ); terr != nil { + return nil, nil, &undecodableResult{raw: b, typ: typ, err: terr} } return value, nil, nil } } // recordedResult is a journaled service result's JSON, which replayService settles so -// the transform decodes it as the live run decoded the ServiceFn's value. -type recordedResult json.RawMessage +// the transform decodes it as the live run decoded the ServiceFn's value, and which a +// RunService settles for a parked result. +type recordedResult struct { + raw json.RawMessage + // typ is a parked result's Go type, as typeName gives it; empty for a settled one, + // whose JSON is already the decoded value's. + typ string +} + +// typeName returns the name reflect gives v's type, the form a parked result records. +func typeName(v any) string { return fmt.Sprint(reflect.TypeOf(v)) } + +// decodedAsRecorded is decodedAsPassed for a parked result, which records its type's +// name rather than a value: it reports, as an ErrEventDataEncoding error, a value the +// decoder returned as another type than typ. As live, nil and JSON's generic form are +// always accepted. An empty typ, a settled result's, accepts anything. +func decodedAsRecorded(value any, typ string) error { + if typ == "" || value == nil || genericJSON(value) { + return nil + } + if got := typeName(value); got != typ { + return fmt.Errorf("%w: a %s service result decodes as %s; decode it as %s with WithEventDataDecoder", + ErrEventDataEncoding, typ, got, typ) + } + return nil +} // undecodableResult is the abort replayableOutcome returns for a result that won't -// decode for its onDone event, or decodes live as another type than the ServiceFn -// returned. The ServiceRunner fires no event for it and hands it back in the settle's -// FireResult.Err, where serviceStep and replayService find it and fail the step. err -// wraps ErrEventDataEncoding. -type undecodableResult struct{ err error } +// decode for its onDone event, or decodes as another type than the ServiceFn returned. +// The ServiceRunner fires no event for it and hands it back in the settle's +// FireResult.Err, where serviceStep finds it and parks the result, and replayService +// finds it and fails the replay. err wraps ErrEventDataEncoding. +type undecodableResult struct { + err error + raw json.RawMessage // the result's JSON, which a park records + typ string // the result's Go type, as typeName gives it +} func (u *undecodableResult) Error() string { return u.err.Error() } func (u *undecodableResult) Unwrap() error { return u.err } @@ -356,7 +474,7 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * // that won't decode aborts the settle before it fires anything. var result any if len(sp.Result) > 0 { - result = recordedResult(sp.Result) + result = recordedResult{raw: sp.Result} } res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { @@ -368,6 +486,19 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * return res.Effects, nil } +// replayPark checks a recorded parked result against the replay: its service must be in +// flight, as it was when the live run parked it, and stays in flight. It settles +// nothing, so no Assign runs. It reports an error for a corrupt or mis-ordered journal. +func replayPark[S comparable, E comparable, C any](svc *state.ServiceRunner[S, E, C], entry state.JournalEntry) error { + if svc == nil { + return errors.New("durable: replaying a parked service result requires a service registry (supply WithServiceRegistry)") + } + if !svc.HasPending(entry.CorrelationID) { + return fmt.Errorf("durable: replaying parked service %q: no in-flight service", entry.CorrelationID) + } + return nil +} + // serviceEntries returns the recorded service outcomes carried by a single // Record, in recorded (settle) order, so replay re-settles them in the order the // live run produced them. diff --git a/durable/service_encode_internal_test.go b/durable/service_encode_internal_test.go index f794cfce..86dbce1c 100644 --- a/durable/service_encode_internal_test.go +++ b/durable/service_encode_internal_test.go @@ -66,7 +66,7 @@ func TestServiceResult_JournalBytesAreTheDecodedValuesEncoding(t *testing.T) { } // Replay decodes either encoding to the value live saw. for _, b := range [][]byte{first, sp.Result} { - replayed, _, rabort := replayableOutcome(tc.decode)("done", recordedResult(b), nil) + replayed, _, rabort := replayableOutcome(tc.decode)("done", recordedResult{raw: b}, nil) if rabort != nil { t.Fatalf("replaying %s: %v", b, rabort) } diff --git a/durable/service_outcome_test.go b/durable/service_outcome_test.go index 72d5054b..3cc8a6b9 100644 --- a/durable/service_outcome_test.go +++ b/durable/service_outcome_test.go @@ -3,7 +3,6 @@ package durable_test import ( "context" "encoding/json" - "errors" "strings" "testing" @@ -35,13 +34,18 @@ func outcomeMachine(fn state.ServiceFn[svcCtx], seen *[]string) *state.Machine[s Quench() } -// TestRunService_ResultThatEncodesButWontDecode_FailsTheStep proves a result whose -// JSON form a replay can't decode, such as a number beyond float64, fails RunService -// with ErrHandleFailed and records nothing, before any onDone or onError Assign runs, -// so the instance stays recoverable with the service in flight again. -func TestRunService_ResultThatEncodesButWontDecode_FailsTheStep(t *testing.T) { +// TestRunService_ResultThatEncodesButWontDecode_ParksIt proves a result whose JSON +// form won't decode, such as a number beyond float64, makes RunService return +// ErrEventDataEncoding before any onDone or onError Assign runs, and parks the result: +// the Handle stays healthy, and the recovered instance has the service parked, so +// RunService doesn't run it again. +func TestRunService_ResultThatEncodesButWontDecode_ParksIt(t *testing.T) { ctx := context.Background() - fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return json.RawMessage("1e400"), nil } + var runs int + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + runs++ + return json.RawMessage("1e400"), nil + } var seen []string m := outcomeMachine(fn, &seen) st := durable.NewMemStore() @@ -51,21 +55,25 @@ func TestRunService_ResultThatEncodesButWontDecode_FailsTheStep(t *testing.T) { t.Fatalf("Start: %v", err) } id := state.InvokeID("outcome", "loading", 0) - if _, _, err = h.RunService(ctx, id); !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEventDataEncoding) { - t.Fatalf("RunService: %v, want ErrHandleFailed wrapping ErrEventDataEncoding", err) - } + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) if len(seen) != 0 { t.Fatalf("an Assign saw %q, want none to run for a result that won't decode", seen) } - if _, recs, lerr := st.Load(ctx, "i1"); lerr != nil || len(recs) != 0 { - t.Fatalf("journal holds %d Records (err %v), want none", len(recs), lerr) + if _, recs, lerr := st.Load(ctx, "i1"); lerr != nil || len(recs) != 1 { + t.Fatalf("journal holds %d Records (err %v), want the one that parks the result", len(recs), lerr) } recovered, err := durable.Recover(ctx, m, st, "i1", opts...) if err != nil { t.Fatalf("Recover: %v, want the instance recoverable", err) } if got := recovered.Instance().Current(); got != "loading" { - t.Fatalf("recovered in %q, want loading with the service in flight again", got) + t.Fatalf("recovered in %q, want loading with the service parked", got) + } + _, ran, err = recovered.RunService(ctx, id) + mustPark(t, "RunService after Recover", ran, err) + if runs != 1 || len(seen) != 0 { + t.Fatalf("the ServiceFn ran %d times and Assigns saw %q; want 1 and none", runs, seen) } } diff --git a/durable/store.go b/durable/store.go index 44885977..e647cf34 100644 --- a/durable/store.go +++ b/durable/store.go @@ -59,7 +59,11 @@ type Record struct { // Effects are the effects the step emitted, each with its deterministically // stamped EffectID, persisted before dispatch so a crash between persist and - // dispatch is recoverable: Recover re-dispatches, deduped by EffectID. + // dispatch is recoverable: Recover re-dispatches, deduped by EffectID. A + // KindedEffect whose payload won't marshal is recorded by EffectID and Kind with no + // Payload, as an effect that isn't a KindedEffect is, and its Meta holds the + // marshal error's message under "crucible.marshalError". Replay re-derives every + // effect's value, so none is rebuilt from its envelope. Effects []state.EffectEnvelope // Snapshot, when non-empty, is a full marshaled state.Snapshot captured at diff --git a/durable/undecodable_result_loop_test.go b/durable/undecodable_result_loop_test.go deleted file mode 100644 index 0046218f..00000000 --- a/durable/undecodable_result_loop_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package durable_test - -import ( - "context" - "errors" - "sync/atomic" - "testing" - - "github.com/stablekernel/crucible/durable" - "github.com/stablekernel/crucible/state" -) - -// countingFetch returns a scnUser and counts its runs, standing in for a ServiceFn -// with a side effect. -func countingFetch(runs *int64) state.ServiceFn[svcCtx] { - return func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { - atomic.AddInt64(runs, 1) - return scnUser{ID: 7, Name: "ann"}, nil - } -} - -// TestServiceResult_UndecodableResult_RerunsTheServiceOnEveryRecover pins the current -// behavior for a result the decoder for its onDone event refuses: the step fails, -// nothing is recorded and the service stays in flight, so every Recover re-arms it and -// every RunService runs the ServiceFn again, repeating its side effect, and fails -// again, until the decoder is fixed. No Assign runs. This is a known loop the review -// raised (an undecodable service result loops forever); the options for ending it are -// compared on the exp/f2-* branches, and this test changes when one is chosen. -func TestServiceResult_UndecodableResult_RerunsTheServiceOnEveryRecover(t *testing.T) { - for _, tc := range []struct { - name string - opt durable.Option[string, string, svcCtx] - }{ - {"decoder fails", failingDecoder()}, - {"type mismatch", decoderFor("done", durable.DecodeAs[scnAdmin])}, - } { - t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - var runs int64 - fn := countingFetch(&runs) - a := &scnAssigns{} - m := scnMachine(fn, a) - store := durable.NewMemStore() - opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), tc.opt} - id := state.InvokeID(scnInvoke, "loading", 0) - h := scnStart(t, m, store, opts...) - if _, ran, err := h.RunService(ctx, id); !ran || !errors.Is(err, durable.ErrEventDataEncoding) { - t.Fatalf("RunService = %v, %v; want ran and ErrEventDataEncoding", ran, err) - } - for i := 1; i <= 3; i++ { - rh, err := durable.Recover(ctx, m, store, "scn-1", opts...) - if err != nil { - t.Fatalf("Recover %d: %v", i, err) - } - if _, ran, err := rh.RunService(ctx, id); !ran || !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEventDataEncoding) { - t.Fatalf("RunService after Recover %d = %v, %v; want ran and ErrHandleFailed wrapping ErrEventDataEncoding", i, ran, err) - } - } - if got := atomic.LoadInt64(&runs); got != 4 { - t.Fatalf("the ServiceFn ran %d times, want 4: once live and once after each of 3 Recovers", got) - } - if a.done != 0 || a.fails != 0 { - t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) - } - if _, recs, err := store.Load(ctx, "scn-1"); err != nil || len(recs) != 0 { - t.Fatalf("the journal holds %d Records (err %v), want none", len(recs), err) - } - }) - } -} diff --git a/durable/unmarshalable_effect_test.go b/durable/unmarshalable_effect_test.go index b05f473e..f3823c3c 100644 --- a/durable/unmarshalable_effect_test.go +++ b/durable/unmarshalable_effect_test.go @@ -3,6 +3,8 @@ package durable_test import ( "context" "errors" + "fmt" + "strings" "sync/atomic" "testing" "time" @@ -88,47 +90,76 @@ func (b *badRun) recover(t *testing.T) *durable.Handle[string, string, svcCtx] { return h } -func mustHandleFailed(t *testing.T, what string, err error) { +// metaMarshalError is the EffectEnvelope.Meta key that holds why an effect's payload is +// missing. +const metaMarshalError = "crucible.marshalError" + +// mustRecordBadByIDAndKind fails unless the journal's Record at step names the +// badEffect it emitted by id and kind, with no payload and its marshal error in Meta. +func mustRecordBadByIDAndKind(t *testing.T, st *durable.MemStore, id durable.InstanceID, step int) { t.Helper() - if !errors.Is(err, durable.ErrHandleFailed) { - t.Fatalf("%s: %v, want ErrHandleFailed", what, err) + _, recs, err := st.Load(context.Background(), id) + if err != nil { + t.Fatalf("Load: %v", err) + } + for _, rec := range recs { + if rec.Step != step { + continue + } + if len(rec.Effects) != 1 { + t.Fatalf("step %d records %d effects, want the badEffect", step, len(rec.Effects)) + } + env := rec.Effects[0] + wantID := fmt.Sprintf("%d#0#demo.bad", step) + if env.Kind != "demo.bad" || env.EffectID != wantID || len(env.Payload) != 0 { + t.Fatalf("step %d records %+v, want kind demo.bad, id %s and no payload", step, env, wantID) + } + if msg, _ := env.Meta[metaMarshalError].(string); !strings.Contains(msg, "unsupported type") { + t.Fatalf("step %d Meta = %v, want the marshal error under %q", step, env.Meta, metaMarshalError) + } + return } + t.Fatalf("no Record at step %d", step) } -// TestUnmarshalableEffect_NoHandler_FailsEveryStep pins the current behavior for a -// KindedEffect with no JSON form when no WithEffectHandler is wired: every step records -// its effects, so the step fails the Handle and records nothing. A Fire failed this way -// before this change; a Tick and a RunService didn't, since they recorded no effects. -// Now they fail and loop: Recover re-arms the timer, and re-arms the service, whose -// ServiceFn runs again. The review raised this (an unmarshalable KindedEffect with no -// handler); the options are compared on the exp/f3-* branches, and this test changes -// when one is chosen. -func TestUnmarshalableEffect_NoHandler_FailsEveryStep(t *testing.T) { +// TestUnmarshalableEffect_NoHandler_RecordsByIDAndKind: with no WithEffectHandler wired, +// a step that emits a KindedEffect with no JSON form records it by id and kind, as an +// effect that isn't a KindedEffect is, with its marshal error in Meta, and succeeds, +// whatever the step: Fire, a Tick's timer, a RunService's settle, or the stateless +// Runner.Fire. On main a Fire failed this way; a Tick and a RunService didn't, since +// they recorded no effects. The ServiceFn runs once. +func TestUnmarshalableEffect_NoHandler_RecordsByIDAndKind(t *testing.T) { ctx := context.Background() t.Run("Fire", func(t *testing.T) { b := newBadRun() - _, err := b.start(t).Fire(ctx, "go") - mustHandleFailed(t, "Fire", err) - if got := b.recover(t).Instance().Current(); got != "idle" { - t.Fatalf("recovered in %q, want idle: the Fire wasn't recorded", got) + if _, err := b.start(t).Fire(ctx, "go"); err != nil { + t.Fatalf("Fire: %v", err) + } + mustRecordBadByIDAndKind(t, b.store, "bad-1", 0) + if got := b.recover(t).Instance().Current(); got != "done" { + t.Fatalf("recovered in %q, want done", got) } }) + t.Run("stateless Fire", func(t *testing.T) { + b := newBadRun() + b.start(t) + if _, err := durable.NewRunner(b.m, b.store, b.opts...).Fire(ctx, "bad-1", "go"); err != nil { + t.Fatalf("Runner.Fire: %v", err) + } + mustRecordBadByIDAndKind(t, b.store, "bad-1", 0) + }) t.Run("Tick", func(t *testing.T) { b := newBadRun() if _, err := b.start(t).Fire(ctx, "wait"); err != nil { t.Fatalf("Fire wait: %v", err) } b.clk.Advance(2 * time.Second) - for i := 0; i < 3; i++ { - rh := b.recover(t) - _, err := rh.Tick(ctx) - mustHandleFailed(t, "Tick", err) - if got := rh.Instance().Current(); got != "done" { - t.Fatalf("Tick %d left the live instance in %q, want done (applied, then failed)", i, got) - } + if _, err := b.recover(t).Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) } - if got := b.recover(t).Instance().Current(); got != "waiting" { - t.Fatalf("recovered in %q, want waiting: no Tick was recorded", got) + mustRecordBadByIDAndKind(t, b.store, "bad-1", 1) + if got := b.recover(t).Instance().Current(); got != "done" { + t.Fatalf("recovered in %q, want done", got) } }) t.Run("RunService", func(t *testing.T) { @@ -136,19 +167,123 @@ func TestUnmarshalableEffect_NoHandler_FailsEveryStep(t *testing.T) { if _, err := b.start(t).Fire(ctx, "load"); err != nil { t.Fatalf("Fire load: %v", err) } - id := state.InvokeID("bad", "loading", 0) - for i := 0; i < 3; i++ { - _, ran, err := b.recover(t).RunService(ctx, id) - if !ran { - t.Fatalf("RunService %d didn't find the service in flight", i) - } - mustHandleFailed(t, "RunService", err) + if _, ran, err := b.recover(t).RunService(ctx, state.InvokeID("bad", "loading", 0)); !ran || err != nil { + t.Fatalf("RunService = %v, %v; want ran", ran, err) } - if got := atomic.LoadInt64(b.runs); got != 3 { - t.Fatalf("the ServiceFn ran %d times, want 3: once per Recover", got) + mustRecordBadByIDAndKind(t, b.store, "bad-1", 1) + if got := b.recover(t).Instance().Current(); got != "done" { + t.Fatalf("recovered in %q, want done", got) } - if got := b.recover(t).Instance().Current(); got != "loading" { - t.Fatalf("recovered in %q, want loading: no settle was recorded", got) + if got := atomic.LoadInt64(b.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1", got) } }) } + +// badSteps drives b's instance to emit a badEffect through each kind of step, and +// names the effect id each records: Fire, a Tick's timer, a RunService's settle, and +// the stateless Runner.Fire. +var badSteps = []struct { + name string + id string + run func(t *testing.T, b *badRun) error +}{ + {"Fire", "0#0#demo.bad", func(t *testing.T, b *badRun) error { + _, err := b.start(t).Fire(context.Background(), "go") + return err + }}, + {"stateless Fire", "0#0#demo.bad", func(t *testing.T, b *badRun) error { + b.start(t) + _, err := durable.NewRunner(b.m, b.store, b.opts...).Fire(context.Background(), "bad-1", "go") + return err + }}, + {"Tick", "1#0#demo.bad", func(t *testing.T, b *badRun) error { + h := b.start(t) + if _, err := h.Fire(context.Background(), "wait"); err != nil { + t.Fatalf("Fire wait: %v", err) + } + b.clk.Advance(2 * time.Second) + _, err := h.Tick(context.Background()) + return err + }}, + {"RunService", "1#0#demo.bad", func(t *testing.T, b *badRun) error { + h := b.start(t) + if _, err := h.Fire(context.Background(), "load"); err != nil { + t.Fatalf("Fire load: %v", err) + } + _, _, err := h.RunService(context.Background(), state.InvokeID("bad", "loading", 0)) + return err + }}, +} + +// TestUnmarshalableEffect_HandlerAppliesItLiveOnce: with a handler wired, the effect +// reaches it once, live, as the value the action emitted, on every kind of step. +func TestUnmarshalableEffect_HandlerAppliesItLiveOnce(t *testing.T) { + for _, step := range badSteps { + t.Run(step.name, func(t *testing.T) { + b := newBadRun() + rec := newEffectRecorder() + b.opts = append(b.opts, durable.WithEffectHandler[string, string, svcCtx](rec.handler())) + if err := step.run(t, b); err != nil { + t.Fatalf("step: %v", err) + } + b.recover(t) + if n := rec.count(step.id); n != 1 || rec.totalApplies() != 1 { + t.Fatalf("the handler applied %s %d times (%d in all), want once", step.id, n, rec.totalApplies()) + } + if _, ok := rec.values[0].(badEffect); !ok { + t.Fatalf("the handler got %T, want the badEffect", rec.values[0]) + } + }) + } +} + +// TestUnmarshalableEffect_RecoverRetriesItOnce: with a handler wired, an effect whose +// dispatch failed live is applied once by Recover, however often it recovers. +func TestUnmarshalableEffect_RecoverRetriesItOnce(t *testing.T) { + for _, step := range badSteps { + t.Run(step.name, func(t *testing.T) { + b := newBadRun() + live := newEffectRecorder() + live.failKinds["demo.bad"] = errors.New("downstream unavailable") + opts := b.opts + b.opts = append(opts, durable.WithEffectHandler[string, string, svcCtx](live.handler())) + if err := step.run(t, b); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("step: %v, want ErrEffectDispatch", err) + } + healthy := newEffectRecorder() + b.opts = append(opts, durable.WithEffectHandler[string, string, svcCtx](healthy.handler())) + b.recover(t) + b.recover(t) + if n := healthy.count(step.id); n != 1 || healthy.totalApplies() != 1 { + t.Fatalf("Recover applied %s %d times (%d in all), want once", step.id, n, healthy.totalApplies()) + } + if _, ok := healthy.values[0].(badEffect); !ok { + t.Fatalf("Recover's handler got %T, want the badEffect", healthy.values[0]) + } + if got := atomic.LoadInt64(b.runs); got > 1 { + t.Fatalf("the ServiceFn ran %d times, want at most 1", got) + } + }) + } +} + +// TestUnmarshalableEffect_HandlerWiredOnlyAtRecoverSeesIt: a handler wired only at +// Recover applies the effect a step recorded with no handler wired, once. +func TestUnmarshalableEffect_HandlerWiredOnlyAtRecoverSeesIt(t *testing.T) { + for _, step := range badSteps { + t.Run(step.name, func(t *testing.T) { + b := newBadRun() + if err := step.run(t, b); err != nil { + t.Fatalf("step: %v", err) + } + rec := newEffectRecorder() + b.opts = append(b.opts, durable.WithEffectHandler[string, string, svcCtx](rec.handler())) + b.recover(t) + b.recover(t) + if n := rec.count(step.id); n != 1 || rec.totalApplies() != 1 { + t.Fatalf("the Recover-only handler applied %s %d times (%d in all), want once", step.id, n, rec.totalApplies()) + } + }) + } +} From 5d06dec11c3a8403f90d7ffb5ca7bec604e70a23 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 16:29:05 -0700 Subject: [PATCH 07/13] fix(durable): review follow-ups for parked results, journal bytes and format Each finding gets a test that fails if it's real, committed whether it proved or disproved the finding. - durable: a ctx that ended during the ServiceFn parks nothing. A result the decoder refuses fails the Handle and records nothing, as a result that decodes does, and the service runs again after Recover. - durable: the journal records the JSON the live settle decoded, not the decoded value encoded again, so replay decodes the same bytes even for a decoder type whose MarshalJSON and UnmarshalJSON aren't inverses, or whose decoded value won't encode again. The second marshal is gone. - durable: journalFormat is 2. A service result in a Record of format 0 or 1, which an older version wrote when live saw the ServiceFn's own value, replays in JSON's generic form through no decoder, as that version replayed it, so a decoder added for its onDone event can't fail Recover or StateAt on it. - durable: a parked result records its type with its import path, so a decoder for another package's type of the same short name can't settle it. - durable: a result with no JSON form aborts the settle, so no Assign runs, and fails the Handle as before. - durable: a KindedEffect whose MarshalJSON panics is recorded by id and kind with the panic's message under crucible.marshalError, rather than panicking out of a step the kernel applied. A Kind that panics still fails the Handle; a test pins it. - durable: replay decodes each service payload once, and typeName runs only for a park. parkedServices dedupes with a sorted set, as endedServices does. - docs: the Handle that parked a result never sees a fixed decoder; every doc site now says to Recover with the fixed decoder, then call RunService. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 5 +- durable/CHANGELOG.md | 29 +- durable/README.md | 5 +- durable/doc.go | 7 +- durable/effect.go | 17 +- .../internal/typefixture/alpha/model/model.go | 9 + .../internal/typefixture/beta/model/model.go | 10 + durable/journal_format_internal_test.go | 48 +++ durable/journal_format_service_test.go | 94 +++++ durable/result_decoding_scenarios_test.go | 8 +- durable/runner.go | 66 ++-- durable/runner_options.go | 6 +- durable/service.go | 201 +++++++---- durable/service_encode_internal_test.go | 57 ++- durable/service_replay_bench_test.go | 79 +++++ durable/service_result_edges_test.go | 331 ++++++++++++++++++ durable/store.go | 6 +- durable/timetravel.go | 2 +- 18 files changed, 840 insertions(+), 140 deletions(-) create mode 100644 durable/internal/typefixture/alpha/model/model.go create mode 100644 durable/internal/typefixture/beta/model/model.go create mode 100644 durable/journal_format_internal_test.go create mode 100644 durable/journal_format_service_test.go create mode 100644 durable/service_replay_bench_test.go create mode 100644 durable/service_result_edges_test.go diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 427f515f..8c1ecf5b 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -171,8 +171,9 @@ loses precision above 2^53. A decoder that fails, or that returns a type other t one the service returned, parks the result: `RunService` returns `ErrEventDataEncoding` and no `onDone` or `onError` reducer runs, but the journal records the result as the service returned it, and the Handle stays healthy. The service stays in flight and never -runs again: fix the decoder and call `RunService` again, on the same Handle or after -`Recover`, and it settles the parked result. Until then `RunService` keeps returning +runs again. The decoder is part of the Runner's options, so the Handle that parked the +result never sees a fix: `Recover` with the fixed decoder, then call `RunService`, and it +settles the parked result. Until then `RunService` keeps returning `ErrEventDataEncoding`. A checkpoint keeps a parked result, and leaving the state that declares the service discards it. A service error stays message-only, an error with the same message, which `errors.Is` no longer matches. Register the decoder before you record results: replay decodes the journal diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 87a26673..c89beb20 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -17,8 +17,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. onError was refused because a context ended, so `Recover` can tell them from the actors that were running. A Store must keep an empty set distinct from nil. - **`Record.Format`** records the journal format of the version that wrote the Record, - which is 1 from this version. Replay arms the timers and services an actor-driven - parent Fire's state declares only for a Record of format 1 or later. + which is 2 from this version. Replay arms the timers and services an actor-driven + parent Fire's state declares only for a Record of format 1 or later, and decodes a + service result through `WithEventDataDecoder` only for a Record of format 2 or later. **For Store authors:** a Store must keep `Format`, `EndedActors` and `UnstartedActors`, and keep an empty set distinct from nil. A Store that drops them, or maps an empty set to nil as an SQL array column or a protobuf repeated field can, @@ -108,7 +109,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A `KindedEffect` that can't marshal no longer fails a step.** A step records it by its id and kind with no payload, as it records an effect that isn't a `KindedEffect`, and its `EffectEnvelope.Meta` holds the marshal error's message under - `crucible.marshalError`. Replay re-derives the effect's value, so it dispatches as any + `crucible.marshalError`; a `MarshalJSON` that panics is recorded the same way, with the + panic's message there. Replay re-derives the effect's value, so it dispatches as any effect does: once live with a handler wired, once on the `Recover` that retries a failed dispatch, and once on a `Recover` whose handler was wired only then. This relaxes `Fire`, and the stateless `Runner.Fire`, which failed with the marshal error @@ -129,16 +131,25 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. added or changed later changes what replay hands `Assign`. An error stays message-only: an error with the same message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` method that panics settles with a message saying the service error - panicked, its invalid UTF-8 replaced too. The journal records the decoded value. + panicked, its invalid UTF-8 replaced too. The journal records the JSON the live settle + decoded, so replay decodes the same bytes, even for a decoder whose type doesn't + survive a round trip through its own encoding. A result with no JSON form fails the + Handle, as before, but now before any `Assign` runs. A service result in a Record an + older version wrote, of format 0 or 1, replays as that version replayed it: in JSON's + generic form, through no decoder, so a decoder added for its onDone event can't fail + `Recover` on it. - **A service result that won't decode is parked.** `RunService` returns `ErrEventDataEncoding`, not `ErrHandleFailed`, before any onDone or onError `Assign` runs. The decode aborts the settle through `ServiceRunner.WithResultTransform`'s abort return, so no event fires. The journal records the result as the service returned it, - with its Go type, in a Record of its own that settles nothing. The Handle stays - healthy and the service stays in flight, but its `ServiceFn` never runs again: fix the - decoder and call `RunService` again, on a Handle recovered with the fixed decoder, and - it settles the parked result, which the decoder must return as the type the service - returned, or in JSON's generic form, as live. Until then each `RunService` returns + with its Go type and import path, in a Record of its own that settles nothing. The + Handle stays healthy and the service stays in flight, but its `ServiceFn` never runs + again. The Handle that parked it never sees a fixed decoder: `Recover` with the fixed + decoder, then call `RunService`, and it settles the parked result, which the decoder + must return as the type the service returned, or in JSON's generic form, as live. A + ctx that ended during the service parks nothing: `RunService` fails the Handle and + records nothing, as for a result that decodes, and the service runs again after + `Recover`. Until then each `RunService` returns `ErrEventDataEncoding` and records nothing more. `Recover` and `StateAt` replay a parked result as parked, so they don't fail on it, and `StateAt` at that step shows the instance as it was before. A checkpoint carries the parked results in a new diff --git a/durable/README.md b/durable/README.md index 544e94b4..4d3e702c 100644 --- a/durable/README.md +++ b/durable/README.md @@ -182,8 +182,9 @@ loses precision above 2^53. A decoder that fails, or that returns a type other t one the service returned, parks the result: `RunService` returns `ErrEventDataEncoding` and no `onDone` or `onError` reducer runs, but the journal records the result as the service returned it, and the Handle stays healthy. The service stays in flight and never -runs again: fix the decoder and call `RunService` again, on the same Handle or after -`Recover`, and it settles the parked result. Until then `RunService` keeps returning +runs again. The decoder is part of the Runner's options, so the Handle that parked the +result never sees a fix: `Recover` with the fixed decoder, then call `RunService`, and it +settles the parked result. Until then `RunService` keeps returning `ErrEventDataEncoding`. A checkpoint keeps a parked result, and leaving the state that declares the service discards it. A service error stays message-only, an error with the same message, which `errors.Is` no longer matches. Register the decoder before you record results: replay decodes the journal diff --git a/durable/doc.go b/durable/doc.go index 642c82a2..36d77708 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -47,9 +47,10 @@ // map[string]any and a number as a float64. A decoder that fails, or returns a type // other than the service's result, parks the result: RunService returns // ErrEventDataEncoding and no onDone or onError reducer runs, but the result is -// recorded and the Handle stays healthy, so the service never runs again. Fix the -// decoder and call RunService again, on this Handle or a recovered one, and it settles -// the parked result. Leaving the state that declares the service discards it. A +// recorded and the Handle stays healthy, so the service never runs again. The decoder +// is part of the Runner's options, so the Handle that parked the result never sees a +// fix: Recover with the fixed decoder, then call RunService, and it settles the parked +// result. Leaving the state that declares the service discards it. A // service error stays message-only. Register the decoder before recording results: replay decodes the // journal with the recovering Runner's decoder, so one added or changed after results // were recorded changes what replay produces. diff --git a/durable/effect.go b/durable/effect.go index 07220dae..d8269464 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -194,7 +194,8 @@ const metaMarshalError = "crucible.marshalError" // chan, is recorded as a bare effect is, by id and kind with no payload, and its // envelope's Meta holds the marshal error's message under "crucible.marshalError" // (metaMarshalError). Replay re-derives its value, so it dispatches, live and on -// Recover, as any effect does, and the step doesn't fail. +// Recover, as any effect does, and the step doesn't fail. A MarshalJSON that panics is +// recorded the same way, with the panic's message in Meta. func recordEffects(des []dispatchableEffect) []state.EffectEnvelope { if len(des) == 0 { return nil @@ -203,7 +204,7 @@ func recordEffects(des []dispatchableEffect) []state.EffectEnvelope { for _, de := range des { env := state.EffectEnvelope{Kind: effectKind(de.effect), EffectID: de.id} if ke, ok := de.effect.(state.KindedEffect); ok { - marshaled, err := state.MarshalEffect(ke) + marshaled, err := marshalEffect(ke) if err != nil { env.Meta = map[string]any{metaMarshalError: validUTF8(err.Error())} out = append(out, env) @@ -217,6 +218,18 @@ func recordEffects(des []dispatchableEffect) []state.EffectEnvelope { return out } +// marshalEffect is state.MarshalEffect with a panic in the effect's MarshalJSON +// returned as an error, so recordEffects records the effect by id and kind rather than +// let the panic out of a step the kernel has applied. +func marshalEffect(ke state.KindedEffect) (env state.EffectEnvelope, err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("effect payload panicked while encoding: %v", p) + } + }() + return state.MarshalEffect(ke) +} + // dispatchEffects applies a step's domain effects exactly once. It is called // AFTER the step Record (carrying the effect ids) is durably appended, realizing // the write-ahead ordering: for each effect whose id is not already in the diff --git a/durable/internal/typefixture/alpha/model/model.go b/durable/internal/typefixture/alpha/model/model.go new file mode 100644 index 00000000..b33c3fd4 --- /dev/null +++ b/durable/internal/typefixture/alpha/model/model.go @@ -0,0 +1,9 @@ +// Package model is a test fixture: it and the model package under beta each define a +// Result, so a test can tell two types apart that share the short name model.Result. +package model + +// Result is alpha's model.Result. +type Result struct { + ID int64 + Name string +} diff --git a/durable/internal/typefixture/beta/model/model.go b/durable/internal/typefixture/beta/model/model.go new file mode 100644 index 00000000..835600ec --- /dev/null +++ b/durable/internal/typefixture/beta/model/model.go @@ -0,0 +1,10 @@ +// Package model is a test fixture: it and the model package under alpha each define a +// Result, so a test can tell two types apart that share the short name model.Result. +package model + +// Result is beta's model.Result. Its JSON is alpha's, so a decoder for it decodes an +// alpha Result's JSON without error. +type Result struct { + ID int64 + Name string +} diff --git a/durable/journal_format_internal_test.go b/durable/journal_format_internal_test.go new file mode 100644 index 00000000..f2b14772 --- /dev/null +++ b/durable/journal_format_internal_test.go @@ -0,0 +1,48 @@ +package durable + +import ( + "encoding/json" + "testing" +) + +// TestCheckFormat_AcceptsAHigherFormat pins what a downgrade meets. checkFormat, which +// every version since Record.Format has, fails only on a Record of a lower format than +// its checkpoint's, a Store that lost the field. It accepts a Record of a higher +// format, one a newer version wrote, so an older binary replays a newer journal as its +// own format, and no bump of journalFormat can make a binary already built refuse it. +func TestCheckFormat_AcceptsAHigherFormat(t *testing.T) { + for _, tc := range []struct{ checkpoint, rec int }{ + {1, 2}, + {journalFormat, journalFormat + 1}, + {0, journalFormat + 5}, + } { + if err := checkFormat(tc.checkpoint, &Record{Format: tc.rec}); err != nil { + t.Errorf("checkFormat(%d, a Record of format %d) = %v, want nil", tc.checkpoint, tc.rec, err) + } + } + if err := checkFormat(2, &Record{Format: 1}); err == nil { + t.Errorf("checkFormat(2, a Record of format 1) = nil, want the lost-field error") + } +} + +// TestParkedEntry_AnOlderReaderSettlesIt pins what a version from before parking reads +// in a parked entry: its serviceResultPayload has only Result and Error, so the entry +// decodes as a result that settled, and its replay would settle the parked result +// through onDone, running the Assign the park held back. The CHANGELOG says not to +// downgrade a Store this version has written to; this is one reason. +func TestParkedEntry_AnOlderReaderSettlesIt(t *testing.T) { + payload, err := json.Marshal(serviceResultPayload{Result: json.RawMessage(`{"ID":7}`), Parked: true, Type: "x.T"}) + if err != nil { + t.Fatal(err) + } + var older struct { + Result json.RawMessage `json:"result,omitempty"` + Error *string `json:"error,omitempty"` + } + if err := json.Unmarshal(payload, &older); err != nil { + t.Fatalf("an older reader fails on a parked entry: %v", err) + } + if older.Error != nil || string(older.Result) != `{"ID":7}` { + t.Fatalf("an older reader reads %+v, want a settled result", older) + } +} diff --git a/durable/journal_format_service_test.go b/durable/journal_format_service_test.go new file mode 100644 index 00000000..5872e321 --- /dev/null +++ b/durable/journal_format_service_test.go @@ -0,0 +1,94 @@ +package durable_test + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// olderWriter is a MemStore that stamps every Record and checkpoint it keeps with +// format, as a version that wrote that journal format did. The scnUser result these +// tests record needs no rewrite: an older version journaled json.Marshal of the value +// the ServiceFn returned, and for a scnUser those are the bytes this version records. +type olderWriter struct { + *durable.MemStore + format int +} + +func (s *olderWriter) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + rec.Format = s.format + return s.MemStore.Append(ctx, id, rec, opts...) +} + +func (s *olderWriter) Checkpoint(ctx context.Context, id durable.InstanceID, snapshot []byte, throughStep int, opts ...durable.CheckpointOption) error { + var env map[string]json.RawMessage + if err := json.Unmarshal(snapshot, &env); err != nil { + return err + } + delete(env, "format") + if s.format > 0 { + env["format"] = json.RawMessage([]byte{byte('0' + s.format)}) + } + b, err := json.Marshal(env) + if err != nil { + return err + } + return s.MemStore.Checkpoint(ctx, id, b, throughStep, opts...) +} + +// TestJournalFormat_OlderServiceResult_ReplaysAsItDid: a service result an older +// version journaled, at format 0 (the 0.1.0 release) or 1 (before results were decoded +// through WithEventDataDecoder), replays as that version replayed it: in JSON's generic +// form, through no decoder and no type check. A Runner whose decoder covers onDone, and +// fails, returns another type or returns the service's own type, still recovers it, and +// the replayed Assign sees the map the older version's replay gave it. Records this +// version writes, at format 2, decode through the decoder (see +// TestServiceResult_RecordedResultThatWontDecode_RunsNoAssignOnReplay). +func TestJournalFormat_OlderServiceResult_ReplaysAsItDid(t *testing.T) { + decoders := []struct { + name string + opt durable.Option[string, string, svcCtx] + }{ + {"decoder fails", failingDecoder()}, + {"decoder returns another type", decoderFor("done", durable.DecodeAs[scnAdmin])}, + {"decoder returns the service's type", decoderFor("done", durable.DecodeAs[scnUser])}, + } + for _, format := range []int{0, 1} { + for _, dc := range decoders { + t.Run(dc.name+", format "+string(rune('0'+format)), func(t *testing.T) { + ctx := context.Background() + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + a := &scnAssigns{} + m := scnMachine(fn, a) + store := &olderWriter{MemStore: durable.NewMemStore(durable.WithHistory()), format: format} + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + h := scnStart(t, m, store, reg) + mustRunService(t, h, state.InvokeID(scnInvoke, "loading", 0)) + _, recs, err := store.Load(ctx, "scn-1") + if err != nil || len(recs) != 1 || recs[0].Format != format { + t.Fatalf("the journal holds %+v (err %v), want one Record of format %d", recs, err, format) + } + + rh, err := durable.Recover(ctx, m, store, "scn-1", reg, dc.opt) + if err != nil { + t.Fatalf("Recover: %v; the older version recovered this journal", err) + } + if got := rh.Instance().Current(); got != "ready" { + t.Fatalf("recovered in %q, want ready", got) + } + if a.done != 2 { + t.Fatalf("onDone's Assign ran %d times, want 2 (live, replay)", a.done) + } + if _, ok := a.seen[1].(map[string]any); !ok { + t.Fatalf("replay saw %s, want the map[string]any the older version's replay gave", describe(a.seen[1])) + } + if _, err := durable.StateAt(ctx, m, store, "scn-1", 0, reg, dc.opt); err != nil { + t.Fatalf("StateAt: %v", err) + } + }) + } + } +} diff --git a/durable/result_decoding_scenarios_test.go b/durable/result_decoding_scenarios_test.go index 70937acd..1e1f9d68 100644 --- a/durable/result_decoding_scenarios_test.go +++ b/durable/result_decoding_scenarios_test.go @@ -212,8 +212,8 @@ func TestServiceResult_NoDecoder_Int64IsFloat64(t *testing.T) { // TestServiceResult_NoJSONForm_FailsTheHandle: a result with no JSON form fails the // Handle and records nothing, so the recovered instance has the service in flight -// again. The Assign ran live only: the result settled before recording found it had no -// JSON form. +// again. No Assign runs: the settle aborts before it fires anything, as for a result +// that won't decode (see TestServiceResult_NoJSONForm_RunsNoAssign). func TestServiceResult_NoJSONForm_FailsTheHandle(t *testing.T) { a, runErr, recErr := scnRun(t, returning(make(chan int), nil), decoderFor("done", durable.DecodeAs[scnUser])) if !errors.Is(runErr, durable.ErrHandleFailed) { @@ -222,8 +222,8 @@ func TestServiceResult_NoJSONForm_FailsTheHandle(t *testing.T) { if recErr != nil { t.Errorf("Recover: %v", recErr) } - if len(a.seen) != 1 { - t.Errorf("Assign ran %d times, want 1 (live only; nothing recorded)", len(a.seen)) + if len(a.seen) != 0 { + t.Errorf("Assign ran %d times, want 0", len(a.seen)) } } diff --git a/durable/runner.go b/durable/runner.go index 9cadbe12..4a4fe4a7 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "slices" - "strings" "sync" "time" "unicode/utf8" @@ -138,6 +137,8 @@ type Handle[S comparable, E comparable, C any] struct { svc *state.ServiceRunner[S, E, C] // host driver for invoked services, nil when none wired svcBuf *[]state.JournalEntry // recording service-result accumulator, flushed per step + // svcJSON holds the JSON the last successful settle decoded, which the step journals. + svcJSON *settledJSON // settled holds the services that settled in the current activation of the state // that declares them, so a checkpoint records them and Recover doesn't start them. settled map[string]bool @@ -570,7 +571,7 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt // no service entries. starts := inst.StartEffects() svcBuf := make([]state.JournalEntry, 0) - svc := r.newServiceRunner(inst) + svc, svcJSON := r.newServiceRunner(inst) if svc != nil { svc.Absorb(ctx, starts) } @@ -592,6 +593,7 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt timers: map[string]pendingTimer{}, svc: svc, svcBuf: &svcBuf, + svcJSON: svcJSON, actors: actors, actorFires: actorFires, lock: make(chan struct{}, 1), @@ -627,12 +629,13 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt // newServiceRunner builds the kernel host driver for invoked services bound to // inst, or nil when no service registry was wired (a purely event/timer-driven -// machine invokes none). -func (r *Runner[S, E, C]) newServiceRunner(inst *state.Instance[S, E, C]) *state.ServiceRunner[S, E, C] { +// machine invokes none), and the holder of the JSON each settle decodes. +func (r *Runner[S, E, C]) newServiceRunner(inst *state.Instance[S, E, C]) (*state.ServiceRunner[S, E, C], *settledJSON) { if r.cfg.serviceReg == nil { - return nil + return nil, nil } - return state.NewServiceRunner(inst, r.cfg.serviceReg).WithResultTransform(replayableOutcome(r.cfg.eventDataDecoder)) + settled := &settledJSON{} + return state.NewServiceRunner(inst, r.cfg.serviceReg).WithResultTransform(replayableOutcome(r.cfg.eventDataDecoder, settled)), settled } // newActorSystem builds the kernel host driver for child-machine actors bound to @@ -1333,17 +1336,24 @@ func (h *Handle[S, E, C]) parkedServices() []parkedServiceWire { if h.svc == nil || len(h.parked) == 0 { return nil } - var out []parkedServiceWire + var ids []string for _, eff := range h.inst.StartEffects() { - sv, ok := eff.(state.StartService) - if !ok { - continue - } - if p, ok := h.parked[sv.ID]; ok && !slices.ContainsFunc(out, func(w parkedServiceWire) bool { return w.ID == sv.ID }) { - out = append(out, parkedServiceWire{ID: sv.ID, Result: p.raw, Type: p.typ}) + if sv, ok := eff.(state.StartService); ok { + if _, parked := h.parked[sv.ID]; parked { + ids = append(ids, sv.ID) + } } } - slices.SortFunc(out, func(a, b parkedServiceWire) int { return strings.Compare(a.ID, b.ID) }) + if len(ids) == 0 { + return nil + } + slices.Sort(ids) + ids = slices.Compact(ids) + out := make([]parkedServiceWire, len(ids)) + for i, id := range ids { + p := h.parked[id] + out[i] = parkedServiceWire{ID: id, Result: p.raw, Type: p.typ} + } return out } @@ -1461,7 +1471,7 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S restoredStarts := inst.StartEffects() endedServices := env.EndedServices svcBuf := make([]state.JournalEntry, 0) - svc := r.newServiceRunner(inst) + svc, svcJSON := r.newServiceRunner(inst) if svc != nil { svc.Absorb(ctx, startsExcept(restoredStarts, endedServices, func(sv state.StartService) string { return sv.ID })) } @@ -1615,6 +1625,7 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S timers: armed, svc: svc, svcBuf: &svcBuf, + svcJSON: svcJSON, settled: mirror.settled, parked: mirror.parked, actors: actors, @@ -1988,16 +1999,20 @@ func replayRecord[S comparable, E comparable, C any]( stepEffects = res.Effects case hasServiceEntry(rec): for _, entry := range serviceEntries(rec) { - if p, ok := parkedEntry(entry); ok { + sp, err := decodeServiceEntry(entry) + if err != nil { + return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) + } + if sp.Parked { // Live, the result didn't decode, so nothing settled: park it again, for // RunService to settle. Its service stays in flight. - if err := replayPark(svc, entry); err != nil { + if err = replayPark(svc, entry); err != nil { return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) } - mirror.park(entry.CorrelationID, p) + mirror.park(entry.CorrelationID, recordedResult{raw: sp.Result, typ: sp.Type}) continue } - effects, err := replayService(ctx, svc, entry) + effects, err := replayService(ctx, svc, entry, sp, rec.Format >= decodedResultsFormat) if err != nil { return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) } @@ -2065,11 +2080,16 @@ func armsActorFires(rec *Record) bool { } // journalFormat is the format this version writes in every Record. It is -// actorArmsFormat, the first that arms the timers and services an actor-driven parent -// Fire's state declares. +// decodedResultsFormat. actorArmsFormat is the first format that arms the timers and +// services an actor-driven parent Fire's state declares. decodedResultsFormat is the +// first whose service results the live run decoded through WithEventDataDecoder, so +// replay decodes them the same way; it is also the first that can park a result. A +// result in a Record of an older format replays as that version replayed it, in JSON's +// generic form. const ( - actorArmsFormat = 1 - journalFormat = actorArmsFormat + actorArmsFormat = 1 + decodedResultsFormat = 2 + journalFormat = decodedResultsFormat ) // replayArm arms the timers and services a replayed Fire's effects declare, as the live diff --git a/durable/runner_options.go b/durable/runner_options.go index aa2e1d71..2e7e9d38 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -186,9 +186,9 @@ func WithMaxEventDataBytes[S comparable, E comparable, C any](n int) Option[S, E // as a map[string]any and a number as a float64. A decoder that fails on a result, or // returns a type other than the service returned, parks the result: RunService returns // ErrEventDataEncoding before any onDone or onError Assign runs, records the result -// without settling it, and keeps the Handle healthy. Fix the decoder and call -// RunService again, on a recovered Handle, to settle the parked result without running -// the service again (see Handle.RunService). A service error stays message-only and +// without settling it, and keeps the Handle healthy. Recover with the fixed decoder, +// then call RunService, to settle the parked result without running the service again; +// the Handle that parked it never sees the fix (see Handle.RunService). A service error stays message-only and // doesn't reach the decoder. Register the decoder before results are recorded: replay // decodes the journal with the recovering Runner's decoder, so one added or changed // after results were recorded changes what replay produces, and one that can't decode diff --git a/durable/service.go b/durable/service.go index d6e41af7..44180b16 100644 --- a/durable/service.go +++ b/durable/service.go @@ -33,9 +33,9 @@ import ( // The Runner wraps the kernel's reusable host driver, state.ServiceRunner, which // turns StartService / StopService effects into real ServiceFn executions and // re-fires each result through the invocation's onDone / onError event. On the live -// path RunService delegates to ServiceRunner.Tick (resolve + run + settle), reads the -// outcome in its decoded form (see replayableOutcome), and journals it as a -// JournalServiceResult correlated by the invocation id. On recovery the recorded +// path RunService delegates to ServiceRunner.Tick (resolve + run + settle), keeps the +// JSON the settle decoded the outcome from (see replayableOutcome), and journals it as +// a JournalServiceResult correlated by the invocation id. On recovery the recorded // outcomes are replayed in order through ServiceRunner.SettleDone / SettleError — the // same settle methods the live run drove — so the kernel re-fires the identical onDone // / onError event with the identical data and re-derives byte-identical context, @@ -57,10 +57,12 @@ import ( // whichever is present and settles the invocation through the matching seam. type serviceResultPayload struct { // Result is the JSON-encoded successful return of the ServiceFn, present for a - // SettleDone outcome. It is decoded through the Runner's EventDataDecoder for the - // onDone event and re-fired through onDone, so the transition's Assign reads it - // from AssignCtx.Event exactly as on the live run, which settles with the same - // decoded form (see replayableOutcome). + // SettleDone outcome: the bytes the live settle decoded. Replay decodes them through + // the Runner's EventDataDecoder for the onDone event and re-fires them through + // onDone, so the transition's Assign reads from AssignCtx.Event the value the live + // run's decode gave (see replayableOutcome). A Record of a format before + // decodedResultsFormat replays them in JSON's generic form, as the version that + // wrote it did. Result json.RawMessage `json:"result,omitempty"` // Error is the failure message for a SettleError outcome. It is reconstructed // into an error and re-fired through onError so the transition's Assign reads it @@ -71,19 +73,20 @@ type serviceResultPayload struct { // than settling it, and a later RunService settles it once the decoder decodes it. // An older Record never sets it. Parked bool `json:"parked,omitempty"` - // Type is a parked result's Go type, as reflect names it, so the RunService that - // settles it holds the decoder to the type the ServiceFn returned, as the live - // settle did. + // Type is a parked result's Go type, with its import path (see typeName), so the + // RunService that settles it holds the decoder to the type the ServiceFn returned, + // as the live settle did. Type string `json:"type,omitempty"` } -// parkedEntry returns the result a parked entry recorded, or false for any other entry. -func parkedEntry(e state.JournalEntry) (recordedResult, bool) { +// decodeServiceEntry decodes a recorded service outcome's payload, once for replay to +// park or settle it. +func decodeServiceEntry(e state.JournalEntry) (serviceResultPayload, error) { var sp serviceResultPayload - if json.Unmarshal(e.Payload, &sp) != nil || !sp.Parked { - return recordedResult{}, false + if err := json.Unmarshal(e.Payload, &sp); err != nil { + return sp, fmt.Errorf("durable: decoding recorded service %q outcome: %w", e.CorrelationID, err) } - return recordedResult{raw: sp.Result, typ: sp.Type}, true + return sp, nil } // drainServices returns and clears the service outcomes recorded since the last @@ -106,7 +109,8 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // registry supplied with WithServiceRegistry; its result or error is journaled // as a JournalServiceResult correlated by id so recovery replays it without // re-invoking the service. The outcome reaches the machine decoded from its JSON form, -// live as on replay, and the journal holds that form. A result whose onDone event +// live as on replay, and the journal holds the JSON the live settle decoded, so replay +// decodes the same bytes. A result whose onDone event // WithEventDataDecoder covers arrives as the decoder's type; otherwise it arrives in // JSON's generic form: a struct is a map, an integer a float64 that loses precision // above 2^53. An error is message-only: an error with the same message, which @@ -124,17 +128,19 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // ServiceFn returned it, with its Go type, settles nothing, and returns // ErrEventDataEncoding, not ErrHandleFailed, before any onDone or onError Assign runs. // The Handle stays healthy and the service stays in flight, on this Handle and on a -// recovered one, but its ServiceFn never runs again: fix the decoder and call -// RunService again, and it settles the parked result. Until then each RunService +// recovered one, but its ServiceFn never runs again. The decoder is part of the +// Runner's options, so this Handle never sees a fix: Recover with the fixed decoder, +// then call RunService, and it settles the parked result. Until then each RunService // returns ErrEventDataEncoding and records nothing more. A Fire or timer that leaves // the state that declares the service stops it and discards the parked result. A // checkpoint keeps it, and StateAt shows the instance as it was before the park. // -// A result that won't encode, a ctx that ended during the service or while its -// settle's effects were absorbed (even after a clean settle), or a Fire the settle -// drove refused because a context ended (ctx, or one middleware derived from it) -// returns ErrHandleFailed and records nothing: a recovered Handle has the service in -// flight again, and RunService runs it a second time. A KindedEffect the settle emits +// A result that won't encode, a ctx that ended during the service (whether or not the +// result decodes) or while its settle's effects were absorbed (even after a clean +// settle), or a Fire the settle drove refused because a context ended (ctx, or one +// middleware derived from it) returns ErrHandleFailed and records nothing: a recovered +// Handle has the service in flight again, and RunService runs it a second time. A +// result that won't encode runs no Assign, as one that won't decode doesn't. A KindedEffect the settle emits // whose payload won't marshal doesn't fail it: it is recorded by id and kind, as // recordEffects says, and dispatches as any effect does. // A failed Append also returns ErrHandleFailed, but the write may have landed anyway: @@ -205,7 +211,14 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir res := results[0] if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { // The result didn't decode for its onDone event, so the transform aborted the - // settle: nothing fired and no Assign ran. Park it. + // settle: nothing fired and no Assign ran. Park it, unless a context that ended + // during the service may have cut the result short: as for a result that + // settles, that records nothing, and Recover runs the service again. + if !isParked { + if err := stepCanceled(ctx); err != nil { + return state.FireResult[S]{}, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, err)) + } + } return h.parkResult(ctx, id, derr, isParked) } // The service settled: unless the settle's own Fire leaves or re-enters the state @@ -229,7 +242,7 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return res, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, err)) } - entry, err := recordServiceOutcome(id, h.svc.LastResult, h.svc.LastError()) + entry, err := recordServiceOutcome(id, h.svcJSON.raw, h.svc.LastError()) if err != nil { return res, true, nil, nil, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) } @@ -261,9 +274,9 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir // service stays in flight in the ServiceRunner and the Handle stays healthy. It returns // the Record, which the caller finishes as any step's, and an error wrapping // ErrEventDataEncoding that says the result is parked. A result already parked -// (already) is in the journal, so it records nothing. A result with no JSON to record -// fails the Handle, which leaves the service in flight for a recovered Handle to run -// again. +// (already) is in the journal, so it records nothing. A result with no JSON form, which +// the transform aborted with no raw to record, fails the Handle, which leaves the +// service in flight for a recovered Handle to run again. func (h *Handle[S, E, C]) parkResult(ctx context.Context, id string, derr *undecodableResult, already bool) (state.FireResult[S], bool, *Record, []dispatchableEffect, error) { parkErr := fmt.Errorf("durable: settling service %q for %q: its result is parked until RunService can decode it: %w", id, h.id, derr) if already { @@ -306,21 +319,25 @@ func (h *Handle[S, E, C]) noteSettled(id string) { // recorded JSON as a recordedResult, and both get decode's value. With the default // decoder that is JSON's generic form. Live, a decoder that returns a concrete type // other than the result's is refused, though JSON's generic form, including a -// json.Number, is always accepted, so the default keeps working. +// json.Number, is always accepted, so the default keeps working. A settle that +// succeeds leaves in settled the JSON it decoded, which serviceStep journals, so replay +// decodes the bytes live decoded; an error outcome leaves nil there. // // A result that won't decode, or is refused, aborts the settle with an // *undecodableResult before it fires anything, so no onDone or onError Assign runs; // serviceStep sees it in the settle's FireResult.Err and parks the result, and // replayService fails the replay. A parked result settles as a recordedResult that // carries its Go type, and the decoder must return that type, or JSON's generic form, -// as live. A result with no JSON form is left as it is, and fails the Handle when -// recordServiceOutcome can't encode it. +// as live. A result with no JSON form aborts the settle too, with no JSON to park, and +// serviceStep fails the Handle. A legacyResult, which replay passes for a Record of an +// older format, settles as it is, as that version replayed it. // // An error becomes an error with its message as a Store keeps it (see // normalizedErrorMessage), live and on replay. An Error method that panics settles // with a message saying so, normalized the same way. -func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, error) (any, error, error) { +func replayableOutcome[E comparable](decode EventDataDecoder[E], settled *settledJSON) func(E, any, error) (any, error, error) { return func(event E, result any, err error) (any, error, error) { + settled.set(nil) if err != nil { msg, perr := normalizedErrorMessage(err, "service error") if perr != nil { @@ -328,42 +345,101 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E]) func(E, any, er } return nil, errors.New(msg), nil } + if legacy, ok := result.(legacyResult); ok { + return legacy.value, nil, nil + } recorded, replay := result.(recordedResult) - b, typ := []byte(recorded.raw), recorded.typ + b := []byte(recorded.raw) if !replay { var jerr error if b, jerr = marshalPayload(result); jerr != nil { - return result, nil, nil + return nil, nil, &undecodableResult{err: fmt.Errorf("service result won't encode: %w", jerr)} } - typ = typeName(result) } value, derr := decodePayload(eventPayload{data: b}, event, decode) if derr != nil { - return nil, nil, &undecodableResult{raw: b, typ: typ, err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} + return nil, nil, &undecodableResult{raw: b, typ: parkedType(recorded, replay, result), err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} } if !replay { if terr := decodedAsPassed(value, result, "service result", true); terr != nil { - return nil, nil, &undecodableResult{raw: b, typ: typ, err: terr} + return nil, nil, &undecodableResult{raw: b, typ: typeName(result), err: terr} } } else if terr := decodedAsRecorded(value, recorded.typ); terr != nil { - return nil, nil, &undecodableResult{raw: b, typ: typ, err: terr} + return nil, nil, &undecodableResult{raw: b, typ: recorded.typ, err: terr} } + settled.set(b) return value, nil, nil } } +// settledJSON holds the JSON the last successful settle decoded, for serviceStep to +// journal. The transform sets it during the settle, which runs under the Handle's step +// lock live and on replay's one goroutine. A nil settledJSON keeps nothing. +type settledJSON struct{ raw json.RawMessage } + +func (s *settledJSON) set(b []byte) { + if s != nil { + s.raw = b + } +} + +// legacyResult is a service result recorded in a Record of a format before +// decodedResultsFormat, decoded as that version replayed it: into an any, with no +// decoder or type check. The transform settles it as it is. +type legacyResult struct{ value any } + +// parkedType returns the Go type a park records for result: the recorded one on +// replay, else the ServiceFn's value's. +func parkedType(recorded recordedResult, replay bool, result any) string { + if replay { + return recorded.typ + } + return typeName(result) +} + // recordedResult is a journaled service result's JSON, which replayService settles so // the transform decodes it as the live run decoded the ServiceFn's value, and which a // RunService settles for a parked result. type recordedResult struct { raw json.RawMessage // typ is a parked result's Go type, as typeName gives it; empty for a settled one, - // whose JSON is already the decoded value's. + // whose type the live settle already checked. typ string } -// typeName returns the name reflect gives v's type, the form a parked result records. -func typeName(v any) string { return fmt.Sprint(reflect.TypeOf(v)) } +// typeName returns the full name of v's type, the form a parked result records: a +// named type's import path and name, so two packages' model.Result differ, and an +// unnamed type spelled from those (see fullTypeName). +func typeName(v any) string { + t := reflect.TypeOf(v) + if t == nil { + return "" + } + return fullTypeName(t) +} + +// fullTypeName spells t with each named type's import path. A pointer, slice, array or +// map is spelled from its element and key types; any other unnamed type, such as a +// struct literal, as reflect spells it. +func fullTypeName(t reflect.Type) string { + if name := t.Name(); name != "" { + if pkg := t.PkgPath(); pkg != "" { + return pkg + "." + name + } + return name + } + switch t.Kind() { + case reflect.Pointer: + return "*" + fullTypeName(t.Elem()) + case reflect.Slice: + return "[]" + fullTypeName(t.Elem()) + case reflect.Array: + return fmt.Sprintf("[%d]%s", t.Len(), fullTypeName(t.Elem())) + case reflect.Map: + return "map[" + fullTypeName(t.Key()) + "]" + fullTypeName(t.Elem()) + } + return t.String() +} // decodedAsRecorded is decodedAsPassed for a parked result, which records its type's // name rather than a value: it reports, as an ErrEventDataEncoding error, a value the @@ -384,11 +460,12 @@ func decodedAsRecorded(value any, typ string) error { // decode for its onDone event, or decodes as another type than the ServiceFn returned. // The ServiceRunner fires no event for it and hands it back in the settle's // FireResult.Err, where serviceStep finds it and parks the result, and replayService -// finds it and fails the replay. err wraps ErrEventDataEncoding. +// finds it and fails the replay. err wraps ErrEventDataEncoding. For a result with no +// JSON form raw is nil, err says why it won't encode, and serviceStep fails the Handle. type undecodableResult struct { err error raw json.RawMessage // the result's JSON, which a park records - typ string // the result's Go type, as typeName gives it + typ string // the result's Go type, as typeName gives it, for a park } func (u *undecodableResult) Error() string { return u.err.Error() } @@ -421,19 +498,14 @@ func decodedAsPassed(value, passed any, what string, generic bool) error { } // recordServiceOutcome builds the JournalServiceResult entry for a just-settled -// service. It reads the success result through lastResult (valid only during the -// settle's synchronous window) and the error through lastErr, encoding whichever -// the service produced. -func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr error) (state.JournalEntry, error) { +// service: the error lastErr, or, for a success, raw, the JSON the settle decoded (see +// settledJSON), so replay decodes the bytes live decoded. +func recordServiceOutcome(id string, raw json.RawMessage, lastErr error) (state.JournalEntry, error) { var sp serviceResultPayload if lastErr != nil { msg := lastErr.Error() sp.Error = &msg - } else if result, ok := lastResult(); ok { - raw, err := marshalPayload(result) - if err != nil { - return state.JournalEntry{}, fmt.Errorf("marshaling service result: %w", err) - } + } else { sp.Result = raw } payload, err := json.Marshal(sp) @@ -453,16 +525,16 @@ func recordServiceOutcome(id string, lastResult func() (any, bool), lastErr erro // SettleError (with the reconstructed error), so the kernel re-fires the same // onDone / onError event with the same data and re-derives identical context. It // returns the settle's effects, so the caller arms the timers its state declares, as -// the live RunService did. It reports an error only when the recorded entry cannot be -// decoded or names no in-flight service (a corrupt or mis-ordered journal). -func replayService[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], entry state.JournalEntry) ([]state.Effect, error) { +// the live RunService did. sp is entry's decoded payload. A result in a Record of a +// format before decodedResultsFormat (decoded false) replays as the version that wrote +// it replayed it, in JSON's generic form through no decoder: that version's live run +// saw the ServiceFn's own value, which no decoder of this one can be held to. It +// reports an error only when the recorded result cannot be decoded or names no +// in-flight service (a corrupt or mis-ordered journal). +func replayService[S comparable, E comparable, C any](ctx context.Context, svc *state.ServiceRunner[S, E, C], entry state.JournalEntry, sp serviceResultPayload, decoded bool) ([]state.Effect, error) { if svc == nil { return nil, errors.New("durable: replaying a service result requires a service registry (supply WithServiceRegistry)") } - var sp serviceResultPayload - if err := json.Unmarshal(entry.Payload, &sp); err != nil { - return nil, fmt.Errorf("durable: decoding recorded service %q outcome: %w", entry.CorrelationID, err) - } if sp.Error != nil { res, ok := svc.SettleError(ctx, entry.CorrelationID, errors.New(*sp.Error)) if !ok { @@ -473,7 +545,16 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * // The transform decodes the recorded JSON for the onDone event, as it did live. One // that won't decode aborts the settle before it fires anything. var result any - if len(sp.Result) > 0 { + switch { + case !decoded: + var v any + if len(sp.Result) > 0 { + if err := json.Unmarshal(sp.Result, &v); err != nil { + return nil, fmt.Errorf("durable: decoding recorded service %q result: %w", entry.CorrelationID, err) + } + } + result = legacyResult{value: v} + case len(sp.Result) > 0: result = recordedResult{raw: sp.Result} } res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) diff --git a/durable/service_encode_internal_test.go b/durable/service_encode_internal_test.go index 86dbce1c..394aeef6 100644 --- a/durable/service_encode_internal_test.go +++ b/durable/service_encode_internal_test.go @@ -12,29 +12,29 @@ type f8Order struct { ID int64 } -// TestServiceResult_JournalBytesAreTheDecodedValuesEncoding pins why a live service -// result is encoded twice. replayableOutcome encodes the ServiceFn's value to decode -// it; recordServiceOutcome then journals the decoded value's own encoding. Journaling -// the first encoding instead would record different bytes for a struct decoded as a map -// (the keys come out sorted) and for an integer beyond 2^53 decoded as a float64 (the -// journal would keep digits the machine never saw). Replay decodes either to the same -// value, but the journal would no longer hold the form the machine saw, as RunService's -// doc says it does, so the second encode stays. -func TestServiceResult_JournalBytesAreTheDecodedValuesEncoding(t *testing.T) { +// TestServiceResult_JournalBytesAreWhatLiveDecoded pins that a live service result is +// encoded once. replayableOutcome encodes the ServiceFn's value and decodes those bytes; +// recordServiceOutcome journals the same bytes, so replay decodes what live decoded. +// They differ from the decoded value's own encoding for a struct decoded as a map (the +// keys come out sorted) and for an integer beyond 2^53 decoded as a float64 (the journal +// keeps digits the machine never saw), but replay decodes either to the value live saw, +// and only these bytes do for a decoder whose type doesn't survive its own round trip +// (see TestServiceResult_ReplayDecodesWhatLiveDecoded). +func TestServiceResult_JournalBytesAreWhatLiveDecoded(t *testing.T) { asOrder := func(_ string, data []byte) (any, error) { return DecodeAs[f8Order](data) } asInt64 := func(_ string, data []byte) (any, error) { return DecodeAs[int64](data) } cases := []struct { name string result any decode EventDataDecoder[string] - same bool // whether the two encodings match + reencode bool // whether the decoded value's own encoding matches the journal's recorded string // what the journal records }{ {"string", "ok", decodeAny[string], true, `"ok"`}, {"small int, generic", 42, decodeAny[string], true, `42`}, - {"int64 above 2^53, generic", int64(1<<53 + 1), decodeAny[string], false, `9007199254740992`}, + {"int64 above 2^53, generic", int64(1<<53 + 1), decodeAny[string], false, `9007199254740993`}, {"int64 above 2^53, typed", int64(1<<53 + 1), asInt64, true, `9007199254740993`}, - {"struct, generic", f8Order{Zone: "w", ID: 7}, decodeAny[string], false, `{"ID":7,"Zone":"w"}`}, + {"struct, generic", f8Order{Zone: "w", ID: 7}, decodeAny[string], false, `{"Zone":"w","ID":7}`}, {"struct, typed", f8Order{Zone: "w", ID: 7}, asOrder, true, `{"Zone":"w","ID":7}`}, {"map, generic", map[string]any{"b": 1.0, "a": "x"}, decodeAny[string], true, `{"a":"x","b":1}`}, {"slice, generic", []any{"x", 2.5}, decodeAny[string], true, `["x",2.5]`}, @@ -42,15 +42,12 @@ func TestServiceResult_JournalBytesAreTheDecodedValuesEncoding(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - first, err := marshalPayload(tc.result) - if err != nil { - t.Fatalf("encoding the result: %v", err) - } - value, outErr, abort := replayableOutcome(tc.decode)("done", tc.result, nil) + settled := &settledJSON{} + value, outErr, abort := replayableOutcome(tc.decode, settled)("done", tc.result, nil) if outErr != nil || abort != nil { t.Fatalf("the transform returned %v, %v", outErr, abort) } - entry, err := recordServiceOutcome("svc", func() (any, bool) { return value, true }, nil) + entry, err := recordServiceOutcome("svc", settled.raw, nil) if err != nil { t.Fatalf("recordServiceOutcome: %v", err) } @@ -61,18 +58,20 @@ func TestServiceResult_JournalBytesAreTheDecodedValuesEncoding(t *testing.T) { if got := string(sp.Result); got != tc.recorded { t.Fatalf("the journal records %s, want %s", got, tc.recorded) } - if same := string(first) == string(sp.Result); same != tc.same { - t.Fatalf("first encoding %s, journal %s: same = %v, want %v", first, sp.Result, same, tc.same) + reencoded, err := marshalPayload(value) + if err != nil { + t.Fatalf("encoding the decoded value: %v", err) + } + if same := string(reencoded) == string(sp.Result); same != tc.reencode { + t.Fatalf("decoded value's encoding %s, journal %s: same = %v, want %v", reencoded, sp.Result, same, tc.reencode) + } + // Replay decodes the journal to the value live saw. + replayed, _, rabort := replayableOutcome(tc.decode, nil)("done", recordedResult{raw: sp.Result}, nil) + if rabort != nil { + t.Fatalf("replaying %s: %v", sp.Result, rabort) } - // Replay decodes either encoding to the value live saw. - for _, b := range [][]byte{first, sp.Result} { - replayed, _, rabort := replayableOutcome(tc.decode)("done", recordedResult{raw: b}, nil) - if rabort != nil { - t.Fatalf("replaying %s: %v", b, rabort) - } - if describeValue(replayed) != describeValue(value) { - t.Fatalf("replaying %s gives %s, live saw %s", b, describeValue(replayed), describeValue(value)) - } + if describeValue(replayed) != describeValue(value) { + t.Fatalf("replaying %s gives %s, live saw %s", sp.Result, describeValue(replayed), describeValue(value)) } }) } diff --git a/durable/service_replay_bench_test.go b/durable/service_replay_bench_test.go new file mode 100644 index 00000000..87e4de3d --- /dev/null +++ b/durable/service_replay_bench_test.go @@ -0,0 +1,79 @@ +package durable_test + +import ( + "context" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// benchSettles is how many service results the replay benchmark's journal holds. +const benchSettles = 200 + +// pingPongMachine alternates between a and b, each invoking fetch, so every settle +// enters the other state and starts its service: a journal of settled results as long +// as the benchmark wants. +func pingPongMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("pp"). + Service("fetch", fn). + State("a").Invoke("fetch", state.WithInvokeOnDone("next")). + State("b").Invoke("fetch", state.WithInvokeOnDone("next")). + Initial("a"). + Transition("a").On("next").GoTo("b"). + Transition("b").On("next").GoTo("a"). + Quench() +} + +func benchOpts(fn state.ServiceFn[svcCtx]) []durable.Option[string, string, svcCtx] { + return []durable.Option[string, string, svcCtx]{ + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + decoderFor("next", durable.DecodeAs[scnUser]), + } +} + +// runPingPong settles n results on h, alternating the two services. +func runPingPong(b *testing.B, h *durable.Handle[string, string, svcCtx], n int) { + b.Helper() + ids := [2]string{state.InvokeID("pp", "a", 0), state.InvokeID("pp", "b", 0)} + for i := 0; i < n; i++ { + if _, ok, err := h.RunService(context.Background(), ids[i%2]); !ok || err != nil { + b.Fatalf("RunService %d = %v, %v", i, ok, err) + } + } +} + +// BenchmarkRecover_ServiceResults measures Recover replaying benchSettles settled +// service results, each decoded through a decoder for onDone. +func BenchmarkRecover_ServiceResults(b *testing.B) { + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + m := pingPongMachine(fn) + store := durable.NewMemStore() + opts := benchOpts(fn) + h, err := durable.NewRunner(m, store, opts...).Start(context.Background(), "pp-1", svcCtx{}, state.WithInitialState("a")) + if err != nil { + b.Fatalf("Start: %v", err) + } + runPingPong(b, h, benchSettles) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := durable.Recover(context.Background(), m, store, "pp-1", opts...); err != nil { + b.Fatalf("Recover: %v", err) + } + } +} + +// BenchmarkRunService_Settle measures a live RunService that settles a result through +// a decoder for onDone. +func BenchmarkRunService_Settle(b *testing.B) { + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + m := pingPongMachine(fn) + h, err := durable.NewRunner(m, durable.NewMemStore(), benchOpts(fn)...).Start(context.Background(), "pp-1", svcCtx{}, state.WithInitialState("a")) + if err != nil { + b.Fatalf("Start: %v", err) + } + b.ReportAllocs() + b.ResetTimer() + runPingPong(b, h, b.N) +} diff --git a/durable/service_result_edges_test.go b/durable/service_result_edges_test.go new file mode 100644 index 00000000..f613e2d4 --- /dev/null +++ b/durable/service_result_edges_test.go @@ -0,0 +1,331 @@ +package durable_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + + "github.com/stablekernel/crucible/durable" + alpha "github.com/stablekernel/crucible/durable/internal/typefixture/alpha/model" + beta "github.com/stablekernel/crucible/durable/internal/typefixture/beta/model" + "github.com/stablekernel/crucible/state" +) + +// These tests answer review findings on parked service results, the journal's bytes +// for a settled result, a result with no JSON form, and an effect whose MarshalJSON +// panics. Each fails if the defect it names is real. + +// A ctx that ended during the ServiceFn, with a result the decoder refuses, parks +// nothing: as for a result that decodes, RunService fails the Handle with the ctx's +// error and records nothing, and a recovered Handle runs the service again. MemStore's +// Append doesn't check ctx, so only RunService's own check keeps the park out of the +// journal. +func TestParkedResult_CanceledDuringService_RecordsNothing(t *testing.T) { + ctx := context.Background() + turnCtx, cancel := context.WithCancel(ctx) + defer cancel() + var runs int64 + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(&runs, 1) + cancel() // the turn's deadline passes while the call is in flight + return scnUser{ID: 7, Name: "ann"}, nil + } + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + opts := []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), failingDecoder()} + h := scnStart(t, m, store, opts...) + id := state.InvokeID(scnInvoke, "loading", 0) + + _, _, err := h.RunService(turnCtx, id) + if !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, context.Canceled) { + t.Fatalf("RunService = %v, want ErrHandleFailed wrapping context.Canceled", err) + } + if got := tailLen(t, store, "scn-1"); got != 0 { + t.Fatalf("the journal holds %d Records, want none: the canceled turn's result was parked", got) + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } + rh, err := durable.Recover(ctx, m, store, "scn-1", opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + _, ran, err := rh.RunService(ctx, id) + mustPark(t, "RunService after Recover", ran, err) + if got := atomic.LoadInt64(&runs); got != 2 { + t.Fatalf("the ServiceFn ran %d times, want 2: the canceled run recorded nothing", got) + } +} + +// The decoder is part of the Runner's options, so fixing it reaches only a Handle a +// Runner with the fix recovers. On the Handle that parked the result, every RunService +// returns ErrEventDataEncoding; Recover with the fixed decoder, then RunService, settles +// it. +func TestParkedResult_OnlyARecoveredHandleSeesTheFix(t *testing.T) { + r := newParkRun(t) + mustPark(t, "the first RunService", true, r.err) + h, err := durable.Recover(context.Background(), r.m, r.store, "scn-1", r.reg, failingDecoder()) + if err != nil { + t.Fatalf("Recover: %v", err) + } + id := state.InvokeID(scnInvoke, "loading", 0) + for i := 0; i < 3; i++ { + _, ran, rerr := h.RunService(context.Background(), id) + mustPark(t, fmt.Sprintf("RunService %d on the same Handle", i), ran, rerr) + } + fixed, runErr, err := r.recoverAndRun(decoderFor("done", durable.DecodeAs[scnUser])) + if err != nil || runErr != nil { + t.Fatalf("Recover: %v, RunService: %v, with the fixed decoder", err, runErr) + } + if got := fixed.Instance().Current(); got != "ready" { + t.Fatalf("in %q, want ready", got) + } +} + +// A parked result records its Go type with its import path, so a decoder for another +// package's type of the same short name (model.Result) can't settle it. Live, the same +// decoder is refused, since the live check compares reflect types. +func TestParkedResult_TypeCheckKeepsTheImportPath(t *testing.T) { + ctx := context.Background() + var runs int64 + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(&runs, 1) + return alpha.Result{ID: 7, Name: "ann"}, nil + } + id := state.InvokeID(scnInvoke, "loading", 0) + asBeta := decoderFor("done", durable.DecodeAs[beta.Result]) + + t.Run("live", func(t *testing.T) { + a := &scnAssigns{} + h := scnStart(t, scnMachine(fn, a), durable.NewMemStore(), durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), asBeta) + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService with beta's decoder", ran, err) + }) + t.Run("parked", func(t *testing.T) { + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + h := scnStart(t, m, store, reg, failingDecoder()) + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) + rh, err := durable.Recover(ctx, m, store, "scn-1", reg, asBeta) + if err != nil { + t.Fatalf("Recover: %v", err) + } + _, ran, err = rh.RunService(ctx, id) + mustPark(t, "RunService with beta's decoder", ran, err) + if a.done != 0 || a.fails != 0 { + t.Fatalf("an Assign ran: onDone %d (saw %v), onError %d", a.done, a.seen, a.fails) + } + rh, err = durable.Recover(ctx, m, store, "scn-1", reg, decoderFor("done", durable.DecodeAs[alpha.Result])) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, _, err = rh.RunService(ctx, id); err != nil { + t.Fatalf("RunService with alpha's decoder: %v", err) + } + if a.done != 1 || describe(a.seen[0]) != describe(alpha.Result{ID: 7, Name: "ann"}) { + t.Fatalf("onDone's Assign ran %d times and saw %v, want alpha's Result once", a.done, a.seen) + } + }) +} + +// A result with no JSON form runs no Assign: RunService fails the Handle before the +// settle fires anything, records nothing, and a recovered Handle has the service in +// flight again. This holds with the default decoder and with one for onDone. +func TestServiceResult_NoJSONForm_RunsNoAssign(t *testing.T) { + for _, tc := range []struct { + name string + opts []durable.Option[string, string, svcCtx] + }{ + {"default decoder", nil}, + {"decoder for onDone", []durable.Option[string, string, svcCtx]{decoderFor("done", durable.DecodeAs[scnUser])}}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + var runs int64 + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(&runs, 1) + return make(chan int), nil + } + a := &scnAssigns{} + m := scnMachine(fn, a) + store := durable.NewMemStore() + opts := append([]durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))}, tc.opts...) + h := scnStart(t, m, store, opts...) + id := state.InvokeID(scnInvoke, "loading", 0) + if _, _, err := h.RunService(ctx, id); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("RunService = %v, want ErrHandleFailed", err) + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } + if got := tailLen(t, store, "scn-1"); got != 0 { + t.Fatalf("the journal holds %d Records, want none", got) + } + rh, err := durable.Recover(ctx, m, store, "scn-1", opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, ran, err := rh.RunService(ctx, id); !ran || !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("RunService after Recover = %v, %v; want the service run again, failing the Handle", ran, err) + } + if got := atomic.LoadInt64(&runs); got != 2 { + t.Fatalf("the ServiceFn ran %d times, want 2", got) + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("onDone's Assign ran %d times and onError's %d, want neither", a.done, a.fails) + } + }) + } +} + +// unitBug's MarshalJSON writes milliseconds and its UnmarshalJSON reads seconds: an +// asymmetric pair, so decoding its own encoding scales the value a thousandfold. +type unitBug struct{ Seconds int64 } + +func (u unitBug) MarshalJSON() ([]byte, error) { return json.Marshal(u.Seconds * 1000) } + +func (u *unitBug) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &u.Seconds) } + +// sealed encodes as its Token until it is decoded; a decoded sealed won't encode again, +// as a type that keeps a handle it can't serialize won't. +type sealed struct { + Token string + decoded bool +} + +func (s sealed) MarshalJSON() ([]byte, error) { + if s.decoded { + return nil, errors.New("a decoded sealed won't encode again") + } + return json.Marshal(s.Token) +} + +func (s *sealed) UnmarshalJSON(b []byte) error { + s.decoded = true + return json.Unmarshal(b, &s.Token) +} + +// Replay decodes the bytes the live settle decoded, so the Assign sees the same value +// live and after Recover even when the decoder's type doesn't survive a round trip +// through its own encoding: an asymmetric MarshalJSON/UnmarshalJSON pair, or a decoded +// value that won't encode again. Journaling the decoded value's encoding instead +// replays the pair's value scaled again, and fails the Handle for the second, after +// its Assign ran. +func TestServiceResult_ReplayDecodesWhatLiveDecoded(t *testing.T) { + t.Run("asymmetric codec", func(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(unitBug{Seconds: 2}, nil), decoderFor("done", durable.DecodeAs[unitBug])) + mustTwo(t, a, runErr, recErr) + if got := a.seen[0].(unitBug); got.Seconds != 2000 { + t.Fatalf("live saw %+v, want the 2000 its own encoding decodes to", got) + } + }) + t.Run("decoded value won't encode", func(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(sealed{Token: "t"}, nil), decoderFor("done", durable.DecodeAs[sealed])) + mustTwo(t, a, runErr, recErr) + }) + t.Run("int64 above 2^53, no decoder", func(t *testing.T) { + a, runErr, recErr := scnRun(t, returning(int64(1<<53+1), nil)) + mustTwo(t, a, runErr, recErr) + if got, ok := a.seen[0].(float64); !ok || got != float64(1<<53) { + t.Fatalf("live saw %s, want the float64 9007199254740992", describe(a.seen[0])) + } + }) +} + +// panicky is a KindedEffect whose MarshalJSON panics. +type panicky struct{} + +func (panicky) Kind() string { return "demo.panicky" } +func (panicky) MarshalJSON() ([]byte, error) { panic("marshal blew up") } + +// panickyKind is a KindedEffect whose Kind panics. +type panickyKind struct{} + +func (panickyKind) Kind() string { panic("kind blew up") } + +func emitting(eff state.Effect) func(state.ActionCtx[svcCtx]) (state.Effect, error) { + return func(state.ActionCtx[svcCtx]) (state.Effect, error) { return eff, nil } +} + +func panickyMachine(eff state.Effect) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("pnk"). + Action("emit", emitting(eff)). + State("idle"). + State("done").Final(). + Initial("idle"). + Transition("idle").On("go").GoTo("done").Do("emit"). + Quench() +} + +// fireCatching fires event on h and returns the panic that escaped, or the error. +func fireCatching(h *durable.Handle[string, string, svcCtx], event string) (panicked any, err error) { + defer func() { panicked = recover() }() + _, err = h.Fire(context.Background(), event) + return nil, err +} + +// A KindedEffect whose MarshalJSON panics is recorded as one whose marshal fails: by id +// and kind, with the panic's message in Meta under crucible.marshalError. The step +// doesn't panic or fail, and Recover replays it. +func TestUnmarshalableEffect_PanickingMarshal_RecordsByIDAndKind(t *testing.T) { + ctx := context.Background() + m := panickyMachine(panicky{}) + store := durable.NewMemStore() + h, err := durable.NewRunner(m, store).Start(ctx, "pnk-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + p, ferr := fireCatching(h, "go") + if p != nil || ferr != nil { + t.Fatalf("Fire = %v, panic %v; want neither", ferr, p) + } + _, recs, err := store.Load(ctx, "pnk-1") + if err != nil || len(recs) != 1 || len(recs[0].Effects) != 1 { + t.Fatalf("Load = %d Records (err %v), want one with one effect", len(recs), err) + } + env := recs[0].Effects[0] + if env.Kind != "demo.panicky" || env.EffectID != "0#0#demo.panicky" || len(env.Payload) != 0 { + t.Fatalf("recorded %+v, want kind demo.panicky, id 0#0#demo.panicky and no payload", env) + } + if msg, _ := env.Meta[metaMarshalError].(string); !strings.Contains(msg, "marshal blew up") { + t.Fatalf("Meta = %v, want the panic's message under %q", env.Meta, metaMarshalError) + } + rh, err := durable.Recover(ctx, m, store, "pnk-1") + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := rh.Instance().Current(); got != "done" { + t.Fatalf("recovered in %q, want done", got) + } +} + +// Pins current behavior: a KindedEffect whose Kind panics has no kind to record it by, +// so it can't take the id-and-kind fallback. The panic leaves the Fire, and the Handle +// is failed first, as for any panic in host code during a step, so the caller can +// Recover. The kernel applied the transition, but nothing was journaled. +func TestUnmarshalableEffect_PanickingKind_FailsTheHandle(t *testing.T) { + ctx := context.Background() + m := panickyMachine(panickyKind{}) + store := durable.NewMemStore() + h, err := durable.NewRunner(m, store).Start(ctx, "pnk-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if p, _ := fireCatching(h, "go"); p == nil || !strings.Contains(fmt.Sprint(p), "kind blew up") { + t.Fatalf("Fire panicked with %v, want the Kind panic", p) + } + if _, err := h.Fire(ctx, "go"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire after the panic = %v, want ErrHandleFailed", err) + } + if got := tailLen(t, store, "pnk-1"); got != 0 { + t.Fatalf("the journal holds %d Records, want none", got) + } +} diff --git a/durable/store.go b/durable/store.go index e647cf34..c9e11668 100644 --- a/durable/store.go +++ b/durable/store.go @@ -118,8 +118,10 @@ type Record struct { // Format is the journal format of the durable version that wrote the Record: zero on a // Record from before this field, and 1 from the version that arms the timers and // services an actor-driven parent Fire's state declares. Replay arms them only for a - // Record of format 1 or later, since an older run took no clock read for them. A - // Store keeps it as it keeps Step. + // Record of format 1 or later, since an older run took no clock read for them. From + // format 2 the live run decoded each service result through WithEventDataDecoder, + // and replay does too; a result in an older Record replays in JSON's generic form, as + // that version replayed it. A Store keeps it as it keeps Step. Format int `json:",omitempty"` } diff --git a/durable/timetravel.go b/durable/timetravel.go index 9645e45d..18d35c6b 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -283,5 +283,5 @@ func newReplayServiceRunner[S comparable, E comparable, C any]( if cfg.serviceReg == nil { return nil } - return state.NewServiceRunner(inst, cfg.serviceReg).WithResultTransform(replayableOutcome(cfg.eventDataDecoder)) + return state.NewServiceRunner(inst, cfg.serviceReg).WithResultTransform(replayableOutcome(cfg.eventDataDecoder, nil)) } From 2041a20b849a97f5509bbd5bb1bd5db4de34dab3 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 17:03:09 -0700 Subject: [PATCH 08/13] fix(durable): refuse a journal written by a newer version Recover, the stateless Runner.Fire, StateAt and Steps now refuse a journal whose checkpoint or any Record records a journal format above this version's, before they restore or replay anything. The check covers a checkpoint with an empty tail, so each later bump of journalFormat protects a rollback. Formats 0, 1 and 2 still load. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 6 + durable/checkpoint.go | 4 +- durable/journal_format_internal_test.go | 41 ++++-- durable/journal_format_newer_test.go | 188 ++++++++++++++++++++++++ durable/runner.go | 28 +++- durable/store.go | 5 +- durable/timetravel.go | 18 ++- 7 files changed, 268 insertions(+), 22 deletions(-) create mode 100644 durable/journal_format_newer_test.go diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index c89beb20..4c21fde5 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -74,6 +74,12 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. entries as a delivery, skipping its event, settle or tick. It also spawns a running actor as new, so a message the actor sent its parent as it started applies twice. Don't downgrade a Store this version has written to. +- **A journal a newer version wrote is refused.** `Recover`, a stateless + `Runner.Fire`, `StateAt` and `Steps` fail, before they restore or replay anything, + on a journal whose checkpoint or any Record records a journal format above this + version's, with an error naming both formats. A version from this one onward refuses + a newer journal, so a rollback across a later format bump fails cleanly instead of + replaying the journal as its own. Versions already built don't refuse one. - **A journal an older version wrote replays as it did.** Its actor entries carry no refusal, so `Recover` still fails on one whose parent Fire was refused, and an output recorded with a delivery's last Fire still replays into that transition. Only steps diff --git a/durable/checkpoint.go b/durable/checkpoint.go index 6c71c37d..4d04b60d 100644 --- a/durable/checkpoint.go +++ b/durable/checkpoint.go @@ -115,7 +115,9 @@ type checkpointEnvelope struct { ParkedServices []parkedServiceWire `json:"parkedServices,omitempty"` // Format is the journal format of the version that wrote the checkpoint, as // Record.Format is. Zero means a checkpoint from before this field. A Record after - // it with a lower Format means the Store lost the field, and replay fails. + // it with a lower Format means the Store lost the field, and replay fails. A Format + // above this version's journalFormat means a newer version wrote it, and every reader + // refuses the journal. Format int `json:"format,omitempty"` } diff --git a/durable/journal_format_internal_test.go b/durable/journal_format_internal_test.go index f2b14772..3745871b 100644 --- a/durable/journal_format_internal_test.go +++ b/durable/journal_format_internal_test.go @@ -5,24 +5,37 @@ import ( "testing" ) -// TestCheckFormat_AcceptsAHigherFormat pins what a downgrade meets. checkFormat, which -// every version since Record.Format has, fails only on a Record of a lower format than -// its checkpoint's, a Store that lost the field. It accepts a Record of a higher -// format, one a newer version wrote, so an older binary replays a newer journal as its -// own format, and no bump of journalFormat can make a binary already built refuse it. -func TestCheckFormat_AcceptsAHigherFormat(t *testing.T) { - for _, tc := range []struct{ checkpoint, rec int }{ - {1, 2}, - {journalFormat, journalFormat + 1}, - {0, journalFormat + 5}, - } { - if err := checkFormat(tc.checkpoint, &Record{Format: tc.rec}); err != nil { - t.Errorf("checkFormat(%d, a Record of format %d) = %v, want nil", tc.checkpoint, tc.rec, err) - } +// TestCheckFormat_RefusesAHigherFormat pins what a downgrade meets. checkFormat fails +// on a Record of a lower format than its checkpoint's, a Store that lost the field, and +// accepts a higher one, as every version since Record.Format has. checkNotNewer, which +// every reader calls first, refuses a checkpoint or a Record of a format above +// journalFormat, one a newer version wrote, so each later bump of journalFormat makes +// this version refuse that version's journal. A binary built before checkNotNewer +// replays a newer journal as its own format. +func TestCheckFormat_RefusesAHigherFormat(t *testing.T) { + if err := checkFormat(1, &Record{Format: 2}); err != nil { + t.Errorf("checkFormat(1, a Record of format 2) = %v, want nil", err) } if err := checkFormat(2, &Record{Format: 1}); err == nil { t.Errorf("checkFormat(2, a Record of format 1) = nil, want the lost-field error") } + for _, tc := range []struct { + checkpoint int + recs []Record + refused bool + }{ + {0, nil, false}, + {journalFormat, nil, false}, + {0, []Record{{Format: 0}, {Format: 1}, {Format: journalFormat}}, false}, + {journalFormat + 1, nil, true}, + {journalFormat, []Record{{Format: journalFormat}, {Format: journalFormat + 1}}, true}, + {0, []Record{{Format: journalFormat + 5}}, true}, + } { + err := checkNotNewer("x", tc.checkpoint, tc.recs) + if (err != nil) != tc.refused { + t.Errorf("checkNotNewer(%d, %+v) = %v, want refused %v", tc.checkpoint, tc.recs, err, tc.refused) + } + } } // TestParkedEntry_AnOlderReaderSettlesIt pins what a version from before parking reads diff --git a/durable/journal_format_newer_test.go b/durable/journal_format_newer_test.go new file mode 100644 index 00000000..6d6e04d0 --- /dev/null +++ b/durable/journal_format_newer_test.go @@ -0,0 +1,188 @@ +package durable_test + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// newerFormat is a journal format above this version's, as a later version would write. +const newerFormat = 9 + +// newerWriter stamps every Record it keeps with older's format, and the checkpoint too +// when checkpoint is set, as the version that wrote that format would. +type newerWriter struct { + older *olderWriter + checkpoint bool + *durable.MemStore +} + +func (s *newerWriter) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + return s.older.Append(ctx, id, rec, opts...) +} + +func (s *newerWriter) Checkpoint(ctx context.Context, id durable.InstanceID, snapshot []byte, throughStep int, opts ...durable.CheckpointOption) error { + if s.checkpoint { + return s.older.Checkpoint(ctx, id, snapshot, throughStep, opts...) + } + return s.MemStore.Checkpoint(ctx, id, snapshot, throughStep, opts...) +} + +func newWriter(format int, checkpoint bool) *newerWriter { + mem := durable.NewMemStore(durable.WithHistory()) + return &newerWriter{older: &olderWriter{MemStore: mem, format: format}, checkpoint: checkpoint, MemStore: mem} +} + +// journalCase is a journal some version wrote: a Start baseline, and, when step is set, +// one service settlement after it. +type journalCase struct { + name string + checkpoint bool // the baseline records format, not this version's + step bool // the journal holds a Record, of format +} + +// newerCases are the journals a newer version could leave: a newer Record after this +// version's baseline, a newer baseline with an empty tail, and both. +var newerCases = []journalCase{ + {"newer Record", false, true}, + {"newer checkpoint, empty tail", true, false}, + {"newer checkpoint and Record", true, true}, +} + +// writeJournal starts scn-1 on a store that writes format as c says, and settles its +// service once when c.step is set. +func writeJournal(t *testing.T, c journalCase, format int, a *scnAssigns) (*state.Machine[string, string, svcCtx], *newerWriter, durable.Option[string, string, svcCtx]) { + t.Helper() + fn := returning(scnUser{ID: 7, Name: "ann"}, nil) + m := scnMachine(fn, a) + store := newWriter(format, c.checkpoint) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + h := scnStart(t, m, store, reg) + if c.step { + mustRunService(t, h, state.InvokeID(scnInvoke, "loading", 0)) + } + return m, store, reg +} + +// mustBeNewer fails unless err is the refusal of a journal of newerFormat, which names +// the instance and both formats and wraps none of the package's sentinels. +func mustBeNewer(t *testing.T, what string, err error) { + t.Helper() + want := fmt.Sprintf("durable: %q was written by a newer version (journal format %d; this version reads up to 2)", "scn-1", newerFormat) + if err == nil || err.Error() != want { + t.Fatalf("%s = %v, want %q", what, err, want) + } + for _, sentinel := range []error{ + durable.ErrInstanceNotFound, durable.ErrStepOutOfOrder, durable.ErrStepConflict, + durable.ErrFenced, durable.ErrCheckpointNotAdvancing, durable.ErrInstanceExists, + durable.ErrEffectDispatch, durable.ErrStepOutOfRange, durable.ErrEventDataEncoding, + durable.ErrRecordTooLarge, durable.ErrHandleFailed, durable.ErrReentrantStep, + } { + if errors.Is(err, sentinel) { + t.Fatalf("%s's refusal matches %v, want no sentinel", what, sentinel) + } + } +} + +// A version refuses a journal a newer version wrote, a checkpoint or a Record of a +// format above its own, rather than replay it as its own format. Recover refuses it +// before it restores or replays anything, so no Assign runs again. +func TestJournalFormat_NewerJournal_RecoverRefuses(t *testing.T) { + for _, c := range newerCases { + t.Run(c.name, func(t *testing.T) { + a := &scnAssigns{} + m, store, reg := writeJournal(t, c, newerFormat, a) + live := a.done + _, err := durable.Recover(context.Background(), m, store, "scn-1", reg) + mustBeNewer(t, "Recover", err) + if a.done != live { + t.Fatalf("onDone's Assign ran %d times, want only live's %d", a.done, live) + } + }) + } +} + +// The stateless Runner.Fire loads the instance as Recover does, so it refuses a newer +// journal too, and appends nothing to it. +func TestJournalFormat_NewerJournal_StatelessFireRefuses(t *testing.T) { + for _, c := range newerCases { + t.Run(c.name, func(t *testing.T) { + m, store, reg := writeJournal(t, c, newerFormat, &scnAssigns{}) + ctx := context.Background() + _, before, err := store.Load(ctx, "scn-1") + if err != nil { + t.Fatal(err) + } + _, err = durable.NewRunner(m, store, reg).Fire(ctx, "scn-1", "done") + mustBeNewer(t, "Runner.Fire", err) + _, after, err := store.Load(ctx, "scn-1") + if err != nil || len(after) != len(before) { + t.Fatalf("the journal holds %d Records (err %v) after the refused Fire, want %d", len(after), err, len(before)) + } + }) + } +} + +// StateAt refuses a newer journal whatever the step, even the baseline, which a newer +// Record after it doesn't reach. +func TestJournalFormat_NewerJournal_StateAtRefuses(t *testing.T) { + for _, c := range newerCases { + t.Run(c.name, func(t *testing.T) { + m, store, reg := writeJournal(t, c, newerFormat, &scnAssigns{}) + steps := []int{durable.BaselineStep} + if c.step { + steps = append(steps, 0) + } + for _, step := range steps { + _, err := durable.StateAt(context.Background(), m, store, "scn-1", step, reg) + mustBeNewer(t, fmt.Sprintf("StateAt(%d)", step), err) + } + }) + } +} + +// Steps refuses a newer journal, as StateAt does. +func TestJournalFormat_NewerJournal_StepsRefuses(t *testing.T) { + for _, c := range newerCases { + t.Run(c.name, func(t *testing.T) { + _, store, _ := writeJournal(t, c, newerFormat, &scnAssigns{}) + _, err := durable.Steps(context.Background(), store, "scn-1") + mustBeNewer(t, "Steps", err) + }) + } +} + +// The ceiling refuses only a newer format: a journal of format 0 (the 0.1.0 release), 1 +// or 2, this version's, still loads on every read path, with a checkpoint and an empty +// tail or a Record after it. +func TestJournalFormat_OlderAndCurrentJournal_Loads(t *testing.T) { + for _, format := range []int{0, 1, 2} { + for _, c := range []journalCase{ + {"checkpoint, empty tail", true, false}, + {"checkpoint and Record", true, true}, + } { + t.Run(fmt.Sprintf("format %d, %s", format, c.name), func(t *testing.T) { + ctx := context.Background() + m, store, reg := writeJournal(t, c, format, &scnAssigns{}) + if _, err := durable.Recover(ctx, m, store, "scn-1", reg); err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err := durable.StateAt(ctx, m, store, "scn-1", durable.BaselineStep, reg); err != nil { + t.Fatalf("StateAt: %v", err) + } + if _, err := durable.Steps(ctx, store, "scn-1"); err != nil { + t.Fatalf("Steps: %v", err) + } + if !c.step { // the instance is still loading, so its onDone event fires + if _, err := durable.NewRunner(m, store, reg).Fire(ctx, "scn-1", "done"); err != nil { + t.Fatalf("Runner.Fire: %v", err) + } + } + }) + } + } +} diff --git a/durable/runner.go b/durable/runner.go index 4a4fe4a7..5e9fc0a3 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -1419,7 +1419,10 @@ func respawn[S comparable, E comparable, C any](ctx context.Context, actors *sta // spawned drove. A ctx that ends once the actors are spawned, with no Fire refused, // doesn't fail it, as for Start: their behaviors already ran. It applies any recorded // effect not yet marked dispatched, so it returns ErrEffectDispatch and no Handle while -// the effect handler keeps failing. +// the effect handler keeps failing. It refuses a journal a newer version of this module +// wrote, whose checkpoint or any Record records a journal format above this version's, +// before it restores or replays anything; that error wraps none of the package's +// sentinel errors. func Recover[S comparable, E comparable, C any](ctx context.Context, m *state.Machine[S, E, C], st Store, id InstanceID, opts ...Option[S, E, C]) (*Handle[S, E, C], error) { r := NewRunner(m, st, opts...) return r.recover(ctx, id) @@ -1442,6 +1445,9 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // deadlines of the timers armed at the checkpoint, and the next step ordinal. A bare // (pre-envelope) checkpoint yields the snapshot with no persisted timers. env := unmarshalCheckpoint(snapBytes) + if err = checkNotNewer(id, env.Format, tail); err != nil { + return nil, err + } kernelSnap, persistedTimers, checkpointNext := env.Snapshot, env.Timers, env.NextStep snap, err := state.UnmarshalSnapshot[S, E, C](kernelSnap) @@ -2073,13 +2079,31 @@ func checkFormat(checkpoint int, rec *Record) error { return nil } +// checkNotNewer fails when id's checkpoint, of journal format checkpoint, or any of its +// records was written in a journal format above journalFormat: by a newer version, whose +// journal this version can't tell from its own, so replaying it could recover wrong. +// Every reader calls it on what it loaded before it restores or replays anything. The +// error wraps no sentinel. +func checkNotNewer(id InstanceID, checkpoint int, records []Record) error { + format := checkpoint + for i := range records { + format = max(format, records[i].Format) + } + if format > journalFormat { + return fmt.Errorf("durable: %q was written by a newer version (journal format %d; this version reads up to %d)", id, format, journalFormat) + } + return nil +} + // armsActorFires reports whether rec was written by a version that arms the timers and // services an actor-driven parent Fire declares, from the journal Format it records. func armsActorFires(rec *Record) bool { return rec.Format >= actorArmsFormat } -// journalFormat is the format this version writes in every Record. It is +// journalFormat is the format this version writes in every Record and checkpoint, and +// the highest it reads: Recover, the stateless Fire, StateAt and Steps refuse a journal +// whose checkpoint or any Record records a higher one (see checkNotNewer). It is // decodedResultsFormat. actorArmsFormat is the first format that arms the timers and // services an actor-driven parent Fire's state declares. decodedResultsFormat is the // first whose service results the live run decoded through WithEventDataDecoder, so diff --git a/durable/store.go b/durable/store.go index c9e11668..b7df6c50 100644 --- a/durable/store.go +++ b/durable/store.go @@ -121,7 +121,10 @@ type Record struct { // Record of format 1 or later, since an older run took no clock read for them. From // format 2 the live run decoded each service result through WithEventDataDecoder, // and replay does too; a result in an older Record replays in JSON's generic form, as - // that version replayed it. A Store keeps it as it keeps Step. + // that version replayed it. A Store keeps it as it keeps Step. A version reads up to + // the format it writes: Recover, StateAt and Steps refuse a journal with a Record or + // checkpoint of a higher format, one a newer version wrote. Versions built before + // that check don't refuse one. Format int `json:",omitempty"` } diff --git a/durable/timetravel.go b/durable/timetravel.go index 18d35c6b..89eb0699 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -102,7 +102,9 @@ func (v *TimeTravelView[S, E, C]) Step() int { return v.step } // // step must be in [BaselineStep, lastRecordedStep]; a step outside that range // reports ErrStepOutOfRange, an unknown instance reports ErrInstanceNotFound, and an -// ended ctx reports its error. +// ended ctx reports its error. A journal a newer version wrote, whose baseline or any +// Record records a journal format above this version's, is refused whatever the step, +// as Recover refuses it. func StateAt[S comparable, E comparable, C any]( ctx context.Context, m *state.Machine[S, E, C], @@ -120,6 +122,10 @@ func StateAt[S comparable, E comparable, C any]( if baseline == nil { return nil, fmt.Errorf("durable: instance %q has no baseline to reconstruct", id) } + format := unmarshalCheckpoint(baseline).Format + if err = checkNotNewer(id, format, records); err != nil { + return nil, err + } last := lastStep(records) if step < BaselineStep || step > last { @@ -138,7 +144,7 @@ func StateAt[S comparable, E comparable, C any]( return nil, fmt.Errorf("durable: reconstructing %q at step %d: %w", id, step, err) } - if err := replayThrough(ctx, inst, &cfg, bounded, unmarshalCheckpoint(baseline).Format); err != nil { + if err := replayThrough(ctx, inst, &cfg, bounded, format); err != nil { return nil, fmt.Errorf("durable: reconstructing %q at step %d: %w", id, step, err) } return &TimeTravelView[S, E, C]{inst: inst, step: step}, nil @@ -148,12 +154,16 @@ func StateAt[S comparable, E comparable, C any]( // can drive StateAt across the run (for example to render an audit timeline). It // reports only externally fired and service/actor steps — the ordinals a StateAt // target may name — and is read-only. An unknown instance reports -// ErrInstanceNotFound. +// ErrInstanceNotFound, and a journal a newer version wrote is refused, as StateAt +// refuses it. func Steps(ctx context.Context, st Store, id InstanceID) ([]int, error) { - _, records, err := loadHistory(ctx, st, id) + baseline, records, err := loadHistory(ctx, st, id) if err != nil { return nil, err } + if err = checkNotNewer(id, unmarshalCheckpoint(baseline).Format, records); err != nil { + return nil, err + } out := make([]int, 0, len(records)) for i := range records { out = append(out, records[i].Step) From 66b24635c64254b9e3cd508aadfcb8080d2e31a0 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 18:02:17 -0700 Subject: [PATCH 09/13] fix(durable): review follow-ups for replayed effects, parked types, mixed versions and stale settles Each finding has a test that fails if it is real, kept whether it proved or disproved it, and each fix was checked by reverting it. - Recover sends what replaying a step produces by the Record's format, not its Effects envelopes: a Fire's always, a tick's or a service settle's from format 2. A Store that drops Record.Effects no longer loses an effect a crash left unsent; an older version's tick and service effects still aren't sent. - Recover loads the dispatched set once, not once per Record with effects: 1000 tick Records go from about 22 ms to 2 ms on MemStore, 25 ms to 4 ms on FileStore. - A DispatchStore error in Dispatched or MarkDispatched wraps ErrEffectDispatch, as a panic there already did; the ErrEffectDispatch doc now covers the dispatch tracking. - The Record that settles a parked result records its type, and replay holds the decoder to it, as the settle did. - A parked result's type name keeps the import path inside struct literal, func, chan and interface types, with struct tags and unexported fields' packages. - A Record of a lower journal format after a newer checkpoint is reported as an older version appending to a newer journal; a Record with no format still blames the Store. The CHANGELOG says not to run mixed versions on one instance. - A settle that finds its activation stopped or armed again records nothing. The step lock keeps durable's own steps from causing it; an internal test drives the ServiceRunner directly. - The JSON a settle decoded is cleared once the step, or Recover's replay, has used it. - Settled services and parked results are updated together through one helper on the Handle and one on the replay mirror. - Pins: a moved or renamed type leaves a parked result parked under a typed decoder, and JSON's generic form still settles it; a ServiceFn can't change its own activation through the Handle. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 44 ++++- durable/dispatch_store_test.go | 125 +++++++++++-- durable/effect.go | 49 +++-- durable/effect_recover_test.go | 8 +- durable/effects_dropped_store_test.go | 160 ++++++++++++++++ durable/errors.go | 10 +- durable/journal_format_internal_test.go | 2 +- durable/journal_format_mixed_test.go | 104 +++++++++++ durable/parked_type_test.go | 227 +++++++++++++++++++++++ durable/recover_dispatched_bench_test.go | 111 +++++++++++ durable/runner.go | 86 +++++---- durable/service.go | 216 +++++++++++++++++---- durable/service_activation_test.go | 52 ++++++ durable/service_encode_internal_test.go | 2 +- durable/service_step_internal_test.go | 180 ++++++++++++++++++ durable/typename_internal_test.go | 95 ++++++++++ 16 files changed, 1367 insertions(+), 104 deletions(-) create mode 100644 durable/effects_dropped_store_test.go create mode 100644 durable/journal_format_mixed_test.go create mode 100644 durable/parked_type_test.go create mode 100644 durable/recover_dispatched_bench_test.go create mode 100644 durable/service_activation_test.go create mode 100644 durable/service_step_internal_test.go create mode 100644 durable/typename_internal_test.go diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 4c21fde5..75c2c491 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -74,6 +74,15 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. entries as a delivery, skipping its event, settle or tick. It also spawns a running actor as new, so a message the actor sent its parent as it started applies twice. Don't downgrade a Store this version has written to. +- **Don't run mixed versions on one instance.** During a rolling deploy, let only + replicas of one version own a given instance: drain an instance's owner before a + replica of another version recovers it. An older replica that appends to a journal + this version wrote stamps a lower journal format after this version's checkpoint, and + `Recover` then fails, on either version, with an error saying an older version appended + to a newer journal; this version can't replay such a Record by the older format's rules, + because the older version reads the journal differently, settling a parked result + through onDone, for one. A version older than this one doesn't refuse this version's + journal at all, and that can't be fixed for binaries already built. - **A journal a newer version wrote is refused.** `Recover`, a stateless `Runner.Fire`, `StateAt` and `Steps` fail, before they restore or replay anything, on a journal whose checkpoint or any Record records a journal format above this @@ -109,9 +118,10 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. effect stays unmarked in the journal, so no later checkpoint compacts it away, and `Recover` dispatches it again, failing with `ErrEffectDispatch` while the handler still does. A handler that panics in `Start` or `Recover` returns `ErrEffectDispatch` too, and - so does a `DispatchStore` whose `Dispatched` or `MarkDispatched` panics, in a step, - `Recover` or the stateless `Runner.Fire`. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that effect - again. The handler's error now matches through `errors.Is`. + so does a `DispatchStore` whose `Dispatched` or `MarkDispatched` panics or returns an + error, in a step, `Recover` or the stateless `Runner.Fire`, with the Store's error in + the chain. A failed `MarkDispatched` fails the Handle too, and `Recover` applies that + effect again. The handler's error now matches through `errors.Is`. - **A `KindedEffect` that can't marshal no longer fails a step.** A step records it by its id and kind with no payload, as it records an effect that isn't a `KindedEffect`, and its `EffectEnvelope.Meta` holds the marshal error's message under @@ -162,8 +172,15 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. `parkedServices` field of its envelope, so compacting the Record that parked one doesn't lose it. A Fire or timer that leaves the state that declares the service stops it and discards the parked result; entering the state again runs the `ServiceFn` for - the new activation. A parked Record or checkpoint is new to this version: an older - `durable` replays a parked Record as a settled result. + the new activation. The Record that settles a parked result records its Go type too, and + replay holds the decoder to it, as that settle did: `Recover` and `StateAt` fail with + `ErrEventDataEncoding`, running no `Assign`, when the decoder returns another type. The + type is the full name, with every named type's import path, also inside a struct + literal, func or chan type, whose fields' tags are part of it. Moving or renaming the + type after a park leaves the result parked under a decoder for its new name, and, once + settled, fails replay under one; a decoder that returns JSON's generic form for that + event settles or replays it. A parked Record or checkpoint is new to this version: an + older `durable` replays a parked Record as a settled result. - **A step that may not have been recorded fails the Handle.** That call and every later one return `ErrHandleFailed`, wrapping the cause. A step that was recorded while another goroutine failed the Handle also returns `ErrHandleFailed`, saying the step was @@ -210,7 +227,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **`Recover` and `StateAt` fail when a Store loses `Record.Format`.** The Start baseline and every checkpoint record the journal format too, and a Record after one with a lower format fails replay, where it used to replay as an older journal's and silently skip - the timers and services an actor-driven Fire armed. + the timers and services an actor-driven Fire armed. The error says the Store didn't + keep `Record.Format` for a Record with no format, and that an older version appended + it, a mixed-version deploy, for one with a lower format. - **A step that touches no actor doesn't recompute the actor sets.** Each append rebuilt the declared actors from every active state; it now reuses the last sets while the configuration, the running actors and the step's spawns are unchanged. @@ -239,9 +258,16 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. at-least-once: a handler failure, a failed baseline write or overlapping `Start` calls re-apply them, so key any dedupe on `EffectInstance` plus the effect id. Effects from a parent transition an actor drives, including during `Fire`, are still not dispatched. - `Recover` re-dispatches a step's effects only when its Record names them, so it - doesn't send effects from an older version's `Tick` and service steps, which never - dispatched them. + `Recover` re-dispatches what replaying a step produces, as it did for a `Fire`: for a + `Tick` or service step, only in a Record of format 2 or later, so it doesn't send + effects from an older version's `Tick` and service steps, which never dispatched + them. It goes by the Record's format, not its `Effects` envelopes, so a Store that + doesn't keep `Record.Effects` loses no effect a crash left unsent. `Recover` loads the + Store's dispatched set once, at the first Record with effects to send, rather than once + per such Record, which made a long tail of tick Records quadratic in the set's size + (1000 tick Records: about 22 ms down to 2 ms on `MemStore`, 25 ms to 4 ms on + `FileStore`). A mark another owner makes during a handoff while `Recover` replays isn't + seen, so key the handler's dedupe on `EffectInstance` plus the id, as for any handoff. - **A Tick, RunService or DeliverToActor whose context ended partway isn't journaled.** A canceled `RunService`, `Tick` or `DeliverToActor` used to record a result the live instance never applied, which recovery then committed, or for an actor, failed on. Each now returns `ErrHandleFailed` diff --git a/durable/dispatch_store_test.go b/durable/dispatch_store_test.go index 12c9e865..ee262769 100644 --- a/durable/dispatch_store_test.go +++ b/durable/dispatch_store_test.go @@ -11,12 +11,14 @@ import ( ) // dispatchProbeStore wraps a MemStore's dispatch tracking: it counts each call to -// Dispatched and MarkDispatched, and panics in either when told to. +// Dispatched and MarkDispatched, and panics in either, or returns an error, when told +// to. type dispatchProbeStore struct { *durable.MemStore mu sync.Mutex dispatched, marks int panicDispatched, panicMk bool + errDispatched, errMk error } func newDispatchProbeStore(st *durable.MemStore) *dispatchProbeStore { @@ -26,22 +28,28 @@ func newDispatchProbeStore(st *durable.MemStore) *dispatchProbeStore { func (s *dispatchProbeStore) Dispatched(ctx context.Context, id durable.InstanceID) (map[string]bool, error) { s.mu.Lock() s.dispatched++ - p := s.panicDispatched + p, e := s.panicDispatched, s.errDispatched s.mu.Unlock() if p { panic("Dispatched exploded") } + if e != nil { + return nil, e + } return s.MemStore.Dispatched(ctx, id) } func (s *dispatchProbeStore) MarkDispatched(ctx context.Context, id durable.InstanceID, ids ...string) error { s.mu.Lock() s.marks++ - p := s.panicMk + p, e := s.panicMk, s.errMk s.mu.Unlock() if p { panic("MarkDispatched exploded") } + if e != nil { + return e + } return s.MemStore.MarkDispatched(ctx, id, ids...) } @@ -139,12 +147,13 @@ func pingMachine() *state.Machine[string, string, *runCtx] { Quench() } -// TestRecover_LoadsTheDispatchedSetOncePerEffectRecord pins what Recover costs the -// Store: it loads the instance's whole dispatched set once for each tail Record that -// names effects, and marks nothing when every effect already landed. main did the same -// for a Fire's Records; this change adds a tick's and a service settle's at the same -// rate. Loading the set once per Recover would make this 1. -func TestRecover_LoadsTheDispatchedSetOncePerEffectRecord(t *testing.T) { +// TestRecover_LoadsTheDispatchedSetOncePerRecover pins what Recover costs the Store: it +// loads the instance's whole dispatched set once, at the first tail Record with +// effects to send, keeps it up to date as it marks, and marks nothing when every effect +// already landed. main loaded it once for each Fire's Record; with a tick's and a +// service settle's Records sending effects too, that made Recover over a long tail +// quadratic in the set's size (see BenchmarkRecover_TickTailDispatchedSet). +func TestRecover_LoadsTheDispatchedSetOncePerRecover(t *testing.T) { ctx := context.Background() const n = 25 st := newDispatchProbeStore(durable.NewMemStore()) @@ -163,10 +172,104 @@ func TestRecover_LoadsTheDispatchedSetOncePerEffectRecord(t *testing.T) { t.Fatalf("Recover: %v", err) } loads, marks := st.counts() - if got := loads - liveLoads; got != n { - t.Fatalf("Recover over %d effect Records loaded the dispatched set %d times, want %d", n, got, n) + if got := loads - liveLoads; got != 1 { + t.Fatalf("Recover over %d effect Records loaded the dispatched set %d times, want once", n, got) } if got := marks - liveMarks; got != 0 { t.Fatalf("Recover marked %d effects, want none: every one landed live", got) } } + +// errDispatchStore is the error a dispatchProbeStore told to fail returns. +var errDispatchStore = errors.New("dispatch table unavailable") + +// TestDispatchStoreError_ReturnsEffectDispatch proves a DispatchStore whose Dispatched +// or MarkDispatched returns an error fails a live step, Recover and the stateless +// Runner.Fire with ErrEffectDispatch wrapping the Store's error, as one that panics +// does: either way the effect isn't marked, and Recover sends it again. A live step's +// error is ErrHandleFailed too. +func TestDispatchStoreError_ReturnsEffectDispatch(t *testing.T) { + ctx := context.Background() + healthy := durable.WithEffectHandler[string, string, *runCtx](newEffectRecorder().handler()) + cases := []struct { + name string + arm func(*dispatchProbeStore) + }{ + {"MarkDispatched", func(s *dispatchProbeStore) { s.errMk = errDispatchStore }}, + {"Dispatched", func(s *dispatchProbeStore) { s.errDispatched = errDispatchStore }}, + } + want := func(t *testing.T, what string, err error) { + t.Helper() + if !errors.Is(err, durable.ErrEffectDispatch) || !errors.Is(err, errDispatchStore) { + t.Fatalf("%s: %v, want ErrEffectDispatch wrapping the Store's error", what, err) + } + } + for _, tc := range cases { + t.Run(tc.name+"/live step", func(t *testing.T) { + st := newDispatchProbeStore(durable.NewMemStore()) + h, err := durable.NewRunner(effectMachine(), st, healthy).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + tc.arm(st) + _, err = h.Fire(ctx, "go") + want(t, "Fire", err) + if !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire: %v, want ErrHandleFailed", err) + } + }) + t.Run(tc.name+"/Recover", func(t *testing.T) { + st := newDispatchProbeStore(unmarkedFireEffect(t)) + tc.arm(st) + _, err := durable.Recover(ctx, effectMachine(), st, "i1", healthy) + want(t, "Recover", err) + }) + t.Run(tc.name+"/Runner.Fire", func(t *testing.T) { + st := newDispatchProbeStore(unmarkedFireEffect(t)) + tc.arm(st) + _, err := durable.NewRunner(effectMachine(), st, healthy).Fire(ctx, "i1", "bill") + want(t, "Runner.Fire", err) + }) + } +} + +// TestRecover_SendsEachUnmarkedEffectOnceWithOneLoad: with the dispatched set loaded once, +// Recover still sends every unmarked effect, across several Records, once each, and +// skips the ones that landed; a second Recover, which reads the marks the first made, +// sends nothing. +func TestRecover_SendsEachUnmarkedEffectOnceWithOneLoad(t *testing.T) { + ctx := context.Background() + st := newDispatchProbeStore(durable.NewMemStore()) + // With no handler the live run records the effects and sends none. + h, err := durable.NewRunner(effectMachine(), st).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for _, ev := range []string{"go", "bill", "finish"} { + if _, err = h.Fire(ctx, ev); err != nil { + t.Fatalf("Fire %s: %v", ev, err) + } + } + if err = st.MarkDispatched(ctx, "i1", "0#0#demo.notify"); err != nil { // step 0 landed + t.Fatalf("MarkDispatched: %v", err) + } + loads, _ := st.counts() + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](healthy.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + if after, _ := st.counts(); after-loads != 1 { + t.Fatalf("Recover loaded the dispatched set %d times, want once", after-loads) + } + want := []string{"1#0#demo.charge", "1#1#demo.notify", "2#0#demo.notify"} + if got := healthy.ids(); len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] || healthy.totalApplies() != 3 { + t.Fatalf("Recover applied %v (%d in all), want %v once each", got, healthy.totalApplies(), want) + } + again := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](again.handler())); err != nil { + t.Fatalf("second Recover: %v", err) + } + if n := again.totalApplies(); n != 0 { + t.Fatalf("a second Recover applied %v", again.ids()) + } +} diff --git a/durable/effect.go b/durable/effect.go index d8269464..ddf4f388 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -67,7 +67,9 @@ import ( // can run it at once. // - Handler error: the effect is left un-marked, and the Handle fails with // ErrHandleFailed wrapping ErrEffectDispatch, so Recover retries it -// (at-least-once until it succeeds; exactly-once thereafter). +// (at-least-once until it succeeds; exactly-once thereafter). A DispatchStore +// whose Dispatched or MarkDispatched fails leaves it un-marked too, and fails the +// same way. // // The dedup set is held by the Store (DispatchStore) so it shares the Record's // durability and survives checkpoint compaction — a delayed redispatch of an @@ -288,18 +290,30 @@ func (r *Runner[S, E, C]) applyEffect(ctx context.Context, id InstanceID, de dis // (step and emission order reproduce exactly) and applies only those not already // marked, so an effect that landed before the crash is skipped and one recorded // but un-dispatched is applied now. This is the recovery half of the exactly-once -// contract. -func (r *Runner[S, E, C]) dispatchReplayEffects(ctx context.Context, id InstanceID, step int, effects []state.Effect) error { - return r.dispatch(ctx, id, dispatchableEffects(step, effects)) +// contract. seen is the dispatched set one Recover loads once, at the first step +// with effects to send. +func (r *Runner[S, E, C]) dispatchReplayEffects(ctx context.Context, id InstanceID, step int, effects []state.Effect, seen *dispatchedSet) error { + return r.dispatchWith(ctx, id, dispatchableEffects(step, effects), seen) } +// dispatchedSet is a Recover's copy of the instance's dispatched set: nil until the +// first step that has effects to send loads it. +type dispatchedSet struct{ ids map[string]bool } + // dispatch is the shared exactly-once apply loop used by the live path and by // recovery replay: it reads the instance's dispatched set, invokes the handler // for each not-yet-dispatched effect, and marks each id as it succeeds. It is a -// no-op without a handler or a DispatchStore. A panic in the Store's Dispatched or -// MarkDispatched becomes an ErrEffectDispatch error, as applyEffect makes one in the -// handler, so a step and Recover fail the same way. -func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dispatchableEffect) (err error) { +// no-op without a handler or a DispatchStore. An error or a panic in the Store's +// Dispatched or MarkDispatched becomes an ErrEffectDispatch error wrapping it, as +// applyEffect makes one in the handler, so a step and Recover fail the same way. +func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dispatchableEffect) error { + return r.dispatchWith(ctx, id, des, nil) +} + +// dispatchWith is dispatch reading the dispatched set from seen, when it isn't nil, +// loading it there the first time. The ids it marks needn't be added: an effect id +// names one step, and a Recover replays each step once, so it never meets one again. +func (r *Runner[S, E, C]) dispatchWith(ctx context.Context, id InstanceID, des []dispatchableEffect, seen *dispatchedSet) (err error) { ds, ok := r.dispatchStore(des) if !ok { return nil @@ -310,9 +324,19 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis } }() - dispatched, err := ds.Dispatched(ctx, id) - if err != nil { - return fmt.Errorf("durable: loading dispatched set for %q: %w", id, err) + var dispatched map[string]bool + if seen != nil && seen.ids != nil { + dispatched = seen.ids + } else { + if dispatched, err = ds.Dispatched(ctx, id); err != nil { + return fmt.Errorf("%w: loading dispatched set for %q: %w", ErrEffectDispatch, id, err) + } + if seen != nil { + seen.ids = dispatched + if seen.ids == nil { + seen.ids = map[string]bool{} + } + } } for _, de := range des { @@ -323,8 +347,9 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis return err } if err := ds.MarkDispatched(ctx, id, de.id); err != nil { - return fmt.Errorf("durable: marking effect %q dispatched for %q: %w", de.id, id, err) + return fmt.Errorf("%w: marking effect %q dispatched for %q: %w", ErrEffectDispatch, de.id, id, err) } + } return nil } diff --git a/durable/effect_recover_test.go b/durable/effect_recover_test.go index d17f8eae..aba5bcaf 100644 --- a/durable/effect_recover_test.go +++ b/durable/effect_recover_test.go @@ -312,8 +312,9 @@ func TestRecover_HandlerPanic_ReturnsEffectDispatch(t *testing.T) { } // olderVersionJournal copies src's journal for id into a fresh MemStore as a version -// before tick and service effects were dispatched would have left it: their Records -// name no effects, and only the Fire effects are marked dispatched. +// before tick and service effects were dispatched would have left it: the checkpoint +// and every Record are of journal format 1, the tick and service Records name no +// effects, and only the Fire effects are marked dispatched. func olderVersionJournal(t *testing.T, src *durable.MemStore, id durable.InstanceID) *durable.MemStore { t.Helper() ctx := context.Background() @@ -322,7 +323,7 @@ func olderVersionJournal(t *testing.T, src *durable.MemStore, id durable.Instanc t.Fatalf("Load: %v", err) } dst := durable.NewMemStore() - if err = dst.Checkpoint(ctx, id, snap, -1); err != nil { + if err = dst.Checkpoint(ctx, id, withCheckpointFormat(t, snap, 1), -1); err != nil { t.Fatalf("Checkpoint: %v", err) } stripped := 0 @@ -339,6 +340,7 @@ func olderVersionJournal(t *testing.T, src *durable.MemStore, id durable.Instanc } } } + rec.Format = 1 if _, err = dst.Append(ctx, id, rec); err != nil { t.Fatalf("Append: %v", err) } diff --git a/durable/effects_dropped_store_test.go b/durable/effects_dropped_store_test.go new file mode 100644 index 00000000..204ee2ce --- /dev/null +++ b/durable/effects_dropped_store_test.go @@ -0,0 +1,160 @@ +package durable_test + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// effectsDroppingStore is a Store that keeps every Record field but Effects, as a +// custom backend with a schema that has no column for the envelopes would. +type effectsDroppingStore struct{ *durable.MemStore } + +func (s effectsDroppingStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + rec.Effects = nil + return s.MemStore.Append(ctx, id, rec, opts...) +} + +// TestRecover_StoreThatDropsEffects_SendsACrashedStepsEffects answers whether +// exactly-once depends on the Store keeping Record.Effects. A step whose dispatch +// failed live, as a crash between the append and the dispatch leaves it, holds its +// effect unmarked. Recover must send it once, under the id the live run stamped, even +// from a Store that didn't keep the envelopes: replay re-derives the effect, and the +// Record's format says this version dispatched it. Before this change a Fire's effects +// were sent this way whatever the Record held. +func TestRecover_StoreThatDropsEffects_SendsACrashedStepsEffects(t *testing.T) { + ctx := context.Background() + t.Run("fire", func(t *testing.T) { + st := effectsDroppingStore{durable.NewMemStore()} + live := &stepIDHandler{failStep: "0"} + h, err := durable.NewRunner(effectMachine(), st, durable.WithEffectHandler[string, string, *runCtx](live.handle)). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire go: %v, want ErrEffectDispatch", err) + } + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](healthy.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 1) + }) + t.Run("tick", func(t *testing.T) { + st := effectsDroppingStore{durable.NewMemStore()} + clk := state.NewFakeClock(epoch) + live := &stepIDHandler{failStep: "1"} + h, err := durable.NewRunner(timerNotifyMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](live.handle), + durable.WithRunnerClock[string, string, *runCtx](clk)). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + clk.Advance(2 * time.Second) + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Tick: %v, want ErrEffectDispatch", err) + } + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, timerNotifyMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](healthy.handler()), + durable.WithRunnerClock[string, string, *runCtx](clk)); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 1) + }) + t.Run("service", func(t *testing.T) { + st := effectsDroppingStore{durable.NewMemStore()} + var calls int64 + fn := counterService(&calls) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + live := &stepIDHandler{failStep: "1"} + h, err := durable.NewRunner(fetchThenWaitMachine(fn), st, reg, durable.WithEffectHandler[string, string, svcCtx](live.handle)). + Start(ctx, "i1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire start: %v", err) + } + if _, _, err = h.RunService(ctx, state.InvokeID("fetchThenWait", "loading", 0)); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("RunService: %v, want ErrEffectDispatch", err) + } + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, fetchThenWaitMachine(fn), st, "i1", reg, + durable.WithEffectHandler[string, string, svcCtx](healthy.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 1) + if calls != 1 { + t.Fatalf("the service ran %d times, want 1", calls) + } + }) +} + +// withCheckpointFormat returns the checkpoint envelope cp with its journal format set to +// format, as the version that wrote format would have left it. +func withCheckpointFormat(t *testing.T, cp []byte, format int) []byte { + t.Helper() + var env map[string]json.RawMessage + if err := json.Unmarshal(cp, &env); err != nil { + t.Fatalf("decoding the checkpoint envelope: %v", err) + } + env["format"] = json.RawMessage(itoa(format)) + out, err := json.Marshal(env) + if err != nil { + t.Fatalf("encoding the checkpoint envelope: %v", err) + } + return out +} + +func itoa(n int) string { + b, _ := json.Marshal(n) + return string(b) +} + +// TestRecover_OlderVersionsFireEffectIsSentWithoutEnvelopes keeps what every version +// did for a Fire: Recover sends its unmarked effects whatever the Record holds. The +// journal here is of format 1, from a version before tick and service effects were +// dispatched, and its Fire Record names no effects. +func TestRecover_OlderVersionsFireEffectIsSentWithoutEnvelopes(t *testing.T) { + ctx := context.Background() + src := durable.NewMemStore() + live := &stepIDHandler{failStep: "0"} + h, err := durable.NewRunner(effectMachine(), src, durable.WithEffectHandler[string, string, *runCtx](live.handle)). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire go: %v, want ErrEffectDispatch", err) + } + snap, recs, err := src.Load(ctx, "i1") + if err != nil { + t.Fatalf("Load: %v", err) + } + old := durable.NewMemStore() + if err = old.Checkpoint(ctx, "i1", withCheckpointFormat(t, snap, 1), -1); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + for _, rec := range recs { + rec.Format, rec.Effects = 1, nil + if _, err = old.Append(ctx, "i1", rec); err != nil { + t.Fatalf("Append: %v", err) + } + } + healthy := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), old, "i1", durable.WithEffectHandler[string, string, *runCtx](healthy.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + assertAppliedOnceAsLive(t, live, healthy, 1) +} diff --git a/durable/errors.go b/durable/errors.go index 62ef87ae..36431757 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -92,11 +92,13 @@ var ( ErrInstanceExists = errors.New(errorPrefix + errorInstanceExists) // ErrEffectDispatch wraps a failure returned by the caller-supplied effect - // handler (WithEffectHandler) while applying an emitted effect, or a panic in it. The effect was + // handler (WithEffectHandler) while applying an emitted effect, or a panic in it, + // and an error or panic in the Store's dispatch tracking (DispatchStore's + // Dispatched or MarkDispatched), with the cause in its chain. The effect was // persisted (write-ahead) but not marked dispatched, so the Handle fails with // ErrHandleFailed wrapping it, and Recover dispatches it again; a failure in - // Runner.Start leaves no instance. Recover fails with this error while the handler - // still does. Callers match this with errors.Is to tell a dispatch failure from a - // kernel transition or persistence error. + // Runner.Start leaves no instance. Recover fails with this error while the handler, + // or the dispatch tracking, still does. Callers match this with errors.Is to tell a + // dispatch failure from a kernel transition or a Record's persistence error. ErrEffectDispatch = errors.New(errorPrefix + errorEffectDispatch) ) diff --git a/durable/journal_format_internal_test.go b/durable/journal_format_internal_test.go index 3745871b..99c8d13f 100644 --- a/durable/journal_format_internal_test.go +++ b/durable/journal_format_internal_test.go @@ -17,7 +17,7 @@ func TestCheckFormat_RefusesAHigherFormat(t *testing.T) { t.Errorf("checkFormat(1, a Record of format 2) = %v, want nil", err) } if err := checkFormat(2, &Record{Format: 1}); err == nil { - t.Errorf("checkFormat(2, a Record of format 1) = nil, want the lost-field error") + t.Errorf("checkFormat(2, a Record of format 1) = nil, want the older-version error") } for _, tc := range []struct { checkpoint int diff --git a/durable/journal_format_mixed_test.go b/durable/journal_format_mixed_test.go new file mode 100644 index 00000000..fc197437 --- /dev/null +++ b/durable/journal_format_mixed_test.go @@ -0,0 +1,104 @@ +package durable_test + +import ( + "context" + "strings" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These tests answer a review finding on a rolling deploy that runs two versions on one +// instance: an older replica appends a Record of its own, lower journal format after +// this version's checkpoint. Recover must refuse the journal, and say that mixed +// versions wrote it rather than blame the Store. + +// TestRecover_OlderVersionAppendedAfterThisVersionsCheckpoint: this version's Start +// writes a format-2 baseline, and an older replica, which stamps format 1, appends the +// next step. Recover refuses, naming the older version, not a Store that lost +// Record.Format. +func TestRecover_OlderVersionAppendedAfterThisVersionsCheckpoint(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + if _, err := durable.NewRunner(effectMachine(), st).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + // The older replica: a Runner of this version, whose Store stamps format 1 on every + // Record it appends, as that version's Runner did. + if _, err := durable.NewRunner(effectMachine(), &olderWriter{MemStore: st, format: 1}).Fire(ctx, "i1", "go"); err != nil { + t.Fatalf("the older replica's Fire: %v", err) + } + _, err := durable.Recover(ctx, effectMachine(), st, "i1") + if err == nil { + t.Fatal("Recover replayed a format-1 Record after a format-2 checkpoint") + } + if msg := err.Error(); !strings.Contains(msg, "older version") || strings.Contains(msg, "didn't keep Record.Format") { + t.Fatalf("Recover: %v, want it to say an older version appended to this version's journal, not blame the Store", err) + } +} + +// TestRecover_FormatlessRecordAfterThisVersionsCheckpoint keeps the diagnosis for a +// Record with no format at all, which a Store that drops the field gives, as does a +// version from before the field: Recover refuses and names both. +func TestRecover_FormatlessRecordAfterThisVersionsCheckpoint(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + if _, err := durable.NewRunner(effectMachine(), st).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + if _, err := durable.NewRunner(effectMachine(), formatDroppingStore{st}).Fire(ctx, "i1", "go"); err != nil { + t.Fatalf("Fire: %v", err) + } + _, err := durable.Recover(ctx, effectMachine(), st, "i1") + if err == nil || !strings.Contains(err.Error(), "didn't keep Record.Format") { + t.Fatalf("Recover: %v, want it to say the Store may not have kept Record.Format", err) + } +} + +// forkMachine invokes fetch in loading. "next" is declared in loading and in ready, +// with different targets, so a journal replayed from the wrong state still replays, +// to the wrong place. +func forkMachine(fn state.ServiceFn[svcCtx]) *state.Machine[string, string, svcCtx] { + return state.ForgeFor[svcCtx]("fork"). + Service("fetch", fn). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready"). + State("canceled").Final(). + State("shipped").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready"). + Transition("loading").On("fail").GoTo("failed"). + Transition("loading").On("next").GoTo("canceled"). + Transition("ready").On("next").GoTo("shipped"). + Quench() +} + +// TestRecover_OlderVersionAfterAParkIsRefused shows why Recover can't replay an older +// replica's Record by its own format's rules. This version parks a result; the older +// replica, which has no parking, reads the parked entry as a settle, so it moves to +// ready, and its "next" ships the order. Replayed here, the park leaves the instance in +// loading, where "next" cancels it: the two versions disagree about what the journal +// means, so Recover must refuse rather than recover canceled. +func TestRecover_OlderVersionAfterAParkIsRefused(t *testing.T) { + ctx := context.Background() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return scnUser{ID: 7, Name: "ann"}, nil } + m := forkMachine(fn) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + st := durable.NewMemStore() + h, err := durable.NewRunner(m, st, reg, failingDecoder()).Start(ctx, "o1", svcCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, ran, err := h.RunService(ctx, state.InvokeID("fork", "loading", 0)) + mustPark(t, "RunService", ran, err) + // The older replica's step: the "next" it fired from ready, in its format. + if _, err = st.Append(ctx, "o1", durable.Record{Step: 1, Event: []byte(`"next"`), Format: 1}); err != nil { + t.Fatalf("Append: %v", err) + } + rh, err := durable.Recover(ctx, m, st, "o1", reg, failingDecoder()) + if err == nil { + t.Fatalf("Recover replayed the older replica's step into %q; the older replica left the order shipped", rh.Instance().Current()) + } +} diff --git a/durable/parked_type_test.go b/durable/parked_type_test.go new file mode 100644 index 00000000..b175f996 --- /dev/null +++ b/durable/parked_type_test.go @@ -0,0 +1,227 @@ +package durable_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync/atomic" + "testing" + + "github.com/stablekernel/crucible/durable" + alpha "github.com/stablekernel/crucible/durable/internal/typefixture/alpha/model" + beta "github.com/stablekernel/crucible/durable/internal/typefixture/beta/model" + "github.com/stablekernel/crucible/state" +) + +// These tests answer two review findings on the Go type a parked result records: what +// a type moved or renamed after the park does to the result, and whether the Record +// that finally settles a parked result keeps the type, so replay holds the decoder to +// it as the live settle did. + +// alphaResultType is the type name a parked alpha.Result records. +const alphaResultType = "github.com/stablekernel/crucible/durable/internal/typefixture/alpha/model.Result" + +// movedFromType is where alpha.Result lived before the move these tests stand in for. +const movedFromType = "github.com/stablekernel/crucible/durable/internal/typefixture/old/model.Result" + +// parkedAlpha parks an alpha.Result for scn-1 under a decoder that refuses it, and +// returns the machine, its Assigns, the registry option and the store. +func parkedAlpha(t *testing.T) (*state.Machine[string, string, svcCtx], *scnAssigns, durable.Option[string, string, svcCtx], *durable.MemStore, *int64) { + t.Helper() + runs := new(int64) + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + atomic.AddInt64(runs, 1) + return alpha.Result{ID: 7, Name: "ann"}, nil + } + a := &scnAssigns{} + m := scnMachine(fn, a) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + store := durable.NewMemStore() + h := scnStart(t, m, store, reg, failingDecoder()) + _, ran, err := h.RunService(context.Background(), state.InvokeID(scnInvoke, "loading", 0)) + mustPark(t, "RunService", ran, err) + return m, a, reg, store, runs +} + +// rewriteServiceTypes copies scn-1's journal into a fresh MemStore with every recorded +// service entry's Type from replaced by to, as if the type had been moved after the +// version that recorded it. +func rewriteServiceTypes(t *testing.T, src *durable.MemStore, from, to string) *durable.MemStore { + t.Helper() + ctx := context.Background() + snap, recs, err := src.Load(ctx, "scn-1") + if err != nil { + t.Fatalf("Load: %v", err) + } + dst := durable.NewMemStore() + if err = dst.Checkpoint(ctx, "scn-1", snap, -1); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + rewrote := 0 + for _, rec := range recs { + for i, e := range rec.Entries { + if e.Kind != state.JournalServiceResult { + continue + } + var p map[string]any + if err = json.Unmarshal(e.Payload, &p); err != nil { + t.Fatalf("decoding a service entry: %v", err) + } + if p["type"] == from { + p["type"] = to + rewrote++ + } + if rec.Entries[i].Payload, err = json.Marshal(p); err != nil { + t.Fatalf("encoding a service entry: %v", err) + } + } + if _, err = dst.Append(ctx, "scn-1", rec); err != nil { + t.Fatalf("Append: %v", err) + } + } + if rewrote == 0 { + t.Fatalf("no service entry recorded the type %s", from) + } + return dst +} + +// serviceTypes returns the Type of each recorded service entry for scn-1, in order. +func serviceTypes(t *testing.T, st durable.Store) []string { + t.Helper() + _, recs, err := st.Load(context.Background(), "scn-1") + if err != nil { + t.Fatalf("Load: %v", err) + } + var out []string + for _, rec := range recs { + for _, e := range rec.Entries { + if e.Kind != state.JournalServiceResult { + continue + } + var p struct { + Type string `json:"type"` + Parked bool `json:"parked"` + } + if err = json.Unmarshal(e.Payload, &p); err != nil { + t.Fatalf("decoding a service entry: %v", err) + } + out = append(out, p.Type) + } + } + return out +} + +// TestParkedResult_MovedType pins what a type moved, or renamed, after a park does to +// its result. The park recorded the old full name, and settling holds the decoder to +// it, so a decoder for the type in its new home is refused on every RunService: the +// result stays parked, and no Assign runs. The way out is a decoder that gives JSON's +// generic form for the event, which a parked settle always accepts, as live does; the +// ServiceFn never runs again either way. After a park the result's original value is +// gone, so nothing tells a moved type from a wrong decoder's same-shaped one (see +// TestParkedResult_TypeCheckKeepsTheImportPath). +func TestParkedResult_MovedType(t *testing.T) { + ctx := context.Background() + m, a, reg, src, runs := parkedAlpha(t) + store := rewriteServiceTypes(t, src, alphaResultType, movedFromType) + id := state.InvokeID(scnInvoke, "loading", 0) + asAlpha := decoderFor("done", durable.DecodeAs[alpha.Result]) + for i := 0; i < 2; i++ { + h, err := durable.Recover(ctx, m, store, "scn-1", reg, asAlpha) + if err != nil { + t.Fatalf("Recover: %v", err) + } + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService with the moved type's decoder", ran, err) + if !strings.Contains(err.Error(), movedFromType) { + t.Fatalf("RunService: %v, want it to name the recorded type %s", err, movedFromType) + } + } + if a.done != 0 || a.fails != 0 { + t.Fatalf("an Assign ran: onDone %d, onError %d", a.done, a.fails) + } + h, err := durable.Recover(ctx, m, store, "scn-1", reg) // the default decoder + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, _, err = h.RunService(ctx, id); err != nil { + t.Fatalf("RunService with the default decoder: %v, want the generic form to settle it", err) + } + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("in %q, want ready", got) + } + if _, ok := a.seen[0].(map[string]any); a.done != 1 || !ok { + t.Fatalf("onDone's Assign ran %d times and saw %v, want once with JSON's generic form", a.done, a.seen) + } + if got := atomic.LoadInt64(runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1", got) + } +} + +// TestParkedResult_SettledRecordKeepsTheType: the Record that settles a parked result +// records the type the park did, and replay holds the decoder to it, as the live settle +// did. A Recover whose decoder gives another type fails with ErrEventDataEncoding and +// runs no Assign, rather than hand onDone's Assign a value of a type the live run never +// let through. +func TestParkedResult_SettledRecordKeepsTheType(t *testing.T) { + ctx := context.Background() + r := newParkRun(t) + fixed := decoderFor("done", durable.DecodeAs[scnUser]) + if _, runErr, err := r.recoverAndRun(fixed); err != nil || runErr != nil { + t.Fatalf("Recover: %v, RunService: %v, with the fixed decoder", err, runErr) + } + userType := "github.com/stablekernel/crucible/durable_test.scnUser" + if got := serviceTypes(t, r.store); len(got) != 2 || got[0] != userType || got[1] != userType { + t.Fatalf("the service entries record the types %q, want the park's and the settle's both %s", got, userType) + } + done := r.a.done + _, err := durable.Recover(ctx, r.m, r.store, "scn-1", r.reg, decoderFor("done", durable.DecodeAs[scnAdmin])) + if !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Recover with a decoder for another type: %v, want ErrEventDataEncoding", err) + } + if r.a.done != done { + t.Fatalf("replay ran onDone's Assign with %s, a type the live settle refuses", describe(r.a.seen[len(r.a.seen)-1])) + } + // The fixed decoder, and JSON's generic form, still replay it. + if _, err = durable.Recover(ctx, r.m, r.store, "scn-1", r.reg, fixed); err != nil { + t.Fatalf("Recover with the fixed decoder: %v", err) + } + if _, err = durable.Recover(ctx, r.m, r.store, "scn-1", r.reg); err != nil { + t.Fatalf("Recover with the default decoder: %v", err) + } +} + +// TestParkedResult_StructLiteralKeepsTheImportPath: a result whose type is a struct +// literal with a field of alpha's model.Result is parked under that full name, so a +// decoder for the literal with beta's model.Result in its place can't settle it, as for +// the named types (see TestParkedResult_TypeCheckKeepsTheImportPath). +func TestParkedResult_StructLiteralKeepsTheImportPath(t *testing.T) { + ctx := context.Background() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + return struct{ R alpha.Result }{alpha.Result{ID: 7, Name: "ann"}}, nil + } + a := &scnAssigns{} + m := scnMachine(fn, a) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + store := durable.NewMemStore() + id := state.InvokeID(scnInvoke, "loading", 0) + h := scnStart(t, m, store, reg, failingDecoder()) + _, ran, err := h.RunService(ctx, id) + mustPark(t, "RunService", ran, err) + rh, err := durable.Recover(ctx, m, store, "scn-1", reg, decoderFor("done", durable.DecodeAs[struct{ R beta.Result }])) + if err != nil { + t.Fatalf("Recover: %v", err) + } + _, ran, err = rh.RunService(ctx, id) + mustPark(t, "RunService with a decoder for beta's literal", ran, err) + if a.done != 0 { + t.Fatalf("onDone's Assign ran with %v", a.seen) + } + rh, err = durable.Recover(ctx, m, store, "scn-1", reg, decoderFor("done", durable.DecodeAs[struct{ R alpha.Result }])) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, _, err = rh.RunService(ctx, id); err != nil { + t.Fatalf("RunService with a decoder for alpha's literal: %v", err) + } +} diff --git a/durable/recover_dispatched_bench_test.go b/durable/recover_dispatched_bench_test.go new file mode 100644 index 00000000..29da241b --- /dev/null +++ b/durable/recover_dispatched_bench_test.go @@ -0,0 +1,111 @@ +package durable_test + +import ( + "context" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// tickLoopMachine goes from idle to ping on "go", then moves between ping and pong on a +// one-second timer forever, emitting a notify each time, so every Tick records one +// Record that names one effect. +func tickLoopMachine() *state.Machine[string, string, *runCtx] { + return state.ForgeFor[*runCtx]("tickLoop"). + Action("notify", func(state.ActionCtx[*runCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("idle"). + State("ping"). + State("pong"). + Initial("idle"). + Transition("idle").On("go").GoTo("ping"). + Transition("ping").After(time.Second).GoTo("pong").Do("notify"). + Transition("pong").After(time.Second).GoTo("ping").Do("notify"). + Quench() +} + +// tickTail starts i1 on st and ticks it n times, each effect dispatched live, and returns +// the clock and the options Recover needs. +func tickTail(tb testing.TB, st durable.Store, n int) []durable.Option[string, string, *runCtx] { + tb.Helper() + ctx := context.Background() + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{ + durable.WithEffectHandler[string, string, *runCtx](func(context.Context, string, state.Effect) error { return nil }), + durable.WithRunnerClock[string, string, *runCtx](clk), + } + h, err := durable.NewRunner(tickLoopMachine(), st, opts...).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + tb.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "go"); err != nil { + tb.Fatalf("Fire go: %v", err) + } + for i := 0; i < n; i++ { + clk.Advance(time.Second) + if _, err = h.Tick(ctx); err != nil { + tb.Fatalf("Tick %d: %v", i, err) + } + } + return opts +} + +// countingDispatchStore counts a Store's Dispatched and MarkDispatched calls. +type countingDispatchStore struct { + durable.Store + ds durable.DispatchStore + dispatched, marks int +} + +func (s *countingDispatchStore) Dispatched(ctx context.Context, id durable.InstanceID) (map[string]bool, error) { + s.dispatched++ + return s.ds.Dispatched(ctx, id) +} + +func (s *countingDispatchStore) MarkDispatched(ctx context.Context, id durable.InstanceID, ids ...string) error { + s.marks++ + return s.ds.MarkDispatched(ctx, id, ids...) +} + +// BenchmarkRecover_TickTailDispatchedSet measures Recover over a tail of 1000 tick +// Records, each naming one effect that already landed, on MemStore and FileStore, and +// reports the Dispatched calls one Recover makes. +// +// Run with: go test -bench=RecoverTickTail -run=^$ . +func BenchmarkRecover_TickTailDispatchedSet(b *testing.B) { + const n = 1000 + ctx := context.Background() + stores := map[string]func(b *testing.B) (durable.Store, durable.DispatchStore){ + "MemStore": func(*testing.B) (durable.Store, durable.DispatchStore) { + st := durable.NewMemStore() + return st, st + }, + "FileStore": func(b *testing.B) (durable.Store, durable.DispatchStore) { + st, err := durable.NewFileStore(b.TempDir()) + if err != nil { + b.Fatalf("NewFileStore: %v", err) + } + return st, st + }, + } + for _, name := range []string{"MemStore", "FileStore"} { + b.Run(name, func(b *testing.B) { + st, ds := stores[name](b) + cs := &countingDispatchStore{Store: st, ds: ds} + opts := tickTail(b, cs, n) + cs.dispatched, cs.marks = 0, 0 + b.ResetTimer() + for range b.N { + if _, err := durable.Recover(ctx, tickLoopMachine(), cs, "i1", opts...); err != nil { + b.Fatalf("Recover: %v", err) + } + } + b.StopTimer() + b.ReportMetric(float64(cs.dispatched)/float64(b.N), "Dispatched/op") + b.ReportMetric(float64(cs.marks)/float64(b.N), "MarkDispatched/op") + }) + } +} diff --git a/durable/runner.go b/durable/runner.go index 5e9fc0a3..20c5410d 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -137,7 +137,8 @@ type Handle[S comparable, E comparable, C any] struct { svc *state.ServiceRunner[S, E, C] // host driver for invoked services, nil when none wired svcBuf *[]state.JournalEntry // recording service-result accumulator, flushed per step - // svcJSON holds the JSON the last successful settle decoded, which the step journals. + // svcJSON holds the JSON a successful settle decoded, which the step takes and + // journals; it holds nothing between steps. svcJSON *settledJSON // settled holds the services that settled in the current activation of the state // that declares them, so a checkpoint records them and Recover doesn't start them. @@ -358,8 +359,7 @@ func (h *Handle[S, E, C]) drainClock() []state.JournalEntry { // effects is a no-op when that driver is not wired or the effects name none of its // kind. func (h *Handle[S, E, C]) absorbDrivers(ctx context.Context, effects []state.Effect) { - forgetSettled(h.settled, effects) - forgetSettled(h.parked, effects) + h.forgetServices(effects) // Arm or cancel timers only when the effects actually schedule or cancel one: the // kernel Scheduler reads the clock unconditionally on Absorb, so skipping the call // for a timer-free transition keeps a non-timer machine free of clock reads. @@ -382,8 +382,7 @@ func (h *Handle[S, E, C]) absorbDrivers(ctx context.Context, effects []state.Eff // does. Replay arms them the same way, in the same order, so the clock reads match. ctx // is the context of the step that drove the system. func (h *Handle[S, E, C]) absorbParentFire(ctx context.Context, effects []state.Effect) { - forgetSettled(h.settled, effects) - forgetSettled(h.parked, effects) + h.forgetServices(effects) if hasTimerEffect(effects) { h.absorbTimers(ctx, effects) } @@ -450,10 +449,18 @@ func dropUnscheduled[S comparable, E comparable, C any](timers map[string]pendin } } +// forgetServices drops from the settled services and the parked results each service +// effects start or stop (see forgetSettled). +func (h *Handle[S, E, C]) forgetServices(effects []state.Effect) { + forgetSettled(h.settled, effects) + forgetSettled(h.parked, effects) +} + // forgetSettled removes from settled each service effects start or stop: a state that // declares it was entered or exited, so the service is new or gone. It forgets parked // results the same way: a stopped service's parked result is discarded, and a service -// started again runs its ServiceFn. +// started again runs its ServiceFn. The Handle and the replay mirror call it for both +// maps at once, through forgetServices and armMirror.forget. func forgetSettled[V any](settled map[string]V, effects []state.Effect) { if len(settled) == 0 { return @@ -1071,8 +1078,7 @@ func (h *Handle[S, E, C]) tickStep(ctx context.Context) ([]state.FireResult[S], h.timers = map[string]pendingTimer{} } mirrorScheduled(h.timers, h.sched, results[i].Effects) - forgetSettled(h.settled, results[i].Effects) - forgetSettled(h.parked, results[i].Effects) + h.forgetServices(results[i].Effects) } if len(results) > 0 { // Drop from the table the timers the tick fired, the ones the scheduler no longer @@ -1517,6 +1523,8 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // and hadn't ended, recorded alongside ended. Nil means none was recorded. unstarted := env.UnstartedActors + // seen is the dispatched set, loaded once, at the first Record with effects to send. + seen := &dispatchedSet{} for i := range tail { // A canceled context can stop a replayed settle or tick with no error, leaving // state short of the journal, so check it before each Record. @@ -1544,11 +1552,12 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // deduped by the Store's dispatched set: an effect already marked (it landed // before the crash) is skipped; one recorded-but-un-marked (the crash fell between // append and dispatch) is applied now — exactly-once across the crash boundary. - // A Record that names no effects owes none: a tick's or a settlement's from a - // version that didn't dispatch them never applied them, so sending them now - // would apply effects that run never did. - if len(rec.Effects) > 0 { - if err := r.dispatchReplayEffects(ctx, id, rec.Step, stepEffects); err != nil { + // Replay re-derives the effects, so this doesn't depend on the Store keeping + // Record.Effects. A tick's or a settlement's Record from a version that didn't + // dispatch them, of a format before dispatchedStepsFormat, owes none: that run + // never applied them, so sending them now would apply effects it never did. + if sendsReplayedEffects(rec) { + if err := r.dispatchReplayEffects(ctx, id, rec.Step, stepEffects, seen); err != nil { return nil, err } } @@ -1560,6 +1569,9 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S return nil, fmt.Errorf("durable: recovering %q: %w", id, err) } + // The replayed settles left the JSON the last one decoded, which no step journals. + svcJSON.take() + // Drain any clock readings the replay re-recorded after the cursor exhausted // (none in the common case where replay consumes exactly the recorded // readings): they belong to no new step and must not leak into the next Fire. @@ -2022,7 +2034,6 @@ func replayRecord[S comparable, E comparable, C any]( if err != nil { return stepEffects, fmt.Errorf("at step %d: %w", rec.Step, err) } - mirror.unpark(entry.CorrelationID) mirror.settle(entry.CorrelationID) // The settle armed its own services; arm its timers, as RunService did. replayArm(ctx, sched, nil, effects, mirror) @@ -2070,13 +2081,20 @@ func replayActors[S comparable, E comparable, C any]( } // checkFormat fails when rec's journal Format is lower than the checkpoint's that replay -// started from: a version doesn't write an older format than it did, so the Store lost -// the field, and replaying the Record as an older format's would recover wrong. +// started from. A version doesn't write an older format than it did, so either an older +// version appended rec to a journal a newer one wrote, which a rolling deploy that runs +// mixed versions on one instance does, or, for a Record with no format, the Store lost +// the field. Either way the two versions may read the journal differently (an older one +// settles a parked result, for one), so replaying rec by its own format's rules would +// recover wrong. func checkFormat(checkpoint int, rec *Record) error { - if rec.Format < checkpoint { - return fmt.Errorf("durable: step %d records journal format %d after a checkpoint of format %d: the Store didn't keep Record.Format", rec.Step, rec.Format, checkpoint) + switch { + case rec.Format >= checkpoint: + return nil + case rec.Format == 0: + return fmt.Errorf("durable: step %d records no journal format after a checkpoint of format %d: the Store didn't keep Record.Format, or a version from before that field appended the Record (mixed versions on one instance)", rec.Step, checkpoint) } - return nil + return fmt.Errorf("durable: step %d was appended by an older version, of journal format %d, after a checkpoint of format %d: don't run mixed versions on one instance", rec.Step, rec.Format, checkpoint) } // checkNotNewer fails when id's checkpoint, of journal format checkpoint, or any of its @@ -2095,6 +2113,14 @@ func checkNotNewer(id InstanceID, checkpoint int, records []Record) error { return nil } +// sendsReplayedEffects reports whether Recover sends the domain effects replaying rec +// produces: every version dispatched a Fire's, and a tick's or a service settle's since +// dispatchedStepsFormat. It reads the Record's format, not its effect envelopes, so a +// Store that doesn't keep Record.Effects loses no effect. +func sendsReplayedEffects(rec *Record) bool { + return len(rec.Event) > 0 || rec.Format >= dispatchedStepsFormat +} + // armsActorFires reports whether rec was written by a version that arms the timers and // services an actor-driven parent Fire declares, from the journal Format it records. func armsActorFires(rec *Record) bool { @@ -2109,11 +2135,13 @@ func armsActorFires(rec *Record) bool { // first whose service results the live run decoded through WithEventDataDecoder, so // replay decodes them the same way; it is also the first that can park a result. A // result in a Record of an older format replays as that version replayed it, in JSON's -// generic form. +// generic form. dispatchedStepsFormat is the first that dispatches a tick's and a +// service settle's domain effects, so Recover sends them. const ( - actorArmsFormat = 1 - decodedResultsFormat = 2 - journalFormat = decodedResultsFormat + actorArmsFormat = 1 + decodedResultsFormat = 2 + dispatchedStepsFormat = 2 + journalFormat = decodedResultsFormat ) // replayArm arms the timers and services a replayed Fire's effects declare, as the live @@ -2153,15 +2181,8 @@ func (m *armMirror) park(id string, r recordedResult) { m.parked[id] = r } -// unpark notes that a replayed settle settled service id, so no result of it is parked. -func (m *armMirror) unpark(id string) { - if m == nil { - return - } - delete(m.parked, id) -} - -// settle notes that a replayed settle settled the service id. +// settle notes that a replayed settle settled the service id, so no result of it is +// parked, as Handle.markSettled does live. func (m *armMirror) settle(id string) { if m == nil { return @@ -2170,6 +2191,7 @@ func (m *armMirror) settle(id string) { m.settled = map[string]bool{} } m.settled[id] = true + delete(m.parked, id) } // forget drops from the settled services and the parked results each one a replayed diff --git a/durable/service.go b/durable/service.go index 44180b16..08f94dd0 100644 --- a/durable/service.go +++ b/durable/service.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "reflect" + "strconv" + "strings" "github.com/stablekernel/crucible/state" ) @@ -75,7 +77,9 @@ type serviceResultPayload struct { Parked bool `json:"parked,omitempty"` // Type is a parked result's Go type, with its import path (see typeName), so the // RunService that settles it holds the decoder to the type the ServiceFn returned, - // as the live settle did. + // as the live settle did. The Record that settles a parked result records it too, + // so replay holds the decoder to it as that settle did. It is empty for a result + // that settled when it ran, whose type the live settle checked against the value. Type string `json:"type,omitempty"` } @@ -198,17 +202,21 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir // A parked result settles in place of a run: the ServiceFn ran once, and the journal // holds what it returned. parked, isParked := h.parked[id] - var results []state.FireResult[S] + var res state.FireResult[S] + var routed bool if isParked { - res, _ := h.svc.SettleDone(h.stepContext(ctx), id, parked) - results = []state.FireResult[S]{res} + res, routed = h.svc.SettleDone(h.stepContext(ctx), id, parked) } else { - results = h.svc.Tick(h.stepContext(ctx), id) - } - if len(results) == 0 { - return state.FireResult[S]{}, false, nil, nil, nil + results := h.svc.Tick(h.stepContext(ctx), id) + if len(results) == 0 { + return state.FireResult[S]{}, false, nil, nil, nil + } + res = results[0] + routed = !settledNothing(res) } - res := results[0] + // The JSON this settle decoded is the step's alone: take it, so no later step can + // journal it and the Handle doesn't hold it. + raw := h.svcJSON.take() if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { // The result didn't decode for its onDone event, so the transform aborted the // settle: nothing fired and no Assign ran. Park it, unless a context that ended @@ -221,10 +229,17 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir } return h.parkResult(ctx, id, derr, isParked) } + if !routed { + // The activation the ServiceFn ran for was stopped, or stopped and armed again, + // before the settle: nothing fired, so there is nothing to record, and a new + // activation stays in flight. The step lock keeps any other step from doing that + // while this one runs, so only code that drives the ServiceRunner past the Handle + // can. + return state.FireResult[S]{}, true, nil, nil, nil + } // The service settled: unless the settle's own Fire leaves or re-enters the state // that declares it, which the absorb below notes, a checkpoint records it as settled. - h.noteSettled(id) - delete(h.parked, id) + h.markSettled(id) // The settle re-fired the invocation's onDone / onError, which may enter a state // that arms a timer, starts another service, or spawns an actor. Absorb those @@ -242,7 +257,7 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return res, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, err)) } - entry, err := recordServiceOutcome(id, h.svcJSON.raw, h.svc.LastError()) + entry, err := recordServiceOutcome(id, raw, parked.typ, h.svc.LastError()) if err != nil { return res, true, nil, nil, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) } @@ -304,12 +319,24 @@ func (h *Handle[S, E, C]) parkResult(ctx context.Context, id string, derr *undec return state.FireResult[S]{}, true, rec, nil, parkErr } -// noteSettled records that the service id settled in its state's current activation. -func (h *Handle[S, E, C]) noteSettled(id string) { +// settledNothing reports whether res is the one zero FireResult ServiceRunner.Tick +// returns for an activation stopped, or stopped and armed again, while its ServiceFn ran. +// A settle that fired returns its Fire's result, whose Trace names the event and the +// state it fired in, or an error, as fireStep relies on in journaling the Trace's +// event. +func settledNothing[S comparable](res state.FireResult[S]) bool { + return res.Err == nil && res.Effects == nil && res.Trace.Event == "" && res.Trace.FromState == "" +} + +// markSettled records that the service id settled in its state's current activation, so +// no result of it is parked. It and forgetServices are the only writers of settled +// alongside parked, so the two can't disagree. +func (h *Handle[S, E, C]) markSettled(id string) { if h.settled == nil { h.settled = map[string]bool{} } h.settled[id] = true + delete(h.parked, id) } // replayableOutcome returns the transform a ServiceRunner applies to every outcome as @@ -358,14 +385,14 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E], settled *settle } value, derr := decodePayload(eventPayload{data: b}, event, decode) if derr != nil { - return nil, nil, &undecodableResult{raw: b, typ: parkedType(recorded, replay, result), err: fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr)} + derr = fmt.Errorf("%w: service result won't decode for event %v: %w", ErrEventDataEncoding, event, derr) + } else if !replay { + derr = decodedAsPassed(value, result, "service result", true) + } else { + derr = decodedAsRecorded(value, recorded.typ) } - if !replay { - if terr := decodedAsPassed(value, result, "service result", true); terr != nil { - return nil, nil, &undecodableResult{raw: b, typ: typeName(result), err: terr} - } - } else if terr := decodedAsRecorded(value, recorded.typ); terr != nil { - return nil, nil, &undecodableResult{raw: b, typ: recorded.typ, err: terr} + if derr != nil { + return nil, nil, &undecodableResult{raw: b, typ: parkedType(recorded, replay, result), err: derr} } settled.set(b) return value, nil, nil @@ -374,7 +401,8 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E], settled *settle // settledJSON holds the JSON the last successful settle decoded, for serviceStep to // journal. The transform sets it during the settle, which runs under the Handle's step -// lock live and on replay's one goroutine. A nil settledJSON keeps nothing. +// lock live and on replay's one goroutine, and serviceStep takes it once the settle +// returns, so it holds nothing between steps. A nil settledJSON keeps nothing. type settledJSON struct{ raw json.RawMessage } func (s *settledJSON) set(b []byte) { @@ -383,6 +411,16 @@ func (s *settledJSON) set(b []byte) { } } +// take returns the JSON the last settle left and clears it. +func (s *settledJSON) take() json.RawMessage { + if s == nil { + return nil + } + b := s.raw + s.raw = nil + return b +} + // legacyResult is a service result recorded in a Record of a format before // decodedResultsFormat, decoded as that version replayed it: into an any, with no // decoder or type check. The transform settles it as it is. @@ -402,8 +440,9 @@ func parkedType(recorded recordedResult, replay bool, result any) string { // RunService settles for a parked result. type recordedResult struct { raw json.RawMessage - // typ is a parked result's Go type, as typeName gives it; empty for a settled one, - // whose type the live settle already checked. + // typ is a parked result's Go type, as typeName gives it, also when replay settles + // it; empty for a result that settled when it ran, whose type the live settle + // checked against the value. typ string } @@ -418,9 +457,11 @@ func typeName(v any) string { return fullTypeName(t) } -// fullTypeName spells t with each named type's import path. A pointer, slice, array or -// map is spelled from its element and key types; any other unnamed type, such as a -// struct literal, as reflect spells it. +// fullTypeName spells t as reflect does, but with each named type's import path in +// place of its package name, so two packages' model.Result differ wherever they +// appear: in a pointer, slice, array, map or chan, a func's parameters and results, an +// interface's methods, and a struct literal's fields, whose tags and unexported names' +// packages are part of the type too. func fullTypeName(t reflect.Type) string { if name := t.Name(); name != "" { if pkg := t.PkgPath(); pkg != "" { @@ -437,10 +478,121 @@ func fullTypeName(t reflect.Type) string { return fmt.Sprintf("[%d]%s", t.Len(), fullTypeName(t.Elem())) case reflect.Map: return "map[" + fullTypeName(t.Key()) + "]" + fullTypeName(t.Elem()) + case reflect.Chan: + return chanTypeName(t) + case reflect.Func: + return "func" + signatureName(t) + case reflect.Struct: + return structTypeName(t) + case reflect.Interface: + return interfaceTypeName(t) } return t.String() } +// chanTypeName spells a chan type t, parenthesizing a receive-only element of a +// bidirectional chan, as reflect does. +func chanTypeName(t reflect.Type) string { + elem := fullTypeName(t.Elem()) + switch t.ChanDir() { + case reflect.RecvDir: + return "<-chan " + elem + case reflect.SendDir: + return "chan<- " + elem + } + if t.Elem().Kind() == reflect.Chan && t.Elem().Name() == "" && t.Elem().ChanDir() == reflect.RecvDir { + return "chan (" + elem + ")" + } + return "chan " + elem +} + +// signatureName spells a func type t's parameters and results, without "func". +func signatureName(t reflect.Type) string { + var b strings.Builder + b.WriteByte('(') + for i := range t.NumIn() { + if i > 0 { + b.WriteString(", ") + } + if t.IsVariadic() && i == t.NumIn()-1 { + b.WriteString("..." + fullTypeName(t.In(i).Elem())) + continue + } + b.WriteString(fullTypeName(t.In(i))) + } + b.WriteByte(')') + switch n := t.NumOut(); n { + case 0: + case 1: + b.WriteString(" " + fullTypeName(t.Out(0))) + default: + b.WriteString(" (") + for i := range n { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(fullTypeName(t.Out(i))) + } + b.WriteByte(')') + } + return b.String() +} + +// structTypeName spells a struct literal type t field by field: an unexported field's +// name with its package's import path, an embedded field by its type alone, and each +// tag quoted, as reflect does. +func structTypeName(t reflect.Type) string { + if t.NumField() == 0 { + return "struct {}" + } + var b strings.Builder + b.WriteString("struct {") + for i := range t.NumField() { + if i > 0 { + b.WriteByte(';') + } + f := t.Field(i) + b.WriteByte(' ') + if !f.Anonymous { + b.WriteString(qualifiedName(f.PkgPath, f.Name) + " ") + } + b.WriteString(fullTypeName(f.Type)) + if f.Tag != "" { + b.WriteString(" " + strconv.Quote(string(f.Tag))) + } + } + b.WriteString(" }") + return b.String() +} + +// interfaceTypeName spells an interface literal type t method by method, an unexported +// method's name with its package's import path. +func interfaceTypeName(t reflect.Type) string { + if t.NumMethod() == 0 { + return "interface {}" + } + var b strings.Builder + b.WriteString("interface {") + for i := range t.NumMethod() { + if i > 0 { + b.WriteByte(';') + } + m := t.Method(i) + b.WriteString(" " + qualifiedName(m.PkgPath, m.Name) + signatureName(m.Type)) + } + b.WriteString(" }") + return b.String() +} + +// qualifiedName prefixes an unexported name with the import path of its package, pkg, +// which is empty for an exported one. +func qualifiedName(pkg, name string) string { + if pkg == "" { + return name + } + return pkg + "." + name +} + // decodedAsRecorded is decodedAsPassed for a parked result, which records its type's // name rather than a value: it reports, as an ErrEventDataEncoding error, a value the // decoder returned as another type than typ. As live, nil and JSON's generic form are @@ -499,14 +651,15 @@ func decodedAsPassed(value, passed any, what string, generic bool) error { // recordServiceOutcome builds the JournalServiceResult entry for a just-settled // service: the error lastErr, or, for a success, raw, the JSON the settle decoded (see -// settledJSON), so replay decodes the bytes live decoded. -func recordServiceOutcome(id string, raw json.RawMessage, lastErr error) (state.JournalEntry, error) { +// settledJSON), so replay decodes the bytes live decoded, and typ, the Go type a parked +// result recorded, empty for one that settled when it ran. +func recordServiceOutcome(id string, raw json.RawMessage, typ string, lastErr error) (state.JournalEntry, error) { var sp serviceResultPayload if lastErr != nil { msg := lastErr.Error() sp.Error = &msg } else { - sp.Result = raw + sp.Result, sp.Type = raw, typ } payload, err := json.Marshal(sp) if err != nil { @@ -555,7 +708,8 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * } result = legacyResult{value: v} case len(sp.Result) > 0: - result = recordedResult{raw: sp.Result} + // A settled parked result's type holds the decoder to it, as the settle did. + result = recordedResult{raw: sp.Result, typ: sp.Type} } res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { diff --git a/durable/service_activation_test.go b/durable/service_activation_test.go new file mode 100644 index 00000000..e4d52ad2 --- /dev/null +++ b/durable/service_activation_test.go @@ -0,0 +1,52 @@ +package durable_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// TestRunService_ServiceFnCantChangeItsActivation pins why a settle can't find its +// activation gone through durable's API: a ServiceFn runs under RunService's step lock, +// so a Fire it makes to leave the state is refused, with ErrReentrantStep on the step's +// context and with its own context's error on any other, and the settle routes as +// usual. Only code that drives the ServiceRunner past the Handle can stop the activation +// (see TestServiceStep_StaleSettleRecordsNothing). +func TestRunService_ServiceFnCantChangeItsActivation(t *testing.T) { + ctx := context.Background() + var h *durable.Handle[string, string, svcCtx] + var reentrant, other error + fn := func(sctx context.Context, _ state.ServiceCtx[svcCtx]) (any, error) { + _, reentrant = h.Fire(sctx, "next") + octx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + _, other = h.Fire(octx, "next") + return scnUser{ID: 7, Name: "ann"}, nil + } + m := forkMachine(fn) + st := durable.NewMemStore() + var err error + if h, err = durable.NewRunner(m, st, durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))). + Start(ctx, "o1", svcCtx{}, state.WithInitialState("loading")); err != nil { + t.Fatalf("Start: %v", err) + } + if _, ran, rerr := h.RunService(ctx, state.InvokeID("fork", "loading", 0)); !ran || rerr != nil { + t.Fatalf("RunService = %v, %v", ran, rerr) + } + if !errors.Is(reentrant, durable.ErrReentrantStep) { + t.Fatalf("a Fire on the step's context: %v, want ErrReentrantStep", reentrant) + } + if !errors.Is(other, context.DeadlineExceeded) { + t.Fatalf("a Fire on another context: %v, want its deadline", other) + } + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("in %q, want ready: the settle routed", got) + } + if got := tailLen(t, st, "o1"); got != 1 { + t.Fatalf("the journal holds %d Records, want the settle's", got) + } +} diff --git a/durable/service_encode_internal_test.go b/durable/service_encode_internal_test.go index 394aeef6..6b45fd99 100644 --- a/durable/service_encode_internal_test.go +++ b/durable/service_encode_internal_test.go @@ -47,7 +47,7 @@ func TestServiceResult_JournalBytesAreWhatLiveDecoded(t *testing.T) { if outErr != nil || abort != nil { t.Fatalf("the transform returned %v, %v", outErr, abort) } - entry, err := recordServiceOutcome("svc", settled.raw, nil) + entry, err := recordServiceOutcome("svc", settled.raw, "", nil) if err != nil { t.Fatalf("recordServiceOutcome: %v", err) } diff --git a/durable/service_step_internal_test.go b/durable/service_step_internal_test.go new file mode 100644 index 00000000..89edf651 --- /dev/null +++ b/durable/service_step_internal_test.go @@ -0,0 +1,180 @@ +package durable + +import ( + "context" + "testing" + + "github.com/stablekernel/crucible/state" +) + +// These tests answer review findings on serviceStep's bookkeeping: a settle that finds +// its activation gone records nothing, the JSON a settle decoded isn't held past its +// step, and the settled services and parked results are forgotten together. + +type stepCtx struct{} + +// stepMachine invokes fetch in loading; onDone "done" goes to ready, onError "fail" +// to failed. +func stepMachine(fn state.ServiceFn[stepCtx]) *state.Machine[string, string, stepCtx] { + return state.ForgeFor[stepCtx]("step"). + Service("fetch", fn). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready"). + Transition("loading").On("fail").GoTo("failed"). + Quench() +} + +func stepOptions(fn state.ServiceFn[stepCtx]) []Option[string, string, stepCtx] { + return []Option[string, string, stepCtx]{WithServiceRegistry[string, string, stepCtx](state.NewRegistry[stepCtx]().Service("fetch", fn))} +} + +// TestServiceStep_StaleSettleRecordsNothing: when the activation a ServiceFn runs for is +// stopped, or stopped and armed again, before its settle, ServiceRunner.Tick fires +// nothing and returns one zero FireResult. serviceStep must record nothing for it: no +// JournalServiceResult, no settled service, and no JSON left behind, so a Recover still +// finds the service in flight. The step lock keeps durable's own steps from changing the +// activation while a ServiceFn runs (see TestRunService_ServiceFnCantChangeItsActivation), +// so this drives the ServiceRunner directly, as only code inside the package can. +func TestServiceStep_StaleSettleRecordsNothing(t *testing.T) { + for _, tc := range []struct { + name string + rearm bool + }{{"stopped", false}, {"stopped and armed again", true}} { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("step", "loading", 0) + var h *Handle[string, string, stepCtx] + runs := 0 + fn := func(ctx context.Context, _ state.ServiceCtx[stepCtx]) (any, error) { + runs++ + if runs == 1 { + effects := []state.Effect{state.StopService{ID: id}} + if tc.rearm { + effects = append(effects, h.inst.StartEffects()...) + } + h.svc.Absorb(ctx, effects) + } + return "late", nil + } + m := stepMachine(fn) + st := NewMemStore() + var err error + if h, err = NewRunner(m, st, stepOptions(fn)...).Start(ctx, "s1", stepCtx{}, state.WithInitialState("loading")); err != nil { + t.Fatalf("Start: %v", err) + } + if _, ran, rerr := h.RunService(ctx, id); rerr != nil || !ran { + t.Fatalf("RunService = %v, %v; want ran with no error", ran, rerr) + } + if _, recs, _ := st.Load(ctx, "s1"); len(recs) != 0 { + t.Fatalf("the journal holds %d Records, want none: the settle fired nothing", len(recs)) + } + if h.settled[id] { + t.Fatal("the service is noted settled, so a checkpoint would record it settled") + } + if h.svcJSON.raw != nil { + t.Fatalf("the Handle holds the stale settle's JSON %s", h.svcJSON.raw) + } + if got := h.inst.Current(); got != "loading" { + t.Fatalf("in %q, want loading", got) + } + if got := h.svc.HasPending(id); got != tc.rearm { + t.Fatalf("HasPending = %v, want %v", got, tc.rearm) + } + rh, err := Recover(ctx, m, st, "s1", stepOptions(fn)...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if !rh.svc.HasPending(id) { + t.Fatal("the recovered Handle has the service settled; the journal holds no settle") + } + }) + } +} + +// TestServiceStep_ClearsTheSettledJSON: the JSON a settle decoded is the step's alone. +// Once RunService returns, live or for a parked result, and once Recover has replayed +// the journal, the Handle holds none of it. +func TestServiceStep_ClearsTheSettledJSON(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("step", "loading", 0) + fn := func(context.Context, state.ServiceCtx[stepCtx]) (any, error) { return "ok", nil } + m := stepMachine(fn) + st := NewMemStore() + h, err := NewRunner(m, st, stepOptions(fn)...).Start(ctx, "s1", stepCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, _, err = h.RunService(ctx, id); err != nil { + t.Fatalf("RunService: %v", err) + } + if h.svcJSON.raw != nil { + t.Fatalf("after RunService the Handle holds %s", h.svcJSON.raw) + } + rh, err := Recover(ctx, m, st, "s1", stepOptions(fn)...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if rh.svcJSON.raw != nil { + t.Fatalf("after Recover the Handle holds %s", rh.svcJSON.raw) + } + + // A parked result's settle. + refuse := WithEventDataDecoder[string, string, stepCtx](func(e string, data []byte) (any, error) { + if e == "done" { + return nil, ErrEventDataEncoding + } + return DecodeAs[any](data) + }) + st = NewMemStore() + if h, err = NewRunner(m, st, append(stepOptions(fn), refuse)...).Start(ctx, "s1", stepCtx{}, state.WithInitialState("loading")); err != nil { + t.Fatalf("Start: %v", err) + } + if _, _, err = h.RunService(ctx, id); err == nil { + t.Fatal("RunService settled a result the decoder refuses") + } + if rh, err = Recover(ctx, m, st, "s1", stepOptions(fn)...); err != nil { + t.Fatalf("Recover: %v", err) + } + if _, _, err = rh.RunService(ctx, id); err != nil { + t.Fatalf("RunService of the parked result: %v", err) + } + if rh.svcJSON.raw != nil { + t.Fatalf("after a parked result's settle the Handle holds %s", rh.svcJSON.raw) + } +} + +// TestServiceBookkeeping_SettledAndParkedMoveTogether pins that the Handle and the +// replay mirror keep a service out of the parked results once it settles, and forget +// both when a state that declares it is entered or exited, through one helper each. +func TestServiceBookkeeping_SettledAndParkedMoveTogether(t *testing.T) { + start := []state.Effect{state.StartService{ID: "a"}} + stop := []state.Effect{state.StopService{ID: "b"}} + h := &Handle[string, string, stepCtx]{parked: map[string]recordedResult{"a": {}, "b": {}}} + h.markSettled("a") + if _, ok := h.parked["a"]; ok || !h.settled["a"] { + t.Fatalf("after markSettled: settled %v, parked %v; want a settled and not parked", h.settled, h.parked) + } + h.settled["b"] = true // b settled in an earlier activation, then parked in this one + h.forgetServices(start) + h.forgetServices(stop) + if len(h.settled) != 0 || len(h.parked) != 0 { + t.Fatalf("after start and stop effects: settled %v, parked %v; want both empty", h.settled, h.parked) + } + + m := &armMirror{} + m.park("a", recordedResult{}) + m.park("b", recordedResult{}) + m.settle("a") + if _, ok := m.parked["a"]; ok || !m.settled["a"] { + t.Fatalf("after settle: settled %v, parked %v; want a settled and not parked", m.settled, m.parked) + } + m.settled["b"] = true + m.forget(start) + m.forget(stop) + if len(m.settled) != 0 || len(m.parked) != 0 { + t.Fatalf("after start and stop effects: settled %v, parked %v; want both empty", m.settled, m.parked) + } +} diff --git a/durable/typename_internal_test.go b/durable/typename_internal_test.go new file mode 100644 index 00000000..f3513c36 --- /dev/null +++ b/durable/typename_internal_test.go @@ -0,0 +1,95 @@ +package durable + +import ( + "testing" + + alpha "github.com/stablekernel/crucible/durable/internal/typefixture/alpha/model" + beta "github.com/stablekernel/crucible/durable/internal/typefixture/beta/model" +) + +// TestTypeName_UnnamedTypesKeepTheImportPath answers whether a parked result's type +// name tells two packages' types apart inside an unnamed type: a struct literal, func +// or chan spelled from alpha's model.Result must differ from one spelled from beta's, +// as the named types do, or a decoder for the one settles a result parked as the other. +// A struct's tags and its unexported fields' packages are part of its type, so they +// are part of the name too. +func TestTypeName_UnnamedTypesKeepTheImportPath(t *testing.T) { + type tagged struct { + R alpha.Result `json:"r"` + } + cases := []struct { + name string + a, b any + }{ + {"named", alpha.Result{}, beta.Result{}}, + {"struct field", struct{ R alpha.Result }{}, struct{ R beta.Result }{}}, + {"embedded field", struct{ alpha.Result }{}, struct{ beta.Result }{}}, + {"nested struct", struct{ S struct{ R *alpha.Result } }{}, struct{ S struct{ R *beta.Result } }{}}, + {"struct in a slice", []struct{ R alpha.Result }{}, []struct{ R beta.Result }{}}, + {"tag", struct { + R alpha.Result `json:"r"` + }{}, struct { + R alpha.Result `json:"x"` + }{}}, + {"named struct vs literal", tagged{}, struct { + R alpha.Result `json:"r"` + }{}}, + {"func", func(alpha.Result) error { return nil }, func(beta.Result) error { return nil }}, + {"func result", func() alpha.Result { return alpha.Result{} }, func() beta.Result { return beta.Result{} }}, + {"chan", make(chan alpha.Result), make(chan beta.Result)}, + {"chan direction", make(<-chan alpha.Result), make(chan<- alpha.Result)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if a, b := typeName(tc.a), typeName(tc.b); a == b { + t.Fatalf("typeName gives %q for both %T and %T", a, tc.a, tc.b) + } + }) + } +} + +// TestTypeName_SpellsUnnamedTypesAsGoDoes pins the form: reflect's own spelling, with +// every named type's import path in place of its package name, so a type with no +// named parts reads as reflect gives it. +func TestTypeName_SpellsUnnamedTypesAsGoDoes(t *testing.T) { + const a = "github.com/stablekernel/crucible/durable/internal/typefixture/alpha/model.Result" + cases := []struct { + v any + want string + }{ + {struct{}{}, "struct {}"}, + {struct { + A int + B string `json:"b"` + }{}, `struct { A int; B string "json:\"b\"" }`}, + {struct{ R alpha.Result }{}, "struct { R " + a + " }"}, + {struct{ alpha.Result }{}, "struct { " + a + " }"}, + {struct{ r alpha.Result }{}, "struct { github.com/stablekernel/crucible/durable.r " + a + " }"}, + {func(int, ...alpha.Result) (bool, error) { return false, nil }, "func(int, ..." + a + ") (bool, error)"}, + {func() alpha.Result { return alpha.Result{} }, "func() " + a}, + {func() {}, "func()"}, + {make(chan alpha.Result), "chan " + a}, + {make(<-chan alpha.Result), "<-chan " + a}, + {make(chan<- alpha.Result), "chan<- " + a}, + {make(chan (<-chan int)), "chan (<-chan int)"}, + {map[string][]struct{ N int }{}, "map[string][]struct { N int }"}, + } + for _, tc := range cases { + if got := typeName(tc.v); got != tc.want { + t.Errorf("typeName(%T) = %q, want %q", tc.v, got, tc.want) + } + } +} + +// TestTypeName_InterfaceFieldKeepsTheImportPath covers an interface literal inside a +// struct literal, the one place an interface type can appear in a value's type. +func TestTypeName_InterfaceFieldKeepsTheImportPath(t *testing.T) { + a := typeName(struct{ X interface{ M() alpha.Result } }{}) + b := typeName(struct{ X interface{ M() beta.Result } }{}) + if a == b { + t.Fatalf("typeName gives %q for both", a) + } + if got, want := typeName(struct{ X any }{}), "struct { X interface {} }"; got != want { + t.Fatalf("typeName = %q, want %q", got, want) + } +} From c3df07f6f7d8b61acc0a634df0c0d97e0133a784 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 18:40:04 -0700 Subject: [PATCH 10/13] docs(durable): say how to settle a parked result after its type moves Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 5 ++++- durable/doc.go | 5 ++++- durable/runner_options.go | 5 ++++- durable/service.go | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 75c2c491..ad4c53d0 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -162,7 +162,10 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. Handle stays healthy and the service stays in flight, but its `ServiceFn` never runs again. The Handle that parked it never sees a fixed decoder: `Recover` with the fixed decoder, then call `RunService`, and it settles the parked result, which the decoder - must return as the type the service returned, or in JSON's generic form, as live. A + must return as the type the service returned, or in JSON's generic form, as live. + The type is matched by import path and name, so after the type is moved or renamed + only a decoder that returns the generic form settles it: settle parked results before + such a move. A ctx that ended during the service parks nothing: `RunService` fails the Handle and records nothing, as for a result that decodes, and the service runs again after `Recover`. Until then each `RunService` returns diff --git a/durable/doc.go b/durable/doc.go index 36d77708..67c19a98 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -50,7 +50,10 @@ // recorded and the Handle stays healthy, so the service never runs again. The decoder // is part of the Runner's options, so the Handle that parked the result never sees a // fix: Recover with the fixed decoder, then call RunService, and it settles the parked -// result. Leaving the state that declares the service discards it. A +// result. The decoder must return the type the service returned, by import path and +// name, or JSON's generic form, so settle parked results before moving or renaming that +// type; after a move, only a decoder that returns the generic form settles them. +// Leaving the state that declares the service discards it. A // service error stays message-only. Register the decoder before recording results: replay decodes the // journal with the recovering Runner's decoder, so one added or changed after results // were recorded changes what replay produces. diff --git a/durable/runner_options.go b/durable/runner_options.go index 2e7e9d38..6d427561 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -188,7 +188,10 @@ func WithMaxEventDataBytes[S comparable, E comparable, C any](n int) Option[S, E // ErrEventDataEncoding before any onDone or onError Assign runs, records the result // without settling it, and keeps the Handle healthy. Recover with the fixed decoder, // then call RunService, to settle the parked result without running the service again; -// the Handle that parked it never sees the fix (see Handle.RunService). A service error stays message-only and +// the Handle that parked it never sees the fix (see Handle.RunService). The settling +// decoder must return the type the service returned, by import path and name, or JSON's +// generic form, so a type moved or renamed after the park settles only through a decoder +// that returns the generic form. A service error stays message-only and // doesn't reach the decoder. Register the decoder before results are recorded: replay // decodes the journal with the recovering Runner's decoder, so one added or changed // after results were recorded changes what replay produces, and one that can't decode diff --git a/durable/service.go b/durable/service.go index 08f94dd0..657f3e31 100644 --- a/durable/service.go +++ b/durable/service.go @@ -134,7 +134,10 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // The Handle stays healthy and the service stays in flight, on this Handle and on a // recovered one, but its ServiceFn never runs again. The decoder is part of the // Runner's options, so this Handle never sees a fix: Recover with the fixed decoder, -// then call RunService, and it settles the parked result. Until then each RunService +// then call RunService, and it settles the parked result. The decoder must return the +// recorded type, by import path and name, or JSON's generic form: after the type is +// moved or renamed, only a decoder that returns the generic form settles it, so settle +// parked results before such a move. Until then each RunService // returns ErrEventDataEncoding and records nothing more. A Fire or timer that leaves // the state that declares the service stops it and discards the parked result. A // checkpoint keeps it, and StateAt shows the instance as it was before the park. From c9e06a2435ae31a87e75e67c02ac2daa6832237e Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 19:29:20 -0700 Subject: [PATCH 11/13] fix(durable): refuse a lower format mid-tail, record a settled result's type, reread the dispatched set before unmarked effects Each finding has a test that fails if it is real, kept whether it proved or disproved it, and each fix was checked by reverting it. - A Record of a lower journal format than any before it, the checkpoint's or an earlier Record's, is refused as an older version appending to a newer journal, not only one below the checkpoint's. Recover and StateAt check every Record's format before they replay any, so a refused journal sends no effect. - Every successful service settle records the Go type the service returned, and replay holds the decoder to it, as for a settled parked result. JSON's generic form and nil are always accepted, and a Record of format 0 or 1 still replays with no check. Changing the type a decoder returns for an event with recorded results now fails Recover unless it returns the generic form; the RunService doc, package doc and CHANGELOG say so. - Recover reads the dispatched set again before sending a step's effects that its copy doesn't show marked, so a mark a stale owner makes during a handoff is seen, as the per-step load did. A step whose effects are all marked costs no read: 1000 landed tick effects still take one read, and about 2 ms on MemStore. - The tick-tail benchmark also measures a tail with no effect landed, and its run line names it correctly. Signed-off-by: Ray Karnes --- durable/CHANGELOG.md | 36 ++++-- durable/dispatch_store_test.go | 151 +++++++++++++++++------ durable/doc.go | 6 +- durable/effect.go | 54 +++++--- durable/journal_format_internal_test.go | 20 ++- durable/journal_format_midtail_test.go | 118 ++++++++++++++++++ durable/recover_dispatched_bench_test.go | 50 ++++++-- durable/runner.go | 51 +++++--- durable/service.go | 91 ++++++++------ durable/service_encode_internal_test.go | 2 +- durable/service_step_internal_test.go | 4 +- durable/settled_type_test.go | 102 +++++++++++++++ durable/timetravel.go | 6 +- 13 files changed, 549 insertions(+), 142 deletions(-) create mode 100644 durable/journal_format_midtail_test.go create mode 100644 durable/settled_type_test.go diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index ad4c53d0..01ffb493 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -77,9 +77,10 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **Don't run mixed versions on one instance.** During a rolling deploy, let only replicas of one version own a given instance: drain an instance's owner before a replica of another version recovers it. An older replica that appends to a journal - this version wrote stamps a lower journal format after this version's checkpoint, and - `Recover` then fails, on either version, with an error saying an older version appended - to a newer journal; this version can't replay such a Record by the older format's rules, + this version wrote stamps a lower journal format after this version's checkpoint or + Records, and `Recover` and `StateAt` then fail, on either version, with an error saying + an older version appended to a newer journal, before replaying any of it, so a refused + journal sends no effect; this version can't replay such a Record by the older format's rules, because the older version reads the journal differently, settling a parked result through onDone, for one. A version older than this one doesn't refuse this version's journal at all, and that can't be fixed for binaries already built. @@ -142,9 +143,15 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. service returned, or a result whose JSON form won't decode, such as a number beyond `float64`, parks the result (see below). `Recover` and `StateAt` fail with `ErrEventDataEncoding`, running no `Assign`, on a settled result their decoder can't - decode. Replay decodes the journal with the - recovering Runner's decoder, so register a decoder before results are recorded: one - added or changed later changes what replay hands `Assign`. An error stays message-only: + decode. Every successful settle records the Go type the service returned, with its + import path, and replay holds the decoder to it as the live settle did: `Recover` and + `StateAt` fail with `ErrEventDataEncoding`, running no `Assign`, when the decoder + returns another type, such as a pointer to it. JSON's generic form, and nil, are always + accepted. Replay decodes the journal with the recovering Runner's decoder, so register + a decoder before results are recorded: changing the type a decoder returns for an event + whose results are recorded fails `Recover` on them, unless it returns the generic form, + and a decoder added later for the type the service returned changes what replay hands + `Assign` from the generic form to that type. An error stays message-only: an error with the same message, any invalid UTF-8 replaced, which `errors.Is` no longer matches. An `Error` method that panics settles with a message saying the service error panicked, its invalid UTF-8 replaced too. The journal records the JSON the live settle @@ -228,8 +235,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Fixed - **`Recover` and `StateAt` fail when a Store loses `Record.Format`.** The Start baseline - and every checkpoint record the journal format too, and a Record after one with a lower - format fails replay, where it used to replay as an older journal's and silently skip + and every checkpoint record the journal format too, and a Record with a lower format + than the checkpoint or any Record before it fails replay, before any Record replays, where it used to replay as an older journal's and silently skip the timers and services an actor-driven Fire armed. The error says the Store didn't keep `Record.Format` for a Record with no format, and that an older version appended it, a mixed-version deploy, for one with a lower format. @@ -266,11 +273,14 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. effects from an older version's `Tick` and service steps, which never dispatched them. It goes by the Record's format, not its `Effects` envelopes, so a Store that doesn't keep `Record.Effects` loses no effect a crash left unsent. `Recover` loads the - Store's dispatched set once, at the first Record with effects to send, rather than once - per such Record, which made a long tail of tick Records quadratic in the set's size - (1000 tick Records: about 22 ms down to 2 ms on `MemStore`, 25 ms to 4 ms on - `FileStore`). A mark another owner makes during a handoff while `Recover` replays isn't - seen, so key the handler's dedupe on `EffectInstance` plus the id, as for any handoff. + Store's dispatched set at the first Record with effects to send, and reads it again only + before sending a later Record's effects that its copy doesn't show marked, rather than + once per Record with effects, which made a long tail of tick Records whose effects had + landed quadratic in the set's size (1000 tick Records: about 24 ms down to 2 ms on + `MemStore`, 25 ms to 3 ms on `FileStore`). A Record with an effect to send still reads + the set first, so a mark another owner makes during a handoff is seen before that + effect is sent; one made between that read and the send isn't, so key the handler's + dedupe on `EffectInstance` plus the id, as for any handoff. - **A Tick, RunService or DeliverToActor whose context ended partway isn't journaled.** A canceled `RunService`, `Tick` or `DeliverToActor` used to record a result the live instance never applied, which recovery then committed, or for an actor, failed on. Each now returns `ErrHandleFailed` diff --git a/durable/dispatch_store_test.go b/durable/dispatch_store_test.go index ee262769..e977d626 100644 --- a/durable/dispatch_store_test.go +++ b/durable/dispatch_store_test.go @@ -147,36 +147,55 @@ func pingMachine() *state.Machine[string, string, *runCtx] { Quench() } -// TestRecover_LoadsTheDispatchedSetOncePerRecover pins what Recover costs the Store: it -// loads the instance's whole dispatched set once, at the first tail Record with -// effects to send, keeps it up to date as it marks, and marks nothing when every effect -// already landed. main loaded it once for each Fire's Record; with a tick's and a -// service settle's Records sending effects too, that made Recover over a long tail -// quadratic in the set's size (see BenchmarkRecover_TickTailDispatchedSet). -func TestRecover_LoadsTheDispatchedSetOncePerRecover(t *testing.T) { - ctx := context.Background() +// TestRecover_ReadsTheDispatchedSetOnlyForUnmarkedSteps pins what Recover costs the Store: it +// loads the instance's whole dispatched set at the first tail Record with effects to +// send, and reads it again only for a later Record with an effect its copy doesn't show +// marked. When every effect already landed, the common case, that is one load and no +// marks. main loaded it once for each Fire's Record; with a tick's and a service +// settle's Records sending effects too, that made Recover over a long tail of landed +// effects quadratic in the set's size (see BenchmarkRecover_TickTailDispatchedSet). +// When none landed, Recover reads it once per Record, before sending that Record's +// effects, so a mark a stale owner makes during a handoff is seen (see +// TestRecover_SeesAStaleOwnersMarkDuringReplay); it marks each effect then anyway. +func TestRecover_ReadsTheDispatchedSetOnlyForUnmarkedSteps(t *testing.T) { const n = 25 - st := newDispatchProbeStore(durable.NewMemStore()) - handler := durable.WithEffectHandler[string, string, *runCtx](newEffectRecorder().handler()) - h, err := durable.NewRunner(pingMachine(), st, handler).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) - if err != nil { - t.Fatalf("Start: %v", err) - } - for i := 0; i < n; i++ { - if _, err = h.Fire(ctx, "ping"); err != nil { - t.Fatalf("Fire ping %d: %v", i, err) - } - } - liveLoads, liveMarks := st.counts() - if _, err = durable.Recover(ctx, pingMachine(), st, "i1", handler); err != nil { - t.Fatalf("Recover: %v", err) - } - loads, marks := st.counts() - if got := loads - liveLoads; got != 1 { - t.Fatalf("Recover over %d effect Records loaded the dispatched set %d times, want once", n, got) - } - if got := marks - liveMarks; got != 0 { - t.Fatalf("Recover marked %d effects, want none: every one landed live", got) + for _, tc := range []struct { + name string + live bool // a handler sends, and marks, every effect live + loads, marks int + }{ + {"every effect landed", true, 1, 0}, + {"no effect landed", false, n, n}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + st := newDispatchProbeStore(durable.NewMemStore()) + handler := durable.WithEffectHandler[string, string, *runCtx](newEffectRecorder().handler()) + var live []durable.Option[string, string, *runCtx] + if tc.live { + live = append(live, handler) + } + h, err := durable.NewRunner(pingMachine(), st, live...).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for i := 0; i < n; i++ { + if _, err = h.Fire(ctx, "ping"); err != nil { + t.Fatalf("Fire ping %d: %v", i, err) + } + } + liveLoads, liveMarks := st.counts() + if _, err = durable.Recover(ctx, pingMachine(), st, "i1", handler); err != nil { + t.Fatalf("Recover: %v", err) + } + loads, marks := st.counts() + if got := loads - liveLoads; got != tc.loads { + t.Fatalf("Recover over %d effect Records loaded the dispatched set %d times, want %d", n, got, tc.loads) + } + if got := marks - liveMarks; got != tc.marks { + t.Fatalf("Recover marked %d effects, want %d", got, tc.marks) + } + }) } } @@ -233,11 +252,11 @@ func TestDispatchStoreError_ReturnsEffectDispatch(t *testing.T) { } } -// TestRecover_SendsEachUnmarkedEffectOnceWithOneLoad: with the dispatched set loaded once, -// Recover still sends every unmarked effect, across several Records, once each, and -// skips the ones that landed; a second Recover, which reads the marks the first made, -// sends nothing. -func TestRecover_SendsEachUnmarkedEffectOnceWithOneLoad(t *testing.T) { +// TestRecover_SendsEachUnmarkedEffectOnce: Recover sends every unmarked effect, across +// several Records, once each, and skips the ones that landed, loading the dispatched +// set for the first Record and again for each later one with an effect to send; a +// second Recover, which reads the marks the first made, sends nothing and loads it once. +func TestRecover_SendsEachUnmarkedEffectOnce(t *testing.T) { ctx := context.Background() st := newDispatchProbeStore(durable.NewMemStore()) // With no handler the live run records the effects and sends none. @@ -258,18 +277,78 @@ func TestRecover_SendsEachUnmarkedEffectOnceWithOneLoad(t *testing.T) { if _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](healthy.handler())); err != nil { t.Fatalf("Recover: %v", err) } - if after, _ := st.counts(); after-loads != 1 { - t.Fatalf("Recover loaded the dispatched set %d times, want once", after-loads) + // Step 0 landed: one load for it, and one each for steps 1 and 2. + if after, _ := st.counts(); after-loads != 3 { + t.Fatalf("Recover loaded the dispatched set %d times, want 3", after-loads) } want := []string{"1#0#demo.charge", "1#1#demo.notify", "2#0#demo.notify"} if got := healthy.ids(); len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] || healthy.totalApplies() != 3 { t.Fatalf("Recover applied %v (%d in all), want %v once each", got, healthy.totalApplies(), want) } again := newEffectRecorder() + loads, _ = st.counts() if _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](again.handler())); err != nil { t.Fatalf("second Recover: %v", err) } + if after, _ := st.counts(); after-loads != 1 { + t.Fatalf("a second Recover loaded the dispatched set %d times, want once", after-loads) + } if n := again.totalApplies(); n != 0 { t.Fatalf("a second Recover applied %v", again.ids()) } } + +// staleOwnerStore is a MemStore whose first Dispatched call is followed, before Recover +// sees anything more, by a stale owner marking effect: one owner's handoff to another, +// where the stale owner still sends and marks a step's effect while the new owner's +// Recover replays. MarkDispatched isn't fenced, so the mark lands. +type staleOwnerStore struct { + *durable.MemStore + effect string + loads int +} + +func (s *staleOwnerStore) Dispatched(ctx context.Context, id durable.InstanceID) (map[string]bool, error) { + set, err := s.MemStore.Dispatched(ctx, id) + s.loads++ + if s.loads == 1 { + if merr := s.MarkDispatched(ctx, id, s.effect); merr != nil { + return nil, merr + } + } + return set, err +} + +// TestRecover_SeesAStaleOwnersMarkDuringReplay answers a review finding: loading the +// dispatched set once per Recover, and never again, widened the handoff window, so an +// effect a stale owner sent and marked after that load, for a later step, was sent +// again. Recover reads the set again before sending a step's effects that its copy +// doesn't show marked, so it sees the mark, as the per-step load it replaced did. +func TestRecover_SeesAStaleOwnersMarkDuringReplay(t *testing.T) { + ctx := context.Background() + mem := durable.NewMemStore() + // With no handler the live run records the effects and sends none. + h, err := durable.NewRunner(effectMachine(), mem).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + for _, ev := range []string{"go", "bill", "finish"} { + if _, err = h.Fire(ctx, ev); err != nil { + t.Fatalf("Fire %s: %v", ev, err) + } + } + // The stale owner sends and marks step 2's effect once Recover has first loaded the + // set. + st := &staleOwnerStore{MemStore: mem, effect: "2#0#demo.notify"} + rec := newEffectRecorder() + if _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](rec.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.count("2#0#demo.notify"); n != 0 { + t.Fatalf("Recover sent 2#0#demo.notify %d times, though the stale owner marked it before Recover reached step 2", n) + } + want := []string{"0#0#demo.notify", "1#0#demo.charge", "1#1#demo.notify"} + if got := rec.ids(); len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] || rec.totalApplies() != 3 { + t.Fatalf("Recover applied %v (%d in all), want %v once each", got, rec.totalApplies(), want) + } +} diff --git a/durable/doc.go b/durable/doc.go index 67c19a98..205cc7fa 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -55,8 +55,10 @@ // type; after a move, only a decoder that returns the generic form settles them. // Leaving the state that declares the service discards it. A // service error stays message-only. Register the decoder before recording results: replay decodes the -// journal with the recovering Runner's decoder, so one added or changed after results -// were recorded changes what replay produces. +// journal with the recovering Runner's decoder, held to the Go type the service +// returned, which every settle records, so a decoder changed after results were +// recorded to return another type fails Recover, unless it returns JSON's generic form, +// and one added later for the service's type changes what replay produces. // // The kernel stays pure and dependency-free. The durable module is host-side: // database drivers, cloud SDKs, and I/O libraries live in the caller or in out- diff --git a/durable/effect.go b/durable/effect.go index ddf4f388..cb620f17 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -290,14 +290,15 @@ func (r *Runner[S, E, C]) applyEffect(ctx context.Context, id InstanceID, de dis // (step and emission order reproduce exactly) and applies only those not already // marked, so an effect that landed before the crash is skipped and one recorded // but un-dispatched is applied now. This is the recovery half of the exactly-once -// contract. seen is the dispatched set one Recover loads once, at the first step -// with effects to send. +// contract. seen is the copy of the dispatched set one Recover keeps across its steps +// (see dispatchWith). func (r *Runner[S, E, C]) dispatchReplayEffects(ctx context.Context, id InstanceID, step int, effects []state.Effect, seen *dispatchedSet) error { return r.dispatchWith(ctx, id, dispatchableEffects(step, effects), seen) } // dispatchedSet is a Recover's copy of the instance's dispatched set: nil until the -// first step that has effects to send loads it. +// first step that has effects to send loads it, and read again for each later step +// with an effect it doesn't show marked. type dispatchedSet struct{ ids map[string]bool } // dispatch is the shared exactly-once apply loop used by the live path and by @@ -310,9 +311,14 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis return r.dispatchWith(ctx, id, des, nil) } -// dispatchWith is dispatch reading the dispatched set from seen, when it isn't nil, -// loading it there the first time. The ids it marks needn't be added: an effect id -// names one step, and a Recover replays each step once, so it never meets one again. +// dispatchWith is dispatch reading the dispatched set from seen, when it isn't nil. It +// reads the Store only when seen doesn't show every effect in des marked: the first +// time, and for a step with an effect not marked when seen was read. A mark is never +// taken back, so a step seen shows fully marked has nothing to send, and costs no read; +// a step with one to send reads the set fresh, so a mark a stale owner made during a +// handoff since the last read is seen, as a read per step would see it. The ids it +// marks needn't be added: an effect id names one step, and a Recover replays each step +// once, so it never meets one again. func (r *Runner[S, E, C]) dispatchWith(ctx context.Context, id InstanceID, des []dispatchableEffect, seen *dispatchedSet) (err error) { ds, ok := r.dispatchStore(des) if !ok { @@ -324,18 +330,17 @@ func (r *Runner[S, E, C]) dispatchWith(ctx context.Context, id InstanceID, des [ } }() - var dispatched map[string]bool - if seen != nil && seen.ids != nil { - dispatched = seen.ids - } else { - if dispatched, err = ds.Dispatched(ctx, id); err != nil { - return fmt.Errorf("%w: loading dispatched set for %q: %w", ErrEffectDispatch, id, err) - } - if seen != nil { - seen.ids = dispatched - if seen.ids == nil { - seen.ids = map[string]bool{} - } + if seen != nil && allMarked(seen.ids, des) { + return nil + } + dispatched, err := ds.Dispatched(ctx, id) + if err != nil { + return fmt.Errorf("%w: loading dispatched set for %q: %w", ErrEffectDispatch, id, err) + } + if seen != nil { + seen.ids = dispatched + if seen.ids == nil { + seen.ids = map[string]bool{} } } @@ -353,3 +358,16 @@ func (r *Runner[S, E, C]) dispatchWith(ctx context.Context, id InstanceID, des [ } return nil } + +// allMarked reports whether dispatched, which may be nil, marks every effect in des. +func allMarked(dispatched map[string]bool, des []dispatchableEffect) bool { + if dispatched == nil { + return false + } + for _, de := range des { + if !dispatched[de.id] { + return false + } + } + return true +} diff --git a/durable/journal_format_internal_test.go b/durable/journal_format_internal_test.go index 99c8d13f..f94f2069 100644 --- a/durable/journal_format_internal_test.go +++ b/durable/journal_format_internal_test.go @@ -7,7 +7,9 @@ import ( // TestCheckFormat_RefusesAHigherFormat pins what a downgrade meets. checkFormat fails // on a Record of a lower format than its checkpoint's, a Store that lost the field, and -// accepts a higher one, as every version since Record.Format has. checkNotNewer, which +// accepts a higher one, as every version since Record.Format has. checkFormats holds +// each Record to the highest format before it, the checkpoint's or a Record's, so an +// older version's Record is refused anywhere in the tail. checkNotNewer, which // every reader calls first, refuses a checkpoint or a Record of a format above // journalFormat, one a newer version wrote, so each later bump of journalFormat makes // this version refuse that version's journal. A binary built before checkNotNewer @@ -19,6 +21,22 @@ func TestCheckFormat_RefusesAHigherFormat(t *testing.T) { if err := checkFormat(2, &Record{Format: 1}); err == nil { t.Errorf("checkFormat(2, a Record of format 1) = nil, want the older-version error") } + for _, tc := range []struct { + checkpoint int + recs []Record + refused bool + }{ + {0, []Record{{Format: 0}, {Format: 1}, {Format: 1}, {Format: 2}}, false}, + {1, []Record{{Format: 1}, {Format: 2}, {Format: 2}}, false}, + {1, []Record{{Format: 2}, {Format: 1}}, true}, + {0, []Record{{Format: 1}, {Format: 0}}, true}, + {2, []Record{{Format: 1}}, true}, + } { + err := checkFormats(tc.checkpoint, tc.recs) + if (err != nil) != tc.refused { + t.Errorf("checkFormats(%d, %+v) = %v, want refused %v", tc.checkpoint, tc.recs, err, tc.refused) + } + } for _, tc := range []struct { checkpoint int recs []Record diff --git a/durable/journal_format_midtail_test.go b/durable/journal_format_midtail_test.go new file mode 100644 index 00000000..1a06516e --- /dev/null +++ b/durable/journal_format_midtail_test.go @@ -0,0 +1,118 @@ +package durable_test + +import ( + "context" + "strings" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These tests answer a review finding on the journal format check: it compared each +// Record's format only with the checkpoint's, so an older version's Record appended +// after this version's Records, past an older checkpoint, replayed by its own format's +// rules. A journal's format never goes down, so a Record below any format before it, +// the checkpoint's or an earlier Record's, means mixed versions wrote it, and Recover +// and StateAt refuse it before replaying anything. + +// TestRecover_OlderRecordMidTailIsRefused: a format-1 checkpoint, then this version's +// format-2 Record that parks a result, then an older replica's format-1 Fire. The older +// replica reads the park as a settle (see TestRecover_OlderVersionAfterAParkIsRefused), +// so the two versions disagree about the journal, and Recover must refuse it, naming +// the older version, as it does when the checkpoint itself is newer. +func TestRecover_OlderRecordMidTailIsRefused(t *testing.T) { + ctx := context.Background() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { return scnUser{ID: 7, Name: "ann"}, nil } + m := forkMachine(fn) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + st := durable.NewMemStore() + // An older version, of format 1, started the instance and wrote its checkpoint. + older := &olderWriter{MemStore: st, format: 1} + if _, err := durable.NewRunner(m, older, reg, failingDecoder()).Start(ctx, "o1", svcCtx{}, state.WithInitialState("loading")); err != nil { + t.Fatalf("Start: %v", err) + } + // This version recovers it and parks the result, in a format-2 Record. + h, err := durable.Recover(ctx, m, st, "o1", reg, failingDecoder()) + if err != nil { + t.Fatalf("Recover the older checkpoint: %v", err) + } + _, ran, err := h.RunService(ctx, state.InvokeID("fork", "loading", 0)) + mustPark(t, "RunService", ran, err) + _, recs, err := st.Load(ctx, "o1") + if err != nil || len(recs) != 1 || recs[0].Format != 2 { + t.Fatalf("the journal holds %+v (err %v), want the park's one format-2 Record", recs, err) + } + // The older replica fires "next" after it, in its format. + if _, err = st.Append(ctx, "o1", durable.Record{Step: recs[0].Step + 1, Event: []byte(`"next"`), Format: 1}); err != nil { + t.Fatalf("Append: %v", err) + } + + rh, err := durable.Recover(ctx, m, st, "o1", reg, failingDecoder()) + if err == nil { + t.Fatalf("Recover replayed a format-1 Record after a format-2 one, into %q", rh.Instance().Current()) + } + if msg := err.Error(); !strings.Contains(msg, "older version") || !strings.Contains(msg, "don't run mixed versions") { + t.Fatalf("Recover: %v, want the mixed-version error", err) + } + if _, err = durable.StateAt(ctx, m, st, "o1", recs[0].Step+1, reg, failingDecoder()); err == nil || !strings.Contains(err.Error(), "older version") { + t.Fatalf("StateAt: %v, want the mixed-version error", err) + } +} + +// TestRecover_RefusesAMixedJournalBeforeReplaying: Recover checks every Record's format +// before it replays any, so a journal it refuses sends no effect. This version's Fire +// records an effect no handler sent, and an older replica appends after it; the effect +// would be sent if Recover only refused on reaching the older Record. +func TestRecover_RefusesAMixedJournalBeforeReplaying(t *testing.T) { + ctx := context.Background() + st := durable.NewMemStore() + h, err := durable.NewRunner(effectMachine(), st).Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + // No handler: step 0's effect is recorded and not sent. + if _, err = h.Fire(ctx, "go"); err != nil { + t.Fatalf("Fire go: %v", err) + } + if _, err = durable.NewRunner(effectMachine(), &olderWriter{MemStore: st, format: 1}).Fire(ctx, "i1", "bill"); err != nil { + t.Fatalf("the older replica's Fire: %v", err) + } + rec := newEffectRecorder() + _, err = durable.Recover(ctx, effectMachine(), st, "i1", durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if err == nil || !strings.Contains(err.Error(), "older version") { + t.Fatalf("Recover: %v, want the mixed-version error", err) + } + if n := rec.totalApplies(); n != 0 { + t.Fatalf("Recover sent %v before refusing the journal, want nothing sent", rec.ids()) + } +} + +// TestRecover_NonDecreasingFormatsLoad: a journal whose formats never go down loads, +// whatever they are: an older version's checkpoint and Records, then this version's. +func TestRecover_NonDecreasingFormatsLoad(t *testing.T) { + ctx := context.Background() + for _, formats := range [][]int{{0, 0, 2}, {0, 1, 2}, {1, 1, 1}, {1, 2, 2}, {0, 2, 2}} { + st := durable.NewMemStore() + r := durable.NewRunner(effectMachine(), &olderWriter{MemStore: st, format: formats[0]}) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("%v: Start: %v", formats, err) + } + if _, err := durable.NewRunner(effectMachine(), &olderWriter{MemStore: st, format: formats[1]}).Fire(ctx, "i1", "go"); err != nil { + t.Fatalf("%v: Fire go: %v", formats, err) + } + if _, err := durable.NewRunner(effectMachine(), &olderWriter{MemStore: st, format: formats[2]}).Fire(ctx, "i1", "bill"); err != nil { + t.Fatalf("%v: Fire bill: %v", formats, err) + } + rh, err := durable.Recover(ctx, effectMachine(), st, "i1") + if err != nil { + t.Fatalf("%v: Recover: %v", formats, err) + } + if got := rh.Instance().Current(); got != "billed" { + t.Fatalf("%v: recovered in %q, want billed", formats, got) + } + if _, err = durable.StateAt(ctx, effectMachine(), st, "i1", 1); err != nil { + t.Fatalf("%v: StateAt: %v", formats, err) + } + } +} diff --git a/durable/recover_dispatched_bench_test.go b/durable/recover_dispatched_bench_test.go index 29da241b..9736661a 100644 --- a/durable/recover_dispatched_bench_test.go +++ b/durable/recover_dispatched_bench_test.go @@ -53,11 +53,13 @@ func tickTail(tb testing.TB, st durable.Store, n int) []durable.Option[string, s return opts } -// countingDispatchStore counts a Store's Dispatched and MarkDispatched calls. +// countingDispatchStore counts a Store's Dispatched and MarkDispatched calls. While +// dropMarks is set it marks nothing, so a tail recorded then has every effect unmarked. type countingDispatchStore struct { durable.Store ds durable.DispatchStore dispatched, marks int + dropMarks bool } func (s *countingDispatchStore) Dispatched(ctx context.Context, id durable.InstanceID) (map[string]bool, error) { @@ -67,14 +69,20 @@ func (s *countingDispatchStore) Dispatched(ctx context.Context, id durable.Insta func (s *countingDispatchStore) MarkDispatched(ctx context.Context, id durable.InstanceID, ids ...string) error { s.marks++ + if s.dropMarks { + return nil + } return s.ds.MarkDispatched(ctx, id, ids...) } // BenchmarkRecover_TickTailDispatchedSet measures Recover over a tail of 1000 tick -// Records, each naming one effect that already landed, on MemStore and FileStore, and -// reports the Dispatched calls one Recover makes. +// Records, each naming one effect, on MemStore and FileStore, and reports the +// Dispatched and MarkDispatched calls one Recover makes: with every effect already +// landed ("marked"), the common case, and with none landed ("unmarked"), where Recover +// sends and marks every one, so the set grows as it replays. An unmarked Recover marks +// its tail, so each one gets a tail of its own, recorded outside the timer. // -// Run with: go test -bench=RecoverTickTail -run=^$ . +// Run with: go test -bench=Recover_TickTail -run=^$ . func BenchmarkRecover_TickTailDispatchedSet(b *testing.B) { const n = 1000 ctx := context.Background() @@ -91,12 +99,18 @@ func BenchmarkRecover_TickTailDispatchedSet(b *testing.B) { return st, st }, } - for _, name := range []string{"MemStore", "FileStore"} { - b.Run(name, func(b *testing.B) { - st, ds := stores[name](b) - cs := &countingDispatchStore{Store: st, ds: ds} - opts := tickTail(b, cs, n) - cs.dispatched, cs.marks = 0, 0 + // tail records a tail of n ticks on a new store of kind, with every effect marked or + // none, and returns the store, counting from zero, and the options Recover needs. + tail := func(b *testing.B, kind string, marked bool) (*countingDispatchStore, []durable.Option[string, string, *runCtx]) { + st, ds := stores[kind](b) + cs := &countingDispatchStore{Store: st, ds: ds, dropMarks: !marked} + opts := tickTail(b, cs, n) + cs.dispatched, cs.marks, cs.dropMarks = 0, 0, false + return cs, opts + } + for _, kind := range []string{"MemStore", "FileStore"} { + b.Run(kind+"/marked", func(b *testing.B) { + cs, opts := tail(b, kind, true) b.ResetTimer() for range b.N { if _, err := durable.Recover(ctx, tickLoopMachine(), cs, "i1", opts...); err != nil { @@ -107,5 +121,21 @@ func BenchmarkRecover_TickTailDispatchedSet(b *testing.B) { b.ReportMetric(float64(cs.dispatched)/float64(b.N), "Dispatched/op") b.ReportMetric(float64(cs.marks)/float64(b.N), "MarkDispatched/op") }) + b.Run(kind+"/unmarked", func(b *testing.B) { + var dispatched, marks int + b.ResetTimer() + for range b.N { + b.StopTimer() + cs, opts := tail(b, kind, false) + b.StartTimer() + if _, err := durable.Recover(ctx, tickLoopMachine(), cs, "i1", opts...); err != nil { + b.Fatalf("Recover: %v", err) + } + dispatched, marks = dispatched+cs.dispatched, marks+cs.marks + } + b.StopTimer() + b.ReportMetric(float64(dispatched)/float64(b.N), "Dispatched/op") + b.ReportMetric(float64(marks)/float64(b.N), "MarkDispatched/op") + }) } } diff --git a/durable/runner.go b/durable/runner.go index 20c5410d..4c4c065d 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -1454,6 +1454,11 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S if err = checkNotNewer(id, env.Format, tail); err != nil { return nil, err } + // Refuse a journal mixed versions wrote before replaying any of it, so a refused one + // sends no effect. + if err = checkFormats(env.Format, tail); err != nil { + return nil, fmt.Errorf("durable: recovering %q: %w", id, err) + } kernelSnap, persistedTimers, checkpointNext := env.Snapshot, env.Timers, env.NextStep snap, err := state.UnmarshalSnapshot[S, E, C](kernelSnap) @@ -1523,7 +1528,10 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // and hadn't ended, recorded alongside ended. Nil means none was recorded. unstarted := env.UnstartedActors - // seen is the dispatched set, loaded once, at the first Record with effects to send. + // seen is Recover's copy of the dispatched set: loaded at the first Record with + // effects to send, and read again only for a Record with an effect it doesn't show + // marked, so a mark a stale owner makes during a handoff is seen before that effect + // is sent (see dispatchWith). seen := &dispatchedSet{} for i := range tail { // A canceled context can stop a replayed settle or tick with no error, leaving @@ -1535,9 +1543,6 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // The mirror tracks the still-pending timers in armed as replay arms, cancels and // fires them, with deadlines drawn from the recorded clock reads, not the live // wall clock. - if err := checkFormat(env.Format, rec); err != nil { - return nil, fmt.Errorf("durable: recovering %q: %w", id, err) - } stepEffects, err := replayRecord(ctx, inst, &r.cfg, sched, svc, rec, mirror, armsActorFires(rec)) if err != nil { return nil, fmt.Errorf("durable: recovering %q: %w", id, err) @@ -2080,21 +2085,37 @@ func replayActors[S comparable, E comparable, C any]( return nil } -// checkFormat fails when rec's journal Format is lower than the checkpoint's that replay -// started from. A version doesn't write an older format than it did, so either an older -// version appended rec to a journal a newer one wrote, which a rolling deploy that runs -// mixed versions on one instance does, or, for a Record with no format, the Store lost -// the field. Either way the two versions may read the journal differently (an older one -// settles a parked result, for one), so replaying rec by its own format's rules would -// recover wrong. -func checkFormat(checkpoint int, rec *Record) error { +// checkFormats fails when a Record in records has a lower journal Format than the +// checkpoint's that replay starts from, or than any Record before it (see checkFormat). +// A journal's format never goes down, so this catches an older version's Record +// appended after a newer one's anywhere in the tail, not only after a newer checkpoint. +// Every replay calls it on the Records it will replay before it replays any. +func checkFormats(checkpoint int, records []Record) error { + highest := checkpoint + for i := range records { + if err := checkFormat(highest, &records[i]); err != nil { + return err + } + highest = max(highest, records[i].Format) + } + return nil +} + +// checkFormat fails when rec's journal Format is lower than highest, the highest format +// the journal records before it: the checkpoint's, or an earlier Record's. A version +// doesn't write an older format than it did, so either an older version appended rec to +// a journal a newer one wrote, which a rolling deploy that runs mixed versions on one +// instance does, or, for a Record with no format, the Store lost the field. Either way +// the two versions may read the journal differently (an older one settles a parked +// result, for one), so replaying rec by its own format's rules would recover wrong. +func checkFormat(highest int, rec *Record) error { switch { - case rec.Format >= checkpoint: + case rec.Format >= highest: return nil case rec.Format == 0: - return fmt.Errorf("durable: step %d records no journal format after a checkpoint of format %d: the Store didn't keep Record.Format, or a version from before that field appended the Record (mixed versions on one instance)", rec.Step, checkpoint) + return fmt.Errorf("durable: step %d records no journal format after one of format %d: the Store didn't keep Record.Format, or a version from before that field appended the Record (mixed versions on one instance)", rec.Step, highest) } - return fmt.Errorf("durable: step %d was appended by an older version, of journal format %d, after a checkpoint of format %d: don't run mixed versions on one instance", rec.Step, rec.Format, checkpoint) + return fmt.Errorf("durable: step %d was appended by an older version, of journal format %d, after one of format %d: don't run mixed versions on one instance", rec.Step, rec.Format, highest) } // checkNotNewer fails when id's checkpoint, of journal format checkpoint, or any of its diff --git a/durable/service.go b/durable/service.go index 657f3e31..6f05b469 100644 --- a/durable/service.go +++ b/durable/service.go @@ -75,11 +75,12 @@ type serviceResultPayload struct { // than settling it, and a later RunService settles it once the decoder decodes it. // An older Record never sets it. Parked bool `json:"parked,omitempty"` - // Type is a parked result's Go type, with its import path (see typeName), so the - // RunService that settles it holds the decoder to the type the ServiceFn returned, - // as the live settle did. The Record that settles a parked result records it too, - // so replay holds the decoder to it as that settle did. It is empty for a result - // that settled when it ran, whose type the live settle checked against the value. + // Type is a result's Go type, with its import path (see typeName): the type the + // ServiceFn returned. A parked result records it so the RunService that settles it + // holds the decoder to that type, as the live settle did, and every successful + // settle records it, the one that settles a parked result too, so replay holds the + // decoder to it as the settle did. It is empty for an error, and for a result + // recorded before this field, which replays with no type check. Type string `json:"type,omitempty"` } @@ -118,8 +119,13 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // WithEventDataDecoder covers arrives as the decoder's type; otherwise it arrives in // JSON's generic form: a struct is a map, an integer a float64 that loses precision // above 2^53. An error is message-only: an error with the same message, which -// errors.Is no longer matches. Register the decoder before results are recorded: a -// decoder added or changed later changes what replay produces. +// errors.Is no longer matches. The journal records the Go type the service returned, +// and replay holds the decoder to it, as the live settle did: a decoder that returns +// another type fails Recover and StateAt with ErrEventDataEncoding, though JSON's +// generic form and nil are always accepted. Register the decoder before results are +// recorded: changing the type a decoder returns for an event whose results are +// recorded fails Recover on them, unless it returns the generic form, and a decoder +// added later for the service's type changes what replay produces. // // It returns the routed FireResult and true, or the zero result and false when id // names no in-flight service (already settled, stopped, or never started). It returns @@ -219,7 +225,7 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir } // The JSON this settle decoded is the step's alone: take it, so no later step can // journal it and the Handle doesn't hold it. - raw := h.svcJSON.take() + settled := h.svcJSON.take() if derr := (*undecodableResult)(nil); errors.As(res.Err, &derr) { // The result didn't decode for its onDone event, so the transform aborted the // settle: nothing fired and no Assign ran. Park it, unless a context that ended @@ -260,7 +266,7 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir return res, true, nil, nil, h.fail(fmt.Errorf("durable: settling service %q for %q: %w", id, h.id, err)) } - entry, err := recordServiceOutcome(id, raw, parked.typ, h.svc.LastError()) + entry, err := recordServiceOutcome(id, settled, h.svc.LastError()) if err != nil { return res, true, nil, nil, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) } @@ -350,8 +356,9 @@ func (h *Handle[S, E, C]) markSettled(id string) { // decoder that is JSON's generic form. Live, a decoder that returns a concrete type // other than the result's is refused, though JSON's generic form, including a // json.Number, is always accepted, so the default keeps working. A settle that -// succeeds leaves in settled the JSON it decoded, which serviceStep journals, so replay -// decodes the bytes live decoded; an error outcome leaves nil there. +// succeeds leaves in settled the JSON it decoded and the result's Go type, which +// serviceStep journals, so replay decodes the bytes live decoded and holds the decoder +// to the type live did; an error outcome leaves nothing there. // // A result that won't decode, or is refused, aborts the settle with an // *undecodableResult before it fires anything, so no onDone or onError Assign runs; @@ -367,7 +374,7 @@ func (h *Handle[S, E, C]) markSettled(id string) { // with a message saying so, normalized the same way. func replayableOutcome[E comparable](decode EventDataDecoder[E], settled *settledJSON) func(E, any, error) (any, error, error) { return func(event E, result any, err error) (any, error, error) { - settled.set(nil) + settled.set(recordedResult{}) if err != nil { msg, perr := normalizedErrorMessage(err, "service error") if perr != nil { @@ -395,33 +402,34 @@ func replayableOutcome[E comparable](decode EventDataDecoder[E], settled *settle derr = decodedAsRecorded(value, recorded.typ) } if derr != nil { - return nil, nil, &undecodableResult{raw: b, typ: parkedType(recorded, replay, result), err: derr} + return nil, nil, &undecodableResult{raw: b, typ: resultType(recorded, replay, result), err: derr} } - settled.set(b) + settled.set(recordedResult{raw: b, typ: resultType(recorded, replay, result)}) return value, nil, nil } } -// settledJSON holds the JSON the last successful settle decoded, for serviceStep to -// journal. The transform sets it during the settle, which runs under the Handle's step -// lock live and on replay's one goroutine, and serviceStep takes it once the settle -// returns, so it holds nothing between steps. A nil settledJSON keeps nothing. -type settledJSON struct{ raw json.RawMessage } +// settledJSON holds the JSON the last successful settle decoded and the result's Go +// type, for serviceStep to journal. The transform sets it during the settle, which runs +// under the Handle's step lock live and on replay's one goroutine, and serviceStep +// takes it once the settle returns, so it holds nothing between steps. A nil +// settledJSON keeps nothing. +type settledJSON struct{ recordedResult } -func (s *settledJSON) set(b []byte) { +func (s *settledJSON) set(r recordedResult) { if s != nil { - s.raw = b + s.recordedResult = r } } -// take returns the JSON the last settle left and clears it. -func (s *settledJSON) take() json.RawMessage { +// take returns what the last settle left and clears it. +func (s *settledJSON) take() recordedResult { if s == nil { - return nil + return recordedResult{} } - b := s.raw - s.raw = nil - return b + r := s.recordedResult + s.recordedResult = recordedResult{} + return r } // legacyResult is a service result recorded in a Record of a format before @@ -429,9 +437,9 @@ func (s *settledJSON) take() json.RawMessage { // decoder or type check. The transform settles it as it is. type legacyResult struct{ value any } -// parkedType returns the Go type a park records for result: the recorded one on -// replay, else the ServiceFn's value's. -func parkedType(recorded recordedResult, replay bool, result any) string { +// resultType returns the Go type a park, or a successful settle, records for result: +// the recorded one on replay, else the ServiceFn's value's. +func resultType(recorded recordedResult, replay bool, result any) string { if replay { return recorded.typ } @@ -443,9 +451,9 @@ func parkedType(recorded recordedResult, replay bool, result any) string { // RunService settles for a parked result. type recordedResult struct { raw json.RawMessage - // typ is a parked result's Go type, as typeName gives it, also when replay settles - // it; empty for a result that settled when it ran, whose type the live settle - // checked against the value. + // typ is the result's Go type, as typeName gives it, which a park and every + // successful settle record; empty for a result recorded before settles recorded it, + // which replay doesn't type-check. typ string } @@ -596,10 +604,11 @@ func qualifiedName(pkg, name string) string { return pkg + "." + name } -// decodedAsRecorded is decodedAsPassed for a parked result, which records its type's +// decodedAsRecorded is decodedAsPassed for a recorded result, which records its type's // name rather than a value: it reports, as an ErrEventDataEncoding error, a value the // decoder returned as another type than typ. As live, nil and JSON's generic form are -// always accepted. An empty typ, a settled result's, accepts anything. +// always accepted. An empty typ, a result recorded before settles recorded the type, +// accepts anything. func decodedAsRecorded(value any, typ string) error { if typ == "" || value == nil || genericJSON(value) { return nil @@ -653,16 +662,16 @@ func decodedAsPassed(value, passed any, what string, generic bool) error { } // recordServiceOutcome builds the JournalServiceResult entry for a just-settled -// service: the error lastErr, or, for a success, raw, the JSON the settle decoded (see -// settledJSON), so replay decodes the bytes live decoded, and typ, the Go type a parked -// result recorded, empty for one that settled when it ran. -func recordServiceOutcome(id string, raw json.RawMessage, typ string, lastErr error) (state.JournalEntry, error) { +// service: the error lastErr, or, for a success, settled, the JSON the settle decoded +// and the result's Go type (see settledJSON), so replay decodes the bytes live decoded +// and holds the decoder to the type live did. +func recordServiceOutcome(id string, settled recordedResult, lastErr error) (state.JournalEntry, error) { var sp serviceResultPayload if lastErr != nil { msg := lastErr.Error() sp.Error = &msg } else { - sp.Result, sp.Type = raw, typ + sp.Result, sp.Type = settled.raw, settled.typ } payload, err := json.Marshal(sp) if err != nil { @@ -711,7 +720,7 @@ func replayService[S comparable, E comparable, C any](ctx context.Context, svc * } result = legacyResult{value: v} case len(sp.Result) > 0: - // A settled parked result's type holds the decoder to it, as the settle did. + // The recorded type holds the decoder to it, as the settle did. result = recordedResult{raw: sp.Result, typ: sp.Type} } res, ok := svc.SettleDone(ctx, entry.CorrelationID, result) diff --git a/durable/service_encode_internal_test.go b/durable/service_encode_internal_test.go index 6b45fd99..e433bbce 100644 --- a/durable/service_encode_internal_test.go +++ b/durable/service_encode_internal_test.go @@ -47,7 +47,7 @@ func TestServiceResult_JournalBytesAreWhatLiveDecoded(t *testing.T) { if outErr != nil || abort != nil { t.Fatalf("the transform returned %v, %v", outErr, abort) } - entry, err := recordServiceOutcome("svc", settled.raw, "", nil) + entry, err := recordServiceOutcome("svc", settled.take(), nil) if err != nil { t.Fatalf("recordServiceOutcome: %v", err) } diff --git a/durable/service_step_internal_test.go b/durable/service_step_internal_test.go index 89edf651..0417c7e5 100644 --- a/durable/service_step_internal_test.go +++ b/durable/service_step_internal_test.go @@ -74,8 +74,8 @@ func TestServiceStep_StaleSettleRecordsNothing(t *testing.T) { if h.settled[id] { t.Fatal("the service is noted settled, so a checkpoint would record it settled") } - if h.svcJSON.raw != nil { - t.Fatalf("the Handle holds the stale settle's JSON %s", h.svcJSON.raw) + if h.svcJSON.raw != nil || h.svcJSON.typ != "" { + t.Fatalf("the Handle holds the stale settle's JSON %s (%s)", h.svcJSON.raw, h.svcJSON.typ) } if got := h.inst.Current(); got != "loading" { t.Fatalf("in %q, want loading", got) diff --git a/durable/settled_type_test.go b/durable/settled_type_test.go new file mode 100644 index 00000000..e6513fad --- /dev/null +++ b/durable/settled_type_test.go @@ -0,0 +1,102 @@ +package durable_test + +import ( + "context" + "errors" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These tests answer a review finding on a result that settles when its service runs: +// its Record kept no Go type, only a parked result's did, so replay let the decoder +// return any type, though the live settle held it to the type the ServiceFn returned. +// Every successful settle in a Record of this format now records the type, and replay +// holds the decoder to it as for a settled parked result. + +// settledUser runs the scn service live, settling a scnUser under live, and returns +// what Recover needs. +func settledUser(t *testing.T, fn state.ServiceFn[svcCtx], live ...durable.Option[string, string, svcCtx]) (*state.Machine[string, string, svcCtx], *scnAssigns, durable.Option[string, string, svcCtx], *durable.MemStore) { + t.Helper() + a := &scnAssigns{} + m := scnMachine(fn, a) + reg := durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)) + store := durable.NewMemStore() + h := scnStart(t, m, store, append([]durable.Option[string, string, svcCtx]{reg}, live...)...) + mustRunService(t, h, state.InvokeID(scnInvoke, "loading", 0)) + return m, a, reg, store +} + +// TestServiceResult_SettledRecordKeepsTheType: a scnUser settled live under a decoder +// for it records its type, and a Recover whose decoder returns another type, a pointer +// to it or another struct, fails with ErrEventDataEncoding and runs no Assign, as for a +// settled parked result (see TestParkedResult_SettledRecordKeepsTheType). The same +// decoder, and JSON's generic form, still replay it. +func TestServiceResult_SettledRecordKeepsTheType(t *testing.T) { + ctx := context.Background() + userType := "github.com/stablekernel/crucible/durable_test.scnUser" + for _, tc := range []struct { + name string + live []durable.Option[string, string, svcCtx] + }{ + {"decoded live as its type", []durable.Option[string, string, svcCtx]{decoderFor("done", durable.DecodeAs[scnUser])}}, + {"decoded live as JSON's generic form", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + m, a, reg, store := settledUser(t, returning(scnUser{ID: 7, Name: "ann"}, nil), tc.live...) + if got := serviceTypes(t, store); len(got) != 1 || got[0] != userType { + t.Fatalf("the settle records the type %q, want %s", got, userType) + } + for _, wrong := range []struct { + name string + opt durable.Option[string, string, svcCtx] + }{ + {"a pointer to it", decoderFor("done", durable.DecodeAs[*scnUser])}, + {"another struct", decoderFor("done", durable.DecodeAs[scnAdmin])}, + } { + done := a.done + _, err := durable.Recover(ctx, m, store, "scn-1", reg, wrong.opt) + if !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Recover with a decoder for %s: %v, want ErrEventDataEncoding", wrong.name, err) + } + if a.done != done { + t.Fatalf("replay ran onDone's Assign with %s, a type the live settle refuses", describe(a.seen[len(a.seen)-1])) + } + } + for _, right := range []struct { + name string + opt []durable.Option[string, string, svcCtx] + }{ + {"its own type", []durable.Option[string, string, svcCtx]{decoderFor("done", durable.DecodeAs[scnUser])}}, + {"JSON's generic form", nil}, + } { + if _, err := durable.Recover(ctx, m, store, "scn-1", append([]durable.Option[string, string, svcCtx]{reg}, right.opt...)...); err != nil { + t.Fatalf("Recover with a decoder for %s: %v", right.name, err) + } + } + }) + } +} + +// TestServiceResult_SettledNilResultReplays: a nil result settles and replays under any +// decoder, as live: it arrives as nil whatever the decoder (see +// TestServiceResult_NilResult_ArrivesNil). +func TestServiceResult_SettledNilResultReplays(t *testing.T) { + ctx := context.Background() + m, a, reg, store := settledUser(t, returning(nil, nil), decoderFor("done", durable.DecodeAs[scnUser])) + for _, opt := range []durable.Option[string, string, svcCtx]{ + decoderFor("done", durable.DecodeAs[scnUser]), + decoderFor("done", durable.DecodeAs[scnAdmin]), + decoderFor("done", durable.DecodeAs[any]), + } { + if _, err := durable.Recover(ctx, m, store, "scn-1", reg, opt); err != nil { + t.Fatalf("Recover: %v", err) + } + } + for i, v := range a.seen { + if v != nil { + t.Fatalf("run %d saw %s, want nil", i, describe(v)) + } + } +} diff --git a/durable/timetravel.go b/durable/timetravel.go index 89eb0699..70027ee4 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -241,6 +241,9 @@ func replayThrough[S comparable, E comparable, C any]( bounded []Record, format int, ) error { + if err := checkFormats(format, bounded); err != nil { + return err + } sched := state.NewScheduler(inst) svc := newReplayServiceRunner(inst, cfg) @@ -256,9 +259,6 @@ func replayThrough[S comparable, E comparable, C any]( } // A domain effect is never dispatched here: a time-travel read applies no // side effect. - if err := checkFormat(format, &bounded[i]); err != nil { - return err - } if _, err := replayRecord(ctx, inst, cfg, sched, svc, &bounded[i], nil, armsActorFires(&bounded[i])); err != nil { return err } From b45dbabda6a88520cd3ee59d9099e3e534653ac7 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 19:44:10 -0700 Subject: [PATCH 12/13] fix(durable,state): Tick returns nothing for a stale settle; refuse an effect handler on a Store without dispatch tracking - state: ServiceRunner.Tick returns an empty slice when the activation its ServiceFn ran for was stopped, or stopped and armed again, before the settle, as for an id not in flight. An abort still returns its one-element slice with the abort in Err. A non-empty slice now always means the settle fired or was aborted, whatever FireResult the Fire returned. - durable: serviceStep reads a stale settle from Tick's empty result, and settledNothing, which guessed it from a zero FireResult, is gone. A middleware that returns a zero FireResult for the onDone settle no longer leaves the service dropped live but unrecorded, which made Recover run the ServiceFn again. - durable: an effect handler with a Store that isn't a DispatchStore is refused with ErrEffectDispatch by Start, Recover and the stateless Runner.Fire, before any Store read or write or handler call. On main it silently skipped every effect. The no-DispatchStore no-op branches in dispatch are gone; no handler still dispatches nothing. Each change has a test that fails without it, checked by reverting it. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 3 + durable/CHANGELOG.md | 9 ++ durable/README.md | 3 + durable/dispatch_store_required_test.go | 159 ++++++++++++++++++++++ durable/doc.go | 6 + durable/effect.go | 51 ++++--- durable/effect_test.go | 25 ---- durable/errors.go | 8 +- durable/runner.go | 15 +- durable/runner_options.go | 5 +- durable/service.go | 23 +--- durable/service_step_internal_test.go | 71 +++++++++- state/CHANGELOG.md | 5 +- state/runner.go | 20 ++- state/settle_stale_test.go | 10 +- 15 files changed, 336 insertions(+), 77 deletions(-) create mode 100644 durable/dispatch_store_required_test.go diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 8c1ecf5b..04ac535c 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -36,6 +36,9 @@ implement the `Store` interface out of tree. handler returned, or a handoff between owners, where it can apply twice, so keep handlers idempotent on `EffectInstance` plus the `EffectID`. Each effect carries a deterministic `EffectID` and is deduplicated through the `Store`'s dispatch set. + The handler needs a `Store` that implements `DispatchStore`, as `MemStore` and + `FileStore` do: with any other `Store`, `Start`, `Recover` and the stateless + `Runner.Fire` refuse the configuration with `ErrEffectDispatch`. - **Durability across restart.** Every `Fire` step is write-ahead appended to the `Store` before it is acknowledged, so a crash after a successful `Fire` never loses the step. Periodic checkpoints bound the tail that recovery replays. diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 01ffb493..3105b2fd 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -67,6 +67,15 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed +- **An effect handler needs a `DispatchStore`.** This is a behavior change. On main, a + Runner with `WithEffectHandler` and a Store that doesn't implement `DispatchStore` + silently skipped every effect: `Fire`'s, and `Recover`'s re-send, never reached the + handler. Now `Start`, `Recover` and the stateless `Runner.Fire` refuse that + configuration with an error wrapping `ErrEffectDispatch`, before they write to the + Store or call the handler. A Store with no handler wired is unaffected, and + `MemStore` and `FileStore` implement `DispatchStore`. A Store author whose backend + doesn't track dispatched effects either implements `Dispatched` and + `MarkDispatched` or drops the handler. - **A journal this version writes isn't for an older one.** An actor entry can now say its parent Fire was refused, which an older `durable` fails `Recover` on, or, for an entry with no event, replays as the zero event. An older `durable` also ignores the diff --git a/durable/README.md b/durable/README.md index 4d3e702c..42448a59 100644 --- a/durable/README.md +++ b/durable/README.md @@ -37,6 +37,9 @@ backends implement the `Store` interface out of tree. handler returned, or a handoff between owners, where it can apply twice, so keep handlers idempotent on `EffectInstance` plus the `EffectID`. Each effect carries a deterministic `EffectID` and is deduplicated through the `Store`'s dispatch set. + The handler needs a `Store` that implements `DispatchStore`, as `MemStore` and + `FileStore` do: with any other `Store`, `Start`, `Recover` and the stateless + `Runner.Fire` refuse the configuration with `ErrEffectDispatch`. - **Durability across restart**: every `Fire` step is write-ahead appended to the `Store` before it is acknowledged, so a crash after a successful `Fire` never loses the step. Periodic checkpoints bound the tail recovery replays. diff --git a/durable/dispatch_store_required_test.go b/durable/dispatch_store_required_test.go new file mode 100644 index 00000000..4c032ec3 --- /dev/null +++ b/durable/dispatch_store_required_test.go @@ -0,0 +1,159 @@ +package durable_test + +import ( + "context" + "errors" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// noDispatchStore wraps a Store and hides its DispatchStore methods, so the Runner +// can't track which effects it applied. It counts the writes that reach the Store. +type noDispatchStore struct { + durable.Store + writes int +} + +func (s *noDispatchStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + s.writes++ + return s.Store.Append(ctx, id, rec, opts...) +} + +func (s *noDispatchStore) Checkpoint(ctx context.Context, id durable.InstanceID, snapshot []byte, throughStep int, opts ...durable.CheckpointOption) error { + s.writes++ + return s.Store.Checkpoint(ctx, id, snapshot, throughStep, opts...) +} + +// TestEffectHandler_RefusesAStoreWithoutDispatch proves an effect handler wired with +// a Store that isn't a DispatchStore is refused with ErrEffectDispatch, before any +// Store write or handler call, by Start, Recover and the stateless Fire. On main the +// handler was silently skipped. +func TestEffectHandler_RefusesAStoreWithoutDispatch(t *testing.T) { + ctx := context.Background() + + t.Run("Start", func(t *testing.T) { + mem := durable.NewMemStore() + st := &noDispatchStore{Store: mem} + rec := newEffectRecorder() + h, err := durable.NewRunner(entryTwoEffectsMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](rec.handler())). + Start(ctx, "i1", &runCtx{}, state.WithInitialState("ready")) + if !errors.Is(err, durable.ErrEffectDispatch) || h != nil { + t.Fatalf("Start = %v, %v; want no Handle and ErrEffectDispatch", h, err) + } + if st.writes != 0 || rec.totalApplies() != 0 { + t.Fatalf("Start wrote %d times and applied %d effects, want none", st.writes, rec.totalApplies()) + } + if _, _, lerr := mem.Load(ctx, "i1"); !errors.Is(lerr, durable.ErrInstanceNotFound) { + t.Fatalf("Load after the refused Start = %v, want ErrInstanceNotFound", lerr) + } + }) + + // started returns a Store holding an instance started with no handler, after fired + // Fires, so with "go" it holds an effect not yet sent. + started := func(t *testing.T, fired ...string) *noDispatchStore { + t.Helper() + st := &noDispatchStore{Store: durable.NewMemStore()} + r := durable.NewRunner(effectMachine(), st) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start with no handler: %v", err) + } + for _, ev := range fired { + if _, err := r.Fire(ctx, "i1", ev); err != nil { + t.Fatalf("Fire %s with no handler: %v", ev, err) + } + } + st.writes = 0 + return st + } + + for _, tc := range []struct { + name string + fired []string + }{{"no effect pending", nil}, {"an effect pending", []string{"go"}}} { + t.Run("Recover/"+tc.name, func(t *testing.T) { + st := started(t, tc.fired...) + rec := newEffectRecorder() + h, err := durable.Recover(ctx, effectMachine(), st, "i1", + durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if !errors.Is(err, durable.ErrEffectDispatch) || h != nil { + t.Fatalf("Recover = %v, %v; want no Handle and ErrEffectDispatch", h, err) + } + if st.writes != 0 || rec.totalApplies() != 0 { + t.Fatalf("Recover wrote %d times and applied %d effects, want none", st.writes, rec.totalApplies()) + } + }) + + t.Run("stateless Fire/"+tc.name, func(t *testing.T) { + st := started(t, tc.fired...) + _, before, _ := st.Load(ctx, "i1") + rec := newEffectRecorder() + ev := "go" + if len(tc.fired) > 0 { + ev = "bill" + } + _, err := durable.NewRunner(effectMachine(), st, + durable.WithEffectHandler[string, string, *runCtx](rec.handler())). + Fire(ctx, "i1", ev) + if !errors.Is(err, durable.ErrEffectDispatch) || errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire = %v, want ErrEffectDispatch refusing it, not a failed Handle", err) + } + if st.writes != 0 || rec.totalApplies() != 0 { + t.Fatalf("Fire wrote %d times and applied %d effects, want none", st.writes, rec.totalApplies()) + } + if _, tail, _ := st.Load(ctx, "i1"); len(tail) != len(before) { + t.Fatalf("the journal holds %d Records after the refused Fire, want %d", len(tail), len(before)) + } + }) + } + + t.Run("no handler", func(t *testing.T) { + st := started(t, "go") + h, err := durable.Recover(ctx, effectMachine(), st, "i1") + if err != nil { + t.Fatalf("Recover with no handler: %v", err) + } + if _, err := h.Fire(ctx, "bill"); err != nil { + t.Fatalf("Fire with no handler: %v", err) + } + if got := h.Instance().Current(); got != "billed" { + t.Fatalf("in %q, want billed", got) + } + }) +} + +// TestEffectHandler_DispatchStoresUnaffected proves MemStore and FileStore, which +// implement DispatchStore, take an effect handler: Start, the stateless Fire and +// Recover all run, and each effect applies once. +func TestEffectHandler_DispatchStoresUnaffected(t *testing.T) { + ctx := context.Background() + fs, err := durable.NewFileStore(t.TempDir()) + if err != nil { + t.Fatalf("NewFileStore: %v", err) + } + for _, tc := range []struct { + name string + st durable.Store + }{{"MemStore", durable.NewMemStore()}, {"FileStore", fs}} { + t.Run(tc.name, func(t *testing.T) { + rec := newEffectRecorder() + r := durable.NewRunner(effectMachine(), tc.st, + durable.WithEffectHandler[string, string, *runCtx](rec.handler())) + if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")); err != nil { + t.Fatalf("Start: %v", err) + } + if _, err := r.Fire(ctx, "i1", "go"); err != nil { + t.Fatalf("Fire: %v", err) + } + if _, err := durable.Recover(ctx, effectMachine(), tc.st, "i1", + durable.WithEffectHandler[string, string, *runCtx](rec.handler())); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := rec.totalApplies(); n != 1 { + t.Fatalf("applied %d effects, want the go step's one, once", n) + } + }) + } +} diff --git a/durable/doc.go b/durable/doc.go index 205cc7fa..141d9a0b 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -154,6 +154,12 @@ // recovery redispatches it; a crash after dispatch finds the id marked and skips // it. // +// The dispatched set lives in a Store that implements DispatchStore, as MemStore and +// FileStore do. A Runner with an effect handler requires one: Start, Recover and the +// stateless Runner.Fire refuse any other Store with ErrEffectDispatch before writing +// anything or calling the handler. Without a handler, any Store works and no effect is +// applied. +// // Kernel driver effects (services, timers, actors) are absorbed by their host // drivers and never reach the handler. // diff --git a/durable/effect.go b/durable/effect.go index cb620f17..ec4e266b 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -105,8 +105,11 @@ func EffectInstance(ctx context.Context) (InstanceID, bool) { // tracks which effect ids an instance has applied, so a (re)dispatch skips the // ones that already landed. It is an additive, optional capability — the core // Store interface is unchanged; a backend opts in by implementing these two -// methods, and the Runner dispatches effects only against a Store that does (the -// in-tree MemStore does). A persistent backend marks an id in the same +// methods. A Runner with an effect handler (WithEffectHandler) requires it: Start, +// Recover and the stateless Runner.Fire refuse a Store that doesn't implement it with +// ErrEffectDispatch, before any Store write or handler call. A Runner with no handler +// dispatches nothing and needs neither method. The in-tree MemStore and FileStore +// implement it. A persistent backend marks an id in the same // transaction that records its side-effect acknowledgement to keep the guarantee // tight. type DispatchStore interface { @@ -238,28 +241,39 @@ func marshalEffect(ke state.KindedEffect) (env state.EffectEnvelope, err error) // dispatched set it invokes the handler and, on success, marks the id dispatched. // A handler error stops dispatch and is returned wrapped in ErrEffectDispatch // with the failing effect left un-marked for a later recovery to retry. With no -// handler wired, or a Store that is not a DispatchStore, dispatch is a no-op (the -// ids are still recorded for a handler-equipped recovery). +// handler wired, dispatch is a no-op (the ids are still recorded for a +// handler-equipped recovery). A handler is only ever wired with a DispatchStore: +// Start and Recover refuse one without (see checkDispatchStore). func (h *Handle[S, E, C]) dispatchEffects(ctx context.Context, des []dispatchableEffect) error { return h.runner.dispatch(ctx, h.id, des) } -// dispatchStore returns the Store's dispatch tracking when des has effects to apply: a -// handler is wired and the Store is a DispatchStore. Otherwise dispatch is a no-op, and -// it returns false. -func (r *Runner[S, E, C]) dispatchStore(des []dispatchableEffect) (DispatchStore, bool) { - if len(des) == 0 || r.cfg.effectHandler == nil { - return nil, false +// checkDispatchStore refuses an effect handler wired with a Store that isn't a +// DispatchStore, which couldn't track which effects were applied. Start and recover, +// which Recover and the stateless Fire share, call it before they touch the Store, so +// every Handle that dispatches has dispatch tracking. The error wraps +// ErrEffectDispatch. +func (r *Runner[S, E, C]) checkDispatchStore(id InstanceID) error { + if r.cfg.effectHandler == nil { + return nil + } + if _, ok := r.store.(DispatchStore); !ok { + return fmt.Errorf("%w: instance %q: an effect handler needs a Store that implements DispatchStore to track the effects it applied, and %T doesn't", ErrEffectDispatch, id, r.store) } - ds, ok := r.store.(DispatchStore) - return ds, ok + return nil +} + +// dispatching reports whether des has effects to apply: a handler is wired and des +// isn't empty. Otherwise dispatch is a no-op. +func (r *Runner[S, E, C]) dispatching(des []dispatchableEffect) bool { + return len(des) > 0 && r.cfg.effectHandler != nil } // applyStartEffects invokes the handler for the initial entry's domain effects. Like -// dispatch it is a no-op without a handler or a DispatchStore. It marks nothing, because -// the instance doesn't exist yet and nothing replays these effects. +// dispatch it is a no-op without a handler. It marks nothing, because the instance +// doesn't exist yet and nothing replays these effects. func (r *Runner[S, E, C]) applyStartEffects(ctx context.Context, id InstanceID, des []dispatchableEffect) error { - if _, ok := r.dispatchStore(des); !ok { + if !r.dispatching(des) { return nil } for _, de := range des { @@ -304,7 +318,7 @@ type dispatchedSet struct{ ids map[string]bool } // dispatch is the shared exactly-once apply loop used by the live path and by // recovery replay: it reads the instance's dispatched set, invokes the handler // for each not-yet-dispatched effect, and marks each id as it succeeds. It is a -// no-op without a handler or a DispatchStore. An error or a panic in the Store's +// no-op without a handler. An error or a panic in the Store's // Dispatched or MarkDispatched becomes an ErrEffectDispatch error wrapping it, as // applyEffect makes one in the handler, so a step and Recover fail the same way. func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dispatchableEffect) error { @@ -320,8 +334,7 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis // marks needn't be added: an effect id names one step, and a Recover replays each step // once, so it never meets one again. func (r *Runner[S, E, C]) dispatchWith(ctx context.Context, id InstanceID, des []dispatchableEffect, seen *dispatchedSet) (err error) { - ds, ok := r.dispatchStore(des) - if !ok { + if !r.dispatching(des) { return nil } defer func() { @@ -329,6 +342,8 @@ func (r *Runner[S, E, C]) dispatchWith(ctx context.Context, id InstanceID, des [ err = fmt.Errorf("%w: effect dispatch for %q panicked: %v", ErrEffectDispatch, id, p) } }() + // Start and recover refused a handler without dispatch tracking. + ds := r.store.(DispatchStore) if seen != nil && allMarked(seen.ids, des) { return nil diff --git a/durable/effect_test.go b/durable/effect_test.go index 1be259b5..62a8e1f8 100644 --- a/durable/effect_test.go +++ b/durable/effect_test.go @@ -407,31 +407,6 @@ func TestRunner_EffectDispatch_BareEffect_StampedByType(t *testing.T) { } } -// nonDispatchStore wraps a Store WITHOUT implementing DispatchStore, so the -// Runner records effect ids but cannot dedup; dispatch must degrade to a no-op -// rather than panic. -type nonDispatchStore struct{ durable.Store } - -// TestRunner_EffectDispatch_NonDispatchStore_NoOp proves a Store that does not -// implement DispatchStore leaves dispatch inert (the ids are still recorded for a -// dispatch-capable backend on recovery). -func TestRunner_EffectDispatch_NonDispatchStore_NoOp(t *testing.T) { - ctx := context.Background() - st := nonDispatchStore{Store: durable.NewMemStore()} - rec := newEffectRecorder() - r := durable.NewRunner(effectMachine(), st, - durable.WithEffectHandler[string, string, *runCtx](rec.handler())) - if _, err := r.Start(ctx, "i1", &runCtx{}, state.WithInitialState("idle")); err != nil { - t.Fatalf("Start: %v", err) - } - if _, err := r.Fire(ctx, "i1", "go"); err != nil { - t.Fatalf("Fire go: %v", err) - } - if n := rec.totalApplies(); n != 0 { - t.Fatalf("non-dispatch store applied %d effects, want 0 (inert)", n) - } -} - // timerNotifyMachine emits a kernel driver effect (a delayed `after` transition) on // the same step a domain effect fires, so the dispatcher must route the driver // effect to the Scheduler and only the domain effect to the handler. The timer diff --git a/durable/errors.go b/durable/errors.go index 36431757..d5b25527 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -98,7 +98,11 @@ var ( // persisted (write-ahead) but not marked dispatched, so the Handle fails with // ErrHandleFailed wrapping it, and Recover dispatches it again; a failure in // Runner.Start leaves no instance. Recover fails with this error while the handler, - // or the dispatch tracking, still does. Callers match this with errors.Is to tell a - // dispatch failure from a kernel transition or a Record's persistence error. + // or the dispatch tracking, still does. Start, Recover and the stateless Runner.Fire + // also return it, before writing anything or calling the handler, for an effect + // handler wired with a Store that isn't a DispatchStore, which has no dispatch + // tracking: that is a configuration error, so retrying doesn't help. Callers match + // this with errors.Is to tell a dispatch failure from a kernel transition or a + // Record's persistence error. ErrEffectDispatch = errors.New(errorPrefix + errorEffectDispatch) ) diff --git a/durable/runner.go b/durable/runner.go index 4c4c065d..2e6946cf 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -523,7 +523,12 @@ func (h *Handle[S, E, C]) ID() InstanceID { return h.id } // effect, as does a crash before the baseline. Overlapping Starts on one id, including a // stale owner's, each apply them before the Store refuses all but one. To dedupe, key on // EffectInstance plus the effect id. A handler wired only at Recover never sees them. +// With an effect handler wired, Start refuses a Store that isn't a DispatchStore with +// ErrEffectDispatch, before it reads or writes the Store or calls the handler. func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opts ...state.CastOption[S]) (*Handle[S, E, C], error) { + if err := r.checkDispatchStore(id); err != nil { + return nil, err + } if _, _, err := r.store.Load(ctx, id); err == nil { return nil, fmt.Errorf("%w: %q", ErrInstanceExists, id) } else if !errors.Is(err, ErrInstanceNotFound) { @@ -830,7 +835,8 @@ func parentFireErrs[S comparable](fires []state.ParentFire[S]) []error { // Fire drives one event through a durable instance identified by id, loading and // replaying it from the Store first, then recording the step. Handle.Fire describes // how a state.WithEventData payload is handled, and when it's refused. Loading can -// return ErrEffectDispatch before the event fires, as Recover can. It is the +// return ErrEffectDispatch before the event fires, as Recover can, including its +// refusal of an effect handler with a Store that isn't a DispatchStore. It is the // stateless entry point (no Handle required); for a hot path that fires many // events in sequence, hold a Handle from Start or Recover and use Handle.Fire to // avoid reloading between steps. Calls on one id aren't serialized, and must not overlap @@ -1425,7 +1431,9 @@ func respawn[S comparable, E comparable, C any](ctx context.Context, actors *sta // spawned drove. A ctx that ends once the actors are spawned, with no Fire refused, // doesn't fail it, as for Start: their behaviors already ran. It applies any recorded // effect not yet marked dispatched, so it returns ErrEffectDispatch and no Handle while -// the effect handler keeps failing. It refuses a journal a newer version of this module +// the effect handler keeps failing. With an effect handler wired, it refuses a Store +// that isn't a DispatchStore with ErrEffectDispatch before it loads anything, whether +// or not an effect is waiting. It refuses a journal a newer version of this module // wrote, whose checkpoint or any Record records a journal format above this version's, // before it restores or replays anything; that error wraps none of the package's // sentinel errors. @@ -1439,6 +1447,9 @@ func Recover[S comparable, E comparable, C any](ctx context.Context, m *state.Ma // reproducing the live driver sequence (Fire+Absorb for external steps, Tick for // scheduler barriers), then continue live against the recording clock. func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S, E, C], error) { + if err := r.checkDispatchStore(id); err != nil { + return nil, err + } snapBytes, tail, err := r.store.Load(ctx, id) if err != nil { return nil, fmt.Errorf("durable: loading instance %q: %w", id, err) diff --git a/durable/runner_options.go b/durable/runner_options.go index 6d427561..426db610 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -129,7 +129,10 @@ func WithActorPalette[S comparable, E comparable, C any](palette map[string]stat // WithEffectHandler binds the seam the Runner calls to apply each emitted domain // effect — an email, a charge, a published message: any side effect // a transition emits as a domain Effect value. Effects from a parent transition an actor -// drives aren't dispatched, and a Store that isn't a DispatchStore applies none. A +// drives aren't dispatched. The handler needs a Store that is a DispatchStore, as +// MemStore and FileStore are: with any other Store, Start, Recover and the stateless +// Runner.Fire refuse the configuration with ErrEffectDispatch before they write to the +// Store or call the handler. A // durable effect from a step is applied // exactly once over the whole lifetime of an instance (the live run plus any // number of recoveries), except that it can apply twice after a crash or failed mark diff --git a/durable/service.go b/durable/service.go index 6f05b469..e914bb44 100644 --- a/durable/service.go +++ b/durable/service.go @@ -212,16 +212,16 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir // holds what it returned. parked, isParked := h.parked[id] var res state.FireResult[S] - var routed bool + routed := true if isParked { res, routed = h.svc.SettleDone(h.stepContext(ctx), id, parked) - } else { - results := h.svc.Tick(h.stepContext(ctx), id) - if len(results) == 0 { - return state.FireResult[S]{}, false, nil, nil, nil - } + } else if results := h.svc.Tick(h.stepContext(ctx), id); len(results) > 0 { res = results[0] - routed = !settledNothing(res) + } else { + // HasPending found the service in flight, so the ServiceFn ran: Tick returns + // nothing only when its activation was stopped, or stopped and armed again, + // before the settle. + routed = false } // The JSON this settle decoded is the step's alone: take it, so no later step can // journal it and the Handle doesn't hold it. @@ -328,15 +328,6 @@ func (h *Handle[S, E, C]) parkResult(ctx context.Context, id string, derr *undec return state.FireResult[S]{}, true, rec, nil, parkErr } -// settledNothing reports whether res is the one zero FireResult ServiceRunner.Tick -// returns for an activation stopped, or stopped and armed again, while its ServiceFn ran. -// A settle that fired returns its Fire's result, whose Trace names the event and the -// state it fired in, or an error, as fireStep relies on in journaling the Trace's -// event. -func settledNothing[S comparable](res state.FireResult[S]) bool { - return res.Err == nil && res.Effects == nil && res.Trace.Event == "" && res.Trace.FromState == "" -} - // markSettled records that the service id settled in its state's current activation, so // no result of it is parked. It and forgetServices are the only writers of settled // alongside parked, so the two can't disagree. diff --git a/durable/service_step_internal_test.go b/durable/service_step_internal_test.go index 0417c7e5..3bd4fe1c 100644 --- a/durable/service_step_internal_test.go +++ b/durable/service_step_internal_test.go @@ -33,7 +33,7 @@ func stepOptions(fn state.ServiceFn[stepCtx]) []Option[string, string, stepCtx] // TestServiceStep_StaleSettleRecordsNothing: when the activation a ServiceFn runs for is // stopped, or stopped and armed again, before its settle, ServiceRunner.Tick fires -// nothing and returns one zero FireResult. serviceStep must record nothing for it: no +// nothing and returns nothing. serviceStep must record nothing for it: no // JournalServiceResult, no settled service, and no JSON left behind, so a Recover still // finds the service in flight. The step lock keeps durable's own steps from changing the // activation while a ServiceFn runs (see TestRunService_ServiceFnCantChangeItsActivation), @@ -94,6 +94,75 @@ func TestServiceStep_StaleSettleRecordsNothing(t *testing.T) { } } +// TestServiceStep_ZeroFireResultSettleIsRecorded: a middleware may return a zero +// FireResult for a Fire it ran. When that Fire is the onDone settle, the settle still +// happened: the runner dropped the service and the instance moved. serviceStep must +// record it as any settle, so the live Handle and the journal agree: the step is +// recorded, the service is noted settled, and Recover replays the result to the same +// state without running the ServiceFn again. +func TestServiceStep_ZeroFireResultSettleIsRecorded(t *testing.T) { + ctx := context.Background() + id := state.InvokeID("step", "loading", 0) + runs := 0 + fn := func(context.Context, state.ServiceCtx[stepCtx]) (any, error) { + runs++ + return "ok", nil + } + hide := func(next state.FireFunc[string, string, stepCtx]) state.FireFunc[string, string, stepCtx] { + return func(ctx context.Context, event string) state.FireResult[string] { + res := next(ctx, event) + if event == "done" { + return state.FireResult[string]{} + } + return res + } + } + m := state.ForgeFor[stepCtx]("step"). + Use(hide). + Service("fetch", fn). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done"), state.WithInvokeOnError("fail")). + State("ready").Final(). + State("failed").Final(). + Initial("loading"). + Transition("loading").On("done").GoTo("ready"). + Transition("loading").On("fail").GoTo("failed"). + Quench() + st := NewMemStore() + h, err := NewRunner(m, st, stepOptions(fn)...).Start(ctx, "s1", stepCtx{}, state.WithInitialState("loading")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, ran, rerr := h.RunService(ctx, id); rerr != nil || !ran { + t.Fatalf("RunService = %v, %v; want ran with no error", ran, rerr) + } + if got := h.inst.Current(); got != "ready" || h.svc.HasPending(id) { + t.Fatalf("live: in %q with the service pending %v; want ready and settled", got, h.svc.HasPending(id)) + } + if !h.settled[id] { + t.Fatal("the live Handle doesn't note the service settled") + } + _, recs, err := st.Load(ctx, "s1") + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(recs) != 1 { + t.Fatalf("the journal holds %d Records, want the settle's one", len(recs)) + } + rh, err := Recover(ctx, m, st, "s1", stepOptions(fn)...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := rh.inst.Current(); got != "ready" || rh.svc.HasPending(id) { + t.Fatalf("recovered: in %q with the service pending %v; want ready and settled, as live", got, rh.svc.HasPending(id)) + } + if _, ran, rerr := rh.RunService(ctx, id); ran || rerr != nil { + t.Fatalf("the recovered RunService = %v, %v; want nothing in flight", ran, rerr) + } + if runs != 1 { + t.Fatalf("the ServiceFn ran %d times, want once", runs) + } +} + // TestServiceStep_ClearsTheSettledJSON: the JSON a settle decoded is the step's alone. // Once RunService returns, live or for a parked result, and once Recover has replayed // the journal, the Handle holds none of it. diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index a54c6ba3..283431af 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -86,8 +86,9 @@ counts as an additive (minor) versus breaking (major) change. Use the `SettleDone` or `SettleError` whose result transform was running, or a `Tick` whose `ServiceFn` was running, while the service was stopped and armed again under the same invoke id, settled the new activation with the old outcome. It now returns false, or - one zero `FireResult` from `Tick`, fires nothing, and leaves the new activation in - flight. + an empty slice from `Tick`, as for an id not in flight, fires nothing, and leaves the + new activation in flight. A `Tick` that returns a `FireResult` has fired its event or + been aborted, whatever that `FireResult` holds. ## [1.0.0] — 2026-06-13 diff --git a/state/runner.go b/state/runner.go index 71eb2938..dcde8b3b 100644 --- a/state/runner.go +++ b/state/runner.go @@ -317,8 +317,10 @@ func (r *ServiceRunner[S, E, C]) applyTransform(rs running[E], result any, err e // returned in the slice. When a WithResultTransform function aborts the settle, the // slice holds one FireResult whose Err is the abort error, no event fires, and the // service stays in flight. A service stopped while its ServiceFn ran settles nothing, -// even if it was armed again under the same id: the slice holds one zero FireResult, -// and the new activation stays in flight. +// even if it was armed again under the same id: Tick returns an empty slice, as for +// an id not in flight, and the new activation stays in flight. So a non-empty slice +// always means the settle fired its event or was aborted, whatever FireResult the +// Fire returned. func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResult[S] { r.mu.Lock() rs, ok := r.running[id] @@ -329,14 +331,22 @@ func (r *ServiceRunner[S, E, C]) Tick(ctx context.Context, id string) []FireResu // Settle the activation the ServiceFn runs for, not one armed under id meanwhile. fn := r.resolve(rs.src.Name) if fn == nil { - res, _ := r.settleActivation(ctx, id, rs, nil, &UnboundRefError{Kind: "service", Name: rs.src.Name}) - return []FireResult[S]{res} + return tickResult(r.settleActivation(ctx, id, rs, nil, &UnboundRefError{Kind: "service", Name: rs.src.Name})) } out, err := fn(ctx, ServiceCtx[C]{Entity: r.inst.entity, Params: rs.src.Params, Input: rs.input}) if err != nil { out = nil } - res, _ := r.settleActivation(ctx, id, rs, out, err) + return tickResult(r.settleActivation(ctx, id, rs, out, err)) +} + +// tickResult is Tick's return for a settle: the settle's FireResult when it fired or +// was aborted, which leaves the abort in Err, and nothing when it found its +// activation gone. +func tickResult[S comparable](res FireResult[S], settled bool) []FireResult[S] { + if !settled && res.Err == nil { + return nil + } return []FireResult[S]{res} } diff --git a/state/settle_stale_test.go b/state/settle_stale_test.go index 523e9d00..d66f4b2c 100644 --- a/state/settle_stale_test.go +++ b/state/settle_stale_test.go @@ -76,7 +76,9 @@ func TestSettle_StaleTransformDoesNotSettleARearmedActivation(t *testing.T) { // TestTick_StaleServiceRunDoesNotSettleARearmedActivation proves the same for the // ServiceFn's own run: a Tick whose ServiceFn ran for one activation can't settle a -// later one armed under the same InvokeID while it ran. +// later one armed under the same InvokeID while it ran. It returns nil, as for an id +// not in flight, so a caller can't mistake it for a settle whose Fire returned a zero +// FireResult. func TestTick_StaleServiceRunDoesNotSettleARearmedActivation(t *testing.T) { ctx := context.Background() id := state.InvokeID("loader", "loading", 0) @@ -104,10 +106,8 @@ func TestTick_StaleServiceRunDoesNotSettleARearmedActivation(t *testing.T) { close(release) got := <-done - for _, res := range got { - if res.NewState != "" { - t.Fatalf("the stale Tick fired %+v, want nothing fired", got) - } + if got != nil { + t.Fatalf("the stale Tick returned %+v, want nil: it settled nothing", got) } if inst.Current() != "loading" || len(entity.notes) != 0 { t.Fatalf("the stale Tick moved the instance to %q with notes %q, want loading and none", inst.Current(), entity.notes) From 2fada7fcfb8d6bb9342cab03c173aad6b25501e3 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 16 Nov 2026 20:37:10 -0700 Subject: [PATCH 13/13] fix(durable): a timer keeps a parked result; document RunService's bool for a stale settle - A Tick's timer transition no longer discards a parked result, live or on replay. Tick hands the transition's effects only to the scheduler, so the ServiceRunner keeps the service in flight; discarding the parked result made the next RunService run the ServiceFn again. Tick still forgets settled services, so Recover starts the service of a state a timer re-entered, as TestRecover_StartsAServiceThatNeverSettled pins. #236 stops the service and then forgets both. - A Recover from a checkpoint taken after the timer left the state still finds the service neither in flight nor parked, while live and the journal's replay keep it in flight: a skipped test pins the agreement #236 brings. - RunService's doc says the bool means the service was in flight and ran, not that it settled: a settle that finds its activation stopped or armed again returns true with the zero result and records nothing. The doc no longer says a timer that leaves the state discards the parked result. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 5 +- durable/CHANGELOG.md | 13 +- durable/parked_tick_test.go | 217 ++++++++++++++++++++++ durable/runner.go | 31 +++- durable/service.go | 27 ++- durable/service_step_internal_test.go | 10 +- 6 files changed, 283 insertions(+), 20 deletions(-) create mode 100644 durable/parked_tick_test.go diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 04ac535c..32016c53 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -177,8 +177,9 @@ service returned it, and the Handle stays healthy. The service stays in flight a runs again. The decoder is part of the Runner's options, so the Handle that parked the result never sees a fix: `Recover` with the fixed decoder, then call `RunService`, and it settles the parked result. Until then `RunService` keeps returning -`ErrEventDataEncoding`. A checkpoint keeps a parked result, and leaving the state that -declares the service discards it. A service error stays message-only, an error with the same message, which `errors.Is` no +`ErrEventDataEncoding`. A checkpoint keeps a parked result, and a `Fire` that leaves the +state that declares the service discards it; a timer that leaves it doesn't yet, since +`Tick` doesn't stop the service. A service error stays message-only, an error with the same message, which `errors.Is` no longer matches. Register the decoder before you record results: replay decodes the journal with the recovering Runner's decoder, so a decoder added or changed after results were recorded changes what replay hands the reducer, and one that can't decode a settled result diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 3105b2fd..a7ca72e2 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -189,10 +189,15 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. parked result as parked, so they don't fail on it, and `StateAt` at that step shows the instance as it was before. A checkpoint carries the parked results in a new `parkedServices` field of its envelope, so compacting the Record that parked one - doesn't lose it. A Fire or timer that leaves the state that declares the service stops - it and discards the parked result; entering the state again runs the `ServiceFn` for - the new activation. The Record that settles a parked result records its Go type too, and - replay holds the decoder to it, as that settle did: `Recover` and `StateAt` fail with + doesn't lose it. A Fire that leaves the state that declares the service stops it and + discards the parked result; entering the state again runs the `ServiceFn` for the new + activation. A timer that leaves or re-enters the state doesn't stop the service yet + (#236): `Tick` hands a timer's effects only to the scheduler, so the service stays in + flight and durable keeps its parked result, which the next `RunService` settles rather + than run the `ServiceFn` again. A `Recover` from a checkpoint taken after the timer + left the state finds the service neither in flight nor parked. The Record that settles + a parked result records its Go type too, and replay holds the decoder to it, as that + settle did: `Recover` and `StateAt` fail with `ErrEventDataEncoding`, running no `Assign`, when the decoder returns another type. The type is the full name, with every named type's import path, also inside a struct literal, func or chan type, whose fields' tags are part of it. Moving or renaming the diff --git a/durable/parked_tick_test.go b/durable/parked_tick_test.go new file mode 100644 index 00000000..e5acd735 --- /dev/null +++ b/durable/parked_tick_test.go @@ -0,0 +1,217 @@ +package durable_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// These tests pin what a Tick does to a parked result: nothing yet. Tick hands a timer +// transition's effects only to the scheduler, so a timer that leaves or re-enters the +// state that declares a parked service doesn't stop the service, and the ServiceRunner +// keeps it in flight. Durable keeps its parked result with it, so RunService settles +// the parked result rather than run the ServiceFn again. #236 stops the service and +// discards the parked result together. + +// timerParkMachine invokes fn in "loading", which "begin" enters from "idle" (Start +// arms no timer). Its after timer leaves for "timedOut", or with reenter re-enters +// "loading". +func timerParkMachine(fn state.ServiceFn[svcCtx], a *scnAssigns, reenter bool) *state.Machine[string, string, svcCtx] { + f := state.ForgeFor[svcCtx]("park"). + Service("fetch", fn). + Reducer("noteDone", func(in state.AssignCtx[svcCtx]) svcCtx { + a.done++ + a.seen = append(a.seen, in.Event) + return in.Entity + }). + State("idle"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("done")). + State("ready").Final(). + State("timedOut"). + Initial("idle"). + Transition("idle").On("begin").GoTo("loading"). + Transition("loading").On("done").GoTo("ready").Assign("noteDone") + if reenter { + f = f.Transition("loading").After(5 * time.Second).On("redo").GoTo("loading").Reenter() + } else { + f = f.Transition("loading").After(5 * time.Second).On("timeout").GoTo("timedOut") + } + return f.Quench() +} + +// parkProbe is what a RunService of a parked service reports: whether it found the +// service in flight, and whether it found the result still parked +// (ErrEventDataEncoding, with the Handle healthy). +type parkProbe struct { + inFlight, parked bool +} + +func probePark(t *testing.T, h *durable.Handle[string, string, svcCtx], id string) parkProbe { + t.Helper() + _, ran, err := h.RunService(context.Background(), id) + if err != nil && !errors.Is(err, durable.ErrEventDataEncoding) || errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("RunService = %v, %v; want nil or ErrEventDataEncoding", ran, err) + } + return parkProbe{inFlight: ran, parked: ran && errors.Is(err, durable.ErrEventDataEncoding)} +} + +// timerPark is a run that parks the result of loading's service, then Ticks its after +// timer. +type timerPark struct { + m *state.Machine[string, string, svcCtx] + a *scnAssigns + store *durable.MemStore + runs *int64 + live *durable.Handle[string, string, svcCtx] + broken, fixed []durable.Option[string, string, svcCtx] +} + +const timerParkID = "p-1" + +// newTimerPark parks the result, Ticks the timer, and fails unless the timer fired. +// With every above zero, every step checkpoints, so the Tick's Record carries one. +func newTimerPark(t *testing.T, reenter bool, every int) *timerPark { + t.Helper() + ctx := context.Background() + r := &timerPark{a: &scnAssigns{}, store: durable.NewMemStore(), runs: new(int64)} + fn := countingUsers(r.runs) + r.m = timerParkMachine(fn, r.a, reenter) + clk := state.NewFakeClock(epoch) + base := []durable.Option[string, string, svcCtx]{ + durable.WithRunnerClock[string, string, svcCtx](clk), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn)), + } + if every > 0 { + base = append(base, durable.WithCheckpointEvery[string, string, svcCtx](every)) + } + r.broken = append(append([]durable.Option[string, string, svcCtx]{}, base...), failingDecoder()) + r.fixed = append(append([]durable.Option[string, string, svcCtx]{}, base...), decoderFor("done", durable.DecodeAs[scnUser])) + + h, err := durable.NewRunner(r.m, r.store, r.broken...).Start(ctx, timerParkID, svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "begin"); err != nil { + t.Fatalf("Fire begin: %v", err) + } + _, ran, err := h.RunService(ctx, timerParkSvc) + mustPark(t, "RunService", ran, err) + clk.Advance(6 * time.Second) + if results, terr := h.Tick(ctx); terr != nil || len(results) != 1 { + t.Fatalf("Tick = %d results, %v; want the timer to fire", len(results), terr) + } + want := "timedOut" + if reenter { + want = "loading" + } + if got := h.Instance().Current(); got != want { + t.Fatalf("after the Tick in %q, want %q", got, want) + } + r.live = h + return r +} + +var timerParkSvc = state.InvokeID("park", "loading", 0) + +func (r *timerPark) recover(t *testing.T, opts []durable.Option[string, string, svcCtx]) *durable.Handle[string, string, svcCtx] { + t.Helper() + h, err := durable.Recover(context.Background(), r.m, r.store, timerParkID, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + return h +} + +// mustRunOnce fails unless the ServiceFn ran once: the parked result stands for its run. +func (r *timerPark) mustRunOnce(t *testing.T) { + t.Helper() + if got := atomic.LoadInt64(r.runs); got != 1 { + t.Fatalf("the ServiceFn ran %d times, want 1: the parked result stands for its run", got) + } +} + +// TestParkedResult_TimerLeavingTheStateKeepsItParked pins that an after timer leaving +// the state that declares a parked service discards nothing, as the ServiceRunner, which +// still holds the service in flight, doesn't: RunService finds the result parked rather +// than run the ServiceFn again, live and on a Handle recovered from the journal, which +// agree, and settles it once the decoder is fixed. +func TestParkedResult_TimerLeavingTheStateKeepsItParked(t *testing.T) { + r := newTimerPark(t, false, 0) + want := parkProbe{inFlight: true, parked: true} + if got := probePark(t, r.live, timerParkSvc); got != want { + t.Errorf("live after the Tick: %+v, want %+v", got, want) + } + if got := probePark(t, r.recover(t, r.broken), timerParkSvc); got != want { + t.Errorf("recovered from the journal: %+v, want %+v, as live", got, want) + } + r.mustRunOnce(t) + + // The fixed decoder settles the parked result, without running the ServiceFn. + fh := r.recover(t, r.fixed) + if got := probePark(t, fh, timerParkSvc); got != (parkProbe{inFlight: true}) { + t.Errorf("with the fixed decoder: %+v, want the parked result settled", got) + } + if got := probePark(t, fh, timerParkSvc); got != (parkProbe{}) { + t.Errorf("after the settle: %+v, want nothing in flight", got) + } + r.mustRunOnce(t) +} + +// TestParkedResult_TimerLeavingTheStateCheckpointMatchesLive pins that a Handle +// recovered from a checkpoint the Tick took agrees with the live Handle and with one +// recovered from the journal. It doesn't yet: a checkpoint records the services of the +// states the instance is in, so a Recover from it starts none for the state the timer +// left, and finds nothing in flight or parked, while live and the journal's replay keep +// the service in flight and its result parked, because Tick doesn't stop it. No Handle +// runs the ServiceFn again. #236 stops the service, and then all three find nothing in +// flight. +func TestParkedResult_TimerLeavingTheStateCheckpointMatchesLive(t *testing.T) { + t.Skip("Tick hands a timer transition's effects only to the scheduler, not the service and actor drivers; #236 fixes it") + r := newTimerPark(t, false, 1) + live := probePark(t, r.live, timerParkSvc) + if got := probePark(t, r.recover(t, r.broken), timerParkSvc); got != live { + t.Errorf("recovered from the Tick's checkpoint: %+v, live %+v; want them to agree", got, live) + } + r.mustRunOnce(t) +} + +// TestParkedResult_TimerReenteringTheStateKeepsItParked pins that an after timer that +// re-enters the state that declares a parked service keeps the parked result too, and +// that the live Handle, one recovered from the journal and one recovered from the +// Tick's checkpoint all agree: the service in flight, its result parked, the ServiceFn +// not run again. A checkpoint records the parked results of the states the instance is +// in, so it keeps this one. The fixed decoder then settles the parked result. #236 +// makes the re-entry start a new activation, whose ServiceFn runs, as a Fire's does. +func TestParkedResult_TimerReenteringTheStateKeepsItParked(t *testing.T) { + for _, every := range []int{0, 1} { + name := "journal" + if every > 0 { + name = "checkpoint every step" + } + t.Run(name, func(t *testing.T) { + r := newTimerPark(t, true, every) + want := parkProbe{inFlight: true, parked: true} + if got := probePark(t, r.live, timerParkSvc); got != want { + t.Errorf("live after the Tick: %+v, want %+v", got, want) + } + if got := probePark(t, r.recover(t, r.broken), timerParkSvc); got != want { + t.Errorf("recovered: %+v, want %+v, as live", got, want) + } + r.mustRunOnce(t) + + fh := r.recover(t, r.fixed) + if got := probePark(t, fh, timerParkSvc); got != (parkProbe{inFlight: true}) { + t.Errorf("with the fixed decoder: %+v, want the parked result settled", got) + } + if got := fh.Instance().Current(); got != "ready" || r.a.done != 1 { + t.Errorf("after the settle in %q with onDone run %d times, want ready and once", got, r.a.done) + } + r.mustRunOnce(t) + }) + } +} diff --git a/durable/runner.go b/durable/runner.go index 2e6946cf..3c4d0974 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -456,11 +456,23 @@ func (h *Handle[S, E, C]) forgetServices(effects []state.Effect) { forgetSettled(h.parked, effects) } +// forgetTickedSettles drops from the settled services each service a Tick's timer +// transition starts or stops, so Recover starts the service of a state the timer +// re-entered. It keeps the parked results: Tick doesn't hand the transition's effects to +// the ServiceRunner, so a parked service stays in flight there, and RunService settles +// its parked result rather than run the ServiceFn again. #236 stops the service, and +// then forgets both together, through forgetServices. A parked service is never +// settled, so the two maps still can't both hold an id. +func (h *Handle[S, E, C]) forgetTickedSettles(effects []state.Effect) { + forgetSettled(h.settled, effects) +} + // forgetSettled removes from settled each service effects start or stop: a state that // declares it was entered or exited, so the service is new or gone. It forgets parked // results the same way: a stopped service's parked result is discarded, and a service // started again runs its ServiceFn. The Handle and the replay mirror call it for both -// maps at once, through forgetServices and armMirror.forget. +// maps at once, through forgetServices and armMirror.forget, except for a Tick's timer +// transitions, which forget only settled services (see forgetTickedSettles). func forgetSettled[V any](settled map[string]V, effects []state.Effect) { if len(settled) == 0 { return @@ -1084,7 +1096,7 @@ func (h *Handle[S, E, C]) tickStep(ctx context.Context) ([]state.FireResult[S], h.timers = map[string]pendingTimer{} } mirrorScheduled(h.timers, h.sched, results[i].Effects) - h.forgetServices(results[i].Effects) + h.forgetTickedSettles(results[i].Effects) } if len(results) > 0 { // Drop from the table the timers the tick fired, the ones the scheduler no longer @@ -2245,16 +2257,27 @@ func (m *armMirror) arm(effects []state.Effect) { mirrorArmed(m.armed, effects, m.clock.lastRead()) } +// forgetTickedSettles drops from the settled services each one a replayed Tick's timer +// transition starts or stops, and keeps the parked results, as the live Tick does (see +// Handle.forgetTickedSettles). +func (m *armMirror) forgetTickedSettles(effects []state.Effect) { + if m == nil { + return + } + forgetSettled(m.settled, effects) +} + // mirrorTick arms in m any timer a replayed tick's chained transition scheduled, at the // deadline the scheduler holds, as the live Tick does, then drops the timers the tick -// fired: the ones sched no longer holds. +// fired: the ones sched no longer holds. It forgets the settles the tick's transitions +// start or stop but keeps parked results, as the live Tick does. func mirrorTick[S comparable, E comparable, C any](m *armMirror, sched *state.Scheduler[S, E, C], results []state.FireResult[S]) { if m == nil { return } for i := range results { mirrorScheduled(m.armed, sched, results[i].Effects) - m.forget(results[i].Effects) + m.forgetTickedSettles(results[i].Effects) } dropUnscheduled(m.armed, sched) } diff --git a/durable/service.go b/durable/service.go index e914bb44..9209fd8c 100644 --- a/durable/service.go +++ b/durable/service.go @@ -128,9 +128,14 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // added later for the service's type changes what replay produces. // // It returns the routed FireResult and true, or the zero result and false when id -// names no in-flight service (already settled, stopped, or never started). It returns -// false and an error when no registry is wired, the Handle has failed, or ctx ended -// before the service started. +// names no in-flight service (already settled, stopped, or never started). The bool +// says whether the service was in flight and ran, not whether it settled: a parked +// result returns true, and so does a settle that finds the activation the ServiceFn +// ran for stopped, or stopped and armed again, while it ran, which returns the zero +// result and no error and records nothing, since nothing fired. The step lock keeps +// durable's own steps from doing that, so only code that drives the ServiceRunner past +// the Handle can. It returns false and an error when no registry is wired, the Handle +// has failed, or ctx ended before the service started. // // Once the service has run, it returns true with any error. A result the decoder for // its onDone event can't decode, such as a number beyond float64, or decodes as a type @@ -144,9 +149,14 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // recorded type, by import path and name, or JSON's generic form: after the type is // moved or renamed, only a decoder that returns the generic form settles it, so settle // parked results before such a move. Until then each RunService -// returns ErrEventDataEncoding and records nothing more. A Fire or timer that leaves -// the state that declares the service stops it and discards the parked result. A -// checkpoint keeps it, and StateAt shows the instance as it was before the park. +// returns ErrEventDataEncoding and records nothing more. A Fire that leaves the state +// that declares the service stops it and discards the parked result. A timer that leaves +// or re-enters that state doesn't yet: Tick doesn't stop the service (#236), so it stays +// in flight with its result parked, and RunService settles that result. A Recover from +// a checkpoint taken after the timer left the state starts only the services of the +// states the instance is in, so it finds the service neither in flight nor parked. A +// checkpoint keeps the parked result, and StateAt shows the instance as it was before +// the park. // // A result that won't encode, a ctx that ended during the service (whether or not the // result decodes) or while its settle's effects were absorbed (even after a clean @@ -243,7 +253,7 @@ func (h *Handle[S, E, C]) serviceStep(ctx context.Context, id string) (state.Fir // before the settle: nothing fired, so there is nothing to record, and a new // activation stays in flight. The step lock keeps any other step from doing that // while this one runs, so only code that drives the ServiceRunner past the Handle - // can. + // can. It reports true: the service was in flight and its ServiceFn ran. return state.FireResult[S]{}, true, nil, nil, nil } // The service settled: unless the settle's own Fire leaves or re-enters the state @@ -330,7 +340,8 @@ func (h *Handle[S, E, C]) parkResult(ctx context.Context, id string, derr *undec // markSettled records that the service id settled in its state's current activation, so // no result of it is parked. It and forgetServices are the only writers of settled -// alongside parked, so the two can't disagree. +// alongside parked, so the two can't disagree. forgetTickedSettles only removes from +// settled, and no id is in both. func (h *Handle[S, E, C]) markSettled(id string) { if h.settled == nil { h.settled = map[string]bool{} diff --git a/durable/service_step_internal_test.go b/durable/service_step_internal_test.go index 3bd4fe1c..726c72c7 100644 --- a/durable/service_step_internal_test.go +++ b/durable/service_step_internal_test.go @@ -65,8 +65,14 @@ func TestServiceStep_StaleSettleRecordsNothing(t *testing.T) { if h, err = NewRunner(m, st, stepOptions(fn)...).Start(ctx, "s1", stepCtx{}, state.WithInitialState("loading")); err != nil { t.Fatalf("Start: %v", err) } - if _, ran, rerr := h.RunService(ctx, id); rerr != nil || !ran { - t.Fatalf("RunService = %v, %v; want ran with no error", ran, rerr) + // RunService reports the service ran, as its doc says: it was in flight and the + // ServiceFn ran, though nothing settled, so the result is the zero one. + res, ran, rerr := h.RunService(ctx, id) + if rerr != nil || !ran || runs != 1 { + t.Fatalf("RunService = %v, %v with the ServiceFn run %d times; want ran with no error, once", ran, rerr, runs) + } + if res.NewState != "" || len(res.Effects) != 0 || res.Err != nil || res.Trace.Event != "" { + t.Fatalf("RunService returned %+v, want the zero FireResult: nothing fired", res) } if _, recs, _ := st.Load(ctx, "s1"); len(recs) != 0 { t.Fatalf("the journal holds %d Records, want none: the settle fired nothing", len(recs))