From fee1643233d118a3cba12e1425fb25eba6d99b56 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Thu, 5 Nov 2026 09:00:40 -0700 Subject: [PATCH 01/10] fix(durable): lock the Handle and refuse steps after a failed Append Fire, Tick, RunService and DeliverToActor now hold a lock for the whole step, so a host's timer loop no longer loses acknowledged steps. A step that applied to the kernel but wasn't appended leaves the instance ahead of the journal, so the Handle refuses further steps with ErrHandleFailed until it's recovered. Relates to #214. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 10 + durable/CHANGELOG.md | 16 ++ durable/README.md | 10 + durable/actor.go | 20 +- durable/errors.go | 10 +- durable/runner.go | 85 ++++++-- durable/runner_test.go | 244 ++++++++++++++++++++++ durable/service.go | 20 +- 8 files changed, 391 insertions(+), 24 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 8dfd056b..4680e2a8 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -125,6 +125,16 @@ Time-travel needs the full record history through the target step. A `Store` opt in by implementing `HistoryStore` (the in-tree `MemStore` does so under `WithHistory`); `StateAt` otherwise falls back to the latest checkpoint plus tail. +## Handles across goroutines + +`Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different +goroutines: each holds the Handle's lock for the whole step. A host callback must not +call back into its Handle. `Instance()` is outside the lock. + +A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` +a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a +newer owner holds the instance, so stop. + ## A note on serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 543dc021..5067e0d0 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -25,9 +25,23 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. a refused payload, and `ErrRecordTooLarge` a Record too large for a `FileStore` line. `Record.EventData` and `Record.EventDataError` hold the payload. +- **`ErrHandleFailed`** reports a Handle out of step with its journal. + ### Changed - Requires Go 1.27.3. +- **A Handle's steps are serialized across goroutines.** `Fire`, `Tick`, `RunService` and + `DeliverToActor` hold a lock for the whole step, so a host's timer loop no longer loses + acknowledged steps. A host callback must not call back into its Handle, or it + deadlocks. `Instance()` isn't covered: read it only while no step is running. A `Tick`, + `RunService` or `DeliverToActor` whose context ended while it waited returns that error + and changes nothing. +- **A step that may not have been recorded fails the Handle.** That call and every later + one return `ErrHandleFailed`, wrapping the cause. `Recover` a new Handle with the same + options to continue. A service whose result wasn't recorded is in flight again, so + `RunService` runs it a second time. An `InvokeActor` actor restarts from its initial + state, and a `Spawn` actor doesn't return. If the cause is `ErrFenced`, a newer owner + holds the instance: stop. - **A `WithEventData` payload reaches `Assign` decoded, live as well as on replay.** A struct arrives as a map and an integer as a `float64`, which loses precision above 2^53, so an assertion to the original @@ -59,6 +73,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Fixed +- **A failed checkpoint no longer makes the next step conflict.** `Tick`, `RunService` + and `DeliverToActor` now advance the next step as soon as the append lands. - **The next step survives a checkpoint that compacts tick barriers.** A checkpoint now records the step that follows it, in a `nextStep` field of its envelope. Recovery used the Trace count, which runs low past a tick barrier, so the next Fire could reuse a diff --git a/durable/README.md b/durable/README.md index 9b9858c6..f720c52f 100644 --- a/durable/README.md +++ b/durable/README.md @@ -136,6 +136,16 @@ opts in by implementing `HistoryStore` (the in-tree `MemStore` does so under `WithHistory`); `StateAt` falls back to `Load` (latest checkpoint plus tail) otherwise. +## Handles across goroutines + +`Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different +goroutines: each holds the Handle's lock for the whole step. A host callback must not +call back into its Handle. `Instance()` is outside the lock. + +A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` +a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a +newer owner holds the instance, so stop. + ## Caveat: serialized payloads Events, service done-data, actor done-data, and actor messages are recorded as their diff --git a/durable/actor.go b/durable/actor.go index 792ba00d..41abf380 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -99,10 +99,21 @@ type actorMessagePayload struct { // re-instantiating the actor. // // It returns whether the actor was found running (false for a ref that names no -// live actor — already settled, stopped, or never spawned), and a persistence -// failure as the error. The kernel transitions the delivery drove do not -// themselves fail. +// live actor — already settled, stopped, or never spawned — or with an error when the +// Handle has failed or ctx has ended), and a persistence failure as the error. The +// kernel transitions the delivery drove do not themselves fail. After a failed Append +// the result wasn't recorded: on a recovered Handle an InvokeActor actor restarts from its +// initial state, and a Spawn actor doesn't return. Resending a recorded delivery repeats +// any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return false, err + } + if err := ctx.Err(); err != nil { + return false, err + } if h.actors == nil { return false, fmt.Errorf("durable: no actor palette wired for %q (supply WithActorPalette)", h.id) } @@ -116,7 +127,7 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef output, settled, settleErr := h.actorOutcome() entries, err := recordActorTransitions(ref.ID, h.inst.History(), before, output, settled, settleErr) if err != nil { - return true, fmt.Errorf("durable: recording actor %q result for %q: %w", ref.ID, h.id, err) + return true, h.fail(fmt.Errorf("durable: recording actor %q result for %q: %w", ref.ID, h.id, err)) } if h.actorBuf != nil { *h.actorBuf = append(*h.actorBuf, entries...) @@ -127,7 +138,6 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef if err := h.persistStep(ctx, step, &rec); err != nil { return true, err } - h.nextStep++ return true, nil } diff --git a/durable/errors.go b/durable/errors.go index d08e5f47..5ac19d7e 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -16,6 +16,7 @@ const ( errorStepOutOfRange = "time-travel step out of range" errorEventDataEncoding = "event data can't be recorded" errorRecordTooLarge = "record too large for the journal" + errorHandleFailed = "handle is out of step with the journal" ) // Sentinel errors a Store reports. Callers match them with errors.Is. @@ -49,9 +50,16 @@ var ( // ErrRecordTooLarge is reported by FileStore.Append for a Record whose journal line // exceeds the store's line limit, which a later Load could not read back, even // without its Snapshot. No journal line is written, but a Handle's kernel has already - // applied the step. + // applied the step, so the Handle fails with ErrHandleFailed. ErrRecordTooLarge = errors.New(errorPrefix + errorRecordTooLarge) + // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService and + // DeliverToActor once a step applied to the live instance but may not have been + // recorded, on that call and every later one. It wraps the cause. Recover a new + // Handle with the same options to continue, unless the cause is ErrFenced: then a + // newer owner holds the instance, so stop. + ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) + // ErrCheckpointNotAdvancing is reported by Store.Checkpoint when throughStep // does not advance beyond the instance's current checkpoint. ErrCheckpointNotAdvancing = errors.New(errorPrefix + errorCheckpointNotAdvancing) diff --git a/durable/runner.go b/durable/runner.go index e77e5a11..442fec6f 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sync" "time" "github.com/stablekernel/crucible/state" @@ -76,11 +77,33 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // clock that journals every reading the scheduler consumes, so timer-driven // transitions are durable and replay wall-clock-independent. Obtain a Handle from // Runner.Start or Recover. +// +// Fire, Tick, RunService and DeliverToActor are safe to call from different +// goroutines: each holds the Handle's lock for the whole step, including the Store +// write, effect dispatch and any service it runs. Any host callback run during a step +// (an effect handler, service, guard, action, reducer or actor behavior) must not call +// back into the same Handle, or it deadlocks. A step waits for the lock without watching +// its context, so a slow service delays other steps past their deadlines; a Tick, +// RunService or DeliverToActor whose context has ended by then returns that error and +// changes nothing, and the kernel refuses a Fire. Instance returns the +// live instance outside that lock, so read it only while no step is running. A +// stateless Runner.Fire builds its own Handle, so it isn't serialized with this one. +// +// A step that applied to the kernel but may not have been recorded leaves the Handle out +// of step with the journal. That call and every later one return ErrHandleFailed, +// wrapping the cause. Recover a new Handle with the same options to continue. A service +// whose result wasn't recorded is in flight again, so RunService runs it a second time. +// A running InvokeActor actor restarts from its initial state on any Recover, losing every +// delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a +// newer owner holds the instance: stop, because recovering won't help. type Handle[S comparable, E comparable, C any] struct { - runner *Runner[S, E, C] - id InstanceID - inst *state.Instance[S, E, C] + runner *Runner[S, E, C] + id InstanceID + inst *state.Instance[S, E, C] + + mu sync.Mutex // serializes the steps, and guards the instance and every field below nextStep int + failed error // the error that took the Handle out of step with the journal, nil until then sched *state.Scheduler[S, E, C] recClock *recordingClock // owns the clock-read accumulator and its mutex @@ -93,6 +116,21 @@ type Handle[S comparable, E comparable, C any] struct { actorBuf *[]state.JournalEntry // recording actor-transition accumulator, flushed per step } +// refuse reports ErrHandleFailed once the Handle is out of step with the journal. +func (h *Handle[S, E, C]) refuse() error { + if h.failed == nil { + return nil + } + return fmt.Errorf("%w: %q: %w", ErrHandleFailed, h.id, h.failed) +} + +// fail records err as the reason the Handle is out of step with the journal, and +// returns the same refusal every later step gets. +func (h *Handle[S, E, C]) fail(err error) error { + h.failed = err + return h.refuse() +} + // drainClock returns and clears the clock readings accumulated since the last // drain, so each recorded step carries exactly the readings consumed during it. // The drain is taken under the recording clock's mutex, so closing a step's @@ -227,7 +265,8 @@ func (h *Handle[S, E, C]) dropFiredTimers(now time.Time) { // Instance returns the underlying kernel Instance the Handle wraps, for reads // such as Configuration, Snapshot, or Current. Drive it through the Handle's Fire // (or the Runner's) so steps continue to be recorded; firing the bare Instance -// bypasses durability. +// bypasses durability. Read it only while no step is running: it's outside the Handle's +// lock. After ErrHandleFailed it shows state the journal may not hold. func (h *Handle[S, E, C]) Instance() *state.Instance[S, E, C] { return h.inst } // ID returns the InstanceID the Handle records under. @@ -343,7 +382,8 @@ func (r *Runner[S, E, C]) newActorSystem(inst *state.Instance[S, E, C]) *state.A // how a state.WithEventData payload is handled, and when it's refused. It is the // stateless entry point (no Handle required); for a hot path that fires many // events in sequence, hold a Handle from Start or Recover and use Handle.Fire to -// avoid reloading between steps. +// avoid reloading between steps. Calls on one id aren't serialized, and must not overlap +// each other or a live Handle. func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts ...state.FireOption) (state.FireResult[S], error) { h, err := r.recover(ctx, id) if err != nil { @@ -370,6 +410,11 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // kernel runs. The payload is read on the calling goroutine with no lock of its own, so // don't mutate it concurrently with Fire. func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireOption) (state.FireResult[S], error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return state.FireResult[S]{}, err + } // Encode the WithEventData payload before the kernel runs, so a payload that // can't be recorded refuses the Fire rather than applying a step it can't record. payload, opts, err := encodeEventData(opts, h.runner.cfg.maxEventData) @@ -395,7 +440,7 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO des := dispatchableEffects(step, res.Effects) effEnvs, err := recordEffects(des) if err != nil { - return res, err + return res, h.fail(err) } rec := Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: h.drainClock(), Effects: effEnvs} // Write-ahead append the step Record (with its effect ids) BEFORE dispatching, but @@ -403,7 +448,7 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO // a crash mid-dispatch leaves the effect recorded in the still-present tail for // recovery to redispatch — never orphaned behind the compaction. if err := h.appendStep(ctx, step, &rec); err != nil { - return res, err + return res, h.fail(err) } h.nextStep++ // Dispatch AFTER the append: a crash in this window leaves the effect recorded @@ -429,6 +474,16 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO // count of timers fired — so recovery re-derives the same timers at their // recorded instants. It returns the FireResults of the timers it fired, in order. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return nil, err + } + // A context that ended in the lock queue must stop here: the scheduler would consume + // a due timer whose Fire the kernel then refuses. + if err := ctx.Err(); err != nil { + return nil, err + } results := h.sched.Tick(ctx) step := h.nextStep @@ -450,20 +505,24 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro if len(rec.Entries) == 0 && rec.TickSteps == 0 { return results, nil } - if err := h.persistStep(ctx, step, &rec); err != nil { + if err := h.appendStep(ctx, step, &rec); err != nil { + return results, h.fail(err) + } + h.nextStep = rec.followingStep() + if err := h.checkpointStep(ctx, step, &rec); err != nil { return results, err } - h.nextStep += len(results) + 1 return results, nil } -// persistStep checkpoints if the policy is due, then write-ahead appends rec -// before the step is acknowledged to the caller. The checkpoint is taken at the -// barrier's own step so a Load after it returns this Snapshot plus the later tail. +// persistStep write-ahead appends rec, advances the next step, then checkpoints if +// the policy is due. A failed append fails the Handle, because the kernel already +// applied the step; a failed checkpoint doesn't, because the Record is durable. func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { if err := h.appendStep(ctx, step, rec); err != nil { - return err + return h.fail(err) } + h.nextStep = rec.followingStep() return h.checkpointStep(ctx, step, rec) } diff --git a/durable/runner_test.go b/durable/runner_test.go index 2d33819a..bc5b3f7b 100644 --- a/durable/runner_test.go +++ b/durable/runner_test.go @@ -4,6 +4,10 @@ import ( "context" "encoding/json" "errors" + "runtime" + "strings" + "sync" + "sync/atomic" "testing" "time" @@ -511,6 +515,21 @@ func quantityMachine() *state.Machine[string, string, *runCtx] { Quench() } +// flakyAppendStore fails the first Append of one step, then delegates. +type flakyAppendStore struct { + durable.Store + failStep int + failed bool +} + +func (s *flakyAppendStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if rec.Step == s.failStep && !s.failed { + s.failed = true + return 0, errors.New("store unavailable") + } + return s.Store.Append(ctx, id, rec, opts...) +} + // TestRecover_TwoHandlesOnOneInstance_JournalMatchesAnAcknowledgedHistory proves // two Handles on one instance (a stalled worker, and the worker that recovered the // instance after it) cannot both have their Fires acknowledged while the journal @@ -552,6 +571,71 @@ func TestRecover_TwoHandlesOnOneInstance_JournalMatchesAnAcknowledgedHistory(t * } } +// TestRunner_FireDuringTimerLoop_LosesNoAcknowledgedSteps proves the "Tick from its +// own timer loop" pattern that drainClock documents holds while requests Fire on +// the same Handle: the recovered instance matches the live one, and every +// acknowledged Fire survives. +func TestRunner_FireDuringTimerLoop_LosesNoAcknowledgedSteps(t *testing.T) { + if runtime.GOMAXPROCS(0) < 2 { + t.Skip("needs two CPUs to interleave Fire and Tick") + } + ctx := context.Background() + m := pollingMachine() + store := durable.NewMemStore() + id := durable.InstanceID("poll-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} + runner := durable.NewRunner(m, store, opts...) + h, err := runner.Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + + const n = 200 + var acked, fired, failed atomic.Int64 + var wg sync.WaitGroup + wg.Add(2) + go func() { // the host's timer loop + defer wg.Done() + for range n { + clk.Advance(time.Second) + results, terr := h.Tick(ctx) + if terr != nil { + failed.Add(1) + } + fired.Add(int64(len(results))) + } + }() + go func() { // request traffic + defer wg.Done() + for range n { + if _, ferr := h.Fire(ctx, "poke"); ferr != nil { + failed.Add(1) + continue + } + acked.Add(1) + } + }() + wg.Wait() + if failed.Load() != 0 || acked.Load() != n || fired.Load() == 0 { + t.Fatalf("failed %d calls, acknowledged %d of %d Fires, fired %d timers", failed.Load(), acked.Load(), n, fired.Load()) + } + + recovered, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got, want := recovered.Instance().Entity().Count, int(acked.Load()); got != want { + t.Fatalf("recovered %d pokes, want %d acknowledged", got, want) + } + if live, got := handleSnapshot(t, h), handleSnapshot(t, recovered); got != live { + t.Fatalf("recovered instance differs from live:\n live: %s\n got: %s", live, got) + } +} + // TestRecover_FireWithEventData_MatchesLive proves an instance driven with // state.WithEventData recovers to the live state, because the Handle journals the data. func TestRecover_FireWithEventData_MatchesLive(t *testing.T) { @@ -629,6 +713,36 @@ func TestRecover_StepAfterTickCheckpoint_IsPersisted(t *testing.T) { } } +// TestRunner_FireAfterFailedAppend_MatchesJournal proves a Fire whose Append failed +// acknowledges nothing further: the Handle refuses, and recovery matches the journal. +func TestRunner_FireAfterFailedAppend_MatchesJournal(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := &flakyAppendStore{Store: durable.NewMemStore(), failStep: 1} + id := durable.InstanceID("poll-1") + h, err := durable.NewRunner(m, store).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) { // step 1: the Append fails + t.Fatalf("Fire(poke) = %v, want ErrHandleFailed", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("second Fire(poke) = %v, want ErrHandleFailed", err) + } + + recovered, err := durable.Recover(ctx, m, store, id) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Entity().Count; got != 0 { + t.Fatalf("recovered %d pokes, want 0: none was recorded", got) + } +} + // lastStep returns the Step of the newest Record in a history-keeping store. func lastStep(t *testing.T, store *durable.MemStore, id durable.InstanceID) int { t.Helper() @@ -725,3 +839,133 @@ func TestRecover_CheckpointWithoutNextStep_UsesTheTraceCount(t *testing.T) { t.Fatalf("poke recorded at step %d, want 3", got) } } + +// TestHandle_FailedAppend_RefusesEveryStep proves the failed Handle stays refused, +// names the cause, and that a recovered Handle continues from the journal. +func TestHandle_FailedAppend_RefusesEveryStep(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := &flakyAppendStore{Store: durable.NewMemStore(), failStep: 1} + id := durable.InstanceID("poll-1") + h, err := durable.NewRunner(m, store).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(poke) = %v, want ErrHandleFailed", err) + } + if _, err = h.Fire(ctx, "poke"); !errors.Is(err, durable.ErrHandleFailed) || !strings.Contains(err.Error(), "store unavailable") { + t.Fatalf("Fire after the failure = %v, want ErrHandleFailed naming the cause", err) + } + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Tick after the failure = %v, want ErrHandleFailed", err) + } + recovered, err := durable.Recover(ctx, m, store, id) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = recovered.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire on the recovered Handle: %v", err) + } +} + +// failingCheckpointStore fails the next Checkpoint after armed is set. +type failingCheckpointStore struct { + durable.Store + armed bool +} + +func (s *failingCheckpointStore) Checkpoint(ctx context.Context, id durable.InstanceID, snapshot []byte, throughStep int, opts ...durable.CheckpointOption) error { + if s.armed { + s.armed = false + return errors.New("checkpoint unavailable") + } + return s.Store.Checkpoint(ctx, id, snapshot, throughStep, opts...) +} + +// TestHandle_FailedCheckpointOnTick_KeepsGoing proves a failed checkpoint doesn't fail +// the Handle or make the next step conflict: the Record is durable. +func TestHandle_FailedCheckpointOnTick_KeepsGoing(t *testing.T) { + ctx := context.Background() + store := &failingCheckpointStore{Store: durable.NewMemStore()} + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{ + durable.WithRunnerClock[string, string, *runCtx](clk), + durable.WithCheckpointEvery[string, string, *runCtx](2), + } + h, err := durable.NewRunner(pollingMachine(), store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + store.armed = true + clk.Advance(time.Second) + if _, err = h.Tick(ctx); err == nil || errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Tick = %v, want the plain checkpoint error", err) + } + if _, err = h.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire after the failed checkpoint: %v", err) + } +} + +// TestHandle_FencedOwner_Stops proves a Handle that lost its instance reports the fence +// through ErrHandleFailed, so a host can tell it must stop rather than recover. +func TestHandle_FencedOwner_Stops(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := durable.NewMemStore() + const id = durable.InstanceID("poll-1") + owner := func(token int64) durable.Option[string, string, *runCtx] { + return durable.WithOwnerFence[string, string, *runCtx](token) + } + old, err := durable.NewRunner(m, store, owner(1)).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = old.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + newer, err := durable.Recover(ctx, m, store, id, owner(2)) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = newer.Fire(ctx, "poke"); err != nil { + t.Fatalf("newer Fire: %v", err) + } + _, err = old.Fire(ctx, "poke") + if !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrFenced) { + t.Fatalf("old Fire = %v, want ErrHandleFailed wrapping ErrFenced", err) + } +} + +// TestHandle_ContextEndedBeforeTheStep_ChangesNothing proves an ended context stops a +// Tick before the scheduler consumes a due timer. +func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := durable.NewMemStore() + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + clk.Advance(time.Second) + ended, cancel := context.WithCancel(ctx) + cancel() + if _, err = h.Tick(ended); !errors.Is(err, context.Canceled) { + t.Fatalf("Tick = %v, want context.Canceled", err) + } + results, err := h.Tick(ctx) + if err != nil || len(results) != 1 { + t.Fatalf("Tick after = (%d results, %v), want the timer still due", len(results), err) + } +} diff --git a/durable/service.go b/durable/service.go index fedfe318..aacb91bc 100644 --- a/durable/service.go +++ b/durable/service.go @@ -84,10 +84,21 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // re-invoking the service. // // It returns the routed FireResult and true, or the zero result and false when id -// names no in-flight service (already settled, stopped, or never started). A -// persistence failure is returned as the error; the kernel transition itself does -// not fail (a settled service always routes onDone or onError). +// names no in-flight service (already settled, stopped, or never started), or false +// and the error when the Handle has failed or ctx has ended. A persistence failure is +// returned as the error; the kernel transition itself does not fail (a settled service +// always routes onDone or onError). After a failed Append the result wasn't recorded, so +// a recovered Handle has the service in flight again, and RunService runs it a second +// time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if err := h.refuse(); err != nil { + return state.FireResult[S]{}, false, err + } + if err := ctx.Err(); err != nil { + return state.FireResult[S]{}, false, err + } if h.svc == nil { return state.FireResult[S]{}, false, fmt.Errorf("durable: no service registry wired for %q (supply WithServiceRegistry)", h.id) } @@ -109,7 +120,7 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire entry, err := recordServiceOutcome(id, h.svc.LastResult, h.svc.LastError()) if err != nil { - return res, true, fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err) + return res, true, h.fail(fmt.Errorf("durable: recording service %q result for %q: %w", id, h.id, err)) } if h.svcBuf != nil { *h.svcBuf = append(*h.svcBuf, entry) @@ -120,7 +131,6 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire if err := h.persistStep(ctx, step, &rec); err != nil { return res, true, err } - h.nextStep++ return res, true, nil } From 9983b629435539d6593019320de6cee7fdd1de74 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Mon, 9 Nov 2026 11:04:01 -0700 Subject: [PATCH 02/10] test(durable): cover the failed-Handle latch on RunService and DeliverToActor RunService and DeliverToActor refuse steps after a failed Append and return an ended context's error, but only Fire and Tick were tested. New tests fail an Append under each and end the context before each, and check nothing ran or was journaled. The Tick context test now also checks that nothing was appended. The RunService and DeliverToActor docs said a failed step returns false; they return true with ErrHandleFailed once the service settled or the actor received the event. The docs now say so, name a context that ends during the Store write as a possible cause, and note that the latch relies on the Append running inline. Signed-off-by: Ray Karnes --- durable/actor.go | 14 +-- durable/errors.go | 2 +- durable/handle_failure_test.go | 164 +++++++++++++++++++++++++++++++++ durable/runner.go | 10 +- durable/runner_test.go | 6 +- durable/service.go | 10 +- 6 files changed, 190 insertions(+), 16 deletions(-) create mode 100644 durable/handle_failure_test.go diff --git a/durable/actor.go b/durable/actor.go index 41abf380..17bed76b 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -98,12 +98,14 @@ type actorMessagePayload struct { // JournalActorMessage correlated by the actor id so recovery replays it without // re-instantiating the actor. // -// It returns whether the actor was found running (false for a ref that names no -// live actor — already settled, stopped, or never spawned — or with an error when the -// Handle has failed or ctx has ended), and a persistence failure as the error. The -// kernel transitions the delivery drove do not themselves fail. After a failed Append -// the result wasn't recorded: on a recovered Handle an InvokeActor actor restarts from its -// initial state, and a Spawn actor doesn't return. Resending a recorded delivery repeats +// It returns false for a ref that names no live actor (already settled, stopped, or +// never spawned), and false with the error when the Handle had already failed or ctx +// has ended. If the actor received the event but the delivery couldn't be recorded or +// appended, it returns true and ErrHandleFailed: the Handle has failed. A failed +// checkpoint returns true and that error, and the Handle carries on. The kernel +// transitions the delivery drove do not themselves fail. When a delivery wasn't +// recorded, on a recovered Handle an InvokeActor actor restarts from its initial state, +// and a Spawn actor doesn't return. Resending a recorded delivery repeats // any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { h.mu.Lock() diff --git a/durable/errors.go b/durable/errors.go index 5ac19d7e..ea357b31 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -19,7 +19,7 @@ const ( errorHandleFailed = "handle is out of step with the journal" ) -// Sentinel errors a Store reports. Callers match them with errors.Is. +// Sentinel errors a Store or a Handle reports. Callers match them with errors.Is. var ( // ErrInstanceNotFound is reported by Store.Load for an instance that has // never been written. diff --git a/durable/handle_failure_test.go b/durable/handle_failure_test.go new file mode 100644 index 00000000..ed57686c --- /dev/null +++ b/durable/handle_failure_test.go @@ -0,0 +1,164 @@ +package durable_test + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// fetchID is the invocation id of the fetch service the single-service machine runs. +var fetchID = state.InvokeID("fetch", "loading", 0) + +// armedAppendStore counts Appends and, once armed, fails the next one. +type armedAppendStore struct { + durable.Store + armed bool + appends int +} + +func (s *armedAppendStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if s.armed { + s.armed = false + return 0, errors.New("store unavailable") + } + s.appends++ + return s.Store.Append(ctx, id, rec, opts...) +} + +// startFetch starts the single-service machine over store and moves it into the +// state that invokes the service. +func startFetch(t *testing.T, store durable.Store, calls *int64) *durable.Handle[string, string, svcCtx] { + t.Helper() + ctx := context.Background() + fn := counterService(calls) + h, err := durable.NewRunner(fetchMachine(fn), store, + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))). + Start(ctx, "fetch-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + return h +} + +// startTwin starts the two-actor machine over store, with childA running. +func startTwin(t *testing.T, store durable.Store, runs *int64) (*durable.Handle[string, string, actorCtx], state.ActorRef) { + t.Helper() + palette := map[string]state.ActorBehavior{"childA": counterChild(runs), "childB": counterChild(runs)} + h, err := durable.NewRunner(twoActorMachine(), store, + durable.WithActorPalette[string, string, actorCtx](palette)). + Start(context.Background(), "twin-1", actorCtx{}, state.WithInitialState("first")) + if err != nil { + t.Fatalf("Start: %v", err) + } + ref, ok := h.ActorRef(state.ActorID("twin", "first", 0)) + if !ok { + t.Fatal("no ref for childA") + } + return h, ref +} + +// TestHandle_RunService_FailedAppend_FailsHandle proves a failed Append of a service +// outcome fails the Handle: RunService reports the settle with ErrHandleFailed, every +// later step is refused, and a recovered Handle runs the service again. +func TestHandle_RunService_FailedAppend_FailsHandle(t *testing.T) { + ctx := context.Background() + var calls int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h := startFetch(t, store, &calls) + + store.armed = true + _, ok, err := h.RunService(ctx, fetchID) + if !ok || !errors.Is(err, durable.ErrHandleFailed) || !strings.Contains(err.Error(), "store unavailable") { + t.Fatalf("RunService = (%v, %v), want true and ErrHandleFailed naming the cause", ok, err) + } + if _, ok, err = h.RunService(ctx, fetchID); ok || !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("RunService after the failure = (%v, %v), want false and ErrHandleFailed", ok, err) + } + if _, err = h.Fire(ctx, "start"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire after the failure = %v, want ErrHandleFailed", err) + } + + fn := counterService(&calls) + rh, err := durable.Recover(ctx, fetchMachine(fn), store, "fetch-1", + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := rh.Instance().Current(); got != "loading" { + t.Fatalf("recovered state = %q, want loading (the outcome wasn't recorded)", got) + } + mustRunService(t, rh, fetchID) + if got := atomic.LoadInt64(&calls); got != 2 { + t.Fatalf("service ran %d times, want 2 (once live, once after recovery)", got) + } +} + +// TestHandle_DeliverToActor_FailedAppend_FailsHandle proves a failed Append of an +// actor delivery fails the Handle: DeliverToActor reports the delivery with +// ErrHandleFailed, and every later step is refused. +func TestHandle_DeliverToActor_FailedAppend_FailsHandle(t *testing.T) { + ctx := context.Background() + var runs int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h, ref := startTwin(t, store, &runs) + + store.armed = true + delivered, err := h.DeliverToActor(ctx, ref, "finish") + if !delivered || !errors.Is(err, durable.ErrHandleFailed) || !strings.Contains(err.Error(), "store unavailable") { + t.Fatalf("DeliverToActor = (%v, %v), want true and ErrHandleFailed naming the cause", delivered, err) + } + if delivered, err = h.DeliverToActor(ctx, ref, "finish"); delivered || !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("DeliverToActor after the failure = (%v, %v), want false and ErrHandleFailed", delivered, err) + } + if _, err = h.Tick(ctx); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Tick after the failure = %v, want ErrHandleFailed", err) + } +} + +// TestHandle_RunService_ContextEnded_ChangesNothing proves an ended context stops +// RunService before the service runs or anything is journaled. +func TestHandle_RunService_ContextEnded_ChangesNothing(t *testing.T) { + var calls int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h := startFetch(t, store, &calls) + before := store.appends + + ended, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok, err := h.RunService(ended, fetchID); ok || !errors.Is(err, context.Canceled) { + t.Fatalf("RunService = (%v, %v), want false and context.Canceled", ok, err) + } + if got := atomic.LoadInt64(&calls); got != 0 || store.appends != before { + t.Fatalf("after an ended context: %d service runs and %d new appends, want 0 and 0", got, store.appends-before) + } + mustRunService(t, h, fetchID) +} + +// TestHandle_DeliverToActor_ContextEnded_ChangesNothing proves an ended context stops +// DeliverToActor before the actor sees the event or anything is journaled. +func TestHandle_DeliverToActor_ContextEnded_ChangesNothing(t *testing.T) { + var runs int64 + store := &armedAppendStore{Store: durable.NewMemStore()} + h, ref := startTwin(t, store, &runs) + before, state0 := store.appends, h.Instance().Current() + + ended, cancel := context.WithCancel(context.Background()) + cancel() + if delivered, err := h.DeliverToActor(ended, ref, "finish"); delivered || !errors.Is(err, context.Canceled) { + t.Fatalf("DeliverToActor = (%v, %v), want false and context.Canceled", delivered, err) + } + if store.appends != before || h.Instance().Current() != state0 { + t.Fatalf("after an ended context: %d new appends, state %q, want 0 and %q", store.appends-before, h.Instance().Current(), state0) + } + if delivered, err := h.DeliverToActor(context.Background(), ref, "finish"); !delivered || err != nil { + t.Fatalf("DeliverToActor after = (%v, %v), want the actor still running", delivered, err) + } +} diff --git a/durable/runner.go b/durable/runner.go index 442fec6f..9fe3f0ce 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -91,7 +91,8 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, -// wrapping the cause. Recover a new Handle with the same options to continue. A service +// wrapping the cause, which can be a context that ended during the Store write. Recover +// a new Handle with the same options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a @@ -396,7 +397,7 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // it Fires the kernel, appends a Record carrying the driving event at the // step's ordinal (write-ahead, before returning), and — when the // checkpoint policy is due — persists a full Snapshot and compacts the tail. A -// kernel transition error is recorded as a no-op (no step was produced) and +// kernel transition error journals nothing (no step was produced) and is // returned to the caller. // // A state.WithEventData payload reaches the transition's Assign in its JSON-decoded @@ -517,7 +518,10 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro // persistStep write-ahead appends rec, advances the next step, then checkpoints if // the policy is due. A failed append fails the Handle, because the kernel already -// applied the step; a failed checkpoint doesn't, because the Record is durable. +// applied the step; a failed checkpoint doesn't, because the Record is durable. This +// relies on the Append running inline, under the Handle's lock, before the step +// returns: an Append deferred off this goroutine would acknowledge a step whose +// outcome isn't known, and the Handle couldn't fail. func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { if err := h.appendStep(ctx, step, rec); err != nil { return h.fail(err) diff --git a/durable/runner_test.go b/durable/runner_test.go index bc5b3f7b..5318f305 100644 --- a/durable/runner_test.go +++ b/durable/runner_test.go @@ -948,7 +948,7 @@ func TestHandle_FencedOwner_Stops(t *testing.T) { func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { ctx := context.Background() m := pollingMachine() - store := durable.NewMemStore() + store := &armedAppendStore{Store: durable.NewMemStore()} clk := state.NewFakeClock(epoch) opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} h, err := durable.NewRunner(m, store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) @@ -959,11 +959,15 @@ func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { t.Fatalf("Fire(start): %v", err) } clk.Advance(time.Second) + before := store.appends ended, cancel := context.WithCancel(ctx) cancel() if _, err = h.Tick(ended); !errors.Is(err, context.Canceled) { t.Fatalf("Tick = %v, want context.Canceled", err) } + if store.appends != before { + t.Fatalf("Tick on an ended context appended %d Records, want 0", store.appends-before) + } results, err := h.Tick(ctx) if err != nil || len(results) != 1 { t.Fatalf("Tick after = (%d results, %v), want the timer still due", len(results), err) diff --git a/durable/service.go b/durable/service.go index aacb91bc..e0927377 100644 --- a/durable/service.go +++ b/durable/service.go @@ -85,11 +85,11 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // // It returns the routed FireResult and true, or the zero result and false when id // names no in-flight service (already settled, stopped, or never started), or false -// and the error when the Handle has failed or ctx has ended. A persistence failure is -// returned as the error; the kernel transition itself does not fail (a settled service -// always routes onDone or onError). After a failed Append the result wasn't recorded, so -// a recovered Handle has the service in flight again, and RunService runs it a second -// time. +// and the error when the Handle had already failed or ctx has ended. If the service +// settled but its outcome couldn't be recorded or appended, it returns the result, +// true and ErrHandleFailed: the Handle has failed. A failed checkpoint returns the +// result, true and that error, and the Handle carries on. A recovered Handle has an +// unrecorded service in flight again, and RunService runs it a second time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { h.mu.Lock() defer h.mu.Unlock() From a8103dacf6202db9d1c15466523f735a18c0f730 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Tue, 10 Nov 2026 10:17:31 -0700 Subject: [PATCH 03/10] fix(durable): dispatch effects outside the step lock, and close review gaps An effect handler that called back into its own Handle deadlocked, because Fire held the step lock through dispatch. The lock now covers only the kernel step and the Record append. Effects dispatch after it's released, and a checkpoint waits until every step at or before it has dispatched, so compaction never drops an effect still being sent. A failed dispatch fails the Handle, so Recover sends the effect again. Code that has to run under the lock gets a marked context: a service that calls back in with it gets ErrReentrantStep instead of deadlocking. A step waiting for the lock stops when its context ends. Handle.View reads the instance under the lock. Also: - WithEventDataDecoder and DecodeAs decode a WithEventData payload as the caller's type, live and on replay. A payload that decodes as another type refuses the Fire. - StepReporter lets Recover number past every recorded step after a checkpoint from before checkpoints carried the next step. MemStore and FileStore implement it. - FileStore.Append encodes a Record once. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 21 +- durable/CHANGELOG.md | 38 +- durable/README.md | 21 +- durable/actor.go | 8 +- durable/doc.go | 6 +- durable/errors.go | 18 +- durable/event_data_test.go | 91 ++++- durable/filestore.go | 28 +- durable/handle_lock_test.go | 277 +++++++++++++++ durable/memstore.go | 31 +- durable/runner.go | 412 ++++++++++++++++------ durable/runner_options.go | 20 ++ durable/runner_test.go | 46 ++- durable/service.go | 8 +- durable/store.go | 15 +- durable/timetravel.go | 2 +- 16 files changed, 872 insertions(+), 170 deletions(-) create mode 100644 durable/handle_lock_test.go diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 4680e2a8..efba4c94 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -128,8 +128,12 @@ in by implementing `HistoryStore` (the in-tree `MemStore` does so under ## Handles across goroutines `Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different -goroutines: each holds the Handle's lock for the whole step. A host callback must not -call back into its Handle. `Instance()` is outside the lock. +goroutines: each holds the Handle's lock while it applies to the kernel and records the +step. Effects dispatch after the lock is released, so an effect handler can call back +into its Handle. A service, guard, action, reducer or actor behavior runs under the lock +and must not: a service that passes its own context gets `ErrReentrantStep`. A step +waiting for the lock stops when its context ends. Read the instance with `View`, which +takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a @@ -140,10 +144,11 @@ newer owner holds the instance, so stop. Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` observes the JSON-decoded shape on a replayed `onDone`. A `WithEventData` payload is -JSON-decoded live as well as on replay, so a reducer gets the same value both ways: a -struct is a map and an integer a `float64`. An error stays an error with the same -message, with any invalid UTF-8 replaced. `Handle.Fire` refuses a payload it can't +decoded from its JSON form live as well as on replay, so a reducer gets the same value +both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per +event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own +form, so `Handle.Fire` refuses a struct or an integer rather than hand the reducer a map +or a `float64`. An error stays an error with the same message, with any invalid UTF-8 +replaced. `Handle.Fire` refuses a payload that decodes as a different type, one it can't record, or one over `WithMaxEventDataBytes` (8 MiB by default), with -`ErrEventDataEncoding`, and nothing applies. A -typed-codec option to carry the concrete Go value across the journal boundary is -reserved for a later, additive change. +`ErrEventDataEncoding`, and nothing applies. diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 5067e0d0..0061974e 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -27,26 +27,44 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **`ErrHandleFailed`** reports a Handle out of step with its journal. +- **`Handle.View`** reads the instance under the Handle's step lock. **`ErrReentrantStep`** + reports a call back into a Handle from inside one of its steps. + +- **`StepReporter`**, implemented by `MemStore` and `FileStore`, lets `Recover` number the + next step past every recorded one after a checkpoint from before checkpoints carried + it. Without it, such a checkpoint followed by tick barriers made every `Append` + conflict. + +- **`WithEventDataDecoder`** decodes a `WithEventData` payload as the caller's type, + given the event it rode with. `DecodeAs[T]` is the usual body of each case. + ### Changed - Requires Go 1.27.3. - **A Handle's steps are serialized across goroutines.** `Fire`, `Tick`, `RunService` and - `DeliverToActor` hold a lock for the whole step, so a host's timer loop no longer loses - acknowledged steps. A host callback must not call back into its Handle, or it - deadlocks. `Instance()` isn't covered: read it only while no step is running. A `Tick`, - `RunService` or `DeliverToActor` whose context ended while it waited returns that error - and changes nothing. + `DeliverToActor` hold a lock while they apply to the kernel and record the step, so a + host's timer loop no longer loses acknowledged steps. Effects dispatch after the lock + is released, so an effect handler can still call back into its Handle; a checkpoint + waits for every earlier step's effects to dispatch. A service, guard, action, reducer + or actor behavior runs under the lock and must not call back in: a service that passes + its context gets the new `ErrReentrantStep`. A step waiting for the lock returns its + context's error once the context ends, and changes nothing. The new `Handle.View` reads + the instance under the lock; `Instance()` is outside it. +- **A failed effect dispatch fails the Handle.** The effect stays unmarked in the + journal, so no later checkpoint compacts it away, and `Recover` dispatches it again. - **A step that may not have been recorded fails the Handle.** That call and every later one return `ErrHandleFailed`, wrapping the cause. `Recover` a new Handle with the same options to continue. A service whose result wasn't recorded is in flight again, so `RunService` runs it a second time. An `InvokeActor` actor restarts from its initial state, and a `Spawn` actor doesn't return. If the cause is `ErrFenced`, a newer owner holds the instance: stop. -- **A `WithEventData` payload reaches `Assign` decoded, live as well as on replay.** A - struct arrives as a map and an integer as a `float64`, which loses precision above - 2^53, so an assertion to the original - type now fails at `Fire`, not at `Recover`. An error arrives as an error with the same - message, any invalid UTF-8 replaced, and `errors.Is` no longer matches it. +- **A `WithEventData` payload reaches `Assign` decoded, live as well as on replay.** + `WithEventDataDecoder` decodes it as the caller's type, chosen per event, with + `DecodeAs[T]`. A payload that decodes as a different type than the caller passed + refuses the `Fire` with `ErrEventDataEncoding`, so without a decoder a struct or an + integer is refused rather than reaching `Assign` as a map or a `float64`. An error + arrives as an error with the same message, any invalid UTF-8 replaced, and `errors.Is` + no longer matches it. - **`Handle.Fire` can refuse a payload.** One with no JSON form that decodes again, or over `WithMaxEventDataBytes` (8 MiB by default), returns the new `ErrEventDataEncoding` before the kernel runs. Fires of that size used to pass, because the payload wasn't diff --git a/durable/README.md b/durable/README.md index f720c52f..861557dc 100644 --- a/durable/README.md +++ b/durable/README.md @@ -139,8 +139,12 @@ otherwise. ## Handles across goroutines `Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different -goroutines: each holds the Handle's lock for the whole step. A host callback must not -call back into its Handle. `Instance()` is outside the lock. +goroutines: each holds the Handle's lock while it applies to the kernel and records the +step. Effects dispatch after the lock is released, so an effect handler can call back +into its Handle. A service, guard, action, reducer or actor behavior runs under the lock +and must not: a service that passes its own context gets `ErrReentrantStep`. A step +waiting for the lock stops when its context ends. Read the instance with `View`, which +takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a @@ -151,13 +155,14 @@ newer owner holds the instance, so stop. Events, service done-data, actor done-data, and actor messages are recorded as their JSON form. A parent reducer that type-asserts a non-JSON Go type from `AssignCtx.Event` observes the JSON-decoded shape on a replayed `onDone`. A `WithEventData` payload is -JSON-decoded live as well as on replay, so a reducer gets the same value both ways: a -struct is a map and an integer a `float64`. An error stays an error with the same -message, with any invalid UTF-8 replaced. `Handle.Fire` refuses a payload it can't +decoded from its JSON form live as well as on replay, so a reducer gets the same value +both ways. `WithEventDataDecoder` decodes it as the type the caller passed, chosen per +event, for example `durable.DecodeAs[Order]`. Without one, a payload decodes as JSON's own +form, so `Handle.Fire` refuses a struct or an integer rather than hand the reducer a map +or a `float64`. An error stays an error with the same message, with any invalid UTF-8 +replaced. `Handle.Fire` refuses a payload that decodes as a different type, one it can't record, or one over `WithMaxEventDataBytes` (8 MiB by default), with -`ErrEventDataEncoding`, and nothing applies. A -typed-codec option to carry the concrete Go value across the journal boundary is -reserved for a later, additive change. +`ErrEventDataEncoding`, and nothing applies. ## Performance diff --git a/durable/actor.go b/durable/actor.go index 17bed76b..1105ea76 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -108,8 +108,10 @@ type actorMessagePayload struct { // and a Spawn actor doesn't return. Resending a recorded delivery repeats // any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { - h.mu.Lock() - defer h.mu.Unlock() + if err := h.acquire(ctx); err != nil { + return false, err + } + defer h.release() if err := h.refuse(); err != nil { return false, err } @@ -121,7 +123,7 @@ func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef } before := len(h.inst.History()) - delivered := h.actors.Deliver(ctx, ref, event) + delivered := h.actors.Deliver(h.stepContext(ctx), ref, event) if !delivered { return false, nil } diff --git a/durable/doc.go b/durable/doc.go index 00e5a8fa..2a0bff56 100644 --- a/durable/doc.go +++ b/durable/doc.go @@ -34,9 +34,9 @@ // Payloads are serialized at the journal boundary. Event, service done-data, actor // done-data, and actor messages are recorded as their JSON form; a parent reducer that // type-asserts a non-JSON Go type from AssignCtx.Event observes the JSON-decoded shape -// on a replayed onDone. A WithEventData payload is JSON-decoded live as well as on -// replay, so a reducer gets the same value both ways. A typed-codec option to carry the -// concrete Go value across the boundary is reserved for a later, additive change. +// on a replayed onDone. A WithEventData payload is decoded from its JSON form live as +// well as on replay, so a reducer gets the same value both ways; WithEventDataDecoder +// decodes it as the type the caller passed. // // The kernel stays pure and dependency-free. The durable module is host-side: // database drivers, cloud SDKs, and I/O libraries live in the caller or in out- diff --git a/durable/errors.go b/durable/errors.go index ea357b31..c58fabdf 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -17,6 +17,7 @@ const ( errorEventDataEncoding = "event data can't be recorded" errorRecordTooLarge = "record too large for the journal" errorHandleFailed = "handle is out of step with the journal" + errorReentrantStep = "step called from inside a step on the same handle" ) // Sentinel errors a Store or a Handle reports. Callers match them with errors.Is. @@ -53,13 +54,19 @@ var ( // applied the step, so the Handle fails with ErrHandleFailed. ErrRecordTooLarge = errors.New(errorPrefix + errorRecordTooLarge) - // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService and - // DeliverToActor once a step applied to the live instance but may not have been + // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService, + // DeliverToActor and View once a step applied to the live instance but may not have been // recorded, on that call and every later one. It wraps the cause. Recover a new // Handle with the same options to continue, unless the cause is ErrFenced: then a // newer owner holds the instance, so stop. ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) + // ErrReentrantStep is reported by a Handle's Fire, Tick, RunService, DeliverToActor + // and View when called with the context a step passed to host code, such as a + // service: the step holds the Handle's lock, so waiting would deadlock. Nothing + // applied. Effect handlers run after the lock is released and may call back in. + ErrReentrantStep = errors.New(errorPrefix + errorReentrantStep) + // ErrCheckpointNotAdvancing is reported by Store.Checkpoint when throughStep // does not advance beyond the instance's current checkpoint. ErrCheckpointNotAdvancing = errors.New(errorPrefix + errorCheckpointNotAdvancing) @@ -70,8 +77,9 @@ var ( // ErrEffectDispatch wraps a failure returned by the caller-supplied effect // handler (WithEffectHandler) while applying an emitted effect. The effect was - // persisted (write-ahead) but not marked dispatched, so a later recovery - // retries it; callers match this with errors.Is to distinguish a dispatch - // failure from a kernel transition or persistence error. + // persisted (write-ahead) but not marked dispatched, so the Handle fails with + // ErrHandleFailed wrapping it, and Recover dispatches it again. Callers match + // this with errors.Is to tell a dispatch failure from a kernel transition or + // persistence error. ErrEffectDispatch = errors.New(errorPrefix + errorEffectDispatch) ) diff --git a/durable/event_data_test.go b/durable/event_data_test.go index 058bafd1..24ac17a0 100644 --- a/durable/event_data_test.go +++ b/durable/event_data_test.go @@ -147,19 +147,22 @@ func payloadMachine(seen *[]any) *state.Machine[string, string, *runCtx] { Quench() } -// TestFire_EventData_LiveEqualsReplay proves Assign sees one value for each kind of -// payload, whether the step ran live or was replayed. +// TestFire_EventData_LiveEqualsReplay proves a payload reaches Assign as the same +// value and type live and after Recover: decoded by the Runner's EventDataDecoder, or +// in JSON's own form by default. func TestFire_EventData_LiveEqualsReplay(t *testing.T) { type order struct{ Qty int } tests := []struct { name string payload any + decode durable.EventDataDecoder[string] want string }{ - {"an int", 3, "float64:3"}, - {"a large int", int64(1<<53 + 1), "float64:9.007199254740992e+15"}, - {"a struct", order{Qty: 2}, "map[string]interface {}:map[Qty:2]"}, - {"an error", errors.New("card declined"), "*errors.errorString:card declined"}, + {"an int", 3, decodeAll[int], "int:3"}, + {"a large int keeps its precision", int64(1<<53 + 1), decodeAll[int64], "int64:9007199254740993"}, + {"a struct", order{Qty: 2}, decodeAll[order], "durable_test.order:{2}"}, + {"a map by default", map[string]any{"qty": 2.0}, nil, "map[string]interface {}:map[qty:2]"}, + {"an error", errors.New("card declined"), nil, "*errors.errorString:card declined"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -168,14 +171,15 @@ func TestFire_EventData_LiveEqualsReplay(t *testing.T) { m := payloadMachine(&seen) store := durable.NewMemStore() const id = durable.InstanceID("p-1") - h, err := durable.NewRunner(m, store).Start(ctx, id, &runCtx{}, state.WithInitialState("open")) + opt := durable.WithEventDataDecoder[string, string, *runCtx](tt.decode) + h, err := durable.NewRunner(m, store, opt).Start(ctx, id, &runCtx{}, state.WithInitialState("open")) if err != nil { t.Fatalf("Start: %v", err) } if _, err = h.Fire(ctx, "add", state.WithEventData(tt.payload)); err != nil { t.Fatalf("Fire: %v", err) } - if _, err = durable.Recover(ctx, m, store, id); err != nil { + if _, err = durable.Recover(ctx, m, store, id, opt); err != nil { t.Fatalf("Recover: %v", err) } if len(seen) != 2 { @@ -190,6 +194,75 @@ func TestFire_EventData_LiveEqualsReplay(t *testing.T) { } } +// decodeAll decodes every event's payload as a T. +func decodeAll[T any](_ string, data []byte) (any, error) { return durable.DecodeAs[T](data) } + +// TestFire_EventData_DecodedAsAnotherType_IsRefused proves a payload the decoder +// returns as a different type than the caller passed refuses the Fire before the +// kernel runs, so Assign never sees a map where the caller passed a struct. +func TestFire_EventData_DecodedAsAnotherType_IsRefused(t *testing.T) { + type order struct{ Qty int } + tests := []struct { + name string + payload any + decode durable.EventDataDecoder[string] + }{ + {"a struct by default", order{Qty: 2}, nil}, + {"an int by default", 3, nil}, + {"a struct the decoder reads as a pointer", order{Qty: 2}, decodeAll[*order]}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + var seen []any + store := durable.NewMemStore() + h, err := durable.NewRunner(payloadMachine(&seen), store, durable.WithEventDataDecoder[string, string, *runCtx](tt.decode)). + Start(ctx, "p-1", &runCtx{}, state.WithInitialState("open")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, err = h.Fire(ctx, "add", state.WithEventData(tt.payload)) + if !errors.Is(err, durable.ErrEventDataEncoding) { + t.Fatalf("Fire err = %v, want ErrEventDataEncoding", err) + } + if len(seen) != 0 { + t.Fatalf("assign ran %d times, want 0", len(seen)) + } + if _, tail, err := store.Load(ctx, "p-1"); err != nil || len(tail) != 0 { + t.Fatalf("Load: tail %d, err %v; want nothing recorded", len(tail), err) + } + }) + } +} + +// TestFire_EventData_DecoderGetsTheEvent proves the decoder is called with the event +// the payload rode with, live and on replay. +func TestFire_EventData_DecoderGetsTheEvent(t *testing.T) { + ctx := context.Background() + var seen []any + var events []string + decode := func(ev string, data []byte) (any, error) { + events = append(events, ev) + return durable.DecodeAs[int](data) + } + m := payloadMachine(&seen) + store := durable.NewMemStore() + opt := durable.WithEventDataDecoder[string, string, *runCtx](decode) + h, err := durable.NewRunner(m, store, opt).Start(ctx, "p-1", &runCtx{}, state.WithInitialState("open")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "add", state.WithEventData(7)); err != nil { + t.Fatalf("Fire: %v", err) + } + if _, err = durable.Recover(ctx, m, store, "p-1", opt); err != nil { + t.Fatalf("Recover: %v", err) + } + if fmt.Sprint(events) != "[add add]" { + t.Fatalf("decoder saw events %v, want [add add]", events) + } +} + // TestFire_NoEventData_AssignSeesTheEvent proves a Fire with no payload keeps the // event in AssignCtx.Event. func TestFire_NoEventData_AssignSeesTheEvent(t *testing.T) { @@ -305,7 +378,7 @@ func TestFire_EventData_LeavesTheCallersOptionsAlone(t *testing.T) { t.Fatalf("Start: %v", err) } opts := make([]state.FireOption, 1, 2) - opts[0] = state.WithEventData(1) + opts[0] = state.WithEventData("one") if _, err = h.Fire(ctx, "add", opts...); err != nil { t.Fatalf("Fire: %v", err) } diff --git a/durable/filestore.go b/durable/filestore.go index 7f46d6f0..7c435686 100644 --- a/durable/filestore.go +++ b/durable/filestore.go @@ -387,10 +387,14 @@ func (s *FileStore) Append(_ context.Context, id InstanceID, rec Record, opts .. if err = inst.admitFence(id, cfg.fence); err != nil { return 0, err } - key, digest, err := appendKey(cfg, rec) + // Encode the Record once: the same bytes are its digest and, unless trimmed below, + // its journal line. + kept := cloneRecord(rec) + line, err := json.Marshal(kept) if err != nil { - return 0, err + return 0, fmt.Errorf("durable: marshaling record step %d for %q: %w", rec.Step, id, err) } + key, digest := appendKeyOf(cfg, rec.Step, line) if existing, ok := inst.applied[key]; ok { return existing.check(id, rec.Step, digest) } @@ -399,11 +403,6 @@ func (s *FileStore) Append(_ context.Context, id InstanceID, rec Record, opts .. ErrStepOutOfOrder, rec.Step, inst.maxStep, id) } - line, err := json.Marshal(cloneRecord(rec)) - if err != nil { - return 0, fmt.Errorf("durable: marshaling record step %d for %q: %w", rec.Step, id, err) - } - kept := cloneRecord(rec) if len(line) >= maxLineBytes && len(rec.Snapshot) > 0 { // Recovery reads the checkpoint, never a tail Record's Snapshot, so an oversized // Record drops it rather than leaving a line Load can't read. The in-memory tail @@ -464,6 +463,21 @@ func (s *FileStore) Load(_ context.Context, id InstanceID) ([]byte, []Record, er return snapshot, tail, nil } +// LastStep reports the highest Step recorded for id, across its checkpoint and its +// tail, so FileStore is a StepReporter. +func (s *FileStore) LastStep(_ context.Context, id InstanceID) (int, bool, error) { + inst, existed, err := s.instance(id) + if err != nil { + return 0, false, err + } + if !existed { + return 0, false, fmt.Errorf("%w: %q", ErrInstanceNotFound, id) + } + inst.mu.Lock() + defer inst.mu.Unlock() + return inst.maxStep, inst.maxStep != noRecord, nil +} + // Checkpoint implements Store. It atomically persists snapshot at throughStep // (temp+rename) and compacts the on-disk journal to only the post-checkpoint // tail. throughStep must advance beyond the current checkpoint. diff --git a/durable/handle_lock_test.go b/durable/handle_lock_test.go new file mode 100644 index 00000000..c8ad4ae1 --- /dev/null +++ b/durable/handle_lock_test.go @@ -0,0 +1,277 @@ +package durable_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stablekernel/crucible/durable" + "github.com/stablekernel/crucible/state" +) + +// waitFor fails the test if done doesn't close within a few seconds, so a deadlock +// fails the test instead of hanging it. +func waitFor(t *testing.T, done <-chan struct{}, what string) { + t.Helper() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("%s didn't finish: deadlocked", what) + } +} + +// TestHandle_EffectHandlerFiresBack_DoesNotDeadlock proves an effect handler can call +// back into its own Handle: effects dispatch after the step releases the lock. +func TestHandle_EffectHandlerFiresBack_DoesNotDeadlock(t *testing.T) { + ctx := context.Background() + m := effectMachine() + store := durable.NewMemStore() + var h *durable.Handle[string, string, *runCtx] + var firedBack atomic.Bool + var backErr error + handler := func(ctx context.Context, _ string, _ state.Effect) error { + if firedBack.CompareAndSwap(false, true) { + _, backErr = h.Fire(ctx, "bill") + } + return nil + } + opt := durable.WithEffectHandler[string, string, *runCtx](handler) + h, err := durable.NewRunner(m, store, opt).Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + _, err = h.Fire(ctx, "go") + }() + waitFor(t, done, "Fire(go)") + if err != nil || backErr != nil { + t.Fatalf("Fire(go) = %v, the handler's Fire(bill) = %v; want both nil", err, backErr) + } + if got := h.Instance().Current(); got != "billed" { + t.Fatalf("live state %q, want billed", got) + } + recovered, err := durable.Recover(ctx, m, store, "e-1", opt) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := recovered.Instance().Current(); got != "billed" { + t.Fatalf("recovered state %q, want billed", got) + } +} + +// blockingFetch starts the single-service machine with a service that signals entered +// and then waits for release, so a test can hold a step open. +func blockingFetch(t *testing.T, entered chan<- struct{}, release <-chan struct{}) *durable.Handle[string, string, svcCtx] { + t.Helper() + ctx := context.Background() + fn := func(context.Context, state.ServiceCtx[svcCtx]) (any, error) { + entered <- struct{}{} + <-release + return "done", nil + } + h, err := durable.NewRunner(fetchMachine(fn), durable.NewMemStore(), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))). + Start(ctx, "fetch-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + return h +} + +// TestHandle_WaitingStep_StopsAtItsDeadline proves a step waiting for the lock returns +// its context's error when the context ends, instead of waiting out a slow step. +func TestHandle_WaitingStep_StopsAtItsDeadline(t *testing.T) { + entered, release := make(chan struct{}, 1), make(chan struct{}) + h := blockingFetch(t, entered, release) + running := make(chan struct{}) + go func() { + defer close(running) + mustRunService(t, h, fetchID) + }() + <-entered // the service holds the step lock + + waits := map[string]func(context.Context) error{ + "Fire": func(ctx context.Context) error { _, err := h.Fire(ctx, "ok"); return err }, + "Tick": func(ctx context.Context) error { _, err := h.Tick(ctx); return err }, + "View": func(ctx context.Context) error { return h.View(ctx, func(*state.Instance[string, string, svcCtx]) {}) }, + } + for name, wait := range waits { + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + returned := make(chan struct{}) + var err error + go func() { + defer close(returned) + err = wait(ctx) + }() + waitFor(t, returned, name) + cancel() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("%s while the lock is held = %v, want context.DeadlineExceeded", name, err) + } + } + close(release) + waitFor(t, running, "RunService") + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("state %q, want ready: a timed-out step changed the instance", got) + } +} + +// TestHandle_ServiceCallsBackWithItsContext_IsRefused proves a service that calls back +// into its own Handle with the context its step passed gets ErrReentrantStep at once +// instead of deadlocking on the lock its step holds. +func TestHandle_ServiceCallsBackWithItsContext_IsRefused(t *testing.T) { + ctx := context.Background() + var h *durable.Handle[string, string, svcCtx] + var fireErr, viewErr error + fn := func(ctx context.Context, _ state.ServiceCtx[svcCtx]) (any, error) { + _, fireErr = h.Fire(ctx, "ok") + viewErr = h.View(ctx, func(*state.Instance[string, string, svcCtx]) {}) + return "done", nil + } + h, err := durable.NewRunner(fetchMachine(fn), durable.NewMemStore(), + durable.WithServiceRegistry[string, string, svcCtx](fetchRegistry(fn))). + Start(ctx, "fetch-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + mustRunService(t, h, fetchID) + }() + waitFor(t, done, "RunService") + if !errors.Is(fireErr, durable.ErrReentrantStep) || !errors.Is(viewErr, durable.ErrReentrantStep) { + t.Fatalf("the service's Fire = %v and View = %v, want ErrReentrantStep", fireErr, viewErr) + } + if got := h.Instance().Current(); got != "ready" { + t.Fatalf("state %q, want ready", got) + } +} + +// TestHandle_Checkpoint_WaitsForAnEarlierStepsDispatch proves a checkpoint doesn't +// compact a step whose effect another goroutine is still dispatching, and is written +// once that dispatch lands. +func TestHandle_Checkpoint_WaitsForAnEarlierStepsDispatch(t *testing.T) { + ctx := context.Background() + store := durable.NewMemStore() + entered, release := make(chan struct{}), make(chan struct{}) + var waited atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if waited.CompareAndSwap(false, true) { // only step 0's notify waits + close(entered) + <-release + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler), + durable.WithCheckpointEvery[string, string, *runCtx](2)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + first := make(chan struct{}) + var firstErr error + go func() { + defer close(first) + _, firstErr = h.Fire(ctx, "go") // step 0, its notify still dispatching + }() + <-entered + if _, err = h.Fire(ctx, "bill"); err != nil { // step 1, where a checkpoint falls due + t.Fatalf("Fire(bill): %v", err) + } + if _, tail, err := store.Load(ctx, "e-1"); err != nil || len(tail) != 2 { + t.Fatalf("while step 0 dispatches: tail of %d, err %v; want both steps kept", len(tail), err) + } + close(release) + waitFor(t, first, "Fire(go)") + if firstErr != nil { + t.Fatalf("Fire(go): %v", firstErr) + } + if _, tail, err := store.Load(ctx, "e-1"); err != nil || len(tail) != 0 { + t.Fatalf("after step 0 dispatched: tail of %d, err %v; want the checkpoint written", len(tail), err) + } +} + +// TestHandle_FailedDispatch_FailsTheHandle_AndRecoverRedispatches proves a failed +// effect fails the Handle, so no later checkpoint compacts it away, and Recover +// dispatches it again. +func TestHandle_FailedDispatch_FailsTheHandle_AndRecoverRedispatches(t *testing.T) { + ctx := context.Background() + m := effectMachine() + store := durable.NewMemStore() + rec := newEffectRecorder() + rec.failKinds["demo.notify"] = errors.New("mail server down") + opts := []durable.Option[string, string, *runCtx]{ + durable.WithEffectHandler[string, string, *runCtx](rec.handler()), + durable.WithCheckpointEvery[string, string, *runCtx](2), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, err = h.Fire(ctx, "go") + if !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire(go) = %v, want ErrHandleFailed wrapping ErrEffectDispatch", err) + } + if _, err = h.Fire(ctx, "bill"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(bill) after the failure = %v, want ErrHandleFailed", err) + } + delete(rec.failKinds, "demo.notify") + recovered, err := durable.Recover(ctx, m, store, "e-1", opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if got := rec.totalApplies(); got != 1 { + t.Fatalf("Recover applied %d effects, want the failed notify once", got) + } + if _, err = recovered.Fire(ctx, "bill"); err != nil { + t.Fatalf("Fire(bill) on the recovered Handle: %v", err) + } +} + +// TestHandle_View_DoesNotRaceSteps proves View reads under the step lock: run with +// -race, a read outside it would be reported against the concurrent Fires. +func TestHandle_View_DoesNotRaceSteps(t *testing.T) { + ctx := context.Background() + h, err := durable.NewRunner(pollingMachine(), durable.NewMemStore()). + Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + for range 100 { + if _, err := h.Fire(ctx, "poke"); err != nil { + t.Errorf("Fire(poke): %v", err) + return + } + } + }() + var last int + for range 100 { + if err := h.View(ctx, func(inst *state.Instance[string, string, *runCtx]) { + last = inst.Entity().Count + }); err != nil { + t.Fatalf("View: %v", err) + } + } + waitFor(t, done, "the Fires") + if err := h.View(ctx, func(inst *state.Instance[string, string, *runCtx]) { last = inst.Entity().Count }); err != nil || last != 100 { + t.Fatalf("View after the Fires: count %d, err %v; want 100", last, err) + } +} diff --git a/durable/memstore.go b/durable/memstore.go index af746094..9a41e9fd 100644 --- a/durable/memstore.go +++ b/durable/memstore.go @@ -177,6 +177,19 @@ func (s *MemStore) Load(_ context.Context, id InstanceID) ([]byte, []Record, err return snapshot, tail, nil } +// LastStep reports the highest Step recorded for id, across its checkpoint and its +// tail, so MemStore is a StepReporter. +func (s *MemStore) LastStep(_ context.Context, id InstanceID) (int, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + inst, ok := s.instances[id] + if !ok || !inst.written() { + return 0, false, fmt.Errorf("%w: %q", ErrInstanceNotFound, id) + } + return inst.maxStep, inst.maxStep != noRecord, nil +} + // History implements HistoryStore for a MemStore constructed WithHistory: it returns // the instance's start baseline snapshot and its full ordered Record log, including // Records a checkpoint compacted out of the live tail, so a time-travel read can @@ -329,6 +342,15 @@ func appendKey(cfg appendConfig, rec Record) (key, digest string, err error) { return fmt.Sprintf("step:%d", rec.Step), digest, nil } +// appendKeyOf is appendKey for a Record already encoded as encoded, the JSON form of +// its clone, so a Store that writes those bytes doesn't encode the Record twice. +func appendKeyOf(cfg appendConfig, step int, encoded []byte) (key, digest string) { + if cfg.idempotencyKey != "" { + return cfg.idempotencyKey, "" + } + return fmt.Sprintf("step:%d", step), digestOf(encoded) +} + // recordDigest returns a stable digest of rec. It digests the clone because // cloneRecord turns an empty field into nil, so the two compare equal. func recordDigest(rec Record) (string, error) { @@ -336,8 +358,13 @@ func recordDigest(rec Record) (string, error) { if err != nil { return "", fmt.Errorf("durable: digesting record step %d: %w", rec.Step, err) } - sum := sha256.Sum256(b) - return hex.EncodeToString(sum[:]), nil + return digestOf(b), nil +} + +// digestOf returns the digest of a Record's encoded clone. +func digestOf(encoded []byte) string { + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) } // fencedError reports a write whose token is below the highest seen. diff --git a/durable/runner.go b/durable/runner.go index 9fe3f0ce..2d2d7235 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "sync" "time" @@ -79,15 +80,20 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // Runner.Start or Recover. // // Fire, Tick, RunService and DeliverToActor are safe to call from different -// goroutines: each holds the Handle's lock for the whole step, including the Store -// write, effect dispatch and any service it runs. Any host callback run during a step -// (an effect handler, service, guard, action, reducer or actor behavior) must not call -// back into the same Handle, or it deadlocks. A step waits for the lock without watching -// its context, so a slow service delays other steps past their deadlines; a Tick, -// RunService or DeliverToActor whose context has ended by then returns that error and -// changes nothing, and the kernel refuses a Fire. Instance returns the -// live instance outside that lock, so read it only while no step is running. A -// stateless Runner.Fire builds its own Handle, so it isn't serialized with this one. +// goroutines: each holds the Handle's step lock while it applies to the kernel and +// writes its Record, including any service or actor it runs. A step's domain effects +// dispatch after it releases the lock, so an effect handler may call back into the +// Handle. Effects of steps on different goroutines can dispatch in either order; a +// goroutine's own steps dispatch in order. A checkpoint waits until every step at or +// before it has dispatched, so it never compacts an effect still being sent. +// +// Code that runs under the lock (a service, guard, action, reducer or actor behavior) +// must not call back into the same Handle. A call that carries the context the step +// passed to it, as a service gets, returns ErrReentrantStep at once; one without that +// context deadlocks. A step waiting for the lock returns its context's error once the +// context ends, and changes nothing. View reads the instance under the lock; Instance +// returns it outside the lock. A stateless Runner.Fire builds its own Handle, so it +// isn't serialized with this one. // // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, @@ -102,9 +108,10 @@ type Handle[S comparable, E comparable, C any] struct { id InstanceID inst *state.Instance[S, E, C] - mu sync.Mutex // serializes the steps, and guards the instance and every field below + // lock is the step lock: a step holds its one slot while it applies to the instance + // and records, and it guards the instance and every field below except out. + lock chan struct{} nextStep int - failed error // the error that took the Handle out of step with the journal, nil until then sched *state.Scheduler[S, E, C] recClock *recordingClock // owns the clock-read accumulator and its mutex @@ -115,21 +122,109 @@ type Handle[S comparable, E comparable, C any] struct { actors *state.ActorSystem[S, E, C] // host driver for child-machine actors, nil when none wired actorBuf *[]state.JournalEntry // recording actor-transition accumulator, flushed per step + + out outbox // what a step leaves behind once it releases the step lock +} + +// outbox holds what outlives a step's hold on the step lock: the failure latch, the +// steps whose domain effects are still being dispatched, and the checkpoint waiting on +// them. A step's effects dispatch after it releases the step lock, so the checkpoint +// policy can fall due while an earlier step's effect is still unmarked; compacting +// through that step would leave nothing for recovery to redispatch. So a due +// checkpoint waits until every step at or before it has dispatched, and the step that +// finishes last writes it. The mutex is taken inside the step lock, never around it. +type outbox struct { + mu sync.Mutex + failed error // why the Handle is out of step with the journal, nil until then + dispatching map[int]struct{} // steps whose effects are recorded but not yet dispatched + pending *dueCheckpoint // the newest due checkpoint, waiting on dispatching + checkpointed int // the newest step this Handle checkpointed through +} + +// dueCheckpoint is a checkpoint the policy made due at step, with the snapshot the +// step recorded. +type dueCheckpoint struct { + step int + snapshot []byte +} + +// stepKey marks a context passed to host code during a step, so a call from that code +// back into the same Handle fails at once instead of waiting forever for the step lock. +type stepKey struct{} + +// stepMark records the Handles whose steps a context is running inside. +type stepMark struct { + handle any + outer *stepMark +} + +// stepContext returns ctx marked as inside a step of h. +func (h *Handle[S, E, C]) stepContext(ctx context.Context) context.Context { + outer, _ := ctx.Value(stepKey{}).(*stepMark) + return context.WithValue(ctx, stepKey{}, &stepMark{handle: h, outer: outer}) +} + +// inStep reports whether ctx is running inside a step of h. +func (h *Handle[S, E, C]) inStep(ctx context.Context) bool { + m, _ := ctx.Value(stepKey{}).(*stepMark) + for ; m != nil; m = m.outer { + if m.handle == any(h) { + return true + } + } + return false +} + +// acquire takes the step lock, or returns ctx's error if ctx ends first, or +// ErrReentrantStep if ctx is already inside a step of h. +func (h *Handle[S, E, C]) acquire(ctx context.Context) error { + if h.inStep(ctx) { + return fmt.Errorf("%w: %q", ErrReentrantStep, h.id) + } + if err := ctx.Err(); err != nil { + return err + } + select { + case h.lock <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } } +// release gives up the step lock. +func (h *Handle[S, E, C]) release() { <-h.lock } + // refuse reports ErrHandleFailed once the Handle is out of step with the journal. func (h *Handle[S, E, C]) refuse() error { - if h.failed == nil { + h.out.mu.Lock() + defer h.out.mu.Unlock() + return h.refuseLocked() +} + +// refuseLocked is refuse with h.out.mu held. +func (h *Handle[S, E, C]) refuseLocked() error { + if h.out.failed == nil { return nil } - return fmt.Errorf("%w: %q: %w", ErrHandleFailed, h.id, h.failed) + return fmt.Errorf("%w: %q: %w", ErrHandleFailed, h.id, h.out.failed) } -// fail records err as the reason the Handle is out of step with the journal, and -// returns the same refusal every later step gets. +// fail records err as the reason the Handle is out of step with the journal, unless +// an earlier step already failed it, and returns the refusal every later step gets. func (h *Handle[S, E, C]) fail(err error) error { - h.failed = err - return h.refuse() + h.out.mu.Lock() + defer h.out.mu.Unlock() + return h.failLocked(err) +} + +// failLocked is fail with h.out.mu held. A failed Handle writes no more checkpoints. +func (h *Handle[S, E, C]) failLocked(err error) error { + if h.out.failed == nil { + h.out.failed = err + } + h.out.pending = nil + return h.refuseLocked() } // drainClock returns and clears the clock readings accumulated since the last @@ -266,10 +361,28 @@ func (h *Handle[S, E, C]) dropFiredTimers(now time.Time) { // Instance returns the underlying kernel Instance the Handle wraps, for reads // such as Configuration, Snapshot, or Current. Drive it through the Handle's Fire // (or the Runner's) so steps continue to be recorded; firing the bare Instance -// bypasses durability. Read it only while no step is running: it's outside the Handle's -// lock. After ErrHandleFailed it shows state the journal may not hold. +// bypasses durability. It's outside the Handle's step lock, so read it only while no +// step is running, or use View. After ErrHandleFailed it shows state the journal may +// not hold. func (h *Handle[S, E, C]) Instance() *state.Instance[S, E, C] { return h.inst } +// View runs fn with the live instance while it holds the Handle's step lock, so the +// read can't race a step on another goroutine. fn must not call back into the Handle, +// and must not keep the instance past its return. View returns ctx's error if ctx ends +// before the lock is free, ErrReentrantStep if ctx is a step's own context, and +// ErrHandleFailed once the Handle has failed, without running fn. +func (h *Handle[S, E, C]) View(ctx context.Context, fn func(*state.Instance[S, E, C])) error { + if err := h.acquire(ctx); err != nil { + return err + } + defer h.release() + if err := h.refuse(); err != nil { + return err + } + fn(h.inst) + return nil +} + // ID returns the InstanceID the Handle records under. func (h *Handle[S, E, C]) ID() InstanceID { return h.id } @@ -351,6 +464,8 @@ func (r *Runner[S, E, C]) Start(ctx context.Context, id InstanceID, input C, opt svcBuf: &svcBuf, actors: actors, actorBuf: &actorBuf, + lock: make(chan struct{}, 1), + out: outbox{dispatching: map[int]struct{}{}, checkpointed: baselineStep}, }, nil } @@ -400,31 +515,50 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // kernel transition error journals nothing (no step was produced) and is // returned to the caller. // -// A state.WithEventData payload reaches the transition's Assign in its JSON-decoded -// form, live and on replay alike, so both see the same value: a struct arrives as a -// map and an integer as a float64, which loses precision above 2^53. An error arrives +// A state.WithEventData payload reaches the transition's Assign as the Runner's +// EventDataDecoder rebuilds it from the payload's JSON form, live and on replay alike, so +// both see the same value. Decode it as the type the caller passed: a payload that +// decodes as another type refuses the Fire with ErrEventDataEncoding before the kernel +// runs. The default decoder gives JSON's own form, so it refuses a struct or an int +// until WithEventDataDecoder covers that event. An error arrives // as an error with the same message, with any invalid UTF-8 replaced, though errors.Is // no longer matches it. Keep its message, not the error, in context: a checkpoint turns -// an error into a map, and Recover fails on an error-typed field. A payload that can't take this form, an error whose Error panics (such as -// a typed nil), or -// exceeds WithMaxEventDataBytes, refuses the Fire with ErrEventDataEncoding before the +// an error into a map, and Recover fails on an error-typed field. A payload that can't +// take this form, an error whose Error panics (such as a typed nil), or one over +// WithMaxEventDataBytes, also refuses the Fire with ErrEventDataEncoding before the // kernel runs. The payload is read on the calling goroutine with no lock of its own, so // don't mutate it concurrently with Fire. func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireOption) (state.FireResult[S], error) { - h.mu.Lock() - defer h.mu.Unlock() - if err := h.refuse(); err != nil { + if err := h.acquire(ctx); err != nil { return state.FireResult[S]{}, err } + res, rec, des, err := h.fireStep(h.stepContext(ctx), event, opts) + h.release() + if err != nil { + return res, err + } + // Dispatch after releasing the step lock, so an effect handler can call back into + // this Handle. The Record already names every effect (write-ahead), so a crash here + // leaves it for recovery to redispatch, deduped by id. + return res, h.finishStep(ctx, rec, des) +} + +// fireStep is the part of Fire that holds the step lock: it applies the event to the +// kernel and appends the step's Record, naming its domain effects, before any is +// dispatched. ctx is the step's marked context. +func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.FireOption) (state.FireResult[S], *Record, []dispatchableEffect, error) { + if err := h.refuse(); err != nil { + return state.FireResult[S]{}, nil, nil, err + } // Encode the WithEventData payload before the kernel runs, so a payload that // can't be recorded refuses the Fire rather than applying a step it can't record. - payload, opts, err := encodeEventData(opts, h.runner.cfg.maxEventData) + payload, opts, err := encodeEventData(event, opts, h.runner.cfg.maxEventData, h.runner.cfg.eventDataDecoder) if err != nil { - return state.FireResult[S]{}, fmt.Errorf("durable: recording event data for %q: %w", h.id, err) + return state.FireResult[S]{}, nil, nil, fmt.Errorf("durable: recording event data for %q: %w", h.id, err) } res := h.inst.Fire(ctx, event, opts...) if res.Err != nil { - return res, res.Err + return res, nil, nil, res.Err } // Absorb the step's driver effects into every host driver — timers, services, and @@ -441,29 +575,36 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO des := dispatchableEffects(step, res.Effects) effEnvs, err := recordEffects(des) if err != nil { - return res, h.fail(err) - } - rec := Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: h.drainClock(), Effects: effEnvs} - // Write-ahead append the step Record (with its effect ids) BEFORE dispatching, but - // hold off compacting it: dispatch happens between the append and the checkpoint so - // a crash mid-dispatch leaves the effect recorded in the still-present tail for - // recovery to redispatch — never orphaned behind the compaction. - if err := h.appendStep(ctx, step, &rec); err != nil { - return res, h.fail(err) - } - h.nextStep++ - // Dispatch AFTER the append: a crash in this window leaves the effect recorded - // but un-marked, so recovery redispatches it (at-least-once), deduped by id to - // exactly-once once it lands. - if err := h.dispatchEffects(ctx, des); err != nil { - return res, err + return res, nil, nil, h.fail(err) } - // Compact through this step only now that its effects are dispatched, so the - // checkpoint never discards a not-yet-dispatched effect. - if err := h.checkpointStep(ctx, step, &rec); err != nil { - return res, err + rec := &Record{Step: step, Event: []byte(res.Trace.EventPayload), EventData: payload.data, EventDataError: payload.errMsg, Entries: h.drainClock(), Effects: effEnvs} + if err := h.appendStep(ctx, step, rec); err != nil { + return res, nil, nil, h.fail(err) + } + h.nextStep = rec.followingStep() + if len(des) > 0 { + h.out.mu.Lock() + h.out.dispatching[step] = struct{}{} + h.out.mu.Unlock() } - return res, nil + return res, rec, des, nil +} + +// finishStep dispatches a recorded step's domain effects, outside the step lock, then +// offers its checkpoint. A failed dispatch fails the Handle: the effect stays unmarked +// in the journal, and Recover dispatches it again. +func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dispatchableEffect) error { + var dispatchErr error + if len(des) > 0 { + dispatchErr = h.dispatchEffects(ctx, des) + } + h.out.mu.Lock() + defer h.out.mu.Unlock() + delete(h.out.dispatching, rec.Step) + if dispatchErr != nil { + return h.failLocked(dispatchErr) + } + return h.offerCheckpointLocked(ctx, rec) } // Tick advances the Handle's delayed-transition scheduler: it fires every timer @@ -475,17 +616,19 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO // count of timers fired — so recovery re-derives the same timers at their // recorded instants. It returns the FireResults of the timers it fired, in order. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { - h.mu.Lock() - defer h.mu.Unlock() + if err := h.acquire(ctx); err != nil { + return nil, err + } + defer h.release() if err := h.refuse(); err != nil { return nil, err } - // A context that ended in the lock queue must stop here: the scheduler would consume - // a due timer whose Fire the kernel then refuses. + // A context that ended as the lock was taken must stop here: the scheduler would + // consume a due timer whose Fire the kernel then refuses. if err := ctx.Err(); err != nil { return nil, err } - results := h.sched.Tick(ctx) + results := h.sched.Tick(h.stepContext(ctx)) step := h.nextStep rec := Record{Step: step, Tick: true, TickSteps: len(results), Entries: h.drainClock()} @@ -506,28 +649,57 @@ func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], erro if len(rec.Entries) == 0 && rec.TickSteps == 0 { return results, nil } - if err := h.appendStep(ctx, step, &rec); err != nil { - return results, h.fail(err) - } - h.nextStep = rec.followingStep() - if err := h.checkpointStep(ctx, step, &rec); err != nil { + if err := h.persistStep(ctx, step, &rec); err != nil { return results, err } return results, nil } -// persistStep write-ahead appends rec, advances the next step, then checkpoints if -// the policy is due. A failed append fails the Handle, because the kernel already -// applied the step; a failed checkpoint doesn't, because the Record is durable. This -// relies on the Append running inline, under the Handle's lock, before the step -// returns: an Append deferred off this goroutine would acknowledge a step whose -// outcome isn't known, and the Handle couldn't fail. +// persistStep write-ahead appends rec for a step with no domain effects to dispatch, +// advances the next step, then offers its checkpoint. A failed append fails the +// Handle, because the kernel already applied the step; a failed checkpoint doesn't, +// because the Record is durable. This relies on the Append running inline, under the +// step lock, before the step returns: an Append deferred off this goroutine would +// acknowledge a step whose outcome isn't known, and the Handle couldn't fail. func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record) error { if err := h.appendStep(ctx, step, rec); err != nil { return h.fail(err) } h.nextStep = rec.followingStep() - return h.checkpointStep(ctx, step, rec) + h.out.mu.Lock() + defer h.out.mu.Unlock() + return h.offerCheckpointLocked(ctx, rec) +} + +// offerCheckpointLocked makes rec's checkpoint pending when the policy is due at its +// step, then writes the pending checkpoint if no step at or before it is still +// dispatching. Otherwise the step that finishes dispatching last writes it. A failed +// Handle writes none. h.out.mu must be held. +func (h *Handle[S, E, C]) offerCheckpointLocked(ctx context.Context, rec *Record) error { + if h.out.failed != nil { + return nil + } + if h.checkpointDue(rec.Step) && rec.Snapshot != nil && (h.out.pending == nil || rec.Step > h.out.pending.step) { + h.out.pending = &dueCheckpoint{step: rec.Step, snapshot: rec.Snapshot} + } + cp := h.out.pending + if cp == nil { + return nil + } + for step := range h.out.dispatching { + if step <= cp.step { + return nil + } + } + h.out.pending = nil + if cp.step <= h.out.checkpointed { + return nil + } + if err := h.runner.store.Checkpoint(ctx, h.id, cp.snapshot, cp.step, WithCheckpointFence(h.runner.cfg.fence)); err != nil { + return fmt.Errorf("durable: checkpointing step %d for %q: %w", cp.step, h.id, err) + } + h.out.checkpointed = cp.step + return nil } // checkpointDue reports whether the checkpoint policy lands a checkpoint at step. @@ -537,10 +709,10 @@ func (h *Handle[S, E, C]) checkpointDue(step int) bool { // appendStep marshals the due checkpoint snapshot into rec (so the Record carries it) // and write-ahead appends rec, WITHOUT compacting the tail yet. Splitting the append -// from the compaction lets the effect-bearing Fire path dispatch a step's domain -// effects after the durable append but before the checkpoint compacts that step away -// — so a crash between append and dispatch leaves the effect recoverable from the -// still-present tail rather than orphaned behind the compaction. +// from the compaction lets a step's domain effects dispatch after the durable append +// but before any checkpoint compacts that step away — so a crash between append and +// dispatch leaves the effect recoverable from the still-present tail rather than +// orphaned behind the compaction. func (h *Handle[S, E, C]) appendStep(ctx context.Context, step int, rec *Record) error { if h.checkpointDue(step) { snap, err := state.MarshalSnapshot(h.inst.Snapshot()) @@ -562,20 +734,6 @@ func (h *Handle[S, E, C]) appendStep(ctx context.Context, step int, rec *Record) return nil } -// checkpointStep compacts the journal tail through step when the checkpoint policy is -// due, using the snapshot appendStep already stamped into rec. It is called after a -// step's domain effects have been dispatched, so the compaction never discards a -// not-yet-dispatched effect. -func (h *Handle[S, E, C]) checkpointStep(ctx context.Context, step int, rec *Record) error { - if !h.checkpointDue(step) { - return nil - } - if err := h.runner.store.Checkpoint(ctx, h.id, rec.Snapshot, step, WithCheckpointFence(h.runner.cfg.fence)); err != nil { - return fmt.Errorf("durable: checkpointing step %d for %q: %w", step, h.id, err) - } - return nil -} - // Recover reconstructs a durable instance purely from the Store: it loads the // latest checkpoint Snapshot and the journal/effect tail after it, Restores the // snapshot (firing nothing, no IO), and replays the tail's recorded driving @@ -699,7 +857,7 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S if err != nil { return nil, fmt.Errorf("durable: decoding recorded event at step %d for %q: %w", rec.Step, id, err) } - fireOpts, err := replayFireOptions(rec) + fireOpts, err := replayFireOptions(rec, event, r.cfg.eventDataDecoder) if err != nil { return nil, fmt.Errorf("durable: decoding recorded event data at step %d for %q: %w", rec.Step, id, err) } @@ -781,14 +939,24 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S // its own span. When the tail is empty (a fresh checkpoint compacted every // Record), the checkpoint carries the next ordinal. A Start baseline carries none and // has no Traces, so its count of zero is right. A checkpoint from before this field - // also falls back to the Trace count, which can be off (see - // checkpointEnvelope.NextStep). + // falls back to the Trace count, which can be off (see checkpointEnvelope.NextStep): + // a Store that reports its last step raises it past every recorded step, so no + // Append conflicts. A step number inside a compacted tick barrier's span carries + // nothing, so skipping one is harmless. next := checkpointNext - if next == 0 { - next = len(inst.History()) - } if n := len(tail); n > 0 { next = tail[n-1].followingStep() + } else if next == 0 { + next = len(inst.History()) + if sr, ok := r.store.(StepReporter); ok { + last, found, err := sr.LastStep(ctx, id) + if err != nil { + return nil, fmt.Errorf("durable: reading the last step of %q: %w", id, err) + } + if found && last+1 > next { + next = last + 1 + } + } } return &Handle[S, E, C]{ runner: r, @@ -802,6 +970,8 @@ func (r *Runner[S, E, C]) recover(ctx context.Context, id InstanceID) (*Handle[S svcBuf: &svcBuf, actors: actors, actorBuf: &actorBuf, + lock: make(chan struct{}, 1), + out: outbox{dispatching: map[int]struct{}{}, checkpointed: baselineStep}, }, nil } @@ -816,7 +986,11 @@ type eventPayload struct { // run and a replay can't differ. With no payload it returns opts unchanged, because // adding one would replace the event in AssignCtx.Event. It copies opts rather than // appending, which could write into the caller's slice. -func encodeEventData(opts []state.FireOption, maxBytes int) (eventPayload, []state.FireOption, error) { +// +// The live run gets the payload back through the Runner's EventDataDecoder, as replay +// does. A payload that comes back as a different type than the caller passed refuses +// the Fire, so a reducer can't be handed a map where its caller passed a struct. +func encodeEventData[E comparable](event E, opts []state.FireOption, maxBytes int, decode EventDataDecoder[E]) (eventPayload, []state.FireOption, error) { data, ok := state.FireEventData(opts...) if !ok { return eventPayload{}, opts, nil @@ -849,12 +1023,18 @@ func encodeEventData(opts []state.FireOption, maxBytes int) (eventPayload, []sta } else { payload.data = b } - decoded, err := payload.fireOption() + value, err := decodePayload(payload, event, decode) if err != nil { return payload, nil, fmt.Errorf("%w: %w", ErrEventDataEncoding, err) } + if !isErr { + if got, want := reflect.TypeOf(value), reflect.TypeOf(data); got != want { + return payload, nil, fmt.Errorf("%w: a %v payload decodes as %v; decode it as %v with WithEventDataDecoder", + ErrEventDataEncoding, want, got, want) + } + } live := make([]state.FireOption, 0, len(opts)+1) - return payload, append(append(live, opts...), decoded), nil + return payload, append(append(live, opts...), state.WithEventData(value)), nil } // errorMessage returns e.Error(), or an error if it panics, as a typed nil can. @@ -883,30 +1063,48 @@ func eventDataTooLarge(size, maxBytes int) error { return fmt.Errorf("%w: payload is %d bytes, over the %d byte limit", ErrEventDataEncoding, size, maxBytes) } -// fireOption returns the option carrying the payload as a Fire sees it: live, and when -// a replay re-Fires the step. -func (p eventPayload) fireOption() (state.FireOption, error) { +// decode returns the payload as a Fire sees it, live and when a replay re-Fires the +// step: an error payload as an error with its message, and data through decode. +func decodePayload[E comparable](p eventPayload, event E, decode EventDataDecoder[E]) (any, error) { if p.errMsg != nil { - return state.WithEventData(errors.New(*p.errMsg)), nil + return errors.New(*p.errMsg), nil } - var data any - if err := json.Unmarshal(p.data, &data); err != nil { + return decode(event, p.data) +} + +// EventDataDecoder rebuilds a state.WithEventData payload from its recorded JSON form, +// for the live Fire and for every replay of it. It gets the event the payload rode +// with, so one decoder serves a machine whose events carry different payload types. +// Return the payload as the same type the caller passed: a Fire whose payload decodes +// as another type is refused with ErrEventDataEncoding. The default decodes with +// DecodeAs[any]. Supply one with WithEventDataDecoder. +type EventDataDecoder[E comparable] func(event E, data []byte) (any, error) + +// DecodeAs decodes a recorded payload into a T, the usual body of an EventDataDecoder +// case: DecodeAs[Order] hands Assign an Order. DecodeAs[any] gives JSON's own form: a +// struct arrives as map[string]any and a number as float64. +func DecodeAs[T any](data []byte) (any, error) { + var v T + if err := json.Unmarshal(data, &v); err != nil { return nil, err } - return state.WithEventData(data), nil + return v, nil } +// decodeAny is the default EventDataDecoder. +func decodeAny[E comparable](_ E, data []byte) (any, error) { return DecodeAs[any](data) } + // replayFireOptions returns the options that re-Fire rec's event with the payload -// the live Fire carried, or none when it carried none. -func replayFireOptions(rec *Record) ([]state.FireOption, error) { +// the live Fire carried, decoded the same way, or none when it carried none. +func replayFireOptions[E comparable](rec *Record, event E, decode EventDataDecoder[E]) ([]state.FireOption, error) { if len(rec.EventData) == 0 && rec.EventDataError == nil { return nil, nil } - opt, err := eventPayload{data: rec.EventData, errMsg: rec.EventDataError}.fireOption() + value, err := decodePayload(eventPayload{data: rec.EventData, errMsg: rec.EventDataError}, event, decode) if err != nil { return nil, err } - return []state.FireOption{opt}, nil + return []state.FireOption{state.WithEventData(value)}, nil } // baselineStep is the Step of the start baseline checkpoint, recorded before any diff --git a/durable/runner_options.go b/durable/runner_options.go index a84886e4..be8276b1 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -24,6 +24,9 @@ type runnerConfig[S comparable, E comparable, C any] struct { eventCodec EventCodec[E] // maxEventData caps a WithEventData payload's JSON form, in bytes. maxEventData int + // eventDataDecoder rebuilds a recorded WithEventData payload; it defaults to + // DecodeAs[any]. + eventDataDecoder EventDataDecoder[E] // clock is the real time seam wrapped by the recording clock on the live path // and replaced by the replay clock on recovery. It defaults to // state.SystemClock(); a test supplies a deterministic fake. @@ -166,6 +169,20 @@ func WithMaxEventDataBytes[S comparable, E comparable, C any](n int) Option[S, E } } +// WithEventDataDecoder sets how a recorded state.WithEventData payload is decoded for +// the live Fire and every replay of it. Without one, a payload decodes with +// DecodeAs[any], so only a payload JSON already gives back as itself (a string, bool, +// float64, map[string]any or []any) is accepted; a struct, an int or a typed slice is +// refused with ErrEventDataEncoding until the decoder covers its event. A nil decoder is +// ignored. +func WithEventDataDecoder[S comparable, E comparable, C any](d EventDataDecoder[E]) Option[S, E, C] { + return func(c *runnerConfig[S, E, C]) { + if d != nil { + c.eventDataDecoder = d + } + } +} + // WithOwnerFence sets the fence token a Runner carries on every Append and // Checkpoint, so a Handle that has lost the instance to a newer owner gets ErrFenced // instead of committing. Pass the token the host's lease issued; a retry by the same @@ -198,6 +215,9 @@ func resolveRunner[S comparable, E comparable, C any](opts ...Option[S, E, C]) r if c.eventCodec == nil { c.eventCodec = jsonEventCodec[E]{} } + if c.eventDataDecoder == nil { + c.eventDataDecoder = decodeAny[E] + } if c.clock == nil { c.clock = state.SystemClock() } diff --git a/durable/runner_test.go b/durable/runner_test.go index 5318f305..ce222d8a 100644 --- a/durable/runner_test.go +++ b/durable/runner_test.go @@ -811,12 +811,13 @@ func (s oldEnvelopeStore) Load(ctx context.Context, id durable.InstanceID) ([]by } // TestRecover_CheckpointWithoutNextStep_UsesTheTraceCount proves a checkpoint from before -// the field still recovers, from the restored Trace count. +// the field still recovers from the restored Trace count on a Store that can't report +// its last step. func TestRecover_CheckpointWithoutNextStep_UsesTheTraceCount(t *testing.T) { ctx := context.Background() m := pollingMachine() mem := durable.NewMemStore(durable.WithHistory()) - store := oldEnvelopeStore{mem} + store := struct{ durable.Store }{oldEnvelopeStore{mem}} // hides LastStep id := durable.InstanceID("poll-1") opts := []durable.Option[string, string, *runCtx]{durable.WithCheckpointEvery[string, string, *runCtx](3)} h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) @@ -840,6 +841,47 @@ func TestRecover_CheckpointWithoutNextStep_UsesTheTraceCount(t *testing.T) { } } +// TestRecover_CheckpointWithoutNextStep_AfterTicks_NumbersPastTheJournal proves a +// checkpoint from before the field recovers past every recorded step when tick barriers +// left fewer Traces than step numbers, so the next Append doesn't conflict. +func TestRecover_CheckpointWithoutNextStep_AfterTicks_NumbersPastTheJournal(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + mem := durable.NewMemStore(durable.WithHistory()) + store := oldEnvelopeStore{mem} + id := durable.InstanceID("poll-1") + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{ + durable.WithRunnerClock[string, string, *runCtx](clk), + durable.WithCheckpointEvery[string, string, *runCtx](8), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, id, &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + for range 4 { // five Traces, but the checkpoint compacts through step 7 + clk.Advance(time.Second) + if _, err = h.Tick(ctx); err != nil { + t.Fatalf("Tick: %v", err) + } + } + for i := range 2 { // a second Recover must number past the first one's step too + resumed, err := durable.Recover(ctx, m, store, id, opts...) + if err != nil { + t.Fatalf("Recover %d: %v", i, err) + } + if _, err = resumed.Fire(ctx, "poke"); err != nil { + t.Fatalf("Fire(poke) after Recover %d: %v", i, err) + } + if got, want := lastStep(t, mem, id), 8+i; got != want { + t.Fatalf("poke recorded at step %d, want %d", got, want) + } + } +} + // TestHandle_FailedAppend_RefusesEveryStep proves the failed Handle stays refused, // names the cause, and that a recovered Handle continues from the journal. func TestHandle_FailedAppend_RefusesEveryStep(t *testing.T) { diff --git a/durable/service.go b/durable/service.go index e0927377..2a8fd565 100644 --- a/durable/service.go +++ b/durable/service.go @@ -91,8 +91,10 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // result, true and that error, and the Handle carries on. A recovered Handle has an // unrecorded service in flight again, and RunService runs it a second time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { - h.mu.Lock() - defer h.mu.Unlock() + if err := h.acquire(ctx); err != nil { + return state.FireResult[S]{}, false, err + } + defer h.release() if err := h.refuse(); err != nil { return state.FireResult[S]{}, false, err } @@ -106,7 +108,7 @@ func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.Fire return state.FireResult[S]{}, false, nil } - results := h.svc.Tick(ctx, id) + results := h.svc.Tick(h.stepContext(ctx), id) if len(results) == 0 { return state.FireResult[S]{}, false, nil } diff --git a/durable/store.go b/durable/store.go index 31b12a79..de54b19e 100644 --- a/durable/store.go +++ b/durable/store.go @@ -43,8 +43,8 @@ type Record struct { // EventData is the JSON form of the payload the Fire at this Step carried // through state.WithEventData, or empty when it carried none. The live Fire and - // a replay both supply the decoded form, so the transition's Assign sees the same - // value: a struct as a map and an integer as a float64. + // a replay both decode it through the Runner's EventDataDecoder, so the transition's + // Assign sees the same value. EventData []byte `json:",omitempty"` // EventDataError is the message of an error payload, which has no useful JSON @@ -180,3 +180,14 @@ type Store interface { // seen is rejected with ErrFenced. Checkpoint(ctx context.Context, id InstanceID, snapshot []byte, throughStep int, opts ...CheckpointOption) error } + +// StepReporter is an optional Store capability: the highest Step recorded for an +// instance, across its checkpoint and its tail. Recover asks for it when a checkpoint +// from before checkpoints carried the next step leaves no Record to number from, so +// the next step lands past every recorded one. MemStore and FileStore implement it. A +// Store without it falls back to the restored Trace count, which can number a step +// the Store already holds, so every Append then fails with ErrStepOutOfOrder. +type StepReporter interface { + // LastStep returns the highest Step recorded for id, and false when it holds none. + LastStep(ctx context.Context, id InstanceID) (step int, ok bool, err error) +} diff --git a/durable/timetravel.go b/durable/timetravel.go index 4d5be559..ed19ff79 100644 --- a/durable/timetravel.go +++ b/durable/timetravel.go @@ -264,7 +264,7 @@ func replayThrough[S comparable, E comparable, C any]( if err != nil { return fmt.Errorf("decoding recorded event at step %d: %w", rec.Step, err) } - fireOpts, err := replayFireOptions(rec) + fireOpts, err := replayFireOptions(rec, event, cfg.eventDataDecoder) if err != nil { return fmt.Errorf("decoding recorded event data at step %d: %w", rec.Step, err) } From 63f4d2e7cdd505cab232b3f96247b6c5d710a1cb Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Tue, 10 Nov 2026 11:54:02 -0700 Subject: [PATCH 04/10] fix(durable): release the lock on panic, and settle dispatch before failing Review of the outbox change found gaps: - Fire released the step lock without defer, so a panic in host code under the lock left it held. It now releases however the step ends. - An effect handler that panicked left its step dispatching forever, which silently stopped every later checkpoint. A panic now fails the Handle with ErrEffectDispatch, like a returned error. - A call that fails the Handle returns only once no effect is still being sent, so a host that recovers right away can't send one twice. A step that held the lock while another goroutine failed the Handle leaves its effects for Recover instead of sending them. - refuse takes the context, so every step runs one check after taking the lock: the Handle hasn't failed, and the context hasn't ended. - A nil payload stays nil under a typed decoder, and ErrEffectDispatch wraps the handler's error. - Docs: the wait after a failure, handler failures failing the Handle, dispatch order for a step fired from a handler, StepReporter errors. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 9 +- durable/CHANGELOG.md | 10 +- durable/README.md | 9 +- durable/actor.go | 11 +- durable/effect.go | 11 +- durable/errors.go | 12 +- durable/event_data_test.go | 1 + durable/handle_lock_test.go | 203 ++++++++++++++++++++++ durable/memstore.go | 2 +- durable/runner.go | 166 ++++++++++++++---- durable/runner_options.go | 4 +- durable/runner_test.go | 45 +++++ durable/service.go | 11 +- durable/store.go | 4 +- 14 files changed, 438 insertions(+), 60 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index efba4c94..a2ca22bc 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -135,9 +135,12 @@ and must not: a service that passes its own context gets `ErrReentrantStep`. A s waiting for the lock stops when its context ends. Read the instance with `View`, which takes the lock; `Instance()` is outside it. -A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` -a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a -newer owner holds the instance, so stop. +A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an +effect handler that returns an error or panics. A call that returns `ErrHandleFailed` +first waits until no effect of the Handle is still being sent, or its context ends, so +`Recover` right after it can't send an effect twice. `Recover` a new Handle with the same +options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +instance, so stop. ## A note on serialized payloads diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 0061974e..8144e364 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -50,10 +50,14 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. its context gets the new `ErrReentrantStep`. A step waiting for the lock returns its context's error once the context ends, and changes nothing. The new `Handle.View` reads the instance under the lock; `Instance()` is outside it. -- **A failed effect dispatch fails the Handle.** The effect stays unmarked in the - journal, so no later checkpoint compacts it away, and `Recover` dispatches it again. +- **A failed effect dispatch fails the Handle**, and so does an effect handler that + panics, which now returns `ErrEffectDispatch` instead of panicking out of `Fire`. The + effect stays unmarked in the journal, so no later checkpoint compacts it away, and + `Recover` dispatches it again. - **A step that may not have been recorded fails the Handle.** That call and every later - one return `ErrHandleFailed`, wrapping the cause. `Recover` a new Handle with the same + one return `ErrHandleFailed`, wrapping the cause. A call that returns it first waits + until no effect of the Handle is still being sent, or its context ends, so a `Recover` + right after it can't send an effect twice. `Recover` a new Handle with the same options to continue. A service whose result wasn't recorded is in flight again, so `RunService` runs it a second time. An `InvokeActor` actor restarts from its initial state, and a `Spawn` actor doesn't return. If the cause is `ErrFenced`, a newer owner diff --git a/durable/README.md b/durable/README.md index 861557dc..df8e2ab4 100644 --- a/durable/README.md +++ b/durable/README.md @@ -146,9 +146,12 @@ and must not: a service that passes its own context gets `ErrReentrantStep`. A s waiting for the lock stops when its context ends. Read the instance with `View`, which takes the lock; `Instance()` is outside it. -A step that may not have been recorded fails the Handle with `ErrHandleFailed`. `Recover` -a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a -newer owner holds the instance, so stop. +A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an +effect handler that returns an error or panics. A call that returns `ErrHandleFailed` +first waits until no effect of the Handle is still being sent, or its context ends, so +`Recover` right after it can't send an effect twice. `Recover` a new Handle with the same +options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +instance, so stop. ## Caveat: serialized payloads diff --git a/durable/actor.go b/durable/actor.go index 1105ea76..b167600a 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -108,14 +108,17 @@ type actorMessagePayload struct { // and a Spawn actor doesn't return. Resending a recorded delivery repeats // any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { + ok, err := h.deliverToActor(ctx, ref, event) + return ok, h.afterFailure(ctx, err) +} + +// deliverToActor is DeliverToActor without the wait after a failure. +func (h *Handle[S, E, C]) deliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { if err := h.acquire(ctx); err != nil { return false, err } defer h.release() - if err := h.refuse(); err != nil { - return false, err - } - if err := ctx.Err(); err != nil { + if err := h.refuse(ctx); err != nil { return false, err } if h.actors == nil { diff --git a/durable/effect.go b/durable/effect.go index 73a002c0..71da5ff3 100644 --- a/durable/effect.go +++ b/durable/effect.go @@ -59,9 +59,9 @@ import ( // once) and, because it is then marked, never again. // - Crash AFTER dispatch (id marked): recovery re-emits the effect but finds its // id already marked and skips it — so it is not redispatched (exactly once). -// - Handler error: the effect is left un-marked and the error surfaces wrapped in -// ErrEffectDispatch, so a later recovery retries it (at-least-once until it -// succeeds; exactly-once thereafter). +// - Handler error: the effect is left un-marked, and the Handle fails with +// ErrHandleFailed wrapping ErrEffectDispatch, so Recover retries it +// (at-least-once until it succeeds; exactly-once thereafter). // // The dedup set is held by the Store (DispatchStore) so it shares the Record's // durability and survives checkpoint compaction — a delayed redispatch of an @@ -70,7 +70,8 @@ import ( // EffectHandler applies one emitted domain effect, identified by its stamped // EffectID. The Runner calls it exactly once per EffectID over the lifetime of an // instance (see WithEffectHandler). A non-nil error leaves the effect un-marked -// for a later retry and is surfaced to the caller wrapped in ErrEffectDispatch. +// for Recover to retry, and fails the Handle with ErrHandleFailed wrapping +// ErrEffectDispatch and the error. type EffectHandler func(ctx context.Context, effectID string, effect state.Effect) error // DispatchStore is the dedup seam for idempotent effect dispatch: the Store @@ -226,7 +227,7 @@ func (r *Runner[S, E, C]) dispatch(ctx context.Context, id InstanceID, des []dis continue } if err := r.cfg.effectHandler(ctx, de.id, de.effect); err != nil { - return fmt.Errorf("%w: effect %q for %q: %v", ErrEffectDispatch, de.id, id, err) + return fmt.Errorf("%w: effect %q for %q: %w", ErrEffectDispatch, de.id, id, err) } if err := ds.MarkDispatched(ctx, id, de.id); err != nil { return fmt.Errorf("durable: marking effect %q dispatched for %q: %w", de.id, id, err) diff --git a/durable/errors.go b/durable/errors.go index c58fabdf..f437104b 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -43,7 +43,8 @@ var ( ErrFenced = errors.New(errorPrefix + errorFenced) // ErrEventDataEncoding wraps a Handle.Fire refused before the kernel ran, because - // its state.WithEventData payload has no JSON form that decodes again, panics while + // its state.WithEventData payload has no JSON form that decodes again, decodes as a + // different type than the caller passed (see WithEventDataDecoder), panics while // encoding (such as an error that's a typed nil), or exceeds WithMaxEventDataBytes. // Nothing applied, so the Handle stays usable. ErrEventDataEncoding = errors.New(errorPrefix + errorEventDataEncoding) @@ -56,9 +57,10 @@ var ( // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService, // DeliverToActor and View once a step applied to the live instance but may not have been - // recorded, on that call and every later one. It wraps the cause. Recover a new - // Handle with the same options to continue, unless the cause is ErrFenced: then a - // newer owner holds the instance, so stop. + // recorded, or an effect handler failed, on that call and every later one. It wraps the cause. The call returns + // once no effect of the Handle is still being sent, so recovering right away can't + // send one twice. Recover a new Handle with the same options to continue, unless the + // cause is ErrFenced: then a newer owner holds the instance, so stop. ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) // ErrReentrantStep is reported by a Handle's Fire, Tick, RunService, DeliverToActor @@ -76,7 +78,7 @@ var ( ErrInstanceExists = errors.New(errorPrefix + errorInstanceExists) // ErrEffectDispatch wraps a failure returned by the caller-supplied effect - // handler (WithEffectHandler) while applying an emitted effect. The effect was + // handler (WithEffectHandler) while applying an emitted effect, or a panic in it. The effect was // persisted (write-ahead) but not marked dispatched, so the Handle fails with // ErrHandleFailed wrapping it, and Recover dispatches it again. Callers match // this with errors.Is to tell a dispatch failure from a kernel transition or diff --git a/durable/event_data_test.go b/durable/event_data_test.go index 24ac17a0..f1db62f1 100644 --- a/durable/event_data_test.go +++ b/durable/event_data_test.go @@ -162,6 +162,7 @@ func TestFire_EventData_LiveEqualsReplay(t *testing.T) { {"a large int keeps its precision", int64(1<<53 + 1), decodeAll[int64], "int64:9007199254740993"}, {"a struct", order{Qty: 2}, decodeAll[order], "durable_test.order:{2}"}, {"a map by default", map[string]any{"qty": 2.0}, nil, "map[string]interface {}:map[qty:2]"}, + {"a nil payload stays nil under a typed decoder", nil, decodeAll[int], ":"}, {"an error", errors.New("card declined"), nil, "*errors.errorString:card declined"}, } for _, tt := range tests { diff --git a/durable/handle_lock_test.go b/durable/handle_lock_test.go index c8ad4ae1..43cc3495 100644 --- a/durable/handle_lock_test.go +++ b/durable/handle_lock_test.go @@ -275,3 +275,206 @@ func TestHandle_View_DoesNotRaceSteps(t *testing.T) { t.Fatalf("View after the Fires: count %d, err %v; want 100", last, err) } } + +// panicAppendStore panics on the next Append once armed. +type panicAppendStore struct { + durable.Store + armed atomic.Bool +} + +func (s *panicAppendStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if s.armed.CompareAndSwap(true, false) { + panic("store exploded") + } + return s.Store.Append(ctx, id, rec, opts...) +} + +// TestHandle_PanicUnderTheLock_ReleasesIt proves a panic while Fire holds the step +// lock doesn't leave the lock held: the next step still gets it. +func TestHandle_PanicUnderTheLock_ReleasesIt(t *testing.T) { + ctx := context.Background() + store := &panicAppendStore{Store: durable.NewMemStore()} + h, err := durable.NewRunner(pollingMachine(), store).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + store.armed.Store(true) + func() { + defer func() { _ = recover() }() + _, _ = h.Fire(ctx, "start") + }() + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if err := h.View(waitCtx, func(*state.Instance[string, string, *runCtx]) {}); errors.Is(err, context.DeadlineExceeded) { + t.Fatal("View after the panic waited out its deadline: the step lock was left held") + } +} + +// TestHandle_PanickingEffectHandler_FailsTheHandle proves a handler that panics fails +// the Handle like one that returns an error, instead of leaving its step dispatching +// forever, which would silently stop every later checkpoint. +func TestHandle_PanickingEffectHandler_FailsTheHandle(t *testing.T) { + ctx := context.Background() + m := effectMachine() + store := durable.NewMemStore() + var panicked atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if panicked.CompareAndSwap(false, true) { + panic("handler exploded") + } + return nil + } + opts := []durable.Option[string, string, *runCtx]{ + durable.WithEffectHandler[string, string, *runCtx](handler), + durable.WithCheckpointEvery[string, string, *runCtx](2), + } + h, err := durable.NewRunner(m, store, opts...).Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + _, err = h.Fire(ctx, "go") + if !errors.Is(err, durable.ErrHandleFailed) || !errors.Is(err, durable.ErrEffectDispatch) { + t.Fatalf("Fire(go) = %v, want ErrHandleFailed wrapping ErrEffectDispatch", err) + } + recovered, err := durable.Recover(ctx, m, store, "e-1", opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if _, err = recovered.Fire(ctx, "bill"); err != nil { // step 1: a checkpoint falls due + t.Fatalf("Fire(bill) on the recovered Handle: %v", err) + } + if _, tail, err := store.Load(ctx, "e-1"); err != nil || len(tail) != 0 { + t.Fatalf("tail of %d, err %v; want the checkpoint written", len(tail), err) + } +} + +// failingMemStore is a MemStore, dispatch tracking included, that fails the next +// Append once armed. +type failingMemStore struct { + *durable.MemStore + armed atomic.Bool +} + +func (s *failingMemStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if s.armed.CompareAndSwap(true, false) { + return 0, errors.New("store unavailable") + } + return s.MemStore.Append(ctx, id, rec, opts...) +} + +// TestHandle_FailedStep_WaitsForOtherDispatches proves a call that fails the Handle +// returns only once another goroutine's effect has finished sending, so a host that +// recovers right away can't send that effect a second time. +func TestHandle_FailedStep_WaitsForOtherDispatches(t *testing.T) { + ctx := context.Background() + store := &failingMemStore{MemStore: durable.NewMemStore()} + entered, release := make(chan struct{}), make(chan struct{}) + var waited atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if waited.CompareAndSwap(false, true) { + close(entered) + <-release + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + first := make(chan struct{}) + go func() { + defer close(first) + _, _ = h.Fire(ctx, "go") // its notify is still sending + }() + <-entered + store.armed.Store(true) + failed := make(chan error, 1) + go func() { + _, err := h.Fire(ctx, "bill") // its Append fails + failed <- err + }() + select { + case err := <-failed: + t.Fatalf("Fire(bill) returned %v while step 0's effect was still sending", err) + case <-time.After(100 * time.Millisecond): + } + close(release) + select { + case err := <-failed: + if !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(bill) = %v, want ErrHandleFailed", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Fire(bill) never returned after the dispatch finished") + } + waitFor(t, first, "Fire(go)") +} + +// blockingMemStore is a MemStore whose next Append, once armed, signals entered and +// waits for release. +type blockingMemStore struct { + *durable.MemStore + armed atomic.Bool + entered chan struct{} + release chan struct{} +} + +func (s *blockingMemStore) Append(ctx context.Context, id durable.InstanceID, rec durable.Record, opts ...durable.AppendOption) (int64, error) { + if s.armed.CompareAndSwap(true, false) { + close(s.entered) + <-s.release + } + return s.MemStore.Append(ctx, id, rec, opts...) +} + +// TestHandle_StepAfterAFailure_LeavesItsEffectsForRecover proves a step that already +// held the lock when another goroutine's dispatch failed the Handle doesn't send its +// own effects: the failing caller may already be recovering, and Recover sends them. +func TestHandle_StepAfterAFailure_LeavesItsEffectsForRecover(t *testing.T) { + ctx := context.Background() + m := effectMachine() + store := &blockingMemStore{MemStore: durable.NewMemStore(), entered: make(chan struct{}), release: make(chan struct{})} + handlerIn, failNow := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits, then fails + close(handlerIn) + <-failNow + return errors.New("mail server down") + } + calls.Add(1) + return nil + } + opt := durable.WithEffectHandler[string, string, *runCtx](handler) + h, err := durable.NewRunner(m, store, opt).Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstDone := make(chan error, 1) + go func() { _, err := h.Fire(ctx, "go"); firstDone <- err }() + <-handlerIn + store.armed.Store(true) + secondDone := make(chan error, 1) + go func() { _, err := h.Fire(ctx, "bill"); secondDone <- err }() + <-store.entered // step 1 holds the lock, in its Append + close(failNow) + if err := <-firstDone; !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(go) = %v, want ErrHandleFailed", err) + } + close(store.release) + if err := <-secondDone; !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(bill) = %v, want ErrHandleFailed", err) + } + if n := calls.Load(); n != 0 { + t.Fatalf("step 1 sent %d effects after the Handle failed, want 0", n) + } + if _, err = durable.Recover(ctx, m, store.MemStore, "e-1", opt); err != nil { + t.Fatalf("Recover: %v", err) + } + if n := calls.Load(); n != 3 { // step 0's notify, then step 1's charge and notify + t.Fatalf("Recover sent %d effects, want 3", n) + } +} diff --git a/durable/memstore.go b/durable/memstore.go index 9a41e9fd..fd2105ac 100644 --- a/durable/memstore.go +++ b/durable/memstore.go @@ -57,7 +57,7 @@ type memInstance struct { // out of the live tail. It is nil when history is disabled. history []Record // maxStep is the highest Step ever appended (across checkpoint and tail), for - // the strict-ordering check. -1 means nothing appended yet. + // the strict-ordering check. noRecord means nothing appended yet. maxStep int // seq is the monotonic per-instance append sequence; it advances on every // non-idempotent Append. diff --git a/durable/runner.go b/durable/runner.go index 2d2d7235..3a74ab5e 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -83,9 +83,12 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // goroutines: each holds the Handle's step lock while it applies to the kernel and // writes its Record, including any service or actor it runs. A step's domain effects // dispatch after it releases the lock, so an effect handler may call back into the -// Handle. Effects of steps on different goroutines can dispatch in either order; a -// goroutine's own steps dispatch in order. A checkpoint waits until every step at or -// before it has dispatched, so it never compacts an effect still being sent. +// Handle. Effects of steps on different goroutines can dispatch in either order. Steps +// fired one after another dispatch in order, except that a step fired from inside an +// effect handler dispatches before the rest of that handler's step. A checkpoint waits +// until every step at or before it has dispatched, so it never compacts an effect still +// being sent. An effect handler that panics fails the Handle like one that returns an +// error. // // Code that runs under the lock (a service, guard, action, reducer or actor behavior) // must not call back into the same Handle. A call that carries the context the step @@ -97,8 +100,10 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, -// wrapping the cause, which can be a context that ended during the Store write. Recover -// a new Handle with the same options to continue. A service +// wrapping the cause, which can be a context that ended during the Store write. A call +// that returns it waits first until no effect of this Handle is still being sent (or +// its context ends), unless it was made from inside an effect handler. Recover a new +// Handle with the same options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a @@ -132,13 +137,16 @@ type Handle[S comparable, E comparable, C any] struct { // policy can fall due while an earlier step's effect is still unmarked; compacting // through that step would leave nothing for recovery to redispatch. So a due // checkpoint waits until every step at or before it has dispatched, and the step that -// finishes last writes it. The mutex is taken inside the step lock, never around it. +// finishes last writes it. The mutex is taken inside the step lock, never around it. It +// is held across a pending checkpoint's Store write, so a slow Checkpoint delays the +// next step's registration, as it did when checkpoints ran under the step lock. type outbox struct { mu sync.Mutex failed error // why the Handle is out of step with the journal, nil until then dispatching map[int]struct{} // steps whose effects are recorded but not yet dispatched pending *dueCheckpoint // the newest due checkpoint, waiting on dispatching checkpointed int // the newest step this Handle checkpointed through + idle chan struct{} // closed when dispatching empties; nil while it's empty } // dueCheckpoint is a checkpoint the policy made due at step, with the snapshot the @@ -195,11 +203,60 @@ func (h *Handle[S, E, C]) acquire(ctx context.Context) error { // release gives up the step lock. func (h *Handle[S, E, C]) release() { <-h.lock } -// refuse reports ErrHandleFailed once the Handle is out of step with the journal. -func (h *Handle[S, E, C]) refuse() error { +// dispatchKey marks the context an effect handler gets, so a call it makes back into +// the Handle is known to come from inside a dispatch. +type dispatchKey struct{} + +// dispatchContext returns ctx marked as dispatching h's effects. +func (h *Handle[S, E, C]) dispatchContext(ctx context.Context) context.Context { + outer, _ := ctx.Value(dispatchKey{}).(*stepMark) + return context.WithValue(ctx, dispatchKey{}, &stepMark{handle: h, outer: outer}) +} + +// inDispatch reports whether ctx is an effect handler's, dispatching h's effects. +func (h *Handle[S, E, C]) inDispatch(ctx context.Context) bool { + m, _ := ctx.Value(dispatchKey{}).(*stepMark) + for ; m != nil; m = m.outer { + if m.handle == any(h) { + return true + } + } + return false +} + +// afterFailure holds a call that returns ErrHandleFailed until no effect of h is still +// being dispatched, or ctx ends, so a host that recovers a new Handle as soon as the +// call returns can't send an effect a second time while the old one is still sending +// it. A call from inside an effect handler returns at once: its own dispatch is one of +// those still running. +func (h *Handle[S, E, C]) afterFailure(ctx context.Context, err error) error { + if !errors.Is(err, ErrHandleFailed) || h.inDispatch(ctx) { + return err + } h.out.mu.Lock() - defer h.out.mu.Unlock() - return h.refuseLocked() + idle := h.out.idle + h.out.mu.Unlock() + if idle != nil { + select { + case <-idle: + case <-ctx.Done(): + } + } + return err +} + +// refuse reports why a step that holds the lock must not run: ErrHandleFailed once +// the Handle is out of step with the journal, or ctx's error if ctx ended as the lock +// was taken. Either way the step changes nothing. A Tick that ran on an ended context +// would consume a due timer whose Fire the kernel then refuses. +func (h *Handle[S, E, C]) refuse(ctx context.Context) error { + h.out.mu.Lock() + err := h.refuseLocked() + h.out.mu.Unlock() + if err != nil { + return err + } + return ctx.Err() } // refuseLocked is refuse with h.out.mu held. @@ -370,13 +427,19 @@ func (h *Handle[S, E, C]) Instance() *state.Instance[S, E, C] { return h.inst } // read can't race a step on another goroutine. fn must not call back into the Handle, // and must not keep the instance past its return. View returns ctx's error if ctx ends // before the lock is free, ErrReentrantStep if ctx is a step's own context, and -// ErrHandleFailed once the Handle has failed, without running fn. +// ErrHandleFailed once the Handle has failed, without running fn. Like every call that +// returns ErrHandleFailed, it first waits until no effect is still being sent. func (h *Handle[S, E, C]) View(ctx context.Context, fn func(*state.Instance[S, E, C])) error { + return h.afterFailure(ctx, h.view(ctx, fn)) +} + +// view is View without the wait after a failure. +func (h *Handle[S, E, C]) view(ctx context.Context, fn func(*state.Instance[S, E, C])) error { if err := h.acquire(ctx); err != nil { return err } defer h.release() - if err := h.refuse(); err != nil { + if err := h.refuse(ctx); err != nil { return err } fn(h.inst) @@ -532,22 +595,28 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO if err := h.acquire(ctx); err != nil { return state.FireResult[S]{}, err } - res, rec, des, err := h.fireStep(h.stepContext(ctx), event, opts) - h.release() + res, rec, des, err := h.lockedFireStep(ctx, event, opts) if err != nil { - return res, err + return res, h.afterFailure(ctx, err) } // Dispatch after releasing the step lock, so an effect handler can call back into // this Handle. The Record already names every effect (write-ahead), so a crash here // leaves it for recovery to redispatch, deduped by id. - return res, h.finishStep(ctx, rec, des) + return res, h.afterFailure(ctx, h.finishStep(ctx, rec, des)) +} + +// lockedFireStep runs fireStep and releases the step lock however it returns, a panic +// in host code included. +func (h *Handle[S, E, C]) lockedFireStep(ctx context.Context, event E, opts []state.FireOption) (state.FireResult[S], *Record, []dispatchableEffect, error) { + defer h.release() + return h.fireStep(h.stepContext(ctx), event, opts) } // fireStep is the part of Fire that holds the step lock: it applies the event to the // kernel and appends the step's Record, naming its domain effects, before any is // dispatched. ctx is the step's marked context. func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.FireOption) (state.FireResult[S], *Record, []dispatchableEffect, error) { - if err := h.refuse(); err != nil { + if err := h.refuse(ctx); err != nil { return state.FireResult[S]{}, nil, nil, err } // Encode the WithEventData payload before the kernel runs, so a payload that @@ -584,29 +653,57 @@ func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.Fi h.nextStep = rec.followingStep() if len(des) > 0 { h.out.mu.Lock() + defer h.out.mu.Unlock() + // Another goroutine's dispatch may have failed the Handle while this step held + // the lock, and its caller may already be recovering. Leave this step's effects + // unmarked in the journal for that Recover to send, rather than race it. + if err := h.refuseLocked(); err != nil { + return res, nil, nil, err + } + if len(h.out.dispatching) == 0 { + h.out.idle = make(chan struct{}) + } h.out.dispatching[step] = struct{}{} - h.out.mu.Unlock() } return res, rec, des, nil } // finishStep dispatches a recorded step's domain effects, outside the step lock, then // offers its checkpoint. A failed dispatch fails the Handle: the effect stays unmarked -// in the journal, and Recover dispatches it again. +// in the journal, and Recover dispatches it again. A handler that panics fails it the +// same way, rather than leave the step dispatching forever, which would stop every +// later checkpoint. func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dispatchableEffect) error { var dispatchErr error if len(des) > 0 { - dispatchErr = h.dispatchEffects(ctx, des) + dispatchErr = h.dispatchGuarded(h.dispatchContext(ctx), rec.Step, des) } h.out.mu.Lock() defer h.out.mu.Unlock() - delete(h.out.dispatching, rec.Step) + if len(des) > 0 { + delete(h.out.dispatching, rec.Step) + if len(h.out.dispatching) == 0 && h.out.idle != nil { + close(h.out.idle) + h.out.idle = nil + } + } if dispatchErr != nil { return h.failLocked(dispatchErr) } return h.offerCheckpointLocked(ctx, rec) } +// dispatchGuarded dispatches des, turning a panic in the effect handler into an +// ErrEffectDispatch error. +func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []dispatchableEffect) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("%w: step %d for %q: effect handler panicked: %v", ErrEffectDispatch, step, h.id, p) + } + }() + return h.dispatchEffects(ctx, des) +} + // Tick advances the Handle's delayed-transition scheduler: it fires every timer // whose recorded deadline is at or before the recording clock's current time, in // due order, re-firing the delayed events through the durable instance and @@ -616,16 +713,17 @@ func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dis // count of timers fired — so recovery re-derives the same timers at their // recorded instants. It returns the FireResults of the timers it fired, in order. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { + results, err := h.tick(ctx) + return results, h.afterFailure(ctx, err) +} + +// tick is Tick without the wait after a failure. +func (h *Handle[S, E, C]) tick(ctx context.Context) ([]state.FireResult[S], error) { if err := h.acquire(ctx); err != nil { return nil, err } defer h.release() - if err := h.refuse(); err != nil { - return nil, err - } - // A context that ended as the lock was taken must stop here: the scheduler would - // consume a due timer whose Fire the kernel then refuses. - if err := ctx.Err(); err != nil { + if err := h.refuse(ctx); err != nil { return nil, err } results := h.sched.Tick(h.stepContext(ctx)) @@ -1065,10 +1163,18 @@ func eventDataTooLarge(size, maxBytes int) error { // decode returns the payload as a Fire sees it, live and when a replay re-Fires the // step: an error payload as an error with its message, and data through decode. -func decodePayload[E comparable](p eventPayload, event E, decode EventDataDecoder[E]) (any, error) { +func decodePayload[E comparable](p eventPayload, event E, decode EventDataDecoder[E]) (v any, err error) { if p.errMsg != nil { return errors.New(*p.errMsg), nil } + if string(p.data) == "null" { // a nil payload stays nil, whatever the decoder + return nil, nil + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("event data decoder panicked: %v", r) + } + }() return decode(event, p.data) } @@ -1076,7 +1182,9 @@ func decodePayload[E comparable](p eventPayload, event E, decode EventDataDecode // for the live Fire and for every replay of it. It gets the event the payload rode // with, so one decoder serves a machine whose events carry different payload types. // Return the payload as the same type the caller passed: a Fire whose payload decodes -// as another type is refused with ErrEventDataEncoding. The default decodes with +// as another type is refused with ErrEventDataEncoding. Only the top-level type is +// checked, so a []any of structs still arrives as a []any of maps. A nil payload +// arrives as nil without reaching the decoder. The default decodes with // DecodeAs[any]. Supply one with WithEventDataDecoder. type EventDataDecoder[E comparable] func(event E, data []byte) (any, error) diff --git a/durable/runner_options.go b/durable/runner_options.go index be8276b1..7c874b68 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -133,8 +133,8 @@ func WithActorPalette[S comparable, E comparable, C any](palette map[string]stat // number of recoveries), deduped by a deterministic EffectID: the Runner stamps // each emitted effect, write-ahead persists the step Record carrying those ids, // then invokes the handler for every id not already marked dispatched, marking -// each as it succeeds. A handler error is surfaced (wrapped in ErrEffectDispatch) -// and the failing effect is left un-marked, so a later recovery retries it +// each as it succeeds. A handler error or panic fails the Handle with ErrHandleFailed, +// wrapping ErrEffectDispatch, and leaves the effect un-marked, so Recover retries it // (at-least-once until it succeeds; exactly-once once it does). // // The handler receives the stamped EffectID and the live effect value. It is nil diff --git a/durable/runner_test.go b/durable/runner_test.go index ce222d8a..821b7c42 100644 --- a/durable/runner_test.go +++ b/durable/runner_test.go @@ -985,6 +985,51 @@ func TestHandle_FencedOwner_Stops(t *testing.T) { } } +// endsOnSecondCheck is a context that reports itself live the first time its Err is +// read and canceled after, standing in for one that ends just as the step takes the +// lock. +type endsOnSecondCheck struct { + context.Context + checks atomic.Int32 +} + +func (c *endsOnSecondCheck) Err() error { + if c.checks.Add(1) == 1 { + return nil + } + return context.Canceled +} + +// TestHandle_ContextEndingAsTheLockIsTaken_ChangesNothing proves the check after the +// lock is taken stops a Tick whose context ended while it took the lock, before the +// scheduler consumes a due timer. +func TestHandle_ContextEndingAsTheLockIsTaken_ChangesNothing(t *testing.T) { + ctx := context.Background() + m := pollingMachine() + store := &armedAppendStore{Store: durable.NewMemStore()} + clk := state.NewFakeClock(epoch) + opts := []durable.Option[string, string, *runCtx]{durable.WithRunnerClock[string, string, *runCtx](clk)} + h, err := durable.NewRunner(m, store, opts...).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + if _, err = h.Fire(ctx, "start"); err != nil { + t.Fatalf("Fire(start): %v", err) + } + clk.Advance(time.Second) + before := store.appends + if _, err = h.Tick(&endsOnSecondCheck{Context: ctx}); !errors.Is(err, context.Canceled) { + t.Fatalf("Tick = %v, want context.Canceled", err) + } + if store.appends != before { + t.Fatalf("Tick appended %d Records, want 0", store.appends-before) + } + results, err := h.Tick(ctx) + if err != nil || len(results) != 1 { + t.Fatalf("Tick after = (%d results, %v), want the timer still due", len(results), err) + } +} + // TestHandle_ContextEndedBeforeTheStep_ChangesNothing proves an ended context stops a // Tick before the scheduler consumes a due timer. func TestHandle_ContextEndedBeforeTheStep_ChangesNothing(t *testing.T) { diff --git a/durable/service.go b/durable/service.go index 2a8fd565..70976b5d 100644 --- a/durable/service.go +++ b/durable/service.go @@ -91,14 +91,17 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // result, true and that error, and the Handle carries on. A recovered Handle has an // unrecorded service in flight again, and RunService runs it a second time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { + res, ok, err := h.runService(ctx, id) + return res, ok, h.afterFailure(ctx, err) +} + +// runService is RunService without the wait after a failure. +func (h *Handle[S, E, C]) runService(ctx context.Context, id string) (state.FireResult[S], bool, error) { if err := h.acquire(ctx); err != nil { return state.FireResult[S]{}, false, err } defer h.release() - if err := h.refuse(); err != nil { - return state.FireResult[S]{}, false, err - } - if err := ctx.Err(); err != nil { + if err := h.refuse(ctx); err != nil { return state.FireResult[S]{}, false, err } if h.svc == nil { diff --git a/durable/store.go b/durable/store.go index de54b19e..afaf4fd7 100644 --- a/durable/store.go +++ b/durable/store.go @@ -188,6 +188,8 @@ type Store interface { // Store without it falls back to the restored Trace count, which can number a step // the Store already holds, so every Append then fails with ErrStepOutOfOrder. type StepReporter interface { - // LastStep returns the highest Step recorded for id, and false when it holds none. + // LastStep returns the highest Step recorded for id, across its checkpoint and its + // tail, and false when it holds none. Recover calls it only for an instance Load just + // found, and fails on any error it returns. LastStep(ctx context.Context, id InstanceID) (step int, ok bool, err error) } From 82888b01507b934c628989d00796fa798078e34b Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Tue, 10 Nov 2026 13:12:22 -0700 Subject: [PATCH 05/10] fix(durable): add Handle.Drain instead of waiting after a failure A call that failed the Handle waited until no effect was still being sent. An effect handler that called back with a context other than its own, and failed the Handle, waited on its own dispatch forever, and so did every later caller. Calls that fail the Handle now return at once. Handle.Drain waits until no effect is still being sent; call it before Recover. Called from inside a step or an effect handler, it returns ErrReentrantStep instead of waiting. Docs: an action reaches the Handle by returning an effect; a guard, action, reducer or actor behavior that calls back without the step's context deadlocks. Fire's doc covers a failed checkpoint. The payload cap measures an error's JSON form, and a Store panic during dispatch is reported as a dispatch panic, not a handler's. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 11 +- durable/CHANGELOG.md | 20 ++-- durable/README.md | 11 +- durable/actor.go | 6 - durable/errors.go | 14 +-- durable/handle_lock_test.go | 130 ++++++++++++++++++---- durable/runner.go | 75 ++++++------- durable/runner_options.go | 2 +- durable/service.go | 6 - 9 files changed, 178 insertions(+), 97 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index a2ca22bc..6b48fdd5 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -131,15 +131,16 @@ in by implementing `HistoryStore` (the in-tree `MemStore` does so under goroutines: each holds the Handle's lock while it applies to the kernel and records the step. Effects dispatch after the lock is released, so an effect handler can call back into its Handle. A service, guard, action, reducer or actor behavior runs under the lock -and must not: a service that passes its own context gets `ErrReentrantStep`. A step +and must not call back in: to act on the Handle when a step happens, an action returns an +effect, whose handler runs after the lock. A call back that carries the step's context, +as a service gets, returns `ErrReentrantStep`; one without it deadlocks. A step waiting for the lock stops when its context ends. Read the instance with `View`, which takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an -effect handler that returns an error or panics. A call that returns `ErrHandleFailed` -first waits until no effect of the Handle is still being sent, or its context ends, so -`Recover` right after it can't send an effect twice. `Recover` a new Handle with the same -options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +effect handler that returns an error or panics. Before `Recover`, call `Handle.Drain`, +which waits until no effect of the Handle is still being sent, so the recovered Handle +can't send one twice. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. ## A note on serialized payloads diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 8144e364..9e4bf856 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -27,8 +27,12 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **`ErrHandleFailed`** reports a Handle out of step with its journal. -- **`Handle.View`** reads the instance under the Handle's step lock. **`ErrReentrantStep`** - reports a call back into a Handle from inside one of its steps. +- **`Handle.View`** reads the instance under the Handle's step lock. **`Handle.Drain`** + waits until no effect of the Handle is still being sent; call it before `Recover` after + `ErrHandleFailed`. **`ErrReentrantStep`** reports a call back into a Handle that carries + the context one of its steps passed to a callback, such as a service's. A guard, + action, reducer or actor behavior gets no context here, so its call back deadlocks + instead; #245 passes them the step's context. - **`StepReporter`**, implemented by `MemStore` and `FileStore`, lets `Recover` number the next step past every recorded one after a checkpoint from before checkpoints carried @@ -46,8 +50,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. host's timer loop no longer loses acknowledged steps. Effects dispatch after the lock is released, so an effect handler can still call back into its Handle; a checkpoint waits for every earlier step's effects to dispatch. A service, guard, action, reducer - or actor behavior runs under the lock and must not call back in: a service that passes - its context gets the new `ErrReentrantStep`. A step waiting for the lock returns its + or actor behavior runs under the lock and must not call back in; an action reaches the + Handle by returning an effect. A service that calls back with its context gets the + new `ErrReentrantStep`. A step waiting for the lock returns its context's error once the context ends, and changes nothing. The new `Handle.View` reads the instance under the lock; `Instance()` is outside it. - **A failed effect dispatch fails the Handle**, and so does an effect handler that @@ -55,10 +60,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. effect stays unmarked in the journal, so no later checkpoint compacts it away, and `Recover` dispatches it again. - **A step that may not have been recorded fails the Handle.** That call and every later - one return `ErrHandleFailed`, wrapping the cause. A call that returns it first waits - until no effect of the Handle is still being sent, or its context ends, so a `Recover` - right after it can't send an effect twice. `Recover` a new Handle with the same - options to continue. A service whose result wasn't recorded is in flight again, so + one return `ErrHandleFailed`, wrapping the cause. Call `Drain` before `Recover`, so an + effect another goroutine is still sending isn't sent twice. `Recover` a new Handle + with the same options to continue. A service whose result wasn't recorded is in flight again, so `RunService` runs it a second time. An `InvokeActor` actor restarts from its initial state, and a `Spawn` actor doesn't return. If the cause is `ErrFenced`, a newer owner holds the instance: stop. diff --git a/durable/README.md b/durable/README.md index df8e2ab4..3bca58c7 100644 --- a/durable/README.md +++ b/durable/README.md @@ -142,15 +142,16 @@ otherwise. goroutines: each holds the Handle's lock while it applies to the kernel and records the step. Effects dispatch after the lock is released, so an effect handler can call back into its Handle. A service, guard, action, reducer or actor behavior runs under the lock -and must not: a service that passes its own context gets `ErrReentrantStep`. A step +and must not call back in: to act on the Handle when a step happens, an action returns an +effect, whose handler runs after the lock. A call back that carries the step's context, +as a service gets, returns `ErrReentrantStep`; one without it deadlocks. A step waiting for the lock stops when its context ends. Read the instance with `View`, which takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an -effect handler that returns an error or panics. A call that returns `ErrHandleFailed` -first waits until no effect of the Handle is still being sent, or its context ends, so -`Recover` right after it can't send an effect twice. `Recover` a new Handle with the same -options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +effect handler that returns an error or panics. Before `Recover`, call `Handle.Drain`, +which waits until no effect of the Handle is still being sent, so the recovered Handle +can't send one twice. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. ## Caveat: serialized payloads diff --git a/durable/actor.go b/durable/actor.go index b167600a..0af4b7d5 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -108,12 +108,6 @@ type actorMessagePayload struct { // and a Spawn actor doesn't return. Resending a recorded delivery repeats // any message it sends the parent. func (h *Handle[S, E, C]) DeliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { - ok, err := h.deliverToActor(ctx, ref, event) - return ok, h.afterFailure(ctx, err) -} - -// deliverToActor is DeliverToActor without the wait after a failure. -func (h *Handle[S, E, C]) deliverToActor(ctx context.Context, ref state.ActorRef, event any) (bool, error) { if err := h.acquire(ctx); err != nil { return false, err } diff --git a/durable/errors.go b/durable/errors.go index f437104b..06cbd49d 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -55,18 +55,18 @@ var ( // applied the step, so the Handle fails with ErrHandleFailed. ErrRecordTooLarge = errors.New(errorPrefix + errorRecordTooLarge) - // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService, - // DeliverToActor and View once a step applied to the live instance but may not have been - // recorded, or an effect handler failed, on that call and every later one. It wraps the cause. The call returns - // once no effect of the Handle is still being sent, so recovering right away can't - // send one twice. Recover a new Handle with the same options to continue, unless the - // cause is ErrFenced: then a newer owner holds the instance, so stop. + // ErrHandleFailed is reported by a Handle's Fire, Tick, RunService, DeliverToActor + // and View once a step applied to the live instance but may not have been recorded, + // or an effect handler failed, on that call and every later one. It wraps the cause. + // Call Handle.Drain, then Recover a new Handle with the same options to continue, + // unless the cause is ErrFenced: then a newer owner holds the instance, so stop. ErrHandleFailed = errors.New(errorPrefix + errorHandleFailed) // ErrReentrantStep is reported by a Handle's Fire, Tick, RunService, DeliverToActor // and View when called with the context a step passed to host code, such as a // service: the step holds the Handle's lock, so waiting would deadlock. Nothing - // applied. Effect handlers run after the lock is released and may call back in. + // applied. Effect handlers run after the lock is released and may call back in. Drain + // reports it when called from inside a step or an effect handler. ErrReentrantStep = errors.New(errorPrefix + errorReentrantStep) // ErrCheckpointNotAdvancing is reported by Store.Checkpoint when throughStep diff --git a/durable/handle_lock_test.go b/durable/handle_lock_test.go index 43cc3495..e82dd1b3 100644 --- a/durable/handle_lock_test.go +++ b/durable/handle_lock_test.go @@ -362,10 +362,10 @@ func (s *failingMemStore) Append(ctx context.Context, id durable.InstanceID, rec return s.MemStore.Append(ctx, id, rec, opts...) } -// TestHandle_FailedStep_WaitsForOtherDispatches proves a call that fails the Handle -// returns only once another goroutine's effect has finished sending, so a host that -// recovers right away can't send that effect a second time. -func TestHandle_FailedStep_WaitsForOtherDispatches(t *testing.T) { +// TestHandle_Drain_WaitsForOtherDispatches proves a call that fails the Handle returns +// at once, and Drain returns only once another goroutine's effect has finished +// sending, so a host that recovers after Drain can't send that effect a second time. +func TestHandle_Drain_WaitsForOtherDispatches(t *testing.T) { ctx := context.Background() store := &failingMemStore{MemStore: durable.NewMemStore()} entered, release := make(chan struct{}), make(chan struct{}) @@ -391,27 +391,119 @@ func TestHandle_FailedStep_WaitsForOtherDispatches(t *testing.T) { <-entered store.armed.Store(true) failed := make(chan error, 1) - go func() { - _, err := h.Fire(ctx, "bill") // its Append fails - failed <- err - }() + go func() { _, fireErr := h.Fire(ctx, "bill"); failed <- fireErr }() // its Append fails select { case err := <-failed: - t.Fatalf("Fire(bill) returned %v while step 0's effect was still sending", err) + if !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire(bill) = %v, want ErrHandleFailed", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Fire(bill) waited for another goroutine's dispatch; it should return at once") + } + drained := make(chan error, 1) + go func() { drained <- h.Drain(ctx) }() + select { + case err := <-drained: + t.Fatalf("Drain returned %v while step 0's effect was still sending", err) case <-time.After(100 * time.Millisecond): } close(release) select { - case err := <-failed: - if !errors.Is(err, durable.ErrHandleFailed) { - t.Fatalf("Fire(bill) = %v, want ErrHandleFailed", err) + case err := <-drained: + if err != nil { + t.Fatalf("Drain = %v, want nil", err) } case <-time.After(5 * time.Second): - t.Fatal("Fire(bill) never returned after the dispatch finished") + t.Fatal("Drain never returned after the dispatch finished") } waitFor(t, first, "Fire(go)") } +// TestHandle_HandlerFailsTheHandleWithAnotherContext_DoesNotHang proves an effect handler +// that calls back with a context other than its own, and fails the Handle, gets +// ErrHandleFailed back at once instead of waiting on its own dispatch. +func TestHandle_HandlerFailsTheHandleWithAnotherContext_DoesNotHang(t *testing.T) { + ctx := context.Background() + store := &failingMemStore{MemStore: durable.NewMemStore()} + var h *durable.Handle[string, string, *runCtx] + var once atomic.Bool + var backErr error + handler := func(context.Context, string, state.Effect) error { + if once.CompareAndSwap(false, true) { + store.armed.Store(true) + _, backErr = h.Fire(context.Background(), "bill") // not the handler's context + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = h.Fire(ctx, "go") + }() + waitFor(t, done, "Fire(go)") + if !errors.Is(backErr, durable.ErrHandleFailed) { + t.Fatalf("the handler's Fire = %v, want ErrHandleFailed", backErr) + } + if err := h.Drain(ctx); err != nil { + t.Fatalf("Drain after the dispatch finished = %v, want nil", err) + } +} + +// TestHandle_DrainFromAHandler_IsRefused proves Drain called with an effect handler's +// own context returns ErrReentrantStep instead of waiting on its own dispatch. +func TestHandle_DrainFromAHandler_IsRefused(t *testing.T) { + ctx := context.Background() + var h *durable.Handle[string, string, *runCtx] + var drainErr error + var once atomic.Bool + handler := func(ctx context.Context, _ string, _ state.Effect) error { + if once.CompareAndSwap(false, true) { + drainErr = h.Drain(ctx) + } + return nil + } + h, err := durable.NewRunner(effectMachine(), durable.NewMemStore(), + durable.WithEffectHandler[string, string, *runCtx](handler)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = h.Fire(ctx, "go") + }() + waitFor(t, done, "Fire(go)") + if !errors.Is(drainErr, durable.ErrReentrantStep) { + t.Fatalf("Drain from the handler = %v, want ErrReentrantStep", drainErr) + } +} + +// TestHandle_View_AfterAFailure_IsRefused proves View refuses a failed Handle without +// running its function. +func TestHandle_View_AfterAFailure_IsRefused(t *testing.T) { + ctx := context.Background() + store := &failingMemStore{MemStore: durable.NewMemStore()} + h, err := durable.NewRunner(pollingMachine(), store).Start(ctx, "poll-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + store.armed.Store(true) + if _, err = h.Fire(ctx, "start"); !errors.Is(err, durable.ErrHandleFailed) { + t.Fatalf("Fire = %v, want ErrHandleFailed", err) + } + ran := false + if err = h.View(ctx, func(*state.Instance[string, string, *runCtx]) { ran = true }); !errors.Is(err, durable.ErrHandleFailed) || ran { + t.Fatalf("View = %v (ran %v), want ErrHandleFailed without running", err, ran) + } +} + // blockingMemStore is a MemStore whose next Append, once armed, signals entered and // waits for release. type blockingMemStore struct { @@ -454,19 +546,19 @@ func TestHandle_StepAfterAFailure_LeavesItsEffectsForRecover(t *testing.T) { t.Fatalf("Start: %v", err) } firstDone := make(chan error, 1) - go func() { _, err := h.Fire(ctx, "go"); firstDone <- err }() + go func() { _, fireErr := h.Fire(ctx, "go"); firstDone <- fireErr }() <-handlerIn store.armed.Store(true) secondDone := make(chan error, 1) - go func() { _, err := h.Fire(ctx, "bill"); secondDone <- err }() + go func() { _, fireErr := h.Fire(ctx, "bill"); secondDone <- fireErr }() <-store.entered // step 1 holds the lock, in its Append close(failNow) - if err := <-firstDone; !errors.Is(err, durable.ErrHandleFailed) { - t.Fatalf("Fire(go) = %v, want ErrHandleFailed", err) + if got := <-firstDone; !errors.Is(got, durable.ErrHandleFailed) { + t.Fatalf("Fire(go) = %v, want ErrHandleFailed", got) } close(store.release) - if err := <-secondDone; !errors.Is(err, durable.ErrHandleFailed) { - t.Fatalf("Fire(bill) = %v, want ErrHandleFailed", err) + if got := <-secondDone; !errors.Is(got, durable.ErrHandleFailed) { + t.Fatalf("Fire(bill) = %v, want ErrHandleFailed", got) } if n := calls.Load(); n != 0 { t.Fatalf("step 1 sent %d effects after the Handle failed, want 0", n) diff --git a/durable/runner.go b/durable/runner.go index 3a74ab5e..9b61381c 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -91,19 +91,20 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // error. // // Code that runs under the lock (a service, guard, action, reducer or actor behavior) -// must not call back into the same Handle. A call that carries the context the step -// passed to it, as a service gets, returns ErrReentrantStep at once; one without that -// context deadlocks. A step waiting for the lock returns its context's error once the +// must not call back into the same Handle; to act on the Handle when a step happens, an +// action returns an effect, and its handler runs after the lock and may call back. A +// call that carries the context the step passed to the callback, as a service gets, +// returns ErrReentrantStep at once; one without that context deadlocks. A step waiting for the lock returns its context's error once the // context ends, and changes nothing. View reads the instance under the lock; Instance // returns it outside the lock. A stateless Runner.Fire builds its own Handle, so it // isn't serialized with this one. // // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, -// wrapping the cause, which can be a context that ended during the Store write. A call -// that returns it waits first until no effect of this Handle is still being sent (or -// its context ends), unless it was made from inside an effect handler. Recover a new -// Handle with the same options to continue. A service +// wrapping the cause, which can be a context that ended during the Store write. Before +// recovering, call Drain: a step on another goroutine may still be sending an effect, +// and a recovered Handle would send it again. Recover a new Handle with the same +// options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every // delivery so far, and a Spawn actor doesn't return. If the cause is ErrFenced, a @@ -224,25 +225,30 @@ func (h *Handle[S, E, C]) inDispatch(ctx context.Context) bool { return false } -// afterFailure holds a call that returns ErrHandleFailed until no effect of h is still -// being dispatched, or ctx ends, so a host that recovers a new Handle as soon as the -// call returns can't send an effect a second time while the old one is still sending -// it. A call from inside an effect handler returns at once: its own dispatch is one of -// those still running. -func (h *Handle[S, E, C]) afterFailure(ctx context.Context, err error) error { - if !errors.Is(err, ErrHandleFailed) || h.inDispatch(ctx) { - return err +// Drain waits until no domain effect of h is still being dispatched, or ctx ends. +// Call it after ErrHandleFailed and before Recover: a step on another goroutine may +// still be sending an effect it recorded, and a recovered Handle would send it again +// because it isn't marked yet. Drain works on a healthy Handle too. It returns ctx's +// error if ctx ends first, and ErrReentrantStep, without waiting, when called from +// inside a step or an effect handler of h, whose own work it would wait on forever. +// A call from a goroutine an effect handler started, without the handler's context, +// can't be told apart and waits for that handler. +func (h *Handle[S, E, C]) Drain(ctx context.Context) error { + if h.inStep(ctx) || h.inDispatch(ctx) { + return fmt.Errorf("%w: %q: Drain from inside a step or an effect handler", ErrReentrantStep, h.id) } h.out.mu.Lock() idle := h.out.idle h.out.mu.Unlock() - if idle != nil { - select { - case <-idle: - case <-ctx.Done(): - } + if idle == nil { + return nil + } + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() } - return err } // refuse reports why a step that holds the lock must not run: ErrHandleFailed once @@ -427,14 +433,8 @@ func (h *Handle[S, E, C]) Instance() *state.Instance[S, E, C] { return h.inst } // read can't race a step on another goroutine. fn must not call back into the Handle, // and must not keep the instance past its return. View returns ctx's error if ctx ends // before the lock is free, ErrReentrantStep if ctx is a step's own context, and -// ErrHandleFailed once the Handle has failed, without running fn. Like every call that -// returns ErrHandleFailed, it first waits until no effect is still being sent. +// ErrHandleFailed once the Handle has failed, without running fn. func (h *Handle[S, E, C]) View(ctx context.Context, fn func(*state.Instance[S, E, C])) error { - return h.afterFailure(ctx, h.view(ctx, fn)) -} - -// view is View without the wait after a failure. -func (h *Handle[S, E, C]) view(ctx context.Context, fn func(*state.Instance[S, E, C])) error { if err := h.acquire(ctx); err != nil { return err } @@ -576,7 +576,8 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // step's ordinal (write-ahead, before returning), and — when the // checkpoint policy is due — persists a full Snapshot and compacts the tail. A // kernel transition error journals nothing (no step was produced) and is -// returned to the caller. +// returned to the caller. A failed checkpoint returns the result and that error: the +// step is recorded and its effects sent, and the Handle carries on. // // A state.WithEventData payload reaches the transition's Assign as the Runner's // EventDataDecoder rebuilds it from the payload's JSON form, live and on replay alike, so @@ -597,12 +598,12 @@ func (h *Handle[S, E, C]) Fire(ctx context.Context, event E, opts ...state.FireO } res, rec, des, err := h.lockedFireStep(ctx, event, opts) if err != nil { - return res, h.afterFailure(ctx, err) + return res, err } // Dispatch after releasing the step lock, so an effect handler can call back into // this Handle. The Record already names every effect (write-ahead), so a crash here // leaves it for recovery to redispatch, deduped by id. - return res, h.afterFailure(ctx, h.finishStep(ctx, rec, des)) + return res, h.finishStep(ctx, rec, des) } // lockedFireStep runs fireStep and releases the step lock however it returns, a panic @@ -693,12 +694,12 @@ func (h *Handle[S, E, C]) finishStep(ctx context.Context, rec *Record, des []dis return h.offerCheckpointLocked(ctx, rec) } -// dispatchGuarded dispatches des, turning a panic in the effect handler into an -// ErrEffectDispatch error. +// dispatchGuarded dispatches des, turning a panic in the effect handler, or in the +// Store's dispatch tracking, into an ErrEffectDispatch error. func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []dispatchableEffect) (err error) { defer func() { if p := recover(); p != nil { - err = fmt.Errorf("%w: step %d for %q: effect handler panicked: %v", ErrEffectDispatch, step, h.id, p) + err = fmt.Errorf("%w: step %d for %q: effect dispatch panicked: %v", ErrEffectDispatch, step, h.id, p) } }() return h.dispatchEffects(ctx, des) @@ -713,12 +714,6 @@ func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []d // count of timers fired — so recovery re-derives the same timers at their // recorded instants. It returns the FireResults of the timers it fired, in order. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { - results, err := h.tick(ctx) - return results, h.afterFailure(ctx, err) -} - -// tick is Tick without the wait after a failure. -func (h *Handle[S, E, C]) tick(ctx context.Context) ([]state.FireResult[S], error) { if err := h.acquire(ctx); err != nil { return nil, err } diff --git a/durable/runner_options.go b/durable/runner_options.go index 7c874b68..614afe83 100644 --- a/durable/runner_options.go +++ b/durable/runner_options.go @@ -155,7 +155,7 @@ func WithEffectHandler[S comparable, E comparable, C any](h EffectHandler) Optio const defaultMaxEventDataBytes = 8 << 20 // WithMaxEventDataBytes sets the largest state.WithEventData payload Handle.Fire -// accepts, measured as its JSON form, or for an error, its message's. A larger one +// accepts, measured as its JSON form; for an error, its message's JSON form. A larger one // refuses the Fire with ErrEventDataEncoding before the kernel runs. It defaults to // 8 MiB, which leaves a payload room in FileStore's journal line; raise it for a Store // with no such limit. The cap covers the payload only, so a Record can still exceed a diff --git a/durable/service.go b/durable/service.go index 70976b5d..ac5b7c86 100644 --- a/durable/service.go +++ b/durable/service.go @@ -91,12 +91,6 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // result, true and that error, and the Handle carries on. A recovered Handle has an // unrecorded service in flight again, and RunService runs it a second time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { - res, ok, err := h.runService(ctx, id) - return res, ok, h.afterFailure(ctx, err) -} - -// runService is RunService without the wait after a failure. -func (h *Handle[S, E, C]) runService(ctx context.Context, id string) (state.FireResult[S], bool, error) { if err := h.acquire(ctx); err != nil { return state.FireResult[S]{}, false, err } From 0f63aff81a1e0ecf735315212c9faa26b7e371d9 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Tue, 10 Nov 2026 15:42:34 -0700 Subject: [PATCH 06/10] docs(durable): scope Drain to a failed Handle Once a Handle has failed, no step can start sending effects, so Drain covers every one. On a healthy Handle a step still recording can start sending after Drain returns, so the docs say to stop calling it before moving its instance to a new Handle. Also: a typed-nil payload is refused as a type mismatch. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 6 ++++-- durable/CHANGELOG.md | 12 ++++++++---- durable/README.md | 6 ++++-- durable/runner.go | 13 +++++++++---- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 6b48fdd5..a7d66910 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -139,8 +139,10 @@ takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an effect handler that returns an error or panics. Before `Recover`, call `Handle.Drain`, -which waits until no effect of the Handle is still being sent, so the recovered Handle -can't send one twice. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +which waits until no effect of the failed Handle is still being sent, so the recovered +Handle can't send one twice. To move a healthy Handle's instance to a new Handle, stop +calling the old one and let its calls return first: `Drain` alone doesn't cover a step +that is still recording. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. ## A note on serialized payloads diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 9e4bf856..48c2b679 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -27,10 +27,14 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **`ErrHandleFailed`** reports a Handle out of step with its journal. -- **`Handle.View`** reads the instance under the Handle's step lock. **`Handle.Drain`** - waits until no effect of the Handle is still being sent; call it before `Recover` after - `ErrHandleFailed`. **`ErrReentrantStep`** reports a call back into a Handle that carries - the context one of its steps passed to a callback, such as a service's. A guard, +- **`Handle.View`** reads the instance under the Handle's step lock. + +- **`Handle.Drain`** waits until no effect of the Handle is still being sent. Call it + before `Recover` after `ErrHandleFailed`. On a healthy Handle it covers only dispatches + already under way. + +- **`ErrReentrantStep`** reports a call back into a Handle that carries the context one + of its steps passed to a callback, such as a service's. A guard, action, reducer or actor behavior gets no context here, so its call back deadlocks instead; #245 passes them the step's context. diff --git a/durable/README.md b/durable/README.md index 3bca58c7..ae7d8803 100644 --- a/durable/README.md +++ b/durable/README.md @@ -150,8 +150,10 @@ takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an effect handler that returns an error or panics. Before `Recover`, call `Handle.Drain`, -which waits until no effect of the Handle is still being sent, so the recovered Handle -can't send one twice. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +which waits until no effect of the failed Handle is still being sent, so the recovered +Handle can't send one twice. To move a healthy Handle's instance to a new Handle, stop +calling the old one and let its calls return first: `Drain` alone doesn't cover a step +that is still recording. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. ## Caveat: serialized payloads diff --git a/durable/runner.go b/durable/runner.go index 9b61381c..dc588bba 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -103,7 +103,8 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // of step with the journal. That call and every later one return ErrHandleFailed, // wrapping the cause, which can be a context that ended during the Store write. Before // recovering, call Drain: a step on another goroutine may still be sending an effect, -// and a recovered Handle would send it again. Recover a new Handle with the same +// and a recovered Handle would send it again. Recovering a healthy Handle's instance +// instead needs this Handle's calls to have returned first. Recover a new Handle with the same // options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. // A running InvokeActor actor restarts from its initial state on any Recover, losing every @@ -228,7 +229,10 @@ func (h *Handle[S, E, C]) inDispatch(ctx context.Context) bool { // Drain waits until no domain effect of h is still being dispatched, or ctx ends. // Call it after ErrHandleFailed and before Recover: a step on another goroutine may // still be sending an effect it recorded, and a recovered Handle would send it again -// because it isn't marked yet. Drain works on a healthy Handle too. It returns ctx's +// because it isn't marked yet. Once the Handle has failed, no step can start sending, +// so Drain covers every effect. On a healthy Handle it covers only dispatches already +// under way: a step still recording can start sending after Drain returns, so before +// handing the instance to a new Handle, stop driving this one and let its calls return. It returns ctx's // error if ctx ends first, and ErrReentrantStep, without waiting, when called from // inside a step or an effect handler of h, whose own work it would wait on forever. // A call from a goroutine an effect handler started, without the handler's context, @@ -1178,8 +1182,9 @@ func decodePayload[E comparable](p eventPayload, event E, decode EventDataDecode // with, so one decoder serves a machine whose events carry different payload types. // Return the payload as the same type the caller passed: a Fire whose payload decodes // as another type is refused with ErrEventDataEncoding. Only the top-level type is -// checked, so a []any of structs still arrives as a []any of maps. A nil payload -// arrives as nil without reaching the decoder. The default decodes with +// checked, so a []any of structs still arrives as a []any of maps. An untyped nil +// payload arrives as nil without reaching the decoder; a typed nil, such as a nil +// pointer or map, encodes as null and is refused as a type mismatch. The default decodes with // DecodeAs[any]. Supply one with WithEventDataDecoder. type EventDataDecoder[E comparable] func(event E, data []byte) (any, error) From 4626b478b07e18be55decfc166d23fa385c5dc1b Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Wed, 11 Nov 2026 08:02:02 -0700 Subject: [PATCH 07/10] docs(durable): refuse a guard, action or reducer calling back with its Ctx With #245, a guard, action or reducer gets the step's context as its input's Ctx, so a call back into the Handle with it returns ErrReentrantStep instead of deadlocking. Say so in the docs, including that a call back without the step's context, or under middleware that passed down an unrelated one, still deadlocks. Cover Fire, an eventless transition, middleware, Tick, RunService and DeliverToActor in a test. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 13 +- durable/CHANGELOG.md | 16 +-- durable/README.md | 13 +- durable/errors.go | 7 +- durable/handle_lock_test.go | 152 ++++++++++++++++++++++ durable/runner.go | 16 ++- 6 files changed, 189 insertions(+), 28 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index a7d66910..c6378a26 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -130,11 +130,14 @@ in by implementing `HistoryStore` (the in-tree `MemStore` does so under `Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different goroutines: each holds the Handle's lock while it applies to the kernel and records the step. Effects dispatch after the lock is released, so an effect handler can call back -into its Handle. A service, guard, action, reducer or actor behavior runs under the lock -and must not call back in: to act on the Handle when a step happens, an action returns an -effect, whose handler runs after the lock. A call back that carries the step's context, -as a service gets, returns `ErrReentrantStep`; one without it deadlocks. A step -waiting for the lock stops when its context ends. Read the instance with `View`, which +into its Handle. A service, guard, action, reducer or actor behavior runs under the lock, +so it can't act on the Handle itself: to act on the Handle when a step happens, an action +returns an effect, whose handler runs after the lock. If one calls back during a live +step with the step's context, as a service gets and a guard, action or reducer finds in +its input's `Ctx`, the call returns `ErrReentrantStep` instead of waiting. A call back +without that context deadlocks, and so does one under middleware that passed down a +context not derived from the step's. A step waiting for the lock stops when its context +ends. Read the instance with `View`, which takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 48c2b679..aa9f8f92 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -34,9 +34,8 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. already under way. - **`ErrReentrantStep`** reports a call back into a Handle that carries the context one - of its steps passed to a callback, such as a service's. A guard, - action, reducer or actor behavior gets no context here, so its call back deadlocks - instead; #245 passes them the step's context. + of its steps passed to a callback: a service's context, or the `Ctx` a guard, action + or reducer gets (#245). A call back without that context deadlocks instead. - **`StepReporter`**, implemented by `MemStore` and `FileStore`, lets `Recover` number the next step past every recorded one after a checkpoint from before checkpoints carried @@ -54,11 +53,12 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. host's timer loop no longer loses acknowledged steps. Effects dispatch after the lock is released, so an effect handler can still call back into its Handle; a checkpoint waits for every earlier step's effects to dispatch. A service, guard, action, reducer - or actor behavior runs under the lock and must not call back in; an action reaches the - Handle by returning an effect. A service that calls back with its context gets the - new `ErrReentrantStep`. A step waiting for the lock returns its - context's error once the context ends, and changes nothing. The new `Handle.View` reads - the instance under the lock; `Instance()` is outside it. + or actor behavior runs under the lock, so it can't act on the Handle itself; an action + reaches the Handle by returning an effect. A service, guard, action or reducer that + calls back with the context its step passed gets the new `ErrReentrantStep`. A step + waiting for the lock returns its context's error once the context ends, and changes + nothing. The new `Handle.View` reads the instance under the lock; `Instance()` is + outside it. - **A failed effect dispatch fails the Handle**, and so does an effect handler that panics, which now returns `ErrEffectDispatch` instead of panicking out of `Fire`. The effect stays unmarked in the journal, so no later checkpoint compacts it away, and diff --git a/durable/README.md b/durable/README.md index ae7d8803..8539fb73 100644 --- a/durable/README.md +++ b/durable/README.md @@ -141,11 +141,14 @@ otherwise. `Fire`, `Tick`, `RunService` and `DeliverToActor` are safe to call from different goroutines: each holds the Handle's lock while it applies to the kernel and records the step. Effects dispatch after the lock is released, so an effect handler can call back -into its Handle. A service, guard, action, reducer or actor behavior runs under the lock -and must not call back in: to act on the Handle when a step happens, an action returns an -effect, whose handler runs after the lock. A call back that carries the step's context, -as a service gets, returns `ErrReentrantStep`; one without it deadlocks. A step -waiting for the lock stops when its context ends. Read the instance with `View`, which +into its Handle. A service, guard, action, reducer or actor behavior runs under the lock, +so it can't act on the Handle itself: to act on the Handle when a step happens, an action +returns an effect, whose handler runs after the lock. If one calls back during a live +step with the step's context, as a service gets and a guard, action or reducer finds in +its input's `Ctx`, the call returns `ErrReentrantStep` instead of waiting. A call back +without that context deadlocks, and so does one under middleware that passed down a +context not derived from the step's. A step waiting for the lock stops when its context +ends. Read the instance with `View`, which takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an diff --git a/durable/errors.go b/durable/errors.go index 06cbd49d..111a4bd7 100644 --- a/durable/errors.go +++ b/durable/errors.go @@ -64,9 +64,10 @@ var ( // ErrReentrantStep is reported by a Handle's Fire, Tick, RunService, DeliverToActor // and View when called with the context a step passed to host code, such as a - // service: the step holds the Handle's lock, so waiting would deadlock. Nothing - // applied. Effect handlers run after the lock is released and may call back in. Drain - // reports it when called from inside a step or an effect handler. + // service's context or the Ctx a guard, action or reducer gets: the step holds the + // Handle's lock, so waiting would deadlock. Nothing applied. Effect handlers run + // after the lock is released and may call back in. Drain reports it when called + // from inside a step or an effect handler. ErrReentrantStep = errors.New(errorPrefix + errorReentrantStep) // ErrCheckpointNotAdvancing is reported by Store.Checkpoint when throughStep diff --git a/durable/handle_lock_test.go b/durable/handle_lock_test.go index e82dd1b3..ae81f839 100644 --- a/durable/handle_lock_test.go +++ b/durable/handle_lock_test.go @@ -570,3 +570,155 @@ func TestHandle_StepAfterAFailure_LeavesItsEffectsForRecover(t *testing.T) { t.Fatalf("Recover sent %d effects, want 3", n) } } + +// TestHandle_CallbackCallsBackWithItsCtx_IsRefused proves a guard, an action and a +// reducer that call back into their Handle with the Ctx their input carries get +// ErrReentrantStep at once instead of deadlocking on the lock their step holds, on +// each step that runs them: Fire, an eventless transition settling after it, a Fire +// through middleware that derives its context, Tick, RunService and DeliverToActor. +func TestHandle_CallbackCallsBackWithItsCtx_IsRefused(t *testing.T) { + type probe = *state.Builder[string, string, svcCtx] + type handle = *durable.Handle[string, string, svcCtx] + derive := func(next state.FireFunc[string, string, svcCtx]) state.FireFunc[string, string, svcCtx] { + return func(ctx context.Context, event string) state.FireResult[string] { + return next(context.WithValue(ctx, reentryKey{}, "derived"), event) + } + } + clock := state.NewFakeClock(epoch) + var runs int64 + cases := []struct { + name string + build func(probe) probe + opts []durable.Option[string, string, svcCtx] + init string + drive func(t *testing.T, h handle) error + }{ + { + name: "Fire", + build: func(b probe) probe { + return b.State("idle").State("done").Final().Initial("idle"). + Transition("idle").On("go").GoTo("done").When("g").Do("a").Assign("r") + }, + init: "idle", + drive: func(_ *testing.T, h handle) error { _, err := h.Fire(context.Background(), "go"); return err }, + }, + { + name: "eventless after Fire", + build: func(b probe) probe { + return b.State("idle").State("mid").State("done").Final().Initial("idle"). + Transition("idle").On("go").GoTo("mid"). + Transition("mid").Always().GoTo("done").When("g").Do("a").Assign("r") + }, + init: "idle", + drive: func(_ *testing.T, h handle) error { _, err := h.Fire(context.Background(), "go"); return err }, + }, + { + name: "Fire through middleware", + build: func(b probe) probe { + return b.Use(derive).State("idle").State("done").Final().Initial("idle"). + Transition("idle").On("go").GoTo("done").When("g").Do("a").Assign("r") + }, + init: "idle", + drive: func(_ *testing.T, h handle) error { _, err := h.Fire(context.Background(), "go"); return err }, + }, + { + name: "Tick", + build: func(b probe) probe { + return b.State("idle").State("arming").State("done").Final().Initial("idle"). + Transition("idle").On("arm").GoTo("arming"). + Transition("arming").After(5 * time.Second).On("elapsed").GoTo("done").When("g").Do("a").Assign("r") + }, + opts: []durable.Option[string, string, svcCtx]{durable.WithRunnerClock[string, string, svcCtx](clock)}, + init: "idle", + drive: func(_ *testing.T, h handle) error { + ctx := context.Background() + if _, err := h.Fire(ctx, "arm"); err != nil { + return err + } + clock.Advance(6 * time.Second) + _, err := h.Tick(ctx) + return err + }, + }, + { + name: "RunService", + build: func(b probe) probe { + return b.Service("fetch", counterService(&runs)). + State("idle"). + State("loading").Invoke("fetch", state.WithInvokeOnDone("ok")). + State("done").Final().Initial("idle"). + Transition("idle").On("start").GoTo("loading"). + Transition("loading").On("ok").GoTo("done").When("g").Do("a").Assign("r") + }, + opts: []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx]( + state.NewRegistry[svcCtx]().Service("fetch", counterService(&runs)))}, + init: "idle", + drive: func(_ *testing.T, h handle) error { + ctx := context.Background() + if _, err := h.Fire(ctx, "start"); err != nil { + return err + } + _, _, err := h.RunService(ctx, state.InvokeID("probe", "loading", 0)) + return err + }, + }, + { + name: "DeliverToActor", + build: func(b probe) probe { + return b.Actor("spawnChild"). + State("supervising").InvokeActor("spawnChild", state.WithInvokeOnDone("childDone")). + State("done").Final().Initial("supervising"). + Transition("supervising").On("childDone").GoTo("done").When("g").Do("a").Assign("r") + }, + opts: []durable.Option[string, string, svcCtx]{durable.WithActorPalette[string, string, svcCtx](counterPalette(&runs))}, + init: "supervising", + drive: func(t *testing.T, h handle) error { + ref, ok := h.ActorRef(state.ActorID("probe", "supervising", 0)) + if !ok { + t.Fatal("no actor ref for the spawned child") + } + _, err := h.DeliverToActor(context.Background(), ref, "finish") + return err + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var h handle + got := map[string]error{} + view := func(name string, ctx context.Context) { + got[name] = h.View(ctx, func(*state.Instance[string, string, svcCtx]) {}) + } + machine := tc.build(state.ForgeFor[svcCtx]("probe"). + Guard("g", func(in state.GuardCtx[svcCtx]) bool { view("guard", in.Ctx); return true }). + Action("a", func(in state.ActionCtx[svcCtx]) (state.Effect, error) { view("action", in.Ctx); return nil, nil }). + Reducer("r", func(in state.AssignCtx[svcCtx]) svcCtx { view("reducer", in.Ctx); return in.Entity })). + Quench() + var err error + h, err = durable.NewRunner(machine, durable.NewMemStore(), tc.opts...). + Start(context.Background(), "probe-1", svcCtx{}, state.WithInitialState(tc.init)) + if err != nil { + t.Fatalf("Start: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + if err := tc.drive(t, h); err != nil { + t.Errorf("drive: %v", err) + } + }() + waitFor(t, done, tc.name) + for _, name := range []string{"guard", "action", "reducer"} { + if !errors.Is(got[name], durable.ErrReentrantStep) { + t.Errorf("the %s's View = %v, want ErrReentrantStep", name, got[name]) + } + } + if cur := h.Instance().Current(); cur != "done" { + t.Fatalf("state %q, want done", cur) + } + }) + } +} + +// reentryKey is the key the middleware case derives its context with. +type reentryKey struct{} diff --git a/durable/runner.go b/durable/runner.go index dc588bba..836130c1 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -91,13 +91,15 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // error. // // Code that runs under the lock (a service, guard, action, reducer or actor behavior) -// must not call back into the same Handle; to act on the Handle when a step happens, an -// action returns an effect, and its handler runs after the lock and may call back. A -// call that carries the context the step passed to the callback, as a service gets, -// returns ErrReentrantStep at once; one without that context deadlocks. A step waiting for the lock returns its context's error once the -// context ends, and changes nothing. View reads the instance under the lock; Instance -// returns it outside the lock. A stateless Runner.Fire builds its own Handle, so it -// isn't serialized with this one. +// can't act on the Handle itself; to act on the Handle when a step happens, an action +// returns an effect, and its handler runs after the lock and may call back. A call back +// during a live step that carries the step's context, as a service gets and a guard, +// action or reducer finds in its input's Ctx, returns ErrReentrantStep at once. One +// without that context deadlocks, and so does one under middleware that passed down a +// context not derived from the step's. A step waiting for the lock returns its context's +// error once the context ends, and changes nothing. View reads the instance under the +// lock; Instance returns it outside the lock. A stateless Runner.Fire builds its own +// Handle, so it isn't serialized with this one. // // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, From a2541b982f87360f7246576625483bbb9b12ae7b Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Wed, 11 Nov 2026 08:34:45 -0700 Subject: [PATCH 08/10] chore(ci): move to Go 1.27.4 across the repo govulncheck now fails every PR on GO-2026-6859, an HTTP/1 server crash in net/http that Go 1.27.4 fixes. It's reached through grpc's server in transport. Every go.mod and go.work now declares Go 1.27.4; CI reads the version from go.work. No dependency changes. Signed-off-by: Ray Karnes --- CONTRIBUTING.md | 2 +- cluster/CHANGELOG.md | 2 +- cluster/go.mod | 2 +- cmd/crucible/CHANGELOG.md | 2 +- cmd/crucible/go.mod | 2 +- durable/CHANGELOG.md | 2 +- durable/go.mod | 2 +- e2e/go.mod | 2 +- examples/dispatch/CHANGELOG.md | 2 +- examples/dispatch/go.mod | 2 +- examples/fooddelivery/go.mod | 2 +- examples/sinkflow/go.mod | 2 +- examples/sourcedrive/go.mod | 2 +- gen/CHANGELOG.md | 2 +- gen/eject_test.go | 2 +- gen/go.mod | 2 +- go.work | 2 +- magefiles/go.mod | 2 +- sink/CHANGELOG.md | 2 +- sink/bridge/CHANGELOG.md | 2 +- sink/bridge/go.mod | 2 +- sink/cloudwatch/CHANGELOG.md | 2 +- sink/cloudwatch/go.mod | 2 +- sink/dynamo/CHANGELOG.md | 2 +- sink/dynamo/go.mod | 2 +- sink/eventbridge/CHANGELOG.md | 2 +- sink/eventbridge/go.mod | 2 +- sink/file/CHANGELOG.md | 2 +- sink/file/go.mod | 2 +- sink/firehose/CHANGELOG.md | 2 +- sink/firehose/go.mod | 2 +- sink/gcppubsub/CHANGELOG.md | 2 +- sink/gcppubsub/go.mod | 2 +- sink/go.mod | 2 +- sink/http/CHANGELOG.md | 2 +- sink/http/go.mod | 2 +- sink/kafka/CHANGELOG.md | 2 +- sink/kafka/go.mod | 2 +- sink/kinesis/CHANGELOG.md | 2 +- sink/kinesis/go.mod | 2 +- sink/nats/CHANGELOG.md | 2 +- sink/nats/go.mod | 2 +- sink/otel/CHANGELOG.md | 2 +- sink/otel/go.mod | 2 +- sink/prometheus/CHANGELOG.md | 2 +- sink/prometheus/go.mod | 2 +- sink/redis/CHANGELOG.md | 2 +- sink/redis/go.mod | 2 +- sink/s3/CHANGELOG.md | 2 +- sink/s3/go.mod | 2 +- sink/slog/CHANGELOG.md | 2 +- sink/slog/go.mod | 2 +- sink/sns/CHANGELOG.md | 2 +- sink/sns/go.mod | 2 +- sink/sql/CHANGELOG.md | 2 +- sink/sql/go.mod | 2 +- sink/sqs/CHANGELOG.md | 2 +- sink/sqs/go.mod | 2 +- sink/statsd/CHANGELOG.md | 2 +- sink/statsd/go.mod | 2 +- sink/timestream/CHANGELOG.md | 2 +- sink/timestream/go.mod | 2 +- source/CHANGELOG.md | 2 +- source/cloudevents/go.mod | 2 +- source/go.mod | 2 +- source/jetstream/go.mod | 2 +- source/kafka/CHANGELOG.md | 2 +- source/kafka/go.mod | 2 +- source/kafka/integration/go.mod | 2 +- source/redis/go.mod | 2 +- source/statemachine/CHANGELOG.md | 2 +- source/statemachine/go.mod | 2 +- state/CHANGELOG.md | 2 +- state/expr/CHANGELOG.md | 2 +- state/expr/go.mod | 2 +- state/go.mod | 2 +- telemetry/CHANGELOG.md | 2 +- telemetry/datadog/CHANGELOG.md | 2 +- telemetry/datadog/go.mod | 2 +- telemetry/go.mod | 2 +- telemetry/otel/CHANGELOG.md | 2 +- telemetry/otel/go.mod | 2 +- telemetry/slog/README.md | 2 +- telemetry/slog/go.mod | 2 +- tools/docsgen/go.mod | 2 +- transport/README.md | 2 +- transport/go.mod | 2 +- wasm/CHANGELOG.md | 2 +- wasm/go.mod | 2 +- 89 files changed, 89 insertions(+), 89 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5141cf0f..7881c94a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,7 +104,7 @@ let them bring their own? If it forces, redesign it as a seam with a no-op defau Requirements: -- **Go**: 1.27.3, the version every `go.mod` and `go.work` declares. CI reads it from +- **Go**: 1.27.4, the version every `go.mod` and `go.work` declares. CI reads it from `go.work`. - **[Mage](https://magefile.org)**: the build tool. Install with `go install github.com/magefile/mage@latest`, or run targets via diff --git a/cluster/CHANGELOG.md b/cluster/CHANGELOG.md index c18a5829..b2a553bc 100644 --- a/cluster/CHANGELOG.md +++ b/cluster/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/cluster/go.mod b/cluster/go.mod index 5d2a8574..17ac0289 100644 --- a/cluster/go.mod +++ b/cluster/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/cluster -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/state => ../state diff --git a/cmd/crucible/CHANGELOG.md b/cmd/crucible/CHANGELOG.md index b0cb80a8..4fdd3b85 100644 --- a/cmd/crucible/CHANGELOG.md +++ b/cmd/crucible/CHANGELOG.md @@ -9,7 +9,7 @@ versioned independently of the `state` module. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/cmd/crucible/go.mod b/cmd/crucible/go.mod index 16de5a18..f4f1399d 100644 --- a/cmd/crucible/go.mod +++ b/cmd/crucible/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/cmd/crucible -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/gen v0.1.0 diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index aa9f8f92..7f15d978 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -47,7 +47,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. - **A Handle's steps are serialized across goroutines.** `Fire`, `Tick`, `RunService` and `DeliverToActor` hold a lock while they apply to the kernel and record the step, so a host's timer loop no longer loses acknowledged steps. Effects dispatch after the lock diff --git a/durable/go.mod b/durable/go.mod index 48cd8224..5368abcf 100644 --- a/durable/go.mod +++ b/durable/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/durable -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/state => ../state diff --git a/e2e/go.mod b/e2e/go.mod index e15a22c2..da06129c 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/e2e -go 1.27.3 +go 1.27.4 replace ( github.com/stablekernel/crucible/cluster => ../cluster diff --git a/examples/dispatch/CHANGELOG.md b/examples/dispatch/CHANGELOG.md index ff19f846..a1dbc625 100644 --- a/examples/dispatch/CHANGELOG.md +++ b/examples/dispatch/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [0.1.0] diff --git a/examples/dispatch/go.mod b/examples/dispatch/go.mod index b6ec1c8d..35a08e11 100644 --- a/examples/dispatch/go.mod +++ b/examples/dispatch/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/examples/dispatch -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/state => ../../state diff --git a/examples/fooddelivery/go.mod b/examples/fooddelivery/go.mod index 6d32aa8c..6cc3e72e 100644 --- a/examples/fooddelivery/go.mod +++ b/examples/fooddelivery/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/examples/fooddelivery -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/state => ../../state diff --git a/examples/sinkflow/go.mod b/examples/sinkflow/go.mod index ff519e9c..16a6d3b6 100644 --- a/examples/sinkflow/go.mod +++ b/examples/sinkflow/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/examples/sinkflow -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/sink v0.0.0 diff --git a/examples/sourcedrive/go.mod b/examples/sourcedrive/go.mod index 5956e246..49dd4f2c 100644 --- a/examples/sourcedrive/go.mod +++ b/examples/sourcedrive/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/examples/sourcedrive -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/source v0.0.0 diff --git a/gen/CHANGELOG.md b/gen/CHANGELOG.md index 90810235..796d391e 100644 --- a/gen/CHANGELOG.md +++ b/gen/CHANGELOG.md @@ -8,7 +8,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [0.1.0] - 2026-06-13 diff --git a/gen/eject_test.go b/gen/eject_test.go index 90ba21c1..7aa5a557 100644 --- a/gen/eject_test.go +++ b/gen/eject_test.go @@ -175,7 +175,7 @@ func TestEject_CompilesAndWires(t *testing.T) { } tmp := t.TempDir() - gomod := "module genout\n\ngo 1.27.3\n\nrequire github.com/stablekernel/crucible/state v0.0.0\n\nreplace github.com/stablekernel/crucible/state => " + stateAbs + "\n" + gomod := "module genout\n\ngo 1.27.4\n\nrequire github.com/stablekernel/crucible/state v0.0.0\n\nreplace github.com/stablekernel/crucible/state => " + stateAbs + "\n" writeFile(t, filepath.Join(tmp, "go.mod"), gomod) writeFile(t, filepath.Join(tmp, "gen.go"), string(src)) diff --git a/gen/go.mod b/gen/go.mod index f569383e..fe53f6cf 100644 --- a/gen/go.mod +++ b/gen/go.mod @@ -1,5 +1,5 @@ module github.com/stablekernel/crucible/gen -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/state v1.0.0 diff --git a/go.work b/go.work index 4b2b7fea..2d0aeb4a 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.27.3 +go 1.27.4 // Sink and source modules with a non-stdlib dependency are intentionally NOT // listed here: the SDK-backed sink destinations (the AWS family, redis, nats, diff --git a/magefiles/go.mod b/magefiles/go.mod index ee4d7ec2..6ff0e482 100644 --- a/magefiles/go.mod +++ b/magefiles/go.mod @@ -1,5 +1,5 @@ module github.com/stablekernel/crucible/magefiles -go 1.27.3 +go 1.27.4 require github.com/magefile/mage v1.17.2 diff --git a/sink/CHANGELOG.md b/sink/CHANGELOG.md index c5cff260..104ec618 100644 --- a/sink/CHANGELOG.md +++ b/sink/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/bridge/CHANGELOG.md b/sink/bridge/CHANGELOG.md index da7146af..c336c5a4 100644 --- a/sink/bridge/CHANGELOG.md +++ b/sink/bridge/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/bridge/go.mod b/sink/bridge/go.mod index f82c652c..ae99ba19 100644 --- a/sink/bridge/go.mod +++ b/sink/bridge/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/bridge -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/cloudwatch/CHANGELOG.md b/sink/cloudwatch/CHANGELOG.md index 1ae44e30..4c7350c5 100644 --- a/sink/cloudwatch/CHANGELOG.md +++ b/sink/cloudwatch/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/cloudwatch/go.mod b/sink/cloudwatch/go.mod index 4d78060f..5b8680a6 100644 --- a/sink/cloudwatch/go.mod +++ b/sink/cloudwatch/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/cloudwatch -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/sink => ../ diff --git a/sink/dynamo/CHANGELOG.md b/sink/dynamo/CHANGELOG.md index 7768c44d..b6bbeb88 100644 --- a/sink/dynamo/CHANGELOG.md +++ b/sink/dynamo/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/dynamo/go.mod b/sink/dynamo/go.mod index 585d19ca..4f8e8f1f 100644 --- a/sink/dynamo/go.mod +++ b/sink/dynamo/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/dynamo -go 1.27.3 +go 1.27.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.9 diff --git a/sink/eventbridge/CHANGELOG.md b/sink/eventbridge/CHANGELOG.md index 60525647..dfe30795 100644 --- a/sink/eventbridge/CHANGELOG.md +++ b/sink/eventbridge/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/eventbridge/go.mod b/sink/eventbridge/go.mod index 45680836..c2849578 100644 --- a/sink/eventbridge/go.mod +++ b/sink/eventbridge/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/eventbridge -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/sink => ../ diff --git a/sink/file/CHANGELOG.md b/sink/file/CHANGELOG.md index bed5cd47..35356c9e 100644 --- a/sink/file/CHANGELOG.md +++ b/sink/file/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/file/go.mod b/sink/file/go.mod index 97618795..4a638d22 100644 --- a/sink/file/go.mod +++ b/sink/file/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/file -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/sink => ../ diff --git a/sink/firehose/CHANGELOG.md b/sink/firehose/CHANGELOG.md index 2ad55c03..bae3b946 100644 --- a/sink/firehose/CHANGELOG.md +++ b/sink/firehose/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/firehose/go.mod b/sink/firehose/go.mod index 533f11b5..80752dd0 100644 --- a/sink/firehose/go.mod +++ b/sink/firehose/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/firehose -go 1.27.3 +go 1.27.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.9 diff --git a/sink/gcppubsub/CHANGELOG.md b/sink/gcppubsub/CHANGELOG.md index cff9e524..4492232c 100644 --- a/sink/gcppubsub/CHANGELOG.md +++ b/sink/gcppubsub/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/gcppubsub/go.mod b/sink/gcppubsub/go.mod index 7ab976b9..0bdd21a2 100644 --- a/sink/gcppubsub/go.mod +++ b/sink/gcppubsub/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/gcppubsub -go 1.27.3 +go 1.27.4 require ( cloud.google.com/go/pubsub/v2 v2.6.0 diff --git a/sink/go.mod b/sink/go.mod index ca8187f9..292d08d3 100644 --- a/sink/go.mod +++ b/sink/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/telemetry v0.0.0 diff --git a/sink/http/CHANGELOG.md b/sink/http/CHANGELOG.md index 23a40986..a36974ca 100644 --- a/sink/http/CHANGELOG.md +++ b/sink/http/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/http/go.mod b/sink/http/go.mod index c0208a27..5d6b8968 100644 --- a/sink/http/go.mod +++ b/sink/http/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/http -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/kafka/CHANGELOG.md b/sink/kafka/CHANGELOG.md index 4a52a399..ce305334 100644 --- a/sink/kafka/CHANGELOG.md +++ b/sink/kafka/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/kafka/go.mod b/sink/kafka/go.mod index 2db4830e..17aee146 100644 --- a/sink/kafka/go.mod +++ b/sink/kafka/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/kafka -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/kinesis/CHANGELOG.md b/sink/kinesis/CHANGELOG.md index 88d6bb07..8f405fff 100644 --- a/sink/kinesis/CHANGELOG.md +++ b/sink/kinesis/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/kinesis/go.mod b/sink/kinesis/go.mod index beffe1bf..7d27215e 100644 --- a/sink/kinesis/go.mod +++ b/sink/kinesis/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/kinesis -go 1.27.3 +go 1.27.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.9 diff --git a/sink/nats/CHANGELOG.md b/sink/nats/CHANGELOG.md index ebb06378..564ae464 100644 --- a/sink/nats/CHANGELOG.md +++ b/sink/nats/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/nats/go.mod b/sink/nats/go.mod index 0ae915ae..834b60c7 100644 --- a/sink/nats/go.mod +++ b/sink/nats/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/nats -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/sink => ../ diff --git a/sink/otel/CHANGELOG.md b/sink/otel/CHANGELOG.md index 79294627..b32a0ba8 100644 --- a/sink/otel/CHANGELOG.md +++ b/sink/otel/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/otel/go.mod b/sink/otel/go.mod index d56d7797..64188215 100644 --- a/sink/otel/go.mod +++ b/sink/otel/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/otel -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/prometheus/CHANGELOG.md b/sink/prometheus/CHANGELOG.md index d68b150c..fd923f91 100644 --- a/sink/prometheus/CHANGELOG.md +++ b/sink/prometheus/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/prometheus/go.mod b/sink/prometheus/go.mod index c363b630..415b0f6d 100644 --- a/sink/prometheus/go.mod +++ b/sink/prometheus/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/prometheus -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/redis/CHANGELOG.md b/sink/redis/CHANGELOG.md index 2bde1b6e..df795e6c 100644 --- a/sink/redis/CHANGELOG.md +++ b/sink/redis/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/redis/go.mod b/sink/redis/go.mod index b7c84fcf..d6c00b83 100644 --- a/sink/redis/go.mod +++ b/sink/redis/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/redis -go 1.27.3 +go 1.27.4 require ( github.com/redis/go-redis/v9 v9.20.0 diff --git a/sink/s3/CHANGELOG.md b/sink/s3/CHANGELOG.md index 03c2bb8c..eb9c9ea8 100644 --- a/sink/s3/CHANGELOG.md +++ b/sink/s3/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/s3/go.mod b/sink/s3/go.mod index f48adeaf..6d65b914 100644 --- a/sink/s3/go.mod +++ b/sink/s3/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/s3 -go 1.27.3 +go 1.27.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.9 diff --git a/sink/slog/CHANGELOG.md b/sink/slog/CHANGELOG.md index f04ebb0e..ed667361 100644 --- a/sink/slog/CHANGELOG.md +++ b/sink/slog/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/slog/go.mod b/sink/slog/go.mod index 51db8147..10f3b0e3 100644 --- a/sink/slog/go.mod +++ b/sink/slog/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/slog -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/sns/CHANGELOG.md b/sink/sns/CHANGELOG.md index 4ed78dd4..abc50e91 100644 --- a/sink/sns/CHANGELOG.md +++ b/sink/sns/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/sns/go.mod b/sink/sns/go.mod index c054a378..f83373ce 100644 --- a/sink/sns/go.mod +++ b/sink/sns/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/sns -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/sink => ../ diff --git a/sink/sql/CHANGELOG.md b/sink/sql/CHANGELOG.md index bfa6ce74..e2a7953e 100644 --- a/sink/sql/CHANGELOG.md +++ b/sink/sql/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/sql/go.mod b/sink/sql/go.mod index 6b4327e7..55df2fc4 100644 --- a/sink/sql/go.mod +++ b/sink/sql/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/sql -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/sink v0.0.0 diff --git a/sink/sqs/CHANGELOG.md b/sink/sqs/CHANGELOG.md index abb4650b..3f61e8c0 100644 --- a/sink/sqs/CHANGELOG.md +++ b/sink/sqs/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/sqs/go.mod b/sink/sqs/go.mod index 23ef92e4..b01004c8 100644 --- a/sink/sqs/go.mod +++ b/sink/sqs/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/sqs -go 1.27.3 +go 1.27.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.9 diff --git a/sink/statsd/CHANGELOG.md b/sink/statsd/CHANGELOG.md index b1513695..5e06c8e6 100644 --- a/sink/statsd/CHANGELOG.md +++ b/sink/statsd/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/statsd/go.mod b/sink/statsd/go.mod index 46e45e2e..e7b979f4 100644 --- a/sink/statsd/go.mod +++ b/sink/statsd/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/statsd -go 1.27.3 +go 1.27.4 require ( github.com/DataDog/datadog-go/v5 v5.8.3 diff --git a/sink/timestream/CHANGELOG.md b/sink/timestream/CHANGELOG.md index 44c420b7..1e0459b3 100644 --- a/sink/timestream/CHANGELOG.md +++ b/sink/timestream/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/sink/timestream/go.mod b/sink/timestream/go.mod index cb8c4e24..ac404f4e 100644 --- a/sink/timestream/go.mod +++ b/sink/timestream/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/sink/timestream -go 1.27.3 +go 1.27.4 require ( github.com/aws/aws-sdk-go-v2 v1.41.9 diff --git a/source/CHANGELOG.md b/source/CHANGELOG.md index 1aad3706..7e7ce9c1 100644 --- a/source/CHANGELOG.md +++ b/source/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [1.0.1] — 2026-08-23 diff --git a/source/cloudevents/go.mod b/source/cloudevents/go.mod index 560adb72..488d5ad0 100644 --- a/source/cloudevents/go.mod +++ b/source/cloudevents/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/source/cloudevents -go 1.27.3 +go 1.27.4 require ( github.com/cloudevents/sdk-go/v2 v2.16.2 diff --git a/source/go.mod b/source/go.mod index c4bc6673..f9404600 100644 --- a/source/go.mod +++ b/source/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/source -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/telemetry v0.0.0 diff --git a/source/jetstream/go.mod b/source/jetstream/go.mod index db14c83a..a0eed408 100644 --- a/source/jetstream/go.mod +++ b/source/jetstream/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/source/jetstream -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/source => ../ diff --git a/source/kafka/CHANGELOG.md b/source/kafka/CHANGELOG.md index 8f61eb30..41b45ee3 100644 --- a/source/kafka/CHANGELOG.md +++ b/source/kafka/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [1.0.1] — 2026-08-23 diff --git a/source/kafka/go.mod b/source/kafka/go.mod index a70da433..0720fa93 100644 --- a/source/kafka/go.mod +++ b/source/kafka/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/source/kafka -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/source v0.0.0 diff --git a/source/kafka/integration/go.mod b/source/kafka/integration/go.mod index 093e3b47..14674222 100644 --- a/source/kafka/integration/go.mod +++ b/source/kafka/integration/go.mod @@ -7,7 +7,7 @@ module github.com/stablekernel/crucible/source/kafka/integration // `go test -tags integration ./...` from this directory (Docker required; // tests skip cleanly when a daemon is unreachable). -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/source v0.0.0 diff --git a/source/redis/go.mod b/source/redis/go.mod index c6321d40..520006ea 100644 --- a/source/redis/go.mod +++ b/source/redis/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/source/redis -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/source => ../ diff --git a/source/statemachine/CHANGELOG.md b/source/statemachine/CHANGELOG.md index 7d8f79ed..ea7a4305 100644 --- a/source/statemachine/CHANGELOG.md +++ b/source/statemachine/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/source/statemachine/go.mod b/source/statemachine/go.mod index 5af3a2fa..d79f9a47 100644 --- a/source/statemachine/go.mod +++ b/source/statemachine/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/source/statemachine -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/source v0.0.0 diff --git a/state/CHANGELOG.md b/state/CHANGELOG.md index 318610fb..1d55a005 100644 --- a/state/CHANGELOG.md +++ b/state/CHANGELOG.md @@ -32,7 +32,7 @@ counts as an additive (minor) versus breaking (major) change. Use the ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. - **A guard registered with `BindGuard` gets the Fire's context** in `EvalGuard`, where it used to get a background one. An error it returns after that context ended no longer counts as a false verdict: the Fire fails with the context's error at once. Before, diff --git a/state/expr/CHANGELOG.md b/state/expr/CHANGELOG.md index 784c3b2f..43cbbd0e 100644 --- a/state/expr/CHANGELOG.md +++ b/state/expr/CHANGELOG.md @@ -11,7 +11,7 @@ dependency-free, and `expr` depends on `state`, never the reverse. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/state/expr/go.mod b/state/expr/go.mod index 99e3b06c..3044ced1 100644 --- a/state/expr/go.mod +++ b/state/expr/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/state/expr -go 1.27.3 +go 1.27.4 require ( cel.dev/expr v0.25.2 diff --git a/state/go.mod b/state/go.mod index 8ee60942..1f513950 100644 --- a/state/go.mod +++ b/state/go.mod @@ -1,3 +1,3 @@ module github.com/stablekernel/crucible/state -go 1.27.3 +go 1.27.4 diff --git a/telemetry/CHANGELOG.md b/telemetry/CHANGELOG.md index 6d4b071c..1a978248 100644 --- a/telemetry/CHANGELOG.md +++ b/telemetry/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [0.1.0] - 2026-05-30 diff --git a/telemetry/datadog/CHANGELOG.md b/telemetry/datadog/CHANGELOG.md index 1c7aea82..6476899f 100644 --- a/telemetry/datadog/CHANGELOG.md +++ b/telemetry/datadog/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [0.1.0] - 2026-05-30 diff --git a/telemetry/datadog/go.mod b/telemetry/datadog/go.mod index 21dc9a0d..8040bb01 100644 --- a/telemetry/datadog/go.mod +++ b/telemetry/datadog/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/telemetry/datadog -go 1.27.3 +go 1.27.4 require ( github.com/DataDog/datadog-go/v5 v5.8.3 diff --git a/telemetry/go.mod b/telemetry/go.mod index b6464e50..f96de8cc 100644 --- a/telemetry/go.mod +++ b/telemetry/go.mod @@ -1,3 +1,3 @@ module github.com/stablekernel/crucible/telemetry -go 1.27.3 +go 1.27.4 diff --git a/telemetry/otel/CHANGELOG.md b/telemetry/otel/CHANGELOG.md index d42abcd6..aa9ce0fc 100644 --- a/telemetry/otel/CHANGELOG.md +++ b/telemetry/otel/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ## [0.1.0] - 2026-05-30 diff --git a/telemetry/otel/go.mod b/telemetry/otel/go.mod index 91d27ea0..a4038323 100644 --- a/telemetry/otel/go.mod +++ b/telemetry/otel/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/telemetry/otel -go 1.27.3 +go 1.27.4 require ( github.com/stablekernel/crucible/telemetry v0.0.0 diff --git a/telemetry/slog/README.md b/telemetry/slog/README.md index 7724fa71..2e7e6843 100644 --- a/telemetry/slog/README.md +++ b/telemetry/slog/README.md @@ -4,7 +4,7 @@ A standard-library `log/slog` adapter for [`crucible/telemetry`](../README.md). Emits spans and metric instruments as structured log records, with **zero external dependencies**. -Requires Go 1.27.3. +Requires Go 1.27.4. Import path: `github.com/stablekernel/crucible/telemetry/slog` diff --git a/telemetry/slog/go.mod b/telemetry/slog/go.mod index 5bf5b560..017e5cfe 100644 --- a/telemetry/slog/go.mod +++ b/telemetry/slog/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/telemetry/slog -go 1.27.3 +go 1.27.4 require github.com/stablekernel/crucible/telemetry v0.0.0 diff --git a/tools/docsgen/go.mod b/tools/docsgen/go.mod index 662a6c7c..a1fbcd08 100644 --- a/tools/docsgen/go.mod +++ b/tools/docsgen/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/tools/docsgen -go 1.27.3 +go 1.27.4 // The generator builds real example machines and renders their diagrams, and // shells out to gomarkdoc for the API reference. It depends on the local state diff --git a/transport/README.md b/transport/README.md index 25105f94..ad1b9fd2 100644 --- a/transport/README.md +++ b/transport/README.md @@ -9,7 +9,7 @@ time-travel operations between nodes over real gRPC (HTTP/2). Import path: `github.com/stablekernel/crucible/transport` -Requires Go 1.27.3. +Requires Go 1.27.4. ## What it is diff --git a/transport/go.mod b/transport/go.mod index 86c30977..ad7991b1 100644 --- a/transport/go.mod +++ b/transport/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/transport -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/state => ../state diff --git a/wasm/CHANGELOG.md b/wasm/CHANGELOG.md index 04bd105b..83023775 100644 --- a/wasm/CHANGELOG.md +++ b/wasm/CHANGELOG.md @@ -9,7 +9,7 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ### Changed -- Requires Go 1.27.3. +- Requires Go 1.27.4. ### Added diff --git a/wasm/go.mod b/wasm/go.mod index 6e6d0bec..f509c763 100644 --- a/wasm/go.mod +++ b/wasm/go.mod @@ -1,6 +1,6 @@ module github.com/stablekernel/crucible/wasm -go 1.27.3 +go 1.27.4 replace github.com/stablekernel/crucible/state => ../state From b9e60743a9ea835c95b649a8cb4a189d529b0082 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Wed, 11 Nov 2026 10:22:32 -0700 Subject: [PATCH 09/10] fix(durable): drain a step still recording, and keep deferred checkpoints Drain returned while a step that held the lock when the Handle failed was still in its Append, so Recover could load a journal without that step and the next step conflicted. Drain now waits for the step lock before it waits for dispatch, and a step recorded as another goroutine failed the Handle returns ErrHandleFailed saying it was recorded, on Fire, Tick, RunService and DeliverToActor. The docs say to give Drain a context of its own and not to Recover until it returns nil. A checkpoint that waited on an earlier step's dispatch was written with the context of whichever step finished last, so that caller's cancel dropped it and the error went to that caller. Now that caller isn't told and the next step writes the checkpoint. A retry the Store refuses as not advancing counts as written. Any other checkpoint failure is still returned, and only the next step the policy makes due tries again. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 7 +- durable/CHANGELOG.md | 18 +- durable/README.md | 7 +- durable/actor.go | 6 +- durable/handle_lock_test.go | 403 ++++++++++++++++++++++ durable/runner.go | 97 ++++-- durable/service.go | 6 +- 7 files changed, 506 insertions(+), 38 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index c6378a26..80498da5 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -142,8 +142,11 @@ takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an effect handler that returns an error or panics. Before `Recover`, call `Handle.Drain`, -which waits until no effect of the failed Handle is still being sent, so the recovered -Handle can't send one twice. To move a healthy Handle's instance to a new Handle, stop +which waits until no step of the failed Handle is still recording and no effect is still +being sent, so the recovered Handle loads every recorded step and can't send an effect +twice. Give `Drain` a context of its own, not the failed call's, and don't `Recover` +until it returns nil: an error means it didn't finish draining. A step that was recording when another goroutine failed the Handle returns +`ErrHandleFailed` saying it was recorded; `Recover` sends any of its effects. To move a healthy Handle's instance to a new Handle, stop calling the old one and let its calls return first: `Drain` alone doesn't cover a step that is still recording. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 7f15d978..5bd2a790 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -29,9 +29,10 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **`Handle.View`** reads the instance under the Handle's step lock. -- **`Handle.Drain`** waits until no effect of the Handle is still being sent. Call it - before `Recover` after `ErrHandleFailed`. On a healthy Handle it covers only dispatches - already under way. +- **`Handle.Drain`** waits until no step of the Handle is still recording and no effect + is still being sent. Call it before `Recover` after `ErrHandleFailed`, with a context + of its own, and don't `Recover` until it returns nil. On a healthy Handle it covers + only work already under way. - **`ErrReentrantStep`** reports a call back into a Handle that carries the context one of its steps passed to a callback: a service's context, or the `Ctx` a guard, action @@ -64,8 +65,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. effect stays unmarked in the journal, so no later checkpoint compacts it away, and `Recover` dispatches it again. - **A step that may not have been recorded fails the Handle.** That call and every later - one return `ErrHandleFailed`, wrapping the cause. Call `Drain` before `Recover`, so an - effect another goroutine is still sending isn't sent twice. `Recover` a new Handle + one return `ErrHandleFailed`, wrapping the cause. Call `Drain` before `Recover`, so a + step another goroutine is still recording is loaded, and an effect it is still sending + isn't sent twice. `Recover` a new Handle with the same options to continue. A service whose result wasn't recorded is in flight again, so `RunService` runs it a second time. An `InvokeActor` actor restarts from its initial state, and a `Spawn` actor doesn't return. If the cause is `ErrFenced`, a newer owner @@ -105,6 +107,12 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A failed checkpoint no longer makes the next step conflict.** `Tick`, `RunService` and `DeliverToActor` now advance the next step as soon as the append lands. +- **A deferred checkpoint survives an unrelated caller's cancel.** A checkpoint that + waited for an earlier step's effects to dispatch, and was canceled by the context of + the caller whose step finished last, isn't reported to that caller, since its own + step succeeded, and the next step writes it. A retry the Store refuses as not + advancing means the first write landed, and counts as written. Any other checkpoint + failure is returned, and only the next step the policy makes due tries again. - **The next step survives a checkpoint that compacts tick barriers.** A checkpoint now records the step that follows it, in a `nextStep` field of its envelope. Recovery used the Trace count, which runs low past a tick barrier, so the next Fire could reuse a diff --git a/durable/README.md b/durable/README.md index 8539fb73..ee119301 100644 --- a/durable/README.md +++ b/durable/README.md @@ -153,8 +153,11 @@ takes the lock; `Instance()` is outside it. A step that may not have been recorded fails the Handle with `ErrHandleFailed`. So does an effect handler that returns an error or panics. Before `Recover`, call `Handle.Drain`, -which waits until no effect of the failed Handle is still being sent, so the recovered -Handle can't send one twice. To move a healthy Handle's instance to a new Handle, stop +which waits until no step of the failed Handle is still recording and no effect is still +being sent, so the recovered Handle loads every recorded step and can't send an effect +twice. Give `Drain` a context of its own, not the failed call's, and don't `Recover` +until it returns nil: an error means it didn't finish draining. A step that was recording when another goroutine failed the Handle returns +`ErrHandleFailed` saying it was recorded; `Recover` sends any of its effects. To move a healthy Handle's instance to a new Handle, stop calling the old one and let its calls return first: `Drain` alone doesn't cover a step that is still recording. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. diff --git a/durable/actor.go b/durable/actor.go index 0af4b7d5..496a2878 100644 --- a/durable/actor.go +++ b/durable/actor.go @@ -101,8 +101,10 @@ type actorMessagePayload struct { // It returns false for a ref that names no live actor (already settled, stopped, or // never spawned), and false with the error when the Handle had already failed or ctx // has ended. If the actor received the event but the delivery couldn't be recorded or -// appended, it returns true and ErrHandleFailed: the Handle has failed. A failed -// checkpoint returns true and that error, and the Handle carries on. The kernel +// appended, it returns true and ErrHandleFailed: the Handle has failed. If another +// goroutine failed the Handle while this step was recording, it returns true and +// ErrHandleFailed saying the step was recorded. A failed checkpoint returns true and +// that error, and the Handle carries on. The kernel // transitions the delivery drove do not themselves fail. When a delivery wasn't // recorded, on a recovered Handle an InvokeActor actor restarts from its initial state, // and a Spawn actor doesn't return. Resending a recorded delivery repeats diff --git a/durable/handle_lock_test.go b/durable/handle_lock_test.go index ae81f839..484e8d47 100644 --- a/durable/handle_lock_test.go +++ b/durable/handle_lock_test.go @@ -3,6 +3,8 @@ package durable_test import ( "context" "errors" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -722,3 +724,404 @@ func TestHandle_CallbackCallsBackWithItsCtx_IsRefused(t *testing.T) { // reentryKey is the key the middleware case derives its context with. type reentryKey struct{} + +// TestHandle_Drain_WaitsForAStepStillRecording proves Drain doesn't return while a +// step that held the lock when the Handle failed is still in its Append: Recover then +// loads that step, and its caller learns the step was recorded. +func TestHandle_Drain_WaitsForAStepStillRecording(t *testing.T) { + ctx := context.Background() + m := effectMachine() + store := &blockingMemStore{MemStore: durable.NewMemStore(), entered: make(chan struct{}), release: make(chan struct{})} + handlerIn, failNow := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits, then fails + close(handlerIn) + <-failNow + return errors.New("mail server down") + } + return nil + } + opt := durable.WithEffectHandler[string, string, *runCtx](handler) + h, err := durable.NewRunner(m, store, opt).Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(ctx, "go"); firstDone <- fireErr }() + <-handlerIn + store.armed.Store(true) + secondDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(ctx, "bill"); secondDone <- fireErr }() + <-store.entered // step 1 holds the lock, in its Append + close(failNow) + if got := <-firstDone; !errors.Is(got, durable.ErrHandleFailed) { + t.Fatalf("Fire(go) = %v, want ErrHandleFailed", got) + } + drained := make(chan error, 1) + go func() { drained <- h.Drain(ctx) }() + select { + case drainErr := <-drained: + t.Fatalf("Drain returned %v while step 1 was still in its Append", drainErr) + case <-time.After(100 * time.Millisecond): + } + close(store.release) + if drainErr := <-drained; drainErr != nil { + t.Fatalf("Drain: %v", drainErr) + } + got := <-secondDone + if !errors.Is(got, durable.ErrHandleFailed) || !strings.Contains(got.Error(), "step 1 was recorded") { + t.Fatalf("Fire(bill) = %v, want ErrHandleFailed saying step 1 was recorded", got) + } + r, err := durable.Recover(ctx, m, store.MemStore, "e-1", opt) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if cur := r.Instance().Current(); cur != "billed" { + t.Fatalf("recovered state %q, want billed: Recover missed step 1", cur) + } + if _, err := r.Fire(ctx, "finish"); err != nil { + t.Fatalf("Fire(finish) on the recovered Handle: %v", err) + } +} + +// checkpointSpy is a MemStore that records the steps it checkpointed through after +// Start's initial checkpoint and refuses a checkpoint whose context has ended. Once +// armed, it fails the next checkpoint (diskFull first, ahead of the context check), or +// writes it and then reports it canceled (landThenFail, also ahead of the check). +type checkpointSpy struct { + *durable.MemStore + failNext atomic.Bool + diskFull atomic.Bool + landThenFail atomic.Bool + mu sync.Mutex + through []int +} + +func (s *checkpointSpy) Checkpoint(ctx context.Context, id durable.InstanceID, snapshot []byte, throughStep int, opts ...durable.CheckpointOption) error { + if s.diskFull.CompareAndSwap(true, false) { + return errors.New("disk full") + } + if throughStep >= 0 && s.landThenFail.CompareAndSwap(true, false) { + if err := s.MemStore.Checkpoint(context.Background(), id, snapshot, throughStep, opts...); err != nil { + return err + } + s.record(throughStep) + return context.Canceled + } + if err := ctx.Err(); err != nil { + return err + } + if s.failNext.CompareAndSwap(true, false) { + return errors.New("disk full") + } + if err := s.MemStore.Checkpoint(ctx, id, snapshot, throughStep, opts...); err != nil { + return err + } + if throughStep >= 0 { // not Start's initial checkpoint + s.record(throughStep) + } + return nil +} + +func (s *checkpointSpy) record(step int) { + s.mu.Lock() + defer s.mu.Unlock() + s.through = append(s.through, step) +} + +func (s *checkpointSpy) steps() []int { + s.mu.Lock() + defer s.mu.Unlock() + return append([]int(nil), s.through...) +} + +// TestHandle_DeferredCheckpoint_SurvivesTheFinishingCallersCancel proves a checkpoint +// deferred to the step that finishes dispatching last isn't dropped when that step's +// caller cancels, and that caller isn't told: it stays pending, and the next step +// writes it. +func TestHandle_DeferredCheckpoint_SurvivesTheFinishingCallersCancel(t *testing.T) { + ctx := context.Background() + store := &checkpointSpy{MemStore: durable.NewMemStore()} + handlerIn, release := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits + close(handlerIn) + <-release + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler), + durable.WithCheckpointEvery[string, string, *runCtx](2)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstCtx, cancel := context.WithCancel(ctx) + defer cancel() + firstDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(firstCtx, "go"); firstDone <- fireErr }() + <-handlerIn + if _, err := h.Fire(ctx, "bill"); err != nil { // step 1's checkpoint is due, and waits on step 0 + t.Fatalf("Fire(bill): %v", err) + } + if got := store.steps(); len(got) != 0 { + t.Fatalf("checkpointed through %v before step 0 dispatched, want none", got) + } + cancel() + close(release) + if err := <-firstDone; err != nil { + t.Fatalf("Fire(go) = %v, want nil: step 1's checkpoint isn't its caller's", err) + } + if got := store.steps(); len(got) != 0 { + t.Fatalf("checkpointed through %v on a canceled context, want none yet", got) + } + if _, err := h.Fire(ctx, "finish"); err != nil { + t.Fatalf("Fire(finish): %v", err) + } + if got := store.steps(); len(got) != 1 || got[0] != 1 { + t.Fatalf("checkpointed through %v, want [1]: the deferred checkpoint was dropped", got) + } +} + +// TestHandle_FailedCheckpoint_WaitsForTheNextDueStep proves a checkpoint the Store +// refuses isn't retried, or reported, on a step the policy didn't make due: the next +// due step checkpoints instead. +func TestHandle_FailedCheckpoint_WaitsForTheNextDueStep(t *testing.T) { + ctx := context.Background() + store := &checkpointSpy{MemStore: durable.NewMemStore()} + loop := state.ForgeFor[*runCtx]("loop"). + State("a").State("b").Initial("a"). + Transition("a").On("t").GoTo("b"). + Transition("b").On("t").GoTo("a"). + Quench() + h, err := durable.NewRunner(loop, store, durable.WithCheckpointEvery[string, string, *runCtx](2)). + Start(ctx, "l-1", &runCtx{}, state.WithInitialState("a")) + if err != nil { + t.Fatalf("Start: %v", err) + } + fire := func(step int) error { _, err := h.Fire(ctx, "t"); return err } + if err := fire(0); err != nil { + t.Fatalf("step 0: %v", err) + } + store.failNext.Store(true) + if err := fire(1); err == nil { // due + t.Fatal("step 1 = nil, want its checkpoint's error") + } + if err := fire(2); err != nil { // not due + t.Fatalf("step 2 = %v, want nil: it didn't make step 1's checkpoint due", err) + } + if err := fire(3); err != nil { // due + t.Fatalf("step 3: %v", err) + } + if got := store.steps(); len(got) != 1 || got[0] != 3 { + t.Fatalf("checkpointed through %v, want [3]: only the next due step checkpoints", got) + } +} + +// TestHandle_StepRecordedAsTheHandleFails_SaysSo proves each step method that was in +// its Append when another goroutine failed the Handle returns ErrHandleFailed saying +// the step was recorded, a step with no domain effects included, and that Drain then +// Recover load it. +func TestHandle_StepRecordedAsTheHandleFails_SaysSo(t *testing.T) { + type handle = *durable.Handle[string, string, svcCtx] + type probe = *state.Builder[string, string, svcCtx] + clock := state.NewFakeClock(epoch) + var runs int64 + cases := []struct { + name string + build func(probe) probe + opts []durable.Option[string, string, svcCtx] + step func(ctx context.Context, h handle) error + want string + }{ + { + name: "Fire with no effects", + build: func(b probe) probe { + return b.State("hub").State("end").Final().Transition("hub").On("poke").GoTo("end") + }, + step: func(ctx context.Context, h handle) error { _, err := h.Fire(ctx, "poke"); return err }, + }, + { + name: "Tick", + build: func(b probe) probe { + return b.State("hub").State("end").Final().Transition("hub").After(5 * time.Second).On("elapsed").GoTo("end") + }, + opts: []durable.Option[string, string, svcCtx]{durable.WithRunnerClock[string, string, svcCtx](clock)}, + step: func(ctx context.Context, h handle) error { + clock.Advance(6 * time.Second) + _, err := h.Tick(ctx) + return err + }, + }, + { + name: "RunService", + build: func(b probe) probe { + return b.Service("fetch", counterService(&runs)). + State("hub").Invoke("fetch", state.WithInvokeOnDone("ok")).State("end").Final(). + Transition("hub").On("ok").GoTo("end") + }, + opts: []durable.Option[string, string, svcCtx]{durable.WithServiceRegistry[string, string, svcCtx]( + state.NewRegistry[svcCtx]().Service("fetch", counterService(&runs)))}, + step: func(ctx context.Context, h handle) error { + _, _, err := h.RunService(ctx, state.InvokeID("recorded", "hub", 0)) + return err + }, + }, + { + name: "DeliverToActor", + build: func(b probe) probe { + return b.Actor("spawnChild"). + State("hub").InvokeActor("spawnChild", state.WithInvokeOnDone("childDone")).State("end").Final(). + Transition("hub").On("childDone").GoTo("end") + }, + opts: []durable.Option[string, string, svcCtx]{durable.WithActorPalette[string, string, svcCtx](counterPalette(&runs))}, + step: func(ctx context.Context, h handle) error { + ref, ok := h.ActorRef(state.ActorID("recorded", "hub", 0)) + if !ok { + return errors.New("no actor ref for the spawned child") + } + _, err := h.DeliverToActor(ctx, ref, "finish") + return err + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + store := &blockingMemStore{MemStore: durable.NewMemStore(), entered: make(chan struct{}), release: make(chan struct{})} + handlerIn, failNow := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits, then fails + close(handlerIn) + <-failNow + return errors.New("mail server down") + } + return nil + } + machine := tc.build(state.ForgeFor[svcCtx]("recorded"). + Action("notify", func(state.ActionCtx[svcCtx]) (state.Effect, error) { + return notifyEffect{To: "ops@example.com"}, nil + }). + State("idle").Initial("idle"). + Transition("idle").On("go").GoTo("hub").Do("notify")). + Quench() + opts := append([]durable.Option[string, string, svcCtx]{durable.WithEffectHandler[string, string, svcCtx](handler)}, tc.opts...) + h, err := durable.NewRunner(machine, store, opts...).Start(ctx, "r-1", svcCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(ctx, "go"); firstDone <- fireErr }() + <-handlerIn + store.armed.Store(true) + stepDone := make(chan error, 1) + go func() { stepDone <- tc.step(ctx, h) }() + <-store.entered // step 1 holds the lock, in its Append + close(failNow) + if got := <-firstDone; !errors.Is(got, durable.ErrHandleFailed) { + t.Fatalf("Fire(go) = %v, want ErrHandleFailed", got) + } + close(store.release) + got := <-stepDone + if !errors.Is(got, durable.ErrHandleFailed) || !strings.Contains(got.Error(), "step 1 was recorded") { + t.Fatalf("%s = %v, want ErrHandleFailed saying step 1 was recorded", tc.name, got) + } + if drainErr := h.Drain(ctx); drainErr != nil { + t.Fatalf("Drain: %v", drainErr) + } + r, err := durable.Recover(ctx, machine, store.MemStore, "r-1", opts...) + if err != nil { + t.Fatalf("Recover: %v", err) + } + if cur := r.Instance().Current(); cur != "end" { + t.Fatalf("recovered state %q, want end: Recover missed step 1", cur) + } + }) + } +} + +// TestHandle_CheckpointThatLandedDespiteAnError_IsNotRetriedForever proves a deferred +// checkpoint the Store wrote but reported canceled doesn't fail the next step: the +// retry the Store refuses as not advancing counts as written. +func TestHandle_CheckpointThatLandedDespiteAnError_IsNotRetriedForever(t *testing.T) { + ctx := context.Background() + store := &checkpointSpy{MemStore: durable.NewMemStore()} + handlerIn, release := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits + close(handlerIn) + <-release + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler), + durable.WithCheckpointEvery[string, string, *runCtx](2)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstCtx, cancel := context.WithCancel(ctx) + defer cancel() + firstDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(firstCtx, "go"); firstDone <- fireErr }() + <-handlerIn + if _, err := h.Fire(ctx, "bill"); err != nil { // step 1's checkpoint is due, and waits on step 0 + t.Fatalf("Fire(bill): %v", err) + } + cancel() + store.landThenFail.Store(true) + close(release) + if err := <-firstDone; err != nil { + t.Fatalf("Fire(go) = %v, want nil", err) + } + if _, err := h.Fire(ctx, "finish"); err != nil { + t.Fatalf("Fire(finish) = %v, want nil: the checkpoint at step 1 had landed", err) + } + if got := store.steps(); len(got) != 1 || got[0] != 1 { + t.Fatalf("checkpointed through %v, want [1]", got) + } +} + +// TestHandle_DeferredCheckpointStoreError_OnAnEndedContext_IsReported proves the +// finishing caller still hears a deferred checkpoint's Store error when its context +// has ended: only the cancellation itself is kept from it. +func TestHandle_DeferredCheckpointStoreError_OnAnEndedContext_IsReported(t *testing.T) { + ctx := context.Background() + store := &checkpointSpy{MemStore: durable.NewMemStore()} + handlerIn, release := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits + close(handlerIn) + <-release + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler), + durable.WithCheckpointEvery[string, string, *runCtx](2)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstCtx, cancel := context.WithCancel(ctx) + defer cancel() + firstDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(firstCtx, "go"); firstDone <- fireErr }() + <-handlerIn + if _, err := h.Fire(ctx, "bill"); err != nil { + t.Fatalf("Fire(bill): %v", err) + } + cancel() + store.diskFull.Store(true) + close(release) + if err := <-firstDone; err == nil || !strings.Contains(err.Error(), "disk full") { + t.Fatalf("Fire(go) = %v, want the Store's disk-full error", err) + } +} diff --git a/durable/runner.go b/durable/runner.go index 836130c1..0b1b9fdf 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -104,8 +104,8 @@ func NewRunner[S comparable, E comparable, C any](m *state.Machine[S, E, C], st // A step that applied to the kernel but may not have been recorded leaves the Handle out // of step with the journal. That call and every later one return ErrHandleFailed, // wrapping the cause, which can be a context that ended during the Store write. Before -// recovering, call Drain: a step on another goroutine may still be sending an effect, -// and a recovered Handle would send it again. Recovering a healthy Handle's instance +// recovering, call Drain: a step on another goroutine may still be recording, which a +// recovered Handle wouldn't load, or sending an effect, which it would send again. Recovering a healthy Handle's instance // instead needs this Handle's calls to have returned first. Recover a new Handle with the same // options to continue. A service // whose result wasn't recorded is in flight again, so RunService runs it a second time. @@ -158,6 +158,8 @@ type outbox struct { type dueCheckpoint struct { step int snapshot []byte + retry bool // an earlier write failed, so the Store may already hold it + waitDue bool // the Store refused it: wait for a step the policy makes due } // stepKey marks a context passed to host code during a step, so a call from that code @@ -228,21 +230,33 @@ func (h *Handle[S, E, C]) inDispatch(ctx context.Context) bool { return false } -// Drain waits until no domain effect of h is still being dispatched, or ctx ends. -// Call it after ErrHandleFailed and before Recover: a step on another goroutine may -// still be sending an effect it recorded, and a recovered Handle would send it again -// because it isn't marked yet. Once the Handle has failed, no step can start sending, -// so Drain covers every effect. On a healthy Handle it covers only dispatches already -// under way: a step still recording can start sending after Drain returns, so before -// handing the instance to a new Handle, stop driving this one and let its calls return. It returns ctx's -// error if ctx ends first, and ErrReentrantStep, without waiting, when called from -// inside a step or an effect handler of h, whose own work it would wait on forever. +// Drain waits until no step of h is still recording and no domain effect of h is still +// being dispatched, or ctx ends. Call it after ErrHandleFailed and before Recover: a +// step on another goroutine may still be appending its Record, which a recovered +// Handle wouldn't load, or sending an effect it recorded, which a recovered Handle +// would send again because it isn't marked yet. Drain first waits for the step lock, so +// a step that was recording has finished, then for dispatch. Once the Handle has +// failed, no step can start, so Drain covers every step and effect. On a healthy Handle +// it covers only work already under way: a step can start after Drain returns, so +// before handing the instance to a new Handle, stop driving this one and let its calls +// return. It returns ctx's error if ctx ends first, at once if ctx has already ended: +// then nothing is drained, so don't Recover until Drain returns nil. Give it a context +// of its own, not the failed call's. It returns ErrReentrantStep, without waiting, when +// called from inside a step or an effect handler of h, whose own work it would wait on +// forever. // A call from a goroutine an effect handler started, without the handler's context, -// can't be told apart and waits for that handler. +// can't be told apart and waits for that handler; one from a goroutine a service or a +// View callback started, without the step's context, waits for that step. func (h *Handle[S, E, C]) Drain(ctx context.Context) error { if h.inStep(ctx) || h.inDispatch(ctx) { return fmt.Errorf("%w: %q: Drain from inside a step or an effect handler", ErrReentrantStep, h.id) } + // A step that holds the lock may be partway through its Append. Wait it out, so + // Recover loads its Record and any effect it registers is covered below. + if err := h.acquire(ctx); err != nil { + return err + } + h.release() h.out.mu.Lock() idle := h.out.idle h.out.mu.Unlock() @@ -583,7 +597,10 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // checkpoint policy is due — persists a full Snapshot and compacts the tail. A // kernel transition error journals nothing (no step was produced) and is // returned to the caller. A failed checkpoint returns the result and that error: the -// step is recorded and its effects sent, and the Handle carries on. +// step is recorded and its effects sent, the Handle carries on, and the next step the +// policy makes due checkpoints. If another goroutine failed the Handle while this step +// was recording, Fire returns ErrHandleFailed saying the step was recorded; Recover +// sends its effects. // // A state.WithEventData payload reaches the transition's Assign as the Runner's // EventDataDecoder rebuilds it from the payload's JSON form, live and on replay alike, so @@ -658,15 +675,15 @@ func (h *Handle[S, E, C]) fireStep(ctx context.Context, event E, opts []state.Fi return res, nil, nil, h.fail(err) } h.nextStep = rec.followingStep() + h.out.mu.Lock() + defer h.out.mu.Unlock() + // Another goroutine's dispatch may have failed the Handle while this step held the + // lock, and its caller may already be recovering. Leave this step's effects unmarked + // in the journal for that Recover to send, rather than race it. + if err := h.recordedOnFailedLocked(step); err != nil { + return res, nil, nil, err + } if len(des) > 0 { - h.out.mu.Lock() - defer h.out.mu.Unlock() - // Another goroutine's dispatch may have failed the Handle while this step held - // the lock, and its caller may already be recovering. Leave this step's effects - // unmarked in the journal for that Recover to send, rather than race it. - if err := h.refuseLocked(); err != nil { - return res, nil, nil, err - } if len(h.out.dispatching) == 0 { h.out.idle = make(chan struct{}) } @@ -718,7 +735,9 @@ func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []d // timer loop (with a real clock) or a test calls it after advancing a fake clock. // It records one tick barrier — carrying the consumed clock readings and the // count of timers fired — so recovery re-derives the same timers at their -// recorded instants. It returns the FireResults of the timers it fired, in order. +// recorded instants. It returns the FireResults of the timers it fired, in order. If +// another goroutine failed the Handle while this step was recording, it returns them +// with ErrHandleFailed saying the step was recorded. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { if err := h.acquire(ctx); err != nil { return nil, err @@ -767,13 +786,30 @@ func (h *Handle[S, E, C]) persistStep(ctx context.Context, step int, rec *Record h.nextStep = rec.followingStep() h.out.mu.Lock() defer h.out.mu.Unlock() + if err := h.recordedOnFailedLocked(step); err != nil { + return err + } return h.offerCheckpointLocked(ctx, rec) } +// recordedOnFailedLocked reports a step that was recorded while another goroutine failed +// the Handle: ErrHandleFailed, saying the step was recorded and Recover sends any of its +// effects. It returns nil on a healthy Handle. h.out.mu must be held. +func (h *Handle[S, E, C]) recordedOnFailedLocked(step int) error { + if err := h.refuseLocked(); err != nil { + return fmt.Errorf("%w (step %d was recorded; Recover sends any of its effects)", err, step) + } + return nil +} + // offerCheckpointLocked makes rec's checkpoint pending when the policy is due at its // step, then writes the pending checkpoint if no step at or before it is still -// dispatching. Otherwise the step that finishes dispatching last writes it. A failed -// Handle writes none. h.out.mu must be held. +// dispatching. Otherwise the step that finishes dispatching last writes it, with its +// context. When that write fails because the context of a step that didn't make it due +// ended, that caller isn't told, since its own step succeeded, and the next step writes +// it. Any other failure is returned to the caller, and the checkpoint waits for the next +// step the policy makes due, whose newer checkpoint replaces it. A retry the Store +// refuses as not advancing landed the first time, and counts as written. A failed Handle writes none. h.out.mu must be held. func (h *Handle[S, E, C]) offerCheckpointLocked(ctx context.Context, rec *Record) error { if h.out.failed != nil { return nil @@ -782,7 +818,7 @@ func (h *Handle[S, E, C]) offerCheckpointLocked(ctx context.Context, rec *Record h.out.pending = &dueCheckpoint{step: rec.Step, snapshot: rec.Snapshot} } cp := h.out.pending - if cp == nil { + if cp == nil || (cp.waitDue && !h.checkpointDue(rec.Step)) { return nil } for step := range h.out.dispatching { @@ -795,6 +831,17 @@ func (h *Handle[S, E, C]) offerCheckpointLocked(ctx context.Context, rec *Record return nil } if err := h.runner.store.Checkpoint(ctx, h.id, cp.snapshot, cp.step, WithCheckpointFence(h.runner.cfg.fence)); err != nil { + if cp.retry && errors.Is(err, ErrCheckpointNotAdvancing) { + // The failed write landed after all: the Store holds this checkpoint. + h.out.checkpointed = cp.step + return nil + } + cp.retry = true + h.out.pending = cp + if cp.step != rec.Step && ctx.Err() != nil && errors.Is(err, ctx.Err()) { + return nil // not this caller's checkpoint: the next step writes it + } + cp.waitDue = true return fmt.Errorf("durable: checkpointing step %d for %q: %w", cp.step, h.id, err) } h.out.checkpointed = cp.step diff --git a/durable/service.go b/durable/service.go index ac5b7c86..490f8b46 100644 --- a/durable/service.go +++ b/durable/service.go @@ -87,8 +87,10 @@ func (h *Handle[S, E, C]) drainServices() []state.JournalEntry { // names no in-flight service (already settled, stopped, or never started), or false // and the error when the Handle had already failed or ctx has ended. If the service // settled but its outcome couldn't be recorded or appended, it returns the result, -// true and ErrHandleFailed: the Handle has failed. A failed checkpoint returns the -// result, true and that error, and the Handle carries on. A recovered Handle has an +// true and ErrHandleFailed: the Handle has failed. If another goroutine failed the +// Handle while this step was recording, it returns the result, true and ErrHandleFailed +// saying the step was recorded. A failed checkpoint returns the result, true and that +// error, and the Handle carries on. A recovered Handle has an // unrecorded service in flight again, and RunService runs it a second time. func (h *Handle[S, E, C]) RunService(ctx context.Context, id string) (state.FireResult[S], bool, error) { if err := h.acquire(ctx); err != nil { From 0e9d2e755289a5ffcb9812cdb8b933ae6fed6c38 Mon Sep 17 00:00:00 2001 From: Ray Karnes Date: Wed, 11 Nov 2026 12:14:44 -0700 Subject: [PATCH 10/10] fix(durable): fold in the review's notes on Drain and checkpoints The README and overview gave a stale reason for stopping a healthy Handle before handing its instance on: Drain now covers a step still recording, and the reason is that a step can start after Drain returns. The CHANGELOG now says a recorded step on a failed Handle returns ErrHandleFailed, and that a retried deferred checkpoint's failure goes to the step that tried it. Fire and Tick say how a checkpoint failure reaches them. A deferred checkpoint is kept from its finishing caller for any context error the Store reports, not only the one that ended that caller's context. Signed-off-by: Ray Karnes --- docs/src/content/docs/durable/overview.md | 4 +- durable/CHANGELOG.md | 11 +++-- durable/README.md | 4 +- durable/handle_lock_test.go | 51 ++++++++++++++++++++++- durable/runner.go | 25 +++++++---- 5 files changed, 78 insertions(+), 17 deletions(-) diff --git a/docs/src/content/docs/durable/overview.md b/docs/src/content/docs/durable/overview.md index 80498da5..15ea16e8 100644 --- a/docs/src/content/docs/durable/overview.md +++ b/docs/src/content/docs/durable/overview.md @@ -147,8 +147,8 @@ being sent, so the recovered Handle loads every recorded step and can't send an twice. Give `Drain` a context of its own, not the failed call's, and don't `Recover` until it returns nil: an error means it didn't finish draining. A step that was recording when another goroutine failed the Handle returns `ErrHandleFailed` saying it was recorded; `Recover` sends any of its effects. To move a healthy Handle's instance to a new Handle, stop -calling the old one and let its calls return first: `Drain` alone doesn't cover a step -that is still recording. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +calling the old one and let its calls return first: a step can start after `Drain` +returns. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. ## A note on serialized payloads diff --git a/durable/CHANGELOG.md b/durable/CHANGELOG.md index 5bd2a790..08003135 100644 --- a/durable/CHANGELOG.md +++ b/durable/CHANGELOG.md @@ -65,7 +65,9 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. effect stays unmarked in the journal, so no later checkpoint compacts it away, and `Recover` dispatches it again. - **A step that may not have been recorded fails the Handle.** That call and every later - one return `ErrHandleFailed`, wrapping the cause. Call `Drain` before `Recover`, so a + one return `ErrHandleFailed`, wrapping the cause. A step that was recorded while + another goroutine failed the Handle also returns `ErrHandleFailed`, saying the step was + recorded; `Recover` sends any of its effects. Call `Drain` before `Recover`, so a step another goroutine is still recording is loaded, and an effect it is still sending isn't sent twice. `Recover` a new Handle with the same options to continue. A service whose result wasn't recorded is in flight again, so @@ -110,9 +112,10 @@ and this module adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. - **A deferred checkpoint survives an unrelated caller's cancel.** A checkpoint that waited for an earlier step's effects to dispatch, and was canceled by the context of the caller whose step finished last, isn't reported to that caller, since its own - step succeeded, and the next step writes it. A retry the Store refuses as not - advancing means the first write landed, and counts as written. Any other checkpoint - failure is returned, and only the next step the policy makes due tries again. + step succeeded, and the next step writes it; a failure of that retry is returned to + the step that tried it. A retry the Store refuses as not advancing means the first + write landed, and counts as written. Any other checkpoint failure is returned, and + after that only the next step the policy makes due tries again. - **The next step survives a checkpoint that compacts tick barriers.** A checkpoint now records the step that follows it, in a `nextStep` field of its envelope. Recovery used the Trace count, which runs low past a tick barrier, so the next Fire could reuse a diff --git a/durable/README.md b/durable/README.md index ee119301..4f113340 100644 --- a/durable/README.md +++ b/durable/README.md @@ -158,8 +158,8 @@ being sent, so the recovered Handle loads every recorded step and can't send an twice. Give `Drain` a context of its own, not the failed call's, and don't `Recover` until it returns nil: an error means it didn't finish draining. A step that was recording when another goroutine failed the Handle returns `ErrHandleFailed` saying it was recorded; `Recover` sends any of its effects. To move a healthy Handle's instance to a new Handle, stop -calling the old one and let its calls return first: `Drain` alone doesn't cover a step -that is still recording. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the +calling the old one and let its calls return first: a step can start after `Drain` +returns. `Recover` a new Handle with the same options to continue, unless the cause is `ErrFenced`: then a newer owner holds the instance, so stop. ## Caveat: serialized payloads diff --git a/durable/handle_lock_test.go b/durable/handle_lock_test.go index 484e8d47..d0e2d59e 100644 --- a/durable/handle_lock_test.go +++ b/durable/handle_lock_test.go @@ -788,11 +788,13 @@ func TestHandle_Drain_WaitsForAStepStillRecording(t *testing.T) { // checkpointSpy is a MemStore that records the steps it checkpointed through after // Start's initial checkpoint and refuses a checkpoint whose context has ended. Once // armed, it fails the next checkpoint (diskFull first, ahead of the context check), or -// writes it and then reports it canceled (landThenFail, also ahead of the check). +// writes it and then reports it canceled (landThenFail, also ahead of the check), or +// reports an ended context as canceled whatever ended it (canceledAs). type checkpointSpy struct { *durable.MemStore failNext atomic.Bool diskFull atomic.Bool + canceledAs atomic.Bool landThenFail atomic.Bool mu sync.Mutex through []int @@ -810,6 +812,9 @@ func (s *checkpointSpy) Checkpoint(ctx context.Context, id durable.InstanceID, s return context.Canceled } if err := ctx.Err(); err != nil { + if s.canceledAs.CompareAndSwap(true, false) { + return context.Canceled + } return err } if s.failNext.CompareAndSwap(true, false) { @@ -1125,3 +1130,47 @@ func TestHandle_DeferredCheckpointStoreError_OnAnEndedContext_IsReported(t *test t.Fatalf("Fire(go) = %v, want the Store's disk-full error", err) } } + +// TestHandle_DeferredCheckpoint_StoreReportsAnotherContextError_IsNotReported proves a +// deferred checkpoint the Store refuses with a different context error than the one +// that ended the finishing caller's context still isn't reported to that caller. +func TestHandle_DeferredCheckpoint_StoreReportsAnotherContextError_IsNotReported(t *testing.T) { + ctx := context.Background() + store := &checkpointSpy{MemStore: durable.NewMemStore()} + handlerIn, release := make(chan struct{}), make(chan struct{}) + var first atomic.Bool + handler := func(context.Context, string, state.Effect) error { + if first.CompareAndSwap(false, true) { // step 0's notify waits + close(handlerIn) + <-release + } + return nil + } + h, err := durable.NewRunner(effectMachine(), store, + durable.WithEffectHandler[string, string, *runCtx](handler), + durable.WithCheckpointEvery[string, string, *runCtx](2)). + Start(ctx, "e-1", &runCtx{}, state.WithInitialState("idle")) + if err != nil { + t.Fatalf("Start: %v", err) + } + firstCtx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + firstDone := make(chan error, 1) + go func() { _, fireErr := h.Fire(firstCtx, "go"); firstDone <- fireErr }() + <-handlerIn + if _, err := h.Fire(ctx, "bill"); err != nil { // step 1's checkpoint is due, and waits on step 0 + t.Fatalf("Fire(bill): %v", err) + } + <-firstCtx.Done() // ended by its deadline + store.canceledAs.Store(true) + close(release) + if err := <-firstDone; err != nil { + t.Fatalf("Fire(go) = %v, want nil: the Store's context.Canceled for a deadline is still a context error", err) + } + if _, err := h.Fire(ctx, "finish"); err != nil { + t.Fatalf("Fire(finish): %v", err) + } + if got := store.steps(); len(got) != 1 || got[0] != 1 { + t.Fatalf("checkpointed through %v, want [1]", got) + } +} diff --git a/durable/runner.go b/durable/runner.go index 0b1b9fdf..6c4a71c5 100644 --- a/durable/runner.go +++ b/durable/runner.go @@ -598,9 +598,10 @@ func (r *Runner[S, E, C]) Fire(ctx context.Context, id InstanceID, event E, opts // kernel transition error journals nothing (no step was produced) and is // returned to the caller. A failed checkpoint returns the result and that error: the // step is recorded and its effects sent, the Handle carries on, and the next step the -// policy makes due checkpoints. If another goroutine failed the Handle while this step -// was recording, Fire returns ErrHandleFailed saying the step was recorded; Recover -// sends its effects. +// policy makes due checkpoints. A checkpoint this step wrote for an earlier one, which +// failed because ctx ended, isn't reported, since this step succeeded. If another +// goroutine failed the Handle while this step was recording, Fire returns +// ErrHandleFailed saying the step was recorded; Recover sends its effects. // // A state.WithEventData payload reaches the transition's Assign as the Runner's // EventDataDecoder rebuilds it from the payload's JSON form, live and on replay alike, so @@ -735,9 +736,10 @@ func (h *Handle[S, E, C]) dispatchGuarded(ctx context.Context, step int, des []d // timer loop (with a real clock) or a test calls it after advancing a fake clock. // It records one tick barrier — carrying the consumed clock readings and the // count of timers fired — so recovery re-derives the same timers at their -// recorded instants. It returns the FireResults of the timers it fired, in order. If -// another goroutine failed the Handle while this step was recording, it returns them -// with ErrHandleFailed saying the step was recorded. +// recorded instants. It returns the FireResults of the timers it fired, in order. A +// failed checkpoint returns them with that error, and the Handle carries on. If another +// goroutine failed the Handle while this step was recording, it returns them with +// ErrHandleFailed saying the step was recorded. func (h *Handle[S, E, C]) Tick(ctx context.Context) ([]state.FireResult[S], error) { if err := h.acquire(ctx); err != nil { return nil, err @@ -809,7 +811,8 @@ func (h *Handle[S, E, C]) recordedOnFailedLocked(step int) error { // ended, that caller isn't told, since its own step succeeded, and the next step writes // it. Any other failure is returned to the caller, and the checkpoint waits for the next // step the policy makes due, whose newer checkpoint replaces it. A retry the Store -// refuses as not advancing landed the first time, and counts as written. A failed Handle writes none. h.out.mu must be held. +// refuses as not advancing landed the first time, and counts as written. A failed +// Handle writes none. h.out.mu must be held. func (h *Handle[S, E, C]) offerCheckpointLocked(ctx context.Context, rec *Record) error { if h.out.failed != nil { return nil @@ -838,7 +841,7 @@ func (h *Handle[S, E, C]) offerCheckpointLocked(ctx context.Context, rec *Record } cp.retry = true h.out.pending = cp - if cp.step != rec.Step && ctx.Err() != nil && errors.Is(err, ctx.Err()) { + if cp.step != rec.Step && ctx.Err() != nil && isContextErr(err) { return nil // not this caller's checkpoint: the next step writes it } cp.waitDue = true @@ -1292,3 +1295,9 @@ func (jsonEventCodec[E]) Decode(payload []byte) (E, error) { err := json.Unmarshal(payload, &e) return e, err } + +// isContextErr reports whether err is a context's cancellation or deadline, which a +// Store may report for either when the caller's context ends. +func isContextErr(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +}